From ccf32a7e3c3c993af4f25a5e06495e7efcfafcec Mon Sep 17 00:00:00 2001 From: Fredrik Vedvik Date: Wed, 8 Nov 2023 15:20:56 +0100 Subject: [PATCH 1/7] feat(paths): change hard paths into utils.Path --- activities/analyze.go | 5 ++- activities/baton/qc.go | 4 +- activities/files.go | 49 ++++++++++++----------- activities/normalize.go | 15 +++---- activities/preview.go | 13 ++++--- activities/queues.go | 8 ++-- activities/rclone.go | 6 +-- activities/subtrans.go | 13 ++++--- activities/transcode.go | 26 ++++++------- activities/transcribe.go | 23 +++++------ activities/vidispine/files.go | 9 +++-- activities/vidispine/meta.go | 6 +-- cmd/httpin/main.go | 4 +- cmd/httpin/watchers.go | 6 +-- cmd/trigger_ui/main.go | 4 +- cmd/worker/main.go | 13 +++---- common/merge.go | 12 ++++-- common/transcode.go | 38 ++++++++++-------- {utils => environment}/environment.go | 24 ++++++------ {common => environment}/queues.go | 2 +- {utils => paths}/paths.go | 27 +++++++------ {utils => paths}/paths_test.go | 2 +- services/baton/tasks.go | 4 +- services/transcode/audio.go | 52 ++++++++++++++++--------- services/transcode/linear_normalize.go | 20 ++++++---- services/transcode/merge.go | 52 ++++++++++++++++++------- services/transcode/mux.go | 25 +++++++----- services/transcode/playout_mux.go | 25 +++++++----- services/transcode/preview.go | 4 +- services/transcode/video.go | 25 +++++++----- services/transcode/video_test.go | 9 +++-- services/vidispine/export.go | 6 +-- services/vidispine/vsapi/files_paths.go | 5 +-- utils/wfutils/common.go | 6 +-- utils/wfutils/files.go | 43 ++++++++++---------- workflows/export/merge_export_data.go | 29 +++++++------- workflows/export/mux_files.go | 26 +++++++------ workflows/export/prepare_files.go | 23 +++++------ workflows/export/vx_export.go | 13 +++---- workflows/export/vx_export_bmm.go | 17 +++----- workflows/export/vx_export_playout.go | 16 +++----- workflows/export/vx_export_vod.go | 28 +++++++------ workflows/ffmpeg.go | 4 +- workflows/import_subs.go | 7 ++-- workflows/ingest/asset_ingest.go | 28 ++++++------- workflows/ingest/common.go | 3 +- workflows/ingest/masters.go | 23 +++++------ workflows/ingest/raw_material.go | 12 +++--- workflows/normalize_audio.go | 18 ++++++--- workflows/transcode_preview-file.go | 16 +++++--- workflows/transcode_preview-vx.go | 8 ++-- workflows/transcribe-file.go | 19 +++++++-- workflows/transcribe-vx.go | 6 +-- workflows/watch_folder_transcode.go | 33 ++++++++++------ 54 files changed, 512 insertions(+), 402 deletions(-) rename {utils => environment}/environment.go (67%) rename {common => environment}/queues.go (85%) rename {utils => paths}/paths.go (83%) rename {utils => paths}/paths_test.go (97%) diff --git a/activities/analyze.go b/activities/analyze.go index 940f94d4..76671463 100644 --- a/activities/analyze.go +++ b/activities/analyze.go @@ -2,12 +2,13 @@ package activities import ( "context" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/ffmpeg" "go.temporal.io/sdk/activity" ) type AnalyzeFileParams struct { - FilePath string + FilePath paths.Path } type AnalyzeFileResult struct { @@ -19,7 +20,7 @@ func AnalyzeFile(ctx context.Context, input AnalyzeFileParams) (*AnalyzeFileResu logger := activity.GetLogger(ctx) logger.Info("Starting AnalyzeFileActivity") - info, err := ffmpeg.GetStreamInfo(input.FilePath) + info, err := ffmpeg.GetStreamInfo(input.FilePath.LocalPath()) if err != nil { return nil, err } diff --git a/activities/baton/qc.go b/activities/baton/qc.go index 311d77d8..c4605c87 100644 --- a/activities/baton/qc.go +++ b/activities/baton/qc.go @@ -2,14 +2,14 @@ package batonactivities import ( "context" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/baton" - "github.com/bcc-code/bccm-flows/utils" "go.temporal.io/sdk/activity" "time" ) type QCParams struct { - Path utils.Path + Path paths.Path Plan baton.TestPlan } diff --git a/activities/files.go b/activities/files.go index 87a30348..4ef04a4b 100644 --- a/activities/files.go +++ b/activities/files.go @@ -2,23 +2,24 @@ package activities import ( "context" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/paths" + "github.com/samber/lo" "go.temporal.io/sdk/activity" "os" "path/filepath" ) type FileInput struct { - Path string + Path paths.Path } type FileResult struct { - Path string + Path paths.Path } type MoveFileInput struct { - Source string - Destination string + Source paths.Path + Destination paths.Path } func MoveFile(ctx context.Context, input MoveFileInput) (*FileResult, error) { @@ -26,15 +27,15 @@ func MoveFile(ctx context.Context, input MoveFileInput) (*FileResult, error) { activity.RecordHeartbeat(ctx, "MoveFile") log.Info("Starting MoveFileActivity") - err := os.MkdirAll(filepath.Dir(input.Destination), os.ModePerm) + err := os.MkdirAll(filepath.Dir(input.Destination.LocalPath()), os.ModePerm) if err != nil { return nil, err } - err = os.Rename(input.Source, input.Destination) + err = os.Rename(input.Source.LocalPath(), input.Destination.LocalPath()) if err != nil { return nil, err } - _ = os.Chmod(input.Destination, os.ModePerm) + _ = os.Chmod(input.Destination.LocalPath(), os.ModePerm) return &FileResult{ Path: input.Destination, }, nil @@ -45,19 +46,19 @@ func StandardizeFileName(ctx context.Context, input FileInput) (*FileResult, err activity.RecordHeartbeat(ctx, "StandardizeFileName") log.Info("Starting StandardizeFileNameActivity") - path := utils.FixFilename(input.Path) - err := os.Rename(input.Path, path) + path := paths.FixFilename(input.Path.LocalPath()) + err := os.Rename(input.Path.LocalPath(), path) if err != nil { return nil, err } _ = os.Chmod(path, os.ModePerm) return &FileResult{ - Path: path, + Path: paths.MustParsePath(path), }, nil } type CreateFolderInput struct { - Destination string + Destination paths.Path } func CreateFolder(ctx context.Context, input CreateFolderInput) error { @@ -65,15 +66,15 @@ func CreateFolder(ctx context.Context, input CreateFolderInput) error { activity.RecordHeartbeat(ctx, "CreateFolder") log.Info("Starting CreateFolderActivity") - err := os.MkdirAll(input.Destination, os.ModePerm) + err := os.MkdirAll(input.Destination.LocalPath(), os.ModePerm) if err != nil { return err } - return os.Chmod(input.Destination, os.ModePerm) + return os.Chmod(input.Destination.LocalPath(), os.ModePerm) } type WriteFileInput struct { - Path string + Path paths.Path Data []byte } @@ -82,15 +83,15 @@ func WriteFile(ctx context.Context, input WriteFileInput) error { activity.RecordHeartbeat(ctx, "WriteFile") log.Info("Starting WriteFileActivity") - err := os.MkdirAll(filepath.Dir(input.Path), os.ModePerm) + err := os.MkdirAll(filepath.Dir(input.Path.LocalPath()), os.ModePerm) if err != nil { return err } - err = os.WriteFile(input.Path, input.Data, os.ModePerm) + err = os.WriteFile(input.Path.LocalPath(), input.Data, os.ModePerm) if err != nil { return err } - _ = os.Chmod(input.Path, os.ModePerm) + _ = os.Chmod(input.Path.LocalPath(), os.ModePerm) return nil } @@ -99,19 +100,21 @@ func ReadFile(ctx context.Context, input FileInput) ([]byte, error) { activity.RecordHeartbeat(ctx, "ReadFile") log.Info("Starting ReadFileActivity") - return os.ReadFile(input.Path) + return os.ReadFile(input.Path.LocalPath()) } -func ListFiles(ctx context.Context, input FileInput) ([]string, error) { +func ListFiles(ctx context.Context, input FileInput) ([]paths.Path, error) { log := activity.GetLogger(ctx) activity.RecordHeartbeat(ctx, "ListFiles") log.Info("Starting ListFilesActivity") - files, err := filepath.Glob(filepath.Join(input.Path, "*")) + files, err := filepath.Glob(filepath.Join(input.Path.LocalPath(), "*")) if err != nil { return nil, err } - return files, err + return lo.Map(files, func(i string, _ int) paths.Path { + return paths.MustParsePath(i) + }), err } func DeletePath(ctx context.Context, input FileInput) error { @@ -119,5 +122,5 @@ func DeletePath(ctx context.Context, input FileInput) error { activity.RecordHeartbeat(ctx, "DeletePath") log.Info("Starting DeletePathActivity") - return os.RemoveAll(input.Path) + return os.RemoveAll(input.Path.LocalPath()) } diff --git a/activities/normalize.go b/activities/normalize.go index baf1a302..0aa1714c 100644 --- a/activities/normalize.go +++ b/activities/normalize.go @@ -3,13 +3,14 @@ package activities import ( "context" "github.com/bcc-code/bccm-flows/common" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/ffmpeg" "github.com/bcc-code/bccm-flows/services/transcode" "go.temporal.io/sdk/activity" ) type AnalyzeEBUR128Params struct { - FilePath string + FilePath paths.Path TargetLoudness float64 } @@ -21,7 +22,7 @@ func AnalyzeEBUR128Activity(ctx context.Context, input AnalyzeEBUR128Params) (*c stop, progressCallback := registerProgressCallback(ctx) defer close(stop) - analyzeResult, err := ffmpeg.AnalyzeEBUR128(input.FilePath, progressCallback) + analyzeResult, err := ffmpeg.AnalyzeEBUR128(input.FilePath.LocalPath(), progressCallback) if err != nil { return nil, err } @@ -47,8 +48,8 @@ func AnalyzeEBUR128Activity(ctx context.Context, input AnalyzeEBUR128Params) (*c } type AdjustAudioLevelParams struct { - InFilePath string - OutFilePath string + InFilePath paths.Path + OutFilePath paths.Path Adjustment float64 } @@ -67,14 +68,14 @@ func AdjustAudioLevelActivity(ctx context.Context, input *AdjustAudioLevelParams } type NormalizeAudioParams struct { - FilePath string - OutputPath string + FilePath paths.Path + OutputPath paths.Path TargetLUFS float64 PerformOutputAnalysis bool } type NormalizeAudioResult struct { - FilePath string + FilePath paths.Path InputAnalysis *common.AnalyzeEBUR128Result OutputAnalysis *common.AnalyzeEBUR128Result } diff --git a/activities/preview.go b/activities/preview.go index bf96b2c0..201ddb57 100644 --- a/activities/preview.go +++ b/activities/preview.go @@ -3,17 +3,18 @@ package activities import ( "context" "fmt" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/transcode" "go.temporal.io/sdk/activity" ) type TranscodePreviewParams struct { - FilePath string - DestinationDirPath string + FilePath paths.Path + DestinationDirPath paths.Path } type TranscodePreviewResponse struct { - PreviewFilePath string + PreviewFilePath paths.Path AudioOnly bool } @@ -26,8 +27,8 @@ func TranscodePreview(ctx context.Context, input TranscodePreviewParams) (*Trans defer close(stop) result, err := transcode.Preview(transcode.PreviewInput{ - OutputDir: input.DestinationDirPath, - FilePath: input.FilePath, + OutputDir: input.DestinationDirPath.LocalPath(), + FilePath: input.FilePath.LocalPath(), }, progressCallback) if err != nil { fmt.Println(err.Error()) @@ -35,7 +36,7 @@ func TranscodePreview(ctx context.Context, input TranscodePreviewParams) (*Trans } return &TranscodePreviewResponse{ - PreviewFilePath: result.LowResolutionPath, + PreviewFilePath: paths.MustParsePath(result.LowResolutionPath), AudioOnly: result.AudioOnly, }, nil } diff --git a/activities/queues.go b/activities/queues.go index 95aba626..e05223d4 100644 --- a/activities/queues.go +++ b/activities/queues.go @@ -1,11 +1,11 @@ package activities import ( + "github.com/bcc-code/bccm-flows/environment" "reflect" "runtime" "strings" - "github.com/bcc-code/bccm-flows/utils" "github.com/samber/lo" ) @@ -58,10 +58,10 @@ var videoActivities = lo.Map(GetVideoTranscodeActivities(), func(i any, _ int) s func GetQueueForActivity(activity any) string { f := getFunctionName(activity) if lo.Contains(audioActivities, f) { - return utils.GetAudioQueue() + return environment.GetAudioQueue() } if lo.Contains(videoActivities, f) { - return utils.GetTranscodeQueue() + return environment.GetTranscodeQueue() } - return utils.GetWorkerQueue() + return environment.GetWorkerQueue() } diff --git a/activities/rclone.go b/activities/rclone.go index 1fc6aaf9..32292394 100644 --- a/activities/rclone.go +++ b/activities/rclone.go @@ -3,7 +3,7 @@ package activities import ( "context" "fmt" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/paths" "time" "github.com/bcc-code/bccm-flows/services/rclone" @@ -47,8 +47,8 @@ func RcloneCopyDir(ctx context.Context, input RcloneCopyDirInput) (bool, error) } type RcloneMoveFileInput struct { - Source utils.Path - Destination utils.Path + Source paths.Path + Destination paths.Path } func RcloneMoveFile(ctx context.Context, input RcloneMoveFileInput) (bool, error) { diff --git a/activities/subtrans.go b/activities/subtrans.go index 87a24afb..31d81289 100644 --- a/activities/subtrans.go +++ b/activities/subtrans.go @@ -3,6 +3,7 @@ package activities import ( "context" "fmt" + "github.com/bcc-code/bccm-flows/paths" "net/url" "os" "path" @@ -18,7 +19,7 @@ type GetSubtitlesInput struct { SubtransID string Format string ApprovedOnly bool - DestinationFolder string + DestinationFolder paths.Path FilePrefix string } @@ -93,10 +94,10 @@ func GetSubtransIDActivity(ctx context.Context, input *GetSubtransIDInput) (*Get return out, nil } -func GetSubtitlesActivity(ctx context.Context, params GetSubtitlesInput) (map[string]string, error) { +func GetSubtitlesActivity(ctx context.Context, params GetSubtitlesInput) (map[string]paths.Path, error) { client := subtrans.NewClient(os.Getenv("SUBTRANS_BASE_URL"), os.Getenv("SUBTRANS_API_KEY")) - info, err := os.Stat(params.DestinationFolder) + info, err := os.Stat(params.DestinationFolder.LocalPath()) if os.IsNotExist(err) { return nil, err } @@ -116,14 +117,14 @@ func GetSubtitlesActivity(ctx context.Context, params GetSubtitlesInput) (map[st params.FilePrefix = p } - out := map[string]string{} + out := map[string]paths.Path{} for lang, sub := range subs { - path := path.Join(params.DestinationFolder, params.FilePrefix+lang+"."+params.Format) + path := path.Join(params.DestinationFolder.LocalPath(), params.FilePrefix+lang+"."+params.Format) err := os.WriteFile(path, []byte(sub), 0644) if err != nil { return nil, err } - out[lang] = path + out[lang] = paths.MustParsePath(path) } return out, nil diff --git a/activities/transcode.go b/activities/transcode.go index da03f68e..da81176b 100644 --- a/activities/transcode.go +++ b/activities/transcode.go @@ -3,23 +3,23 @@ package activities import ( "context" "fmt" - "github.com/bcc-code/bccm-flows/common" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/ffmpeg" "github.com/bcc-code/bccm-flows/services/transcode" "go.temporal.io/sdk/activity" ) type EncodeParams struct { - FilePath string - OutputDir string + FilePath paths.Path + OutputDir paths.Path Resolution string FrameRate int Bitrate string } type EncodeResult struct { - OutputPath string + OutputPath paths.Path } func TranscodeToProResActivity(ctx context.Context, input EncodeParams) (*EncodeResult, error) { @@ -31,8 +31,8 @@ func TranscodeToProResActivity(ctx context.Context, input EncodeParams) (*Encode defer close(stop) transcodeResult, err := transcode.ProRes(transcode.ProResInput{ - FilePath: input.FilePath, - OutputDir: input.OutputDir, + FilePath: input.FilePath.LocalPath(), + OutputDir: input.OutputDir.LocalPath(), FrameRate: input.FrameRate, Resolution: input.Resolution, }, progressCallback) @@ -42,7 +42,7 @@ func TranscodeToProResActivity(ctx context.Context, input EncodeParams) (*Encode } return &EncodeResult{ - OutputPath: transcodeResult.OutputPath, + OutputPath: paths.MustParsePath(transcodeResult.OutputPath), }, nil } @@ -55,8 +55,8 @@ func TranscodeToH264Activity(ctx context.Context, input EncodeParams) (*EncodeRe defer close(stop) transcodeResult, err := transcode.H264(transcode.EncodeInput{ - FilePath: input.FilePath, - OutputDir: input.OutputDir, + FilePath: input.FilePath.LocalPath(), + OutputDir: input.OutputDir.LocalPath(), FrameRate: input.FrameRate, Resolution: input.Resolution, Bitrate: input.Bitrate, @@ -66,7 +66,7 @@ func TranscodeToH264Activity(ctx context.Context, input EncodeParams) (*EncodeRe } return &EncodeResult{ - OutputPath: transcodeResult.Path, + OutputPath: paths.MustParsePath(transcodeResult.Path), }, nil } @@ -79,8 +79,8 @@ func TranscodeToXDCAMActivity(ctx context.Context, input EncodeParams) (*EncodeR defer close(stop) transcodeResult, err := transcode.XDCAM(transcode.EncodeInput{ - FilePath: input.FilePath, - OutputDir: input.OutputDir, + FilePath: input.FilePath.LocalPath(), + OutputDir: input.OutputDir.LocalPath(), FrameRate: input.FrameRate, Resolution: input.Resolution, Bitrate: input.Bitrate, @@ -90,7 +90,7 @@ func TranscodeToXDCAMActivity(ctx context.Context, input EncodeParams) (*EncodeR } return &EncodeResult{ - OutputPath: transcodeResult.Path, + OutputPath: paths.MustParsePath(transcodeResult.Path), }, nil } diff --git a/activities/transcribe.go b/activities/transcribe.go index 782a8cb6..492bca4f 100644 --- a/activities/transcribe.go +++ b/activities/transcribe.go @@ -2,6 +2,7 @@ package activities import ( "context" + "github.com/bcc-code/bccm-flows/paths" "path/filepath" "time" @@ -10,15 +11,15 @@ import ( ) type TranscribeParams struct { - File string - DestinationPath string + File paths.Path + DestinationPath paths.Path Language string } type TranscribeResponse struct { - JSONPath string - SRTPath string - TXTPath string + JSONPath paths.Path + SRTPath paths.Path + TXTPath paths.Path } // Transcribe is the activity that transcribes a video @@ -32,17 +33,17 @@ func Transcribe( time.Sleep(time.Second * 10) - jobData, err := transcribe.DoTranscribe(ctx, input.File, input.DestinationPath, input.Language) - + jobData, err := transcribe.DoTranscribe(ctx, input.File.LocalPath(), input.DestinationPath.LocalPath(), input.Language) if err != nil { return nil, err } + log.Info("Finished Transcribe") - fileName := filepath.Base(input.File) + fileName := filepath.Base(input.File.LocalPath()) return &TranscribeResponse{ - JSONPath: filepath.Join(jobData.OutputPath, fileName+".json"), - SRTPath: filepath.Join(jobData.OutputPath, fileName+".srt"), - TXTPath: filepath.Join(jobData.OutputPath, fileName+".txt"), + JSONPath: paths.MustParsePath(filepath.Join(jobData.OutputPath, fileName+".json")), + SRTPath: paths.MustParsePath(filepath.Join(jobData.OutputPath, fileName+".srt")), + TXTPath: paths.MustParsePath(filepath.Join(jobData.OutputPath, fileName+".txt")), }, nil } diff --git a/activities/vidispine/files.go b/activities/vidispine/files.go index bc5be1f9..fa4fcabd 100644 --- a/activities/vidispine/files.go +++ b/activities/vidispine/files.go @@ -2,13 +2,14 @@ package vidispine import ( "context" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/vidispine/vsapi" "go.temporal.io/sdk/activity" ) type ImportFileAsShapeParams struct { AssetID string - FilePath string + FilePath paths.Path ShapeTag string } @@ -18,7 +19,7 @@ func ImportFileAsShapeActivity(ctx context.Context, params *ImportFileAsShapePar vsClient := GetClient() - fileID, err := vsClient.RegisterFile(params.FilePath, vsapi.FileStateClosed) + fileID, err := vsClient.RegisterFile(params.FilePath.LocalPath(), vsapi.FileStateClosed) if err != nil { return nil, err } @@ -31,7 +32,7 @@ func ImportFileAsShapeActivity(ctx context.Context, params *ImportFileAsShapePar type ImportSubtitleAsSidecarParams struct { AssetID string - FilePath string + FilePath paths.Path Language string } @@ -45,7 +46,7 @@ func ImportFileAsSidecarActivity(ctx context.Context, params *ImportSubtitleAsSi vsClient := GetClient() - jobID, err := vsClient.AddSidecarToItem(params.AssetID, params.FilePath, params.Language) + jobID, err := vsClient.AddSidecarToItem(params.AssetID, params.FilePath.LocalPath(), params.Language) return &ImportFileAsSidecarResult{ JobID: jobID, }, err diff --git a/activities/vidispine/meta.go b/activities/vidispine/meta.go index 1b7518d0..a5f2325b 100644 --- a/activities/vidispine/meta.go +++ b/activities/vidispine/meta.go @@ -3,7 +3,7 @@ package vidispine import ( "context" "fmt" - + "github.com/bcc-code/bccm-flows/paths" "go.temporal.io/sdk/activity" ) @@ -13,7 +13,7 @@ type GetFileFromVXParams struct { } type GetFileFromVXResult struct { - FilePath string + FilePath paths.Path ShapeTag string } @@ -36,7 +36,7 @@ func GetFileFromVXActivity(ctx context.Context, params GetFileFromVXParams) (*Ge } return &GetFileFromVXResult{ - FilePath: shape.GetPath(), + FilePath: paths.MustParsePath(shape.GetPath()), ShapeTag: tag, }, nil } diff --git a/cmd/httpin/main.go b/cmd/httpin/main.go index edd5a251..52cd6b53 100644 --- a/cmd/httpin/main.go +++ b/cmd/httpin/main.go @@ -3,6 +3,7 @@ package main import ( _ "embed" "fmt" + "github.com/bcc-code/bccm-flows/environment" "github.com/bcc-code/bccm-flows/workflows/ingest" "net/http" "os" @@ -12,7 +13,6 @@ import ( "github.com/bcc-code/bccm-flows/workflows/export" - "github.com/bcc-code/bccm-flows/common" "github.com/bcc-code/bccm-flows/workflows" "github.com/gin-gonic/gin" @@ -34,7 +34,7 @@ func getClient() (client.Client, error) { func getQueue() string { queue := os.Getenv("QUEUE") if queue == "" { - queue = common.QueueWorker + queue = environment.QueueWorker } return queue } diff --git a/cmd/httpin/watchers.go b/cmd/httpin/watchers.go index 4fc4ea06..2535ffd0 100644 --- a/cmd/httpin/watchers.go +++ b/cmd/httpin/watchers.go @@ -3,7 +3,7 @@ package main import ( "context" "fmt" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/environment" "github.com/bcc-code/bccm-flows/workflows" "github.com/bcc-code/bccm-flows/workflows/ingest" "github.com/gin-gonic/gin" @@ -72,7 +72,7 @@ func doTranscode(ctx context.Context, path string) error { workflowOptions := client.StartWorkflowOptions{ ID: uuid.NewString(), - TaskQueue: utils.GetWorkerQueue(), + TaskQueue: environment.GetWorkerQueue(), } _, err = c.ExecuteWorkflow(ctx, workflowOptions, workflows.WatchFolderTranscode, workflows.WatchFolderTranscodeInput{ @@ -90,7 +90,7 @@ func doIngest(ctx context.Context, path string) error { workflowOptions := client.StartWorkflowOptions{ ID: uuid.NewString(), - TaskQueue: utils.GetWorkerQueue(), + TaskQueue: environment.GetWorkerQueue(), } _, err = c.ExecuteWorkflow(ctx, workflowOptions, ingestworkflows.Asset, ingestworkflows.AssetParams{ diff --git a/cmd/trigger_ui/main.go b/cmd/trigger_ui/main.go index 08abf04f..34f748de 100644 --- a/cmd/trigger_ui/main.go +++ b/cmd/trigger_ui/main.go @@ -4,13 +4,13 @@ import ( "encoding/json" "errors" "fmt" + "github.com/bcc-code/bccm-flows/environment" "net/http" "time" "os" bccmflows "github.com/bcc-code/bccm-flows" - "github.com/bcc-code/bccm-flows/common" "github.com/bcc-code/bccm-flows/services/vidispine" "github.com/bcc-code/bccm-flows/services/vidispine/vsapi" "github.com/bcc-code/bccm-flows/services/vidispine/vscommon" @@ -32,7 +32,7 @@ func getTemporalClient() (client.Client, error) { func getQueue() string { queue := os.Getenv("QUEUE") if queue == "" { - queue = common.QueueWorker + queue = environment.QueueWorker } return queue } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 38f07488..98e82839 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -2,6 +2,7 @@ package main import ( batonactivities "github.com/bcc-code/bccm-flows/activities/baton" + "github.com/bcc-code/bccm-flows/environment" "github.com/bcc-code/bccm-flows/workflows/ingest" "log" "os" @@ -11,8 +12,6 @@ import ( "github.com/bcc-code/bccm-flows/workflows/export" "github.com/bcc-code/bccm-flows/activities" - "github.com/bcc-code/bccm-flows/common" - "github.com/bcc-code/bccm-flows/utils" "github.com/bcc-code/bccm-flows/workflows" "github.com/bcc-code/bccm-flows/activities/vidispine" @@ -107,14 +106,14 @@ func main() { MaxConcurrentActivityExecutionSize: activityCount, // Doesn't make sense to have more than one activity running at a time } - registerWorker(c, utils.GetQueue(), workerOptions) + registerWorker(c, environment.GetQueue(), workerOptions) } func registerWorker(c client.Client, queue string, options worker.Options) { w := worker.New(c, queue, options) switch queue { - case common.QueueDebug: + case environment.QueueDebug: w.RegisterActivity(activities.Transcribe) w.RegisterActivity(activities.RcloneCopyDir) w.RegisterActivity(activities.RcloneMoveFile) @@ -139,7 +138,7 @@ func registerWorker(c client.Client, queue string, options worker.Options) { for _, wf := range workerWorkflows { w.RegisterWorkflow(wf) } - case common.QueueWorker: + case environment.QueueWorker: w.RegisterActivity(activities.Transcribe) w.RegisterActivity(activities.RcloneCopyDir) w.RegisterActivity(activities.RcloneMoveFile) @@ -156,11 +155,11 @@ func registerWorker(c client.Client, queue string, options worker.Options) { for _, wf := range workerWorkflows { w.RegisterWorkflow(wf) } - case common.QueueTranscode: + case environment.QueueTranscode: for _, a := range transcodeActivities { w.RegisterActivity(a) } - case common.QueueAudio: + case environment.QueueAudio: for _, a := range audioTranscodeActivities { w.RegisterActivity(a) } diff --git a/common/merge.go b/common/merge.go index 876462e4..fe4bb88d 100644 --- a/common/merge.go +++ b/common/merge.go @@ -1,7 +1,11 @@ package common +import ( + "github.com/bcc-code/bccm-flows/paths" +) + type MergeInputItem struct { - Path string + Path paths.Path Start float64 End float64 Streams []int @@ -10,11 +14,11 @@ type MergeInputItem struct { type MergeInput struct { Title string Items []MergeInputItem - OutputDir string - WorkDir string + OutputDir paths.Path + WorkDir paths.Path Duration float64 } type MergeResult struct { - Path string + Path paths.Path } diff --git a/common/transcode.go b/common/transcode.go index e90a8c19..96cccfd7 100644 --- a/common/transcode.go +++ b/common/transcode.go @@ -1,41 +1,45 @@ package common +import ( + "github.com/bcc-code/bccm-flows/paths" +) + type VideoInput struct { - Path string + Path paths.Path Bitrate string BufferSize string Width int Height int FrameRate int - WatermarkPath string - DestinationPath string + WatermarkPath *paths.Path + DestinationPath paths.Path } type VideoResult struct { - OutputPath string + OutputPath paths.Path } type AudioInput struct { - Path string + Path paths.Path Bitrate string - DestinationPath string + DestinationPath paths.Path } type AudioResult struct { - OutputPath string + OutputPath paths.Path Bitrate string Format string } type MuxInput struct { FileName string - VideoFilePath string - AudioFilePaths map[string]string - SubtitleFilePaths map[string]string - DestinationPath string + VideoFilePath paths.Path + AudioFilePaths map[string]paths.Path + SubtitleFilePaths map[string]paths.Path + DestinationPath paths.Path } type MuxResult struct { - Path string + Path paths.Path } type AnalyzeEBUR128Result struct { @@ -46,13 +50,13 @@ type AnalyzeEBUR128Result struct { } type PlayoutMuxInput struct { - VideoFilePath string - AudioFilePaths map[string]string - SubtitleFilePaths map[string]string - OutputDir string + VideoFilePath paths.Path + AudioFilePaths map[string]paths.Path + SubtitleFilePaths map[string]paths.Path + OutputDir paths.Path FallbackLanguage string } type PlayoutMuxResult struct { - Path string + Path paths.Path } diff --git a/utils/environment.go b/environment/environment.go similarity index 67% rename from utils/environment.go rename to environment/environment.go index a65a7906..989eb0a9 100644 --- a/utils/environment.go +++ b/environment/environment.go @@ -1,10 +1,8 @@ -package utils +package environment import ( "os" "strings" - - "github.com/bcc-code/bccm-flows/common" ) var queue = os.Getenv("QUEUE") @@ -13,28 +11,28 @@ func GetQueue() string { if queue != "" { return queue } - return common.QueueWorker + return QueueWorker } func GetWorkerQueue() string { - if queue == common.QueueDebug { - return common.QueueDebug + if queue == QueueDebug { + return QueueDebug } - return common.QueueWorker + return QueueWorker } func GetTranscodeQueue() string { - if queue == common.QueueDebug { - return common.QueueDebug + if queue == QueueDebug { + return QueueDebug } - return common.QueueTranscode + return QueueTranscode } func GetAudioQueue() string { - if queue == common.QueueDebug { - return common.QueueDebug + if queue == QueueDebug { + return QueueDebug } - return common.QueueAudio + return QueueAudio } var isilonPrefix = os.Getenv("ISILON_PREFIX") diff --git a/common/queues.go b/environment/queues.go similarity index 85% rename from common/queues.go rename to environment/queues.go index adb01c10..610c7777 100644 --- a/common/queues.go +++ b/environment/queues.go @@ -1,4 +1,4 @@ -package common +package environment const ( QueueWorker = "worker" diff --git a/utils/paths.go b/paths/paths.go similarity index 83% rename from utils/paths.go rename to paths/paths.go index 9ad5c157..d8290766 100644 --- a/utils/paths.go +++ b/paths/paths.go @@ -1,7 +1,8 @@ -package utils +package paths import ( "github.com/ansel1/merry/v2" + "github.com/bcc-code/bccm-flows/environment" "github.com/orsinium-labs/enum" "path/filepath" "strings" @@ -59,14 +60,8 @@ type Path struct { Path string } -func (p Path) WorkerPath() string { - switch p.Drive { - case IsilonDrive: - return filepath.Join("/mnt/isilon", p.Path) - case DMZShareDrive: - return filepath.Join("/mnt/dmzshare", p.Path) - } - return "" +func (p Path) LocalPath() string { + return filepath.Join(drivePrefixes[p.Drive].Client, p.Path) } // RcloneFsRemote returns (fs, remote) for rclone usage @@ -97,7 +92,7 @@ func (p Path) FileName() string { } func (p Path) Append(path string) Path { - p.Path = filepath.Join(p.Path, path) + p.Path = filepath.Clean(filepath.Join(p.Path, path)) return p } @@ -108,9 +103,9 @@ type prefix struct { } var drivePrefixes = map[Drive]prefix{ - IsilonDrive: {"/mnt/isilon/", GetIsilonPrefix(), "isilon:isilon/"}, + IsilonDrive: {"/mnt/isilon/", environment.GetIsilonPrefix(), "isilon:isilon/"}, DMZShareDrive: {"/mnt/dmzshare/", "/mnt/dmzshare/", "dmz:dmzshare/"}, - TempDrive: {"/mnt/temp/", GetTempMountPrefix(), "isilon:temp/"}, + TempDrive: {"/mnt/temp/", environment.GetTempMountPrefix(), "isilon:temp/"}, } func ParsePath(path string) (Path, error) { @@ -127,3 +122,11 @@ func ParsePath(path string) (Path, error) { } return Path{}, ErrPathNotValid } + +func MustParsePath(path string) Path { + p, err := ParsePath(path) + if err != nil { + panic(err) + } + return p +} diff --git a/utils/paths_test.go b/paths/paths_test.go similarity index 97% rename from utils/paths_test.go rename to paths/paths_test.go index f42e655c..9515ec57 100644 --- a/utils/paths_test.go +++ b/paths/paths_test.go @@ -1,4 +1,4 @@ -package utils +package paths import ( "github.com/stretchr/testify/assert" diff --git a/services/baton/tasks.go b/services/baton/tasks.go index 21e7d9b6..8bb717f2 100644 --- a/services/baton/tasks.go +++ b/services/baton/tasks.go @@ -2,14 +2,14 @@ package baton import ( "fmt" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/paths" ) type StartTaskResult struct { TaskID string `json:"taskId"` } -func StartTask(client *Client, filePath utils.Path, testPlan TestPlan) (*StartTaskResult, error) { +func StartTask(client *Client, filePath paths.Path, testPlan TestPlan) (*StartTaskResult, error) { req := client.restyClient.R() req.SetQueryParam("mediaFilePath", filePath.BatonPath()) diff --git a/services/transcode/audio.go b/services/transcode/audio.go index 2d70f75e..419a6c3b 100644 --- a/services/transcode/audio.go +++ b/services/transcode/audio.go @@ -2,6 +2,7 @@ package transcode import ( "fmt" + "github.com/bcc-code/bccm-flows/paths" "os" "path/filepath" "strconv" @@ -15,17 +16,17 @@ func AudioAac(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi params := []string{ "-progress", "pipe:1", "-hide_banner", - "-i", input.Path, + "-i", input.Path.LocalPath(), "-c:a", "aac", "-b:a", input.Bitrate, } - outputPath := filepath.Join(input.DestinationPath, filepath.Base(input.Path)) - outputPath = fmt.Sprintf("%s-%s.aac", outputPath[:len(outputPath)-len(filepath.Ext(outputPath))], input.Bitrate) + outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), filepath.Base(input.Path.LocalPath())) + outputFilePath = fmt.Sprintf("%s-%s.aac", outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))], input.Bitrate) - params = append(params, "-y", outputPath) + params = append(params, "-y", outputFilePath) - info, err := ffmpeg.GetStreamInfo(input.Path) + info, err := ffmpeg.GetStreamInfo(input.Path.LocalPath()) if err != nil { return nil, err } @@ -35,7 +36,12 @@ func AudioAac(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi return nil, err } - err = os.Chmod(outputPath, os.ModePerm) + err = os.Chmod(outputFilePath, os.ModePerm) + if err != nil { + return nil, err + } + + outputPath, err := paths.ParsePath(outputFilePath) if err != nil { return nil, err } @@ -51,15 +57,15 @@ func AudioWav(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi params := []string{ "-progress", "pipe:1", "-hide_banner", - "-i", input.Path, + "-i", input.Path.LocalPath(), } - outputPath := filepath.Join(input.DestinationPath, filepath.Base(input.Path)) - outputPath = fmt.Sprintf("%s-%s.wav", outputPath[:len(outputPath)-len(filepath.Ext(outputPath))], input.Bitrate) + outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), filepath.Base(input.Path.LocalPath())) + outputFilePath = fmt.Sprintf("%s-%s.wav", outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))], input.Bitrate) - params = append(params, "-y", outputPath) + params = append(params, "-y", outputFilePath) - info, err := ffmpeg.GetStreamInfo(input.Path) + info, err := ffmpeg.GetStreamInfo(input.Path.LocalPath()) if err != nil { return nil, err } @@ -69,7 +75,12 @@ func AudioWav(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi return nil, err } - err = os.Chmod(outputPath, os.ModePerm) + err = os.Chmod(outputFilePath, os.ModePerm) + if err != nil { + return nil, err + } + + outputPath, err := paths.ParsePath(outputFilePath) if err != nil { return nil, err } @@ -113,17 +124,17 @@ func AudioMP3(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi params := []string{ "-progress", "pipe:1", "-hide_banner", - "-i", input.Path, + "-i", input.Path.LocalPath(), "-c:a", "libmp3lame", "-q:a", fmt.Sprint(getQfactorFromBitrate(input.Bitrate)), } - outputPath := filepath.Join(input.DestinationPath, filepath.Base(input.Path)) - outputPath = fmt.Sprintf("%s-%s.mp3", outputPath[:len(outputPath)-len(filepath.Ext(outputPath))], input.Bitrate) + outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), filepath.Base(input.Path.LocalPath())) + outputFilePath = fmt.Sprintf("%s-%s.mp3", outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))], input.Bitrate) - params = append(params, "-y", outputPath) + params = append(params, "-y", outputFilePath) - info, err := ffmpeg.GetStreamInfo(input.Path) + info, err := ffmpeg.GetStreamInfo(input.Path.LocalPath()) if err != nil { return nil, err } @@ -133,7 +144,12 @@ func AudioMP3(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi return nil, err } - err = os.Chmod(outputPath, os.ModePerm) + err = os.Chmod(outputFilePath, os.ModePerm) + if err != nil { + return nil, err + } + + outputPath, err := paths.ParsePath(outputFilePath) if err != nil { return nil, err } diff --git a/services/transcode/linear_normalize.go b/services/transcode/linear_normalize.go index 90b91615..8914f922 100644 --- a/services/transcode/linear_normalize.go +++ b/services/transcode/linear_normalize.go @@ -2,6 +2,7 @@ package transcode import ( "fmt" + "github.com/bcc-code/bccm-flows/paths" "os" "path/filepath" @@ -12,19 +13,19 @@ import ( // AdjustAudioLevel adjusts the audio level of the input file by the given adjustment in dB // without changing the dynamic range. This function does not protect against clipping! func AdjustAudioLevel(input common.AudioInput, adjustment float64, cb ffmpeg.ProgressCallback) (*common.AudioResult, error) { - outputPath := filepath.Join(input.DestinationPath, filepath.Base(input.Path)) - outputPath = outputPath[:len(outputPath)-len(filepath.Ext(outputPath))] + "_normalized" + filepath.Ext(outputPath) + outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), filepath.Base(input.Path.LocalPath())) + outputFilePath = outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))] + "_normalized" + filepath.Ext(outputFilePath) params := []string{ - "-i", input.Path, + "-i", input.Path.LocalPath(), "-c:v", "copy", "-af", fmt.Sprintf("volume=%.2fdB", adjustment), - outputPath, + outputFilePath, } - params = append(params, "-y", outputPath) + params = append(params, "-y", outputFilePath) - info, err := ffmpeg.GetStreamInfo(input.Path) + info, err := ffmpeg.GetStreamInfo(input.Path.LocalPath()) if err != nil { return nil, err } @@ -34,7 +35,12 @@ func AdjustAudioLevel(input common.AudioInput, adjustment float64, cb ffmpeg.Pro return nil, err } - err = os.Chmod(outputPath, os.ModePerm) + err = os.Chmod(outputFilePath, os.ModePerm) + if err != nil { + return nil, err + } + + outputPath, err := paths.ParsePath(outputFilePath) if err != nil { return nil, err } diff --git a/services/transcode/merge.go b/services/transcode/merge.go index e6227e3f..2837797a 100644 --- a/services/transcode/merge.go +++ b/services/transcode/merge.go @@ -2,6 +2,7 @@ package transcode import ( "fmt" + "github.com/bcc-code/bccm-flows/paths" "log" "math" "os" @@ -21,7 +22,7 @@ func getFramerate(input common.MergeInput) (int, error) { return a.End-a.Start > b.End-b.Start }) - info, err := ffmpeg.GetStreamInfo(longestItem.Path) + info, err := ffmpeg.GetStreamInfo(longestItem.Path.LocalPath()) if err != nil { return 0, err } @@ -39,7 +40,7 @@ func MergeVideo(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac var params []string for _, i := range input.Items { - params = append(params, "-i", utils.IsilonPathFix(i.Path)) + params = append(params, "-i", i.Path.LocalPath()) } var filterComplex string @@ -62,7 +63,7 @@ func MergeVideo(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac // Concatenate the video streams. filterComplex += fmt.Sprintf("concat=n=%d:v=1:a=0[v]", len(input.Items)) - outputPath := filepath.Join(input.OutputDir, filepath.Clean(input.Title)+".mxf") + outputFilePath := filepath.Join(input.OutputDir.LocalPath(), filepath.Clean(input.Title)+".mxf") params = append(params, "-progress", "pipe:1", @@ -80,7 +81,7 @@ func MergeVideo(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac "-color_trc", "bt709", "-colorspace", "bt709", "-y", - outputPath, + outputFilePath, ) _, err = ffmpeg.Do(params, ffmpeg.StreamInfo{ @@ -90,7 +91,12 @@ func MergeVideo(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac return nil, err } - err = os.Chmod(outputPath, os.ModePerm) + err = os.Chmod(outputFilePath, os.ModePerm) + if err != nil { + return nil, err + } + + outputPath, err := paths.ParsePath(outputFilePath) if err != nil { return nil, err } @@ -102,7 +108,7 @@ func MergeVideo(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac // mergeItemToStereoStream takes a merge input item and returns a string that can be used in a filter_complex to merge the audio streams. func mergeItemToStereoStream(index int, tag string, item common.MergeInputItem) (string, error) { - path := utils.IsilonPathFix(item.Path) + path := item.Path.LocalPath() info, _ := ffmpeg.ProbeFile(path) if info == nil || len(info.Streams) == 0 { @@ -158,7 +164,7 @@ func MergeAudio(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac } for _, i := range input.Items { - params = append(params, "-i", utils.IsilonPathFix(i.Path)) + params = append(params, "-i", i.Path.LocalPath()) } params = append(params, @@ -184,12 +190,20 @@ func MergeAudio(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac filterComplex += fmt.Sprintf("concat=n=%d:v=0:a=1 [a]", len(input.Items)) - outputPath := filepath.Join(input.OutputDir, filepath.Clean(input.Title)+".wav") + outputFilePath := filepath.Join(input.OutputDir.LocalPath(), filepath.Clean(input.Title)+".wav") - params = append(params, "-filter_complex", filterComplex, "-map", "[a]", "-y", outputPath) + params = append(params, "-filter_complex", filterComplex, "-map", "[a]", "-y", outputFilePath) log.Default().Println(strings.Join(params, " ")) _, err := ffmpeg.Do(params, ffmpeg.StreamInfo{}, progressCallback) + if err != nil { + return nil, err + } + + outputPath, err := paths.ParsePath(outputFilePath) + if err != nil { + return nil, err + } return &common.MergeResult{ Path: outputPath, @@ -215,9 +229,9 @@ func MergeSubtitles(input common.MergeInput, progressCallback ffmpeg.ProgressCal startAt := 0.0 for index, item := range input.Items { - file := filepath.Join(input.WorkDir, fmt.Sprintf("%s-%d.srt", input.Title, index)) - fileOut := filepath.Join(input.WorkDir, fmt.Sprintf("%s-%d-out.srt", input.Title, index)) - path := utils.IsilonPathFix(item.Path) + file := filepath.Join(input.WorkDir.LocalPath(), fmt.Sprintf("%s-%d.srt", input.Title, index)) + fileOut := filepath.Join(input.WorkDir.LocalPath(), fmt.Sprintf("%s-%d-out.srt", input.Title, index)) + path := item.Path.LocalPath() cmd := exec.Command("ffmpeg", "-i", path, "-ss", fmt.Sprintf("%f", item.Start), "-to", fmt.Sprintf("%f", item.End), "-y", file) @@ -255,7 +269,7 @@ func MergeSubtitles(input common.MergeInput, progressCallback ffmpeg.ProgressCal content += fmt.Sprintf("file '%s'\n", f) } - subtitlesFile := filepath.Join(input.WorkDir, input.Title+"-subtitles.txt") + subtitlesFile := filepath.Join(input.WorkDir.LocalPath(), input.Title+"-subtitles.txt") err := os.WriteFile(subtitlesFile, []byte(content), os.ModePerm) if err != nil { @@ -264,17 +278,25 @@ func MergeSubtitles(input common.MergeInput, progressCallback ffmpeg.ProgressCal concatStr := fmt.Sprintf("concat:%s", strings.Join(files, "|")) - outputPath := filepath.Join(input.OutputDir, filepath.Clean(input.Title)+".srt") + outputFilePath := filepath.Join(input.OutputDir.LocalPath(), filepath.Clean(input.Title)+".srt") params := []string{ "-progress", "pipe:1", "-hide_banner", "-i", concatStr, "-c", "copy", "-y", - outputPath, + outputFilePath, } _, err = ffmpeg.Do(params, ffmpeg.StreamInfo{}, progressCallback) + if err != nil { + return nil, err + } + + outputPath, err := paths.ParsePath(outputFilePath) + if err != nil { + return nil, err + } return &common.MergeResult{ Path: outputPath, diff --git a/services/transcode/mux.go b/services/transcode/mux.go index e1a0ea72..8f72aa80 100644 --- a/services/transcode/mux.go +++ b/services/transcode/mux.go @@ -2,6 +2,7 @@ package transcode import ( "fmt" + "github.com/bcc-code/bccm-flows/paths" "log" "os" "path/filepath" @@ -15,12 +16,12 @@ import ( ) type languageFile struct { - Path string + Path paths.Path Language string } // Order and respect the global language ordering. -func languageFilesForPaths(paths map[string]string) []languageFile { +func languageFilesForPaths(paths map[string]paths.Path) []languageFile { languages := utils.LanguageKeysToOrderedLanguages(lo.Keys(paths)) return lo.Map(languages, func(lang bccmflows.Language, _ int) languageFile { @@ -34,17 +35,17 @@ func languageFilesForPaths(paths map[string]string) []languageFile { // Mux multiplexes specified video, audio and subtitle tracks. func Mux(input common.MuxInput, progressCallback ffmpeg.ProgressCallback) (*common.MuxResult, error) { //Use ffmpeg to mux the video - info, err := ffmpeg.GetStreamInfo(input.VideoFilePath) + info, err := ffmpeg.GetStreamInfo(input.VideoFilePath.LocalPath()) if err != nil { return nil, err } - outputPath := filepath.Join(input.DestinationPath, input.FileName+".mp4") + outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), input.FileName+".mp4") params := []string{ "-progress", "pipe:1", "-hide_banner", - "-i", input.VideoFilePath, + "-i", input.VideoFilePath.LocalPath(), } audioFiles := languageFilesForPaths(input.AudioFilePaths) @@ -52,13 +53,13 @@ func Mux(input common.MuxInput, progressCallback ffmpeg.ProgressCallback) (*comm for _, f := range audioFiles { params = append(params, - "-i", f.Path, + "-i", f.Path.LocalPath(), ) } for _, f := range subtitleFiles { params = append(params, - "-i", f.Path, + "-i", f.Path.LocalPath(), ) } @@ -91,7 +92,7 @@ func Mux(input common.MuxInput, progressCallback ffmpeg.ProgressCallback) (*comm "-c:v", "copy", "-c:a", "copy", "-c:s", "mov_text", - "-y", outputPath, + "-y", outputFilePath, ) _, err = ffmpeg.Do(params, info, progressCallback) @@ -100,10 +101,16 @@ func Mux(input common.MuxInput, progressCallback ffmpeg.ProgressCallback) (*comm return nil, fmt.Errorf("mux failed, %s", strings.Join(params, " ")) } - err = os.Chmod(outputPath, os.ModePerm) + err = os.Chmod(outputFilePath, os.ModePerm) if err != nil { return nil, err } + + outputPath, err := paths.ParsePath(outputFilePath) + if err != nil { + return nil, err + } + return &common.MuxResult{ Path: outputPath, }, nil diff --git a/services/transcode/playout_mux.go b/services/transcode/playout_mux.go index fadb9181..c4f4c477 100644 --- a/services/transcode/playout_mux.go +++ b/services/transcode/playout_mux.go @@ -2,6 +2,7 @@ package transcode import ( "fmt" + "github.com/bcc-code/bccm-flows/paths" "log" "os" "path/filepath" @@ -104,8 +105,8 @@ func generateFFmpegParamsForPlayoutMux(input common.PlayoutMuxInput, outputPath // Inputs ffmpegInputCount := 0 - addInput := func(path string) { - params = append(params, "-i", path) + addInput := func(path paths.Path) { + params = append(params, "-i", path.LocalPath()) ffmpegInputCount++ } addInput(input.VideoFilePath) @@ -130,7 +131,7 @@ func generateFFmpegParamsForPlayoutMux(input common.PlayoutMuxInput, outputPath audioLanguages[i] = &PlayoutLanguageState{ Code: lang, - FilePath: filePath, + FilePath: filePath.LocalPath(), CopyFrom: copyFrom, InputIndex: inputIndex, Stereo: i < 4, @@ -150,7 +151,7 @@ func generateFFmpegParamsForPlayoutMux(input common.PlayoutMuxInput, outputPath return stream } - filterParts := []string{} + var filterParts []string for _, lang := range audioLanguages { if lang.InputIndex == -1 { continue @@ -220,16 +221,16 @@ func generateFFmpegParamsForPlayoutMux(input common.PlayoutMuxInput, outputPath } func PlayoutMux(input common.PlayoutMuxInput, progressCallback ffmpeg.ProgressCallback) (*common.PlayoutMuxResult, error) { - base := filepath.Base(input.VideoFilePath) + base := filepath.Base(input.VideoFilePath.LocalPath()) fileNameWithoutExtension := base[:len(base)-len(filepath.Ext(base))] - outputPath := filepath.Join(input.OutputDir, fileNameWithoutExtension+".mxf") + outputFilePath := filepath.Join(input.OutputDir.LocalPath(), fileNameWithoutExtension+".mxf") - params, err := generateFFmpegParamsForPlayoutMux(input, outputPath) + params, err := generateFFmpegParamsForPlayoutMux(input, outputFilePath) if err != nil { return nil, err } - info, err := ffmpeg.GetStreamInfo(input.VideoFilePath) + info, err := ffmpeg.GetStreamInfo(input.VideoFilePath.LocalPath()) if err != nil { return nil, err } @@ -238,10 +239,16 @@ func PlayoutMux(input common.PlayoutMuxInput, progressCallback ffmpeg.ProgressCa log.Default().Println("mux failed", err) return nil, fmt.Errorf("mux failed, %s", strings.Join(params, " ")) } - err = os.Chmod(outputPath, os.ModePerm) + err = os.Chmod(outputFilePath, os.ModePerm) if err != nil { return nil, err } + + outputPath, err := paths.ParsePath(outputFilePath) + if err != nil { + return nil, err + } + return &common.PlayoutMuxResult{ Path: outputPath, }, nil diff --git a/services/transcode/preview.go b/services/transcode/preview.go index 9e432bb2..bfa10b9d 100644 --- a/services/transcode/preview.go +++ b/services/transcode/preview.go @@ -2,11 +2,11 @@ package transcode import ( "errors" + "github.com/bcc-code/bccm-flows/environment" "os" "path/filepath" "github.com/bcc-code/bccm-flows/services/ffmpeg" - "github.com/bcc-code/bccm-flows/utils" ) type PreviewInput struct { @@ -19,7 +19,7 @@ type PreviewResult struct { AudioOnly bool } -var previewWatermarkPath = utils.GetIsilonPrefix() + "/system/graphics/LOGO_BTV_Preview_960-540.mov" +var previewWatermarkPath = environment.GetIsilonPrefix() + "/system/graphics/LOGO_BTV_Preview_960-540.mov" func Preview(input PreviewInput, progressCallback ffmpeg.ProgressCallback) (*PreviewResult, error) { encoder := os.Getenv("H264_ENCODER") diff --git a/services/transcode/video.go b/services/transcode/video.go index e46ca15d..6b88a309 100644 --- a/services/transcode/video.go +++ b/services/transcode/video.go @@ -3,6 +3,7 @@ package transcode import ( "fmt" "github.com/bcc-code/bccm-flows/common" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/ffmpeg" "os" "path/filepath" @@ -15,12 +16,12 @@ func VideoH264(input common.VideoInput, cb ffmpeg.ProgressCallback) (*common.Vid params := []string{ "-hide_banner", "-progress", "pipe:1", - "-i", input.Path, + "-i", input.Path.LocalPath(), } - if input.WatermarkPath != "" { + if input.WatermarkPath != nil { params = append(params, - "-i", input.WatermarkPath, + "-i", input.WatermarkPath.LocalPath(), ) } @@ -57,7 +58,7 @@ func VideoH264(input common.VideoInput, cb ffmpeg.ProgressCallback) (*common.Vid 1920, 1080) - if input.WatermarkPath != "" { + if input.WatermarkPath != nil { filterComplex += "[main][1:0] overlay=main_w-overlay_w:0 [main];" } @@ -65,7 +66,7 @@ func VideoH264(input common.VideoInput, cb ffmpeg.ProgressCallback) (*common.Vid input.Width, input.Height) - info, err := ffmpeg.GetStreamInfo(input.Path) + info, err := ffmpeg.GetStreamInfo(input.Path.LocalPath()) if err != nil { return nil, err } @@ -85,14 +86,14 @@ func VideoH264(input common.VideoInput, cb ffmpeg.ProgressCallback) (*common.Vid "-map", "[out]", ) - filename := filepath.Base(input.Path) + filename := filepath.Base(input.Path.LocalPath()) filename = filename[:len(filename)-len(filepath.Ext(filename))] + fmt.Sprintf("_%dx%d.mp4", input.Width, input.Height) - outputPath := filepath.Join(input.DestinationPath, filename) + outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), filename) params = append(params, - "-y", outputPath, + "-y", outputFilePath, ) _, err = ffmpeg.Do(params, info, cb) @@ -100,10 +101,16 @@ func VideoH264(input common.VideoInput, cb ffmpeg.ProgressCallback) (*common.Vid return nil, err } - err = os.Chmod(outputPath, os.ModePerm) + err = os.Chmod(outputFilePath, os.ModePerm) if err != nil { return nil, err } + + outputPath, err := paths.ParsePath(outputFilePath) + if err != nil { + return nil, err + } + return &common.VideoResult{ OutputPath: outputPath, }, nil diff --git a/services/transcode/video_test.go b/services/transcode/video_test.go index ff64d125..1a5d3d4b 100644 --- a/services/transcode/video_test.go +++ b/services/transcode/video_test.go @@ -2,6 +2,7 @@ package transcode import ( "github.com/bcc-code/bccm-flows/common" + "github.com/bcc-code/bccm-flows/paths" "github.com/stretchr/testify/assert" "testing" ) @@ -9,14 +10,16 @@ import ( func Test_Video(t *testing.T) { p, stop := printProgress() defer close(stop) + + wm := paths.MustParsePath("/mnt/isilon/system/assets/BTV_LOGO_WATERMARK_BUG_GFX_1080.png") _, err := VideoH264(common.VideoInput{ - Path: "/mnt/isilon/system/tmp/workflows/07c4a523-ee62-481b-b473-80cc82fffb0a/SOTM_7v2123_SEQ.mxf", - DestinationPath: "/mnt/isilon/system/tmp/workflows/07c4a523-ee62-481b-b473-80cc82fffb0a", + Path: paths.MustParsePath("/mnt/isilon/system/tmp/workflows/07c4a523-ee62-481b-b473-80cc82fffb0a/SOTM_7v2123_SEQ.mxf"), + DestinationPath: paths.MustParsePath("/mnt/isilon/system/tmp/workflows/07c4a523-ee62-481b-b473-80cc82fffb0a"), Bitrate: "1M", Width: 1280, Height: 720, FrameRate: 25, - WatermarkPath: "/mnt/isilon/system/assets/BTV_LOGO_WATERMARK_BUG_GFX_1080.png", + WatermarkPath: &wm, }, p) assert.Nil(t, err) } diff --git a/services/vidispine/export.go b/services/vidispine/export.go index 22b99e8e..956753dc 100644 --- a/services/vidispine/export.go +++ b/services/vidispine/export.go @@ -3,13 +3,13 @@ package vidispine import ( "errors" "fmt" + "github.com/bcc-code/bccm-flows/environment" "regexp" "strings" bccmflows "github.com/bcc-code/bccm-flows" "github.com/bcc-code/bccm-flows/services/vidispine/vsapi" "github.com/bcc-code/bccm-flows/services/vidispine/vscommon" - "github.com/bcc-code/bccm-flows/utils" mapset "github.com/deckarep/golang-set/v2" "github.com/orsinium-labs/enum" "github.com/samber/lo" @@ -55,8 +55,8 @@ var ( ExportAudioSourceRelated, ) - EmptyWAVFile = utils.GetIsilonPrefix() + "/system/assets/BlankAudio10h.wav" - EmtpySRTFile = utils.GetIsilonPrefix() + "/system/assets/empty.srt" + EmptyWAVFile = environment.GetIsilonPrefix() + "/system/assets/BlankAudio10h.wav" + EmtpySRTFile = environment.GetIsilonPrefix() + "/system/assets/empty.srt" ) func (s *VidispineService) getClipForAssetOrSubclip( diff --git a/services/vidispine/vsapi/files_paths.go b/services/vidispine/vsapi/files_paths.go index 3303a7a7..34955896 100644 --- a/services/vidispine/vsapi/files_paths.go +++ b/services/vidispine/vsapi/files_paths.go @@ -2,10 +2,9 @@ package vsapi import ( "fmt" + "github.com/bcc-code/bccm-flows/environment" "net/url" "strings" - - "github.com/bcc-code/bccm-flows/utils" ) const DefaultStorageID = "VX-42" @@ -21,7 +20,7 @@ func (c *Client) GetAbsoluteStoragePath(storageID string) (string, error) { for _, m := range result.Result().(*StorageResult).Methods { if strings.HasPrefix(m.URI, "file://") { path := strings.TrimPrefix(m.URI, "file://") - return utils.IsilonPathFix(path), nil + return environment.IsilonPathFix(path), nil } } diff --git a/utils/wfutils/common.go b/utils/wfutils/common.go index 06858039..556a5839 100644 --- a/utils/wfutils/common.go +++ b/utils/wfutils/common.go @@ -1,9 +1,9 @@ package wfutils import ( + "github.com/bcc-code/bccm-flows/environment" "time" - "github.com/bcc-code/bccm-flows/utils" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" ) @@ -23,7 +23,7 @@ func GetDefaultActivityOptions() workflow.ActivityOptions { StartToCloseTimeout: time.Hour * 4, ScheduleToCloseTimeout: time.Hour * 48, HeartbeatTimeout: time.Minute * 1, - TaskQueue: utils.GetWorkerQueue(), + TaskQueue: environment.GetWorkerQueue(), } } @@ -34,6 +34,6 @@ func GetDefaultWorkflowOptions() workflow.ChildWorkflowOptions { MaximumAttempts: 10, MaximumInterval: time.Hour * 1, }, - TaskQueue: utils.GetWorkerQueue(), + TaskQueue: environment.GetWorkerQueue(), } } diff --git a/utils/wfutils/files.go b/utils/wfutils/files.go index 8e322b99..a1a157c4 100644 --- a/utils/wfutils/files.go +++ b/utils/wfutils/files.go @@ -4,20 +4,21 @@ import ( "encoding/xml" "fmt" "github.com/bcc-code/bccm-flows/activities" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/environment" + "github.com/bcc-code/bccm-flows/paths" "github.com/samber/lo" "go.temporal.io/sdk/workflow" "path/filepath" "time" ) -func CreateFolder(ctx workflow.Context, destination string) error { +func CreateFolder(ctx workflow.Context, destination paths.Path) error { return workflow.ExecuteActivity(ctx, activities.CreateFolder, activities.CreateFolderInput{ Destination: destination, }).Get(ctx, nil) } -func StandardizeFileName(ctx workflow.Context, file string) (string, error) { +func StandardizeFileName(ctx workflow.Context, file paths.Path) (paths.Path, error) { var result activities.FileResult err := workflow.ExecuteActivity(ctx, activities.StandardizeFileName, activities.FileInput{ Path: file, @@ -25,27 +26,27 @@ func StandardizeFileName(ctx workflow.Context, file string) (string, error) { return result.Path, err } -func MoveFile(ctx workflow.Context, source, destination string) error { +func MoveFile(ctx workflow.Context, source, destination paths.Path) error { return workflow.ExecuteActivity(ctx, activities.MoveFile, activities.MoveFileInput{ Source: source, Destination: destination, }).Get(ctx, nil) } -func MoveToFolder(ctx workflow.Context, file, folder string) (string, error) { - newPath := filepath.Join(folder, filepath.Base(file)) +func MoveToFolder(ctx workflow.Context, file, folder paths.Path) (paths.Path, error) { + newPath := folder.Append(filepath.Base(file.LocalPath())) err := MoveFile(ctx, file, newPath) return newPath, err } -func WriteFile(ctx workflow.Context, file string, data []byte) error { +func WriteFile(ctx workflow.Context, file paths.Path, data []byte) error { return workflow.ExecuteActivity(ctx, activities.WriteFile, activities.WriteFileInput{ Path: file, Data: data, }).Get(ctx, nil) } -func ReadFile(ctx workflow.Context, file string) ([]byte, error) { +func ReadFile(ctx workflow.Context, file paths.Path) ([]byte, error) { var res []byte err := workflow.ExecuteActivity(ctx, activities.ReadFile, activities.FileInput{ Path: file, @@ -53,15 +54,15 @@ func ReadFile(ctx workflow.Context, file string) ([]byte, error) { return res, err } -func ListFiles(ctx workflow.Context, path string) ([]string, error) { - var res []string +func ListFiles(ctx workflow.Context, path paths.Path) ([]paths.Path, error) { + var res []paths.Path err := workflow.ExecuteActivity(ctx, activities.ListFiles, activities.FileInput{ Path: path, }).Get(ctx, &res) return res, err } -func UnmarshalXMLFile[T any](ctx workflow.Context, file string) (*T, error) { +func UnmarshalXMLFile[T any](ctx workflow.Context, file paths.Path) (*T, error) { var r T res, err := ReadFile(ctx, file) if err != nil { @@ -71,40 +72,40 @@ func UnmarshalXMLFile[T any](ctx workflow.Context, file string) (*T, error) { return &r, err } -func DeletePath(ctx workflow.Context, path string) error { +func DeletePath(ctx workflow.Context, path paths.Path) error { return workflow.ExecuteActivity(ctx, activities.DeletePath, activities.FileInput{ Path: path, }).Get(ctx, nil) } -func GetWorkflowIsilonOutputFolder(ctx workflow.Context, root string) (string, error) { +func GetWorkflowIsilonOutputFolder(ctx workflow.Context, root string) (paths.Path, error) { info := workflow.GetInfo(ctx) date := time.Now() - path := filepath.Join(utils.GetIsilonPrefix(), "Production", root, fmt.Sprintf("%d/%d/%d", date.Year(), date.Month(), date.Day()), info.OriginalRunID) + path := filepath.Join(environment.GetIsilonPrefix(), "Production", root, fmt.Sprintf("%d/%d/%d", date.Year(), date.Month(), date.Day()), info.OriginalRunID) - return path, CreateFolder(ctx, path) + return paths.MustParsePath(path), CreateFolder(ctx, paths.MustParsePath(path)) } -func GetWorkflowMastersOutputFolder(ctx workflow.Context) (string, error) { +func GetWorkflowMastersOutputFolder(ctx workflow.Context) (paths.Path, error) { return GetWorkflowIsilonOutputFolder(ctx, "masters") } -func GetWorkflowRawOutputFolder(ctx workflow.Context) (string, error) { +func GetWorkflowRawOutputFolder(ctx workflow.Context) (paths.Path, error) { return GetWorkflowIsilonOutputFolder(ctx, "raw") } -func GetWorkflowAuxOutputFolder(ctx workflow.Context) (string, error) { +func GetWorkflowAuxOutputFolder(ctx workflow.Context) (paths.Path, error) { return GetWorkflowIsilonOutputFolder(ctx, "aux") } -func GetWorkflowTempFolder(ctx workflow.Context) (string, error) { +func GetWorkflowTempFolder(ctx workflow.Context) (paths.Path, error) { info := workflow.GetInfo(ctx) - path := filepath.Join(utils.GetTempMountPrefix(), "workflows", info.OriginalRunID) + path := filepath.Join(environment.GetTempMountPrefix(), "workflows", info.OriginalRunID) - return path, CreateFolder(ctx, path) + return paths.MustParsePath(path), CreateFolder(ctx, paths.MustParsePath(path)) } // GetMapKeysSafely makes sure that the order of the keys returned are identical to other workflow executions. diff --git a/workflows/export/merge_export_data.go b/workflows/export/merge_export_data.go index de2db33f..4e4939b8 100644 --- a/workflows/export/merge_export_data.go +++ b/workflows/export/merge_export_data.go @@ -3,8 +3,9 @@ package export import ( "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/common" + "github.com/bcc-code/bccm-flows/environment" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/vidispine" - "github.com/bcc-code/bccm-flows/utils" "github.com/bcc-code/bccm-flows/utils/wfutils" "github.com/samber/lo" "go.temporal.io/sdk/workflow" @@ -12,15 +13,15 @@ import ( type MergeExportDataResult struct { Duration float64 - VideoFile string - AudioFiles map[string]string - SubtitleFiles map[string]string + VideoFile paths.Path + AudioFiles map[string]paths.Path + SubtitleFiles map[string]paths.Path } type MergeExportDataParams struct { ExportData *vidispine.ExportData - SubtitlesDir string - TempDir string + SubtitlesDir paths.Path + TempDir paths.Path MakeVideo bool MakeSubtitles bool MakeAudio bool @@ -35,7 +36,7 @@ func MergeExportData(ctx workflow.Context, params MergeExportDataParams) (*Merge mergeInput, audioMergeInputs, subtitleMergeInputs := exportDataToMergeInputs(data, params.TempDir, params.SubtitlesDir) options := wfutils.GetDefaultActivityOptions() - options.TaskQueue = utils.GetTranscodeQueue() + options.TaskQueue = environment.GetTranscodeQueue() ctx = workflow.WithActivityOptions(ctx, options) var audioTasks = map[string]workflow.Future{} @@ -66,7 +67,7 @@ func MergeExportData(ctx workflow.Context, params MergeExportDataParams) (*Merge } - var videoFile string + var videoFile paths.Path if params.MakeVideo { videoTask := wfutils.ExecuteWithQueue(ctx, activities.TranscodeMergeVideo, mergeInput) var result common.MergeResult @@ -77,7 +78,7 @@ func MergeExportData(ctx workflow.Context, params MergeExportDataParams) (*Merge videoFile = result.Path } - var audioFiles = map[string]string{} + var audioFiles = map[string]paths.Path{} { keys, err := wfutils.GetMapKeysSafely(ctx, audioTasks) if err != nil { @@ -94,7 +95,7 @@ func MergeExportData(ctx workflow.Context, params MergeExportDataParams) (*Merge } } - var subtitleFiles = map[string]string{} + var subtitleFiles = map[string]paths.Path{} { keys, err := wfutils.GetMapKeysSafely(ctx, subtitleTasks) if err != nil { @@ -119,7 +120,7 @@ func MergeExportData(ctx workflow.Context, params MergeExportDataParams) (*Merge }, nil } -func exportDataToMergeInputs(data *vidispine.ExportData, tempDir, subtitlesDir string) ( +func exportDataToMergeInputs(data *vidispine.ExportData, tempDir, subtitlesDir paths.Path) ( mergeInput common.MergeInput, audioMergeInputs map[string]*common.MergeInput, subtitleMergeInputs map[string]*common.MergeInput, @@ -136,7 +137,7 @@ func exportDataToMergeInputs(data *vidispine.ExportData, tempDir, subtitlesDir s for _, clip := range data.Clips { mergeInput.Duration += clip.OutSeconds - clip.InSeconds mergeInput.Items = append(mergeInput.Items, common.MergeInputItem{ - Path: clip.VideoFile, + Path: paths.MustParsePath(clip.VideoFile), Start: clip.InSeconds, End: clip.OutSeconds, }) @@ -152,7 +153,7 @@ func exportDataToMergeInputs(data *vidispine.ExportData, tempDir, subtitlesDir s audioMergeInputs[lan].Duration += clip.OutSeconds - clip.InSeconds audioMergeInputs[lan].Items = append(audioMergeInputs[lan].Items, common.MergeInputItem{ - Path: af.File, + Path: paths.MustParsePath(af.File), Start: clip.InSeconds, End: clip.OutSeconds, Streams: af.Streams, @@ -170,7 +171,7 @@ func exportDataToMergeInputs(data *vidispine.ExportData, tempDir, subtitlesDir s subtitleMergeInputs[lan].Duration += clip.OutSeconds - clip.InSeconds subtitleMergeInputs[lan].Items = append(subtitleMergeInputs[lan].Items, common.MergeInputItem{ - Path: sf, + Path: paths.MustParsePath(sf), Start: clip.InSeconds, End: clip.OutSeconds, }) diff --git a/workflows/export/mux_files.go b/workflows/export/mux_files.go index deaeb824..d6ae0030 100644 --- a/workflows/export/mux_files.go +++ b/workflows/export/mux_files.go @@ -2,6 +2,8 @@ package export import ( "fmt" + "github.com/bcc-code/bccm-flows/environment" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/utils/wfutils" "path/filepath" "strings" @@ -26,10 +28,10 @@ const ( ) type MuxFilesParams struct { - VideoFiles map[string]string - AudioFiles map[string]string - SubtitleFiles map[string]string - OutputPath string + VideoFiles map[string]paths.Path + AudioFiles map[string]paths.Path + SubtitleFiles map[string]paths.Path + OutputPath paths.Path WithFiles bool } @@ -45,7 +47,7 @@ func MuxFiles(ctx workflow.Context, params MuxFilesParams) (*MuxFilesResult, err options := wfutils.GetDefaultActivityOptions() ctx = workflow.WithActivityOptions(ctx, options) - ctx = workflow.WithTaskQueue(ctx, utils.GetTranscodeQueue()) + ctx = workflow.WithTaskQueue(ctx, environment.GetTranscodeQueue()) languagesPerQuality := getLanguagesPerQuality(params) streamTasks := startStreamTasks(ctx, params, languagesPerQuality) @@ -76,7 +78,7 @@ func getSubtitlesResult(params MuxFilesParams) []smil.TextStream { for _, language := range subtitleLanguages { path := params.SubtitleFiles[language.ISO6391] subtitles = append(subtitles, smil.TextStream{ - Src: filepath.Base(path), + Src: filepath.Base(path.LocalPath()), SystemLanguage: language.ISO6391, SubtitleName: language.LanguageNameSystem, }) @@ -93,13 +95,13 @@ func startFileTasks(ctx workflow.Context, params MuxFilesParams, languages []bcc key := lang.ISO6391 fileTasks[key] = map[string]workflow.Future{} for _, q := range fileQualities { - base := filepath.Base(params.VideoFiles[q]) + base := filepath.Base(params.VideoFiles[q].LocalPath()) fileName := base[:len(base)-len(filepath.Ext(base))] + "-" + key fileTasks[key][q] = wfutils.ExecuteWithQueue(ctx, activities.TranscodeMux, common.MuxInput{ FileName: fileName, DestinationPath: params.OutputPath, VideoFilePath: params.VideoFiles[q], - AudioFilePaths: map[string]string{key: params.AudioFiles[key]}, + AudioFilePaths: map[string]paths.Path{key: params.AudioFiles[key]}, SubtitleFilePaths: params.SubtitleFiles, }) } @@ -128,7 +130,7 @@ func waitForFileTasks(ctx workflow.Context, params MuxFilesParams, tasks map[str Resolution: q, AudioLanguage: code, Mime: "video/mp4", - Path: filepath.Base(result.Path), + Path: filepath.Base(result.Path.LocalPath()), }) } } @@ -157,13 +159,13 @@ func startStreamTasks(ctx workflow.Context, params MuxFilesParams, languages map for _, q := range streamQualities { path := params.VideoFiles[q] - audioFilePaths := map[string]string{} + audioFilePaths := map[string]paths.Path{} for _, lang := range languages[q] { key := lang.ISO6391 audioFilePaths[key] = params.AudioFiles[key] } - base := filepath.Base(path) + base := filepath.Base(path.LocalPath()) fileName := base[:len(base)-len(filepath.Ext(base))] tasks[q] = workflow.ExecuteActivity(ctx, activities.TranscodeMux, common.MuxInput{ @@ -189,7 +191,7 @@ func waitForStreamTasks(ctx workflow.Context, tasks map[string]workflow.Future, fileLanguages := languages[q] streams = append(streams, smil.Video{ - Src: filepath.Base(result.Path), + Src: filepath.Base(result.Path.LocalPath()), IncludeAudio: fmt.Sprintf("%t", len(fileLanguages) > 0), SystemLanguage: strings.Join(lo.Map(fileLanguages, func(i bccmflows.Language, _ int) string { return i.ISO6391 diff --git a/workflows/export/prepare_files.go b/workflows/export/prepare_files.go index a02143d6..ed9ccb3a 100644 --- a/workflows/export/prepare_files.go +++ b/workflows/export/prepare_files.go @@ -3,24 +3,25 @@ package export import ( "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/common" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/environment" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/utils/wfutils" "go.temporal.io/sdk/workflow" ) type PrepareFilesParams struct { - OutputPath string - VideoFile string - WatermarkPath string - AudioFiles map[string]string + OutputPath paths.Path + VideoFile paths.Path + WatermarkPath *paths.Path + AudioFiles map[string]paths.Path } type PrepareFilesResult struct { - VideoFiles map[string]string - AudioFiles map[string]string + VideoFiles map[string]paths.Path + AudioFiles map[string]paths.Path } -func getVideoQualities(videoFilePath, outputDir, watermarkPath string) map[string]common.VideoInput { +func getVideoQualities(videoFilePath, outputDir paths.Path, watermarkPath *paths.Path) map[string]common.VideoInput { return map[string]common.VideoInput{ r1080p: { Path: videoFilePath, @@ -81,7 +82,7 @@ func PrepareFiles(ctx workflow.Context, params PrepareFilesParams) (*PrepareFile options := wfutils.GetDefaultActivityOptions() ctx = workflow.WithActivityOptions(ctx, options) - ctx = workflow.WithTaskQueue(ctx, utils.GetTranscodeQueue()) + ctx = workflow.WithTaskQueue(ctx, environment.GetTranscodeQueue()) var videoTasks = map[string]workflow.Future{} { @@ -114,7 +115,7 @@ func PrepareFiles(ctx workflow.Context, params PrepareFilesParams) (*PrepareFile } } - var audioFiles = map[string]string{} + var audioFiles = map[string]paths.Path{} { keys, err := wfutils.GetMapKeysSafely(ctx, audioTasks) if err != nil { @@ -131,7 +132,7 @@ func PrepareFiles(ctx workflow.Context, params PrepareFilesParams) (*PrepareFile } } - var videoFiles = map[string]string{} + var videoFiles = map[string]paths.Path{} { keys, err := wfutils.GetMapKeysSafely(ctx, videoTasks) if err != nil { diff --git a/workflows/export/vx_export.go b/workflows/export/vx_export.go index 683966bd..75a3dab9 100644 --- a/workflows/export/vx_export.go +++ b/workflows/export/vx_export.go @@ -2,8 +2,7 @@ package export import ( "fmt" - "path/filepath" - + "github.com/bcc-code/bccm-flows/paths" "github.com/orsinium-labs/enum" avidispine "github.com/bcc-code/bccm-flows/activities/vidispine" @@ -47,8 +46,8 @@ type VXExportChildWorkflowParams struct { ParentParams VXExportParams `json:"parent_params"` ExportData vidispine.ExportData `json:"export_data"` MergeResult MergeExportDataResult - TempDir string - OutputDir string + TempDir paths.Path + OutputDir paths.Path } func formatSecondsToTimestamp(seconds float64) string { @@ -96,13 +95,13 @@ func VXExport(ctx workflow.Context, params VXExportParams) ([]wfutils.ResultOrEr return nil, err } - outputDir := filepath.Join(tempDir, "output") + outputDir := tempDir.Append("output") err = wfutils.CreateFolder(ctx, outputDir) if err != nil { return nil, err } - vodOutputDir := filepath.Join(outputDir, "vod") + vodOutputDir := outputDir.Append("vod") err = wfutils.CreateFolder(ctx, vodOutputDir) if err != nil { return nil, err @@ -141,7 +140,7 @@ func VXExport(ctx workflow.Context, params VXExportParams) ([]wfutils.ResultOrEr return nil, fmt.Errorf("destination not implemented: %s", dest) } - p := filepath.Join(outputDir, dest.Value) + p := outputDir.Append(dest.Value) err = wfutils.CreateFolder(ctx, p) if err != nil { return nil, err diff --git a/workflows/export/vx_export_bmm.go b/workflows/export/vx_export_bmm.go index 5f6452c3..d2d647cd 100644 --- a/workflows/export/vx_export_bmm.go +++ b/workflows/export/vx_export_bmm.go @@ -11,7 +11,6 @@ import ( "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/common" - "github.com/bcc-code/bccm-flows/utils" "github.com/bcc-code/bccm-flows/utils/wfutils" "go.temporal.io/sdk/workflow" ) @@ -46,7 +45,7 @@ func VXExportToBMM(ctx workflow.Context, params VXExportChildWorkflowParams) (*V } // We don't want to upload folders from other workflows that can be triggered at the same export. - outputFolder := path.Join(tempDir, "bmm") + outputFolder := tempDir.Append("bmm") err = wfutils.CreateFolder(ctx, outputFolder) if err != nil { return nil, fmt.Errorf("failed to create output folder: %w", err) @@ -135,19 +134,14 @@ func VXExportToBMM(ctx workflow.Context, params VXExportChildWorkflowParams) (*V return nil, fmt.Errorf("failed to marshal JSON: %w", err) } - err = wfutils.WriteFile(ctx, path.Join(outputFolder, "bmm.json"), marshalled) + err = wfutils.WriteFile(ctx, outputFolder.Append("bmm.json"), marshalled) if err != nil { return nil, fmt.Errorf("failed to write JSON file: %w", err) } - outputPath, err := utils.ParsePath(outputFolder) - if err != nil { - return nil, err - } - ingestFolder := params.ExportData.SafeTitle + "_" + workflow.GetInfo(ctx).OriginalRunID err = workflow.ExecuteActivity(ctx, activities.RcloneCopyDir, activities.RcloneCopyDirInput{ - Source: outputPath.RclonePath(), + Source: outputFolder.RclonePath(), Destination: fmt.Sprintf("bmms3:/int-bmm-mediabanken/" + ingestFolder), }).Get(ctx, nil) if err != nil { @@ -203,8 +197,7 @@ func prepareBMMData(audioFiles map[string][]common.AudioResult, analysis map[str } for lang, variations := range audioFiles { - - langFiles := []BMMAudioFile{} + var langFiles []BMMAudioFile for _, file := range variations { @@ -216,7 +209,7 @@ func prepareBMMData(audioFiles map[string][]common.AudioResult, analysis map[str Bitrate: bitrate, VariableBitrate: true, ChannelCount: 2, - Path: path.Base(file.OutputPath), // This needs to be relative to the resultintg JSON file + Path: path.Base(file.OutputPath.LocalPath()), // This needs to be relative to the resultintg JSON file Lufs: analysis[lang].OutputAnalysis.IntegratedLoudness, DynamicRange: analysis[lang].OutputAnalysis.LoudnessRange, Language: lang, diff --git a/workflows/export/vx_export_playout.go b/workflows/export/vx_export_playout.go index b8f0922a..55fcd1a7 100644 --- a/workflows/export/vx_export_playout.go +++ b/workflows/export/vx_export_playout.go @@ -3,10 +3,9 @@ package export import ( "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/common" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/environment" "github.com/bcc-code/bccm-flows/utils/wfutils" "go.temporal.io/sdk/workflow" - "path/filepath" ) func VXExportToPlayout(ctx workflow.Context, params VXExportChildWorkflowParams) (*VXExportResult, error) { @@ -16,13 +15,13 @@ func VXExportToPlayout(ctx workflow.Context, params VXExportChildWorkflowParams) options := wfutils.GetDefaultActivityOptions() ctx = workflow.WithActivityOptions(ctx, options) - xdcamOutputDir := filepath.Join(params.TempDir, "xdcam_output") + xdcamOutputDir := params.TempDir.Append("xdcam_output") err := wfutils.CreateFolder(ctx, xdcamOutputDir) if err != nil { return nil, err } - options.TaskQueue = utils.GetTranscodeQueue() + options.TaskQueue = environment.GetTranscodeQueue() ctx = workflow.WithActivityOptions(ctx, options) // Transcode video using playout encoding @@ -51,18 +50,13 @@ func VXExportToPlayout(ctx workflow.Context, params VXExportChildWorkflowParams) return nil, err } - options.TaskQueue = utils.GetWorkerQueue() + options.TaskQueue = environment.GetWorkerQueue() ctx = workflow.WithActivityOptions(ctx, options) - outputPath, err := utils.ParsePath(params.OutputDir) - if err != nil { - return nil, err - } - // Rclone to playout destination := "playout:/dropbox" err = workflow.ExecuteActivity(ctx, activities.RcloneCopyDir, activities.RcloneCopyDirInput{ - Source: outputPath.RclonePath(), + Source: params.OutputDir.RclonePath(), Destination: destination, }).Get(ctx, nil) if err != nil { diff --git a/workflows/export/vx_export_vod.go b/workflows/export/vx_export_vod.go index a9dbecfc..73c5200b 100644 --- a/workflows/export/vx_export_vod.go +++ b/workflows/export/vx_export_vod.go @@ -9,7 +9,7 @@ import ( "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/activities/vidispine" "github.com/bcc-code/bccm-flows/common/smil" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/utils/wfutils" "go.temporal.io/sdk/workflow" "path/filepath" @@ -36,16 +36,25 @@ func VXExportToVOD(ctx workflow.Context, params VXExportChildWorkflowParams) (*V Duration: formatSecondsToTimestamp(params.MergeResult.Duration), } - var videoFiles map[string]string - var audioFiles map[string]string + var videoFiles map[string]paths.Path + var audioFiles map[string]paths.Path { var result PrepareFilesResult + var wm *paths.Path + if params.ParentParams.WatermarkPath != "" { + path, err := paths.ParsePath(params.ParentParams.WatermarkPath) + if err != nil { + return nil, err + } + wm = &path + } + ctx = workflow.WithChildOptions(ctx, wfutils.GetDefaultWorkflowOptions()) err := workflow.ExecuteChildWorkflow(ctx, PrepareFiles, PrepareFilesParams{ OutputPath: params.TempDir, VideoFile: params.MergeResult.VideoFile, AudioFiles: params.MergeResult.AudioFiles, - WatermarkPath: params.ParentParams.WatermarkPath, + WatermarkPath: wm, }).Get(ctx, &result) if err != nil { return nil, err @@ -81,7 +90,7 @@ func VXExportToVOD(ctx workflow.Context, params VXExportChildWorkflowParams) (*V xmlData, _ := xml.MarshalIndent(smilData, "", "\t") xmlData = append([]byte("\n"), xmlData...) - err := wfutils.WriteFile(ctx, filepath.Join(params.OutputDir, "aws.smil"), xmlData) + err := wfutils.WriteFile(ctx, params.OutputDir.Append("aws.smil"), xmlData) if err != nil { return nil, err @@ -99,7 +108,7 @@ func VXExportToVOD(ctx workflow.Context, params VXExportChildWorkflowParams) (*V if err != nil { return nil, err } - err = wfutils.WriteFile(ctx, filepath.Join(params.OutputDir, "chapters.json"), marshalled) + err = wfutils.WriteFile(ctx, params.OutputDir.Append("chapters.json"), marshalled) if err != nil { return nil, err } @@ -110,17 +119,14 @@ func VXExportToVOD(ctx workflow.Context, params VXExportChildWorkflowParams) (*V return nil, err } - err = wfutils.WriteFile(ctx, filepath.Join(params.OutputDir, "ingest.json"), marshalled) + err = wfutils.WriteFile(ctx, params.OutputDir.Append("ingest.json"), marshalled) if err != nil { return nil, err } ingestFolder := params.ExportData.SafeTitle + "_" + workflow.GetInfo(ctx).OriginalRunID - outputPath, err := utils.ParsePath(params.OutputDir) - if err != nil { - return nil, err - } + outputPath := params.OutputDir err = workflow.ExecuteActivity(ctx, activities.RcloneCopyDir, activities.RcloneCopyDirInput{ Source: outputPath.RclonePath(), diff --git a/workflows/ffmpeg.go b/workflows/ffmpeg.go index 53393533..1369c652 100644 --- a/workflows/ffmpeg.go +++ b/workflows/ffmpeg.go @@ -2,7 +2,7 @@ package workflows import ( "github.com/bcc-code/bccm-flows/activities" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/environment" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" "time" @@ -26,7 +26,7 @@ func ExecuteFFmpeg( StartToCloseTimeout: time.Hour * 4, ScheduleToCloseTimeout: time.Hour * 48, HeartbeatTimeout: time.Minute * 1, - TaskQueue: utils.GetTranscodeQueue(), + TaskQueue: environment.GetTranscodeQueue(), } ctx = workflow.WithActivityOptions(ctx, options) diff --git a/workflows/import_subs.go b/workflows/import_subs.go index d4bdfc36..409a067a 100644 --- a/workflows/import_subs.go +++ b/workflows/import_subs.go @@ -2,13 +2,14 @@ package workflows import ( "fmt" + "github.com/bcc-code/bccm-flows/environment" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/utils/wfutils" "strings" "time" "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/activities/vidispine" - "github.com/bcc-code/bccm-flows/utils" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" ) @@ -33,7 +34,7 @@ func ImportSubtitlesFromSubtrans( StartToCloseTimeout: time.Hour * 4, ScheduleToCloseTimeout: time.Hour * 48, HeartbeatTimeout: time.Minute * 5, - TaskQueue: utils.GetQueue(), + TaskQueue: environment.GetQueue(), } ctx = workflow.WithActivityOptions(ctx, options) @@ -53,7 +54,7 @@ func ImportSubtitlesFromSubtrans( outputPath, _ := wfutils.GetWorkflowAuxOutputFolder(ctx) - subsList := map[string]string{} + subsList := map[string]paths.Path{} err = workflow.ExecuteActivity(ctx, activities.GetSubtitlesActivity, activities.GetSubtitlesInput{ SubtransID: subtransIDResponse.SubtransID, Format: "srt", diff --git a/workflows/ingest/asset_ingest.go b/workflows/ingest/asset_ingest.go index 1885ab48..023057a4 100644 --- a/workflows/ingest/asset_ingest.go +++ b/workflows/ingest/asset_ingest.go @@ -3,8 +3,8 @@ package ingestworkflows import ( "fmt" "github.com/bcc-code/bccm-flows/activities" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/ingest" - "github.com/bcc-code/bccm-flows/utils" "github.com/bcc-code/bccm-flows/utils/wfutils" "github.com/orsinium-labs/enum" "github.com/samber/lo" @@ -38,7 +38,12 @@ func Asset(ctx workflow.Context, params AssetParams) (*AssetResult, error) { options := wfutils.GetDefaultActivityOptions() ctx = workflow.WithActivityOptions(ctx, options) - metadata, err := wfutils.UnmarshalXMLFile[ingest.Metadata](ctx, params.XMLPath) + xmlPath, err := paths.ParsePath(params.XMLPath) + if err != nil { + return nil, err + } + + metadata, err := wfutils.UnmarshalXMLFile[ingest.Metadata](ctx, xmlPath) if err != nil { return nil, err } @@ -48,8 +53,8 @@ func Asset(ctx workflow.Context, params AssetParams) (*AssetResult, error) { return nil, fmt.Errorf("unsupported order form: %s", metadata.JobProperty.OrderForm) } _, err = wfutils.MoveToFolder(ctx, - params.XMLPath, - filepath.Join(filepath.Dir(params.XMLPath), "processed"), + xmlPath, + xmlPath.Append("processed"), ) if err != nil { return nil, err @@ -60,7 +65,7 @@ func Asset(ctx workflow.Context, params AssetParams) (*AssetResult, error) { return nil, err } - fcOutputDir := filepath.Join(tempDir, "fc") + fcOutputDir := tempDir.Append("fc") err = wfutils.CreateFolder(ctx, fcOutputDir) if err != nil { return nil, err @@ -91,7 +96,7 @@ func Asset(ctx workflow.Context, params AssetParams) (*AssetResult, error) { return &AssetResult{}, nil } -func copyToDir(ctx workflow.Context, dest string, files []ingest.File) error { +func copyToDir(ctx workflow.Context, dest paths.Path, files []ingest.File) error { var dirs []string for _, file := range files { if !lo.Contains(dirs, file.FilePath) { @@ -103,19 +108,14 @@ func copyToDir(ctx workflow.Context, dest string, files []ingest.File) error { return fmt.Errorf("multiple directories not supported: %s", dirs) } - dir, err := utils.ParsePath(filepath.Join("/mnt/dmzshare", "workflow", dirs[0])) - if err != nil { - return err - } - - destPath, err := utils.ParsePath(dest) + dir, err := paths.ParsePath(filepath.Join("/mnt/dmzshare", "workflow", dirs[0])) if err != nil { return err } err = workflow.ExecuteActivity(ctx, activities.RcloneCopyDir, activities.RcloneCopyDirInput{ Source: dir.RclonePath(), - Destination: destPath.RclonePath(), + Destination: dest.RclonePath(), }).Get(ctx, nil) if err != nil { return err @@ -124,7 +124,7 @@ func copyToDir(ctx workflow.Context, dest string, files []ingest.File) error { for _, file := range files { err = wfutils.DeletePath( ctx, - filepath.Join("/mnt/dmzshare", "workflow", file.FilePath, file.FileName), + paths.MustParsePath(filepath.Join("/mnt/dmzshare", "workflow", file.FilePath, file.FileName)), ) if err != nil { return err diff --git a/workflows/ingest/common.go b/workflows/ingest/common.go index c821551f..51e3b6e2 100644 --- a/workflows/ingest/common.go +++ b/workflows/ingest/common.go @@ -2,6 +2,7 @@ package ingestworkflows import ( vsactivity "github.com/bcc-code/bccm-flows/activities/vidispine" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/workflows" "go.temporal.io/sdk/workflow" ) @@ -11,7 +12,7 @@ type importTagResult struct { ImportJobID string } -func importFileAsTag(ctx workflow.Context, tag, path, title string) (*importTagResult, error) { +func importFileAsTag(ctx workflow.Context, tag string, path paths.Path, title string) (*importTagResult, error) { var result vsactivity.CreatePlaceholderResult err := workflow.ExecuteActivity(ctx, vsactivity.CreatePlaceholderActivity, vsactivity.CreatePlaceholderParams{ Title: title, diff --git a/workflows/ingest/masters.go b/workflows/ingest/masters.go index 0a40ebc0..fe9ba708 100644 --- a/workflows/ingest/masters.go +++ b/workflows/ingest/masters.go @@ -5,10 +5,10 @@ import ( "github.com/bcc-code/bccm-flows/activities" batonactivities "github.com/bcc-code/bccm-flows/activities/baton" "github.com/bcc-code/bccm-flows/common" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/baton" "github.com/bcc-code/bccm-flows/services/ingest" "github.com/bcc-code/bccm-flows/services/vidispine/vscommon" - "github.com/bcc-code/bccm-flows/utils" "github.com/bcc-code/bccm-flows/utils/wfutils" "go.temporal.io/sdk/workflow" "path/filepath" @@ -21,14 +21,14 @@ type MasterParams struct { Metadata *ingest.Metadata OrderForm OrderForm - Directory string + Directory paths.Path } type MasterResult struct { Report baton.QCReport AssetID string AnalyzeResult *common.AnalyzeEBUR128Result - Path utils.Path + Path paths.Path } // regexp for making sure the filename does not contain non-alphanumeric characters @@ -91,7 +91,7 @@ func uploadMaster(ctx workflow.Context, params MasterParams) (*MasterResult, err return nil, err } - file := filepath.Join(outputDir, filename) + file := outputDir.Append(filename) err = wfutils.MoveFile(ctx, files[0], file) if err != nil { return nil, err @@ -112,19 +112,14 @@ func uploadMaster(ctx workflow.Context, params MasterParams) (*MasterResult, err return nil, err } - path, err := utils.ParsePath(file) - if err != nil { - return nil, err - } - plan := baton.TestPlanMXF - if filepath.Ext(file) == ".mov" { + if filepath.Ext(file.FileName()) == ".mov" { plan = baton.TestPlanMOV } var report baton.QCReport err = wfutils.ExecuteWithQueue(ctx, batonactivities.QC, batonactivities.QCParams{ - Path: path, + Path: file, Plan: plan, }).Get(ctx, &report) if err != nil { @@ -134,14 +129,14 @@ func uploadMaster(ctx workflow.Context, params MasterParams) (*MasterResult, err return &MasterResult{ Report: report, AssetID: result.AssetID, - Path: path, + Path: file, }, nil } -func analyzeAudioAndSetMetadata(ctx workflow.Context, assetID string, path utils.Path) (*common.AnalyzeEBUR128Result, error) { +func analyzeAudioAndSetMetadata(ctx workflow.Context, assetID string, path paths.Path) (*common.AnalyzeEBUR128Result, error) { var result common.AnalyzeEBUR128Result err := wfutils.ExecuteWithQueue(ctx, activities.AnalyzeEBUR128Activity, activities.AnalyzeEBUR128Params{ - FilePath: path.WorkerPath(), + FilePath: path, TargetLoudness: -24, }).Get(ctx, &result) if err != nil { diff --git a/workflows/ingest/raw_material.go b/workflows/ingest/raw_material.go index 4b18fbd3..91f2df30 100644 --- a/workflows/ingest/raw_material.go +++ b/workflows/ingest/raw_material.go @@ -4,18 +4,18 @@ import ( "fmt" "github.com/bcc-code/bccm-flows/activities" vsactivity "github.com/bcc-code/bccm-flows/activities/vidispine" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/ingest" "github.com/bcc-code/bccm-flows/services/vidispine/vscommon" "github.com/bcc-code/bccm-flows/utils" "github.com/bcc-code/bccm-flows/utils/wfutils" "go.temporal.io/sdk/workflow" - "path/filepath" "strconv" ) type RawMaterialParams struct { Metadata *ingest.Metadata - Directory string + Directory paths.Path } func RawMaterial(ctx workflow.Context, params RawMaterialParams) error { @@ -32,12 +32,12 @@ func RawMaterial(ctx workflow.Context, params RawMaterialParams) error { return err } - var files []string + var files []paths.Path for _, f := range originalFiles { - if !utils.ValidRawFilename(f) { + if !utils.ValidRawFilename(f.LocalPath()) { return fmt.Errorf("invalid filename: %s", f) } - newPath := filepath.Join(outputFolder, filepath.Base(f)) + newPath := outputFolder.Append(f.FileName()) err = wfutils.MoveFile(ctx, f, newPath) if err != nil { return err @@ -50,7 +50,7 @@ func RawMaterial(ctx workflow.Context, params RawMaterialParams) error { for _, file := range files { var result *importTagResult - result, err = importFileAsTag(ctx, "original", file, filepath.Base(file)) + result, err = importFileAsTag(ctx, "original", file, file.FileName()) if err != nil { return err } diff --git a/workflows/normalize_audio.go b/workflows/normalize_audio.go index e7b71177..7d4774da 100644 --- a/workflows/normalize_audio.go +++ b/workflows/normalize_audio.go @@ -1,11 +1,12 @@ package workflows import ( + "github.com/bcc-code/bccm-flows/environment" + "github.com/bcc-code/bccm-flows/paths" "time" "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/common" - "github.com/bcc-code/bccm-flows/utils" "github.com/bcc-code/bccm-flows/utils/wfutils" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" @@ -38,7 +39,7 @@ func NormalizeAudioLevelWorkflow( StartToCloseTimeout: time.Hour * 4, ScheduleToCloseTimeout: time.Hour * 48, HeartbeatTimeout: time.Minute * 1, - TaskQueue: utils.GetWorkerQueue(), + TaskQueue: environment.GetWorkerQueue(), } ctx = workflow.WithActivityOptions(ctx, options) @@ -46,9 +47,14 @@ func NormalizeAudioLevelWorkflow( logger.Info("Starting NormalizeAudio workflow") + filePath, err := paths.ParsePath(params.FilePath) + if err != nil { + return nil, err + } + r128Result := &common.AnalyzeEBUR128Result{} - err := wfutils.ExecuteWithQueue(ctx, activities.AnalyzeEBUR128Activity, activities.AnalyzeEBUR128Params{ - FilePath: params.FilePath, + err = wfutils.ExecuteWithQueue(ctx, activities.AnalyzeEBUR128Activity, activities.AnalyzeEBUR128Params{ + FilePath: filePath, TargetLoudness: params.TargetLUFS, }).Get(ctx, r128Result) if err != nil { @@ -64,14 +70,14 @@ func NormalizeAudioLevelWorkflow( adjustResult := &common.AudioResult{} err = wfutils.ExecuteWithQueue(ctx, activities.AdjustAudioLevelActivity, activities.AdjustAudioLevelParams{ Adjustment: r128Result.SuggestedAdjustment, - InFilePath: params.FilePath, + InFilePath: filePath, OutFilePath: outputFolder, }).Get(ctx, adjustResult) if err != nil { return nil, err } - out.FilePath = adjustResult.OutputPath + out.FilePath = adjustResult.OutputPath.LocalPath() if params.PerformOutputAnalysis { r128Result := &common.AnalyzeEBUR128Result{} diff --git a/workflows/transcode_preview-file.go b/workflows/transcode_preview-file.go index 6600cea1..7ed04c05 100644 --- a/workflows/transcode_preview-file.go +++ b/workflows/transcode_preview-file.go @@ -2,7 +2,8 @@ package workflows import ( "github.com/bcc-code/bccm-flows/activities" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/environment" + "github.com/bcc-code/bccm-flows/paths" "path/filepath" "time" @@ -32,17 +33,22 @@ func TranscodePreviewFile( StartToCloseTimeout: time.Hour * 4, ScheduleToCloseTimeout: time.Hour * 48, HeartbeatTimeout: time.Minute * 1, - TaskQueue: utils.GetTranscodeQueue(), + TaskQueue: environment.GetTranscodeQueue(), } ctx = workflow.WithActivityOptions(ctx, options) logger.Info("Starting TranscodePreviewFile") + filePath, err := paths.ParsePath(params.FilePath) + if err != nil { + return err + } + previewResponse := &activities.TranscodePreviewResponse{} - err := workflow.ExecuteActivity(ctx, activities.TranscodePreview, activities.TranscodePreviewParams{ - FilePath: params.FilePath, - DestinationDirPath: filepath.Dir(params.FilePath), + err = workflow.ExecuteActivity(ctx, activities.TranscodePreview, activities.TranscodePreviewParams{ + FilePath: filePath, + DestinationDirPath: paths.MustParsePath(filepath.Dir(filePath.LocalPath())), }).Get(ctx, previewResponse) if err != nil { diff --git a/workflows/transcode_preview-vx.go b/workflows/transcode_preview-vx.go index 41468ba3..49f34ce7 100644 --- a/workflows/transcode_preview-vx.go +++ b/workflows/transcode_preview-vx.go @@ -3,7 +3,7 @@ package workflows import ( "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/activities/vidispine" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/environment" "github.com/bcc-code/bccm-flows/utils/wfutils" "time" @@ -35,7 +35,7 @@ func TranscodePreviewVX( StartToCloseTimeout: time.Hour * 4, ScheduleToCloseTimeout: time.Hour * 48, HeartbeatTimeout: time.Minute * 1, - TaskQueue: utils.GetWorkerQueue(), + TaskQueue: environment.GetWorkerQueue(), } ctx = workflow.WithActivityOptions(ctx, options) @@ -58,7 +58,7 @@ func TranscodePreviewVX( } previewResponse := &activities.TranscodePreviewResponse{} - ctx = workflow.WithTaskQueue(ctx, utils.GetTranscodeQueue()) + ctx = workflow.WithTaskQueue(ctx, environment.GetTranscodeQueue()) err = workflow.ExecuteActivity(ctx, activities.TranscodePreview, activities.TranscodePreviewParams{ FilePath: shapes.FilePath, DestinationDirPath: destinationPath, @@ -75,7 +75,7 @@ func TranscodePreviewVX( shapeTag = "lowres_watermarked" } - ctx = workflow.WithTaskQueue(ctx, utils.GetWorkerQueue()) + ctx = workflow.WithTaskQueue(ctx, environment.GetWorkerQueue()) err = workflow.ExecuteActivity(ctx, vidispine.ImportFileAsShapeActivity, vidispine.ImportFileAsShapeParams{ AssetID: params.VXID, diff --git a/workflows/transcribe-file.go b/workflows/transcribe-file.go index 67c8b4ae..6b0cd82b 100644 --- a/workflows/transcribe-file.go +++ b/workflows/transcribe-file.go @@ -1,11 +1,12 @@ package workflows import ( + "github.com/bcc-code/bccm-flows/environment" + "github.com/bcc-code/bccm-flows/paths" "time" "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/common" - "github.com/bcc-code/bccm-flows/utils" "github.com/bcc-code/bccm-flows/utils/wfutils" "go.temporal.io/sdk/temporal" @@ -39,7 +40,7 @@ func TranscribeFile( ctx = workflow.WithActivityOptions(ctx, options) - options.TaskQueue = utils.GetAudioQueue() + options.TaskQueue = environment.GetAudioQueue() audioCtx := workflow.WithActivityOptions(ctx, options) logger.Info("Starting TranscribeFile") @@ -49,15 +50,25 @@ func TranscribeFile( return err } + file, err := paths.ParsePath(params.File) + if err != nil { + return err + } + wavFile := common.AudioResult{} workflow.ExecuteActivity(audioCtx, activities.TranscodeToAudioWav, common.AudioInput{ - Path: params.File, + Path: file, DestinationPath: tempFolder, }).Get(ctx, &wavFile) + destination, err := paths.ParsePath(params.DestinationPath) + if err != nil { + return err + } + return workflow.ExecuteActivity(ctx, activities.Transcribe, activities.TranscribeParams{ Language: params.Language, File: wavFile.OutputPath, - DestinationPath: params.DestinationPath, + DestinationPath: destination, }).Get(ctx, nil) } diff --git a/workflows/transcribe-vx.go b/workflows/transcribe-vx.go index 1e85971b..3e5076ab 100644 --- a/workflows/transcribe-vx.go +++ b/workflows/transcribe-vx.go @@ -2,12 +2,12 @@ package workflows import ( "fmt" + "github.com/bcc-code/bccm-flows/environment" "os" "time" "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/common" - "github.com/bcc-code/bccm-flows/utils" "github.com/bcc-code/bccm-flows/utils/wfutils" "github.com/bcc-code/bccm-flows/activities/vidispine" @@ -52,7 +52,7 @@ func TranscribeVX( StartToCloseTimeout: time.Hour * 4, ScheduleToCloseTimeout: time.Hour * 48, HeartbeatTimeout: time.Minute * 1, - TaskQueue: utils.GetAudioQueue(), + TaskQueue: environment.GetAudioQueue(), } ctx = workflow.WithActivityOptions(ctx, options) @@ -134,7 +134,7 @@ func TranscribeVX( return err } - txtValue, err := os.ReadFile(transcriptionJob.TXTPath) + txtValue, err := os.ReadFile(transcriptionJob.TXTPath.LocalPath()) if err != nil { return err } diff --git a/workflows/watch_folder_transcode.go b/workflows/watch_folder_transcode.go index a9dd32c1..45505e36 100644 --- a/workflows/watch_folder_transcode.go +++ b/workflows/watch_folder_transcode.go @@ -4,7 +4,8 @@ import ( "fmt" "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/common" - "github.com/bcc-code/bccm-flows/utils" + "github.com/bcc-code/bccm-flows/environment" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/utils/wfutils" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" @@ -27,19 +28,23 @@ func WatchFolderTranscode(ctx workflow.Context, params WatchFolderTranscodeInput StartToCloseTimeout: time.Hour * 4, ScheduleToCloseTimeout: time.Hour * 48, HeartbeatTimeout: time.Minute * 1, - TaskQueue: utils.GetWorkerQueue(), + TaskQueue: environment.GetWorkerQueue(), } ctx = workflow.WithActivityOptions(ctx, options) logger.Info("Starting WatchFolderTranscode") - path := params.Path - path, err := wfutils.StandardizeFileName(ctx, path) + path, err := paths.ParsePath(params.Path) if err != nil { return err } - processingFolder, err := utils.GetSiblingFolder(path, "processing") + path, err = wfutils.StandardizeFileName(ctx, path) + if err != nil { + return err + } + processingFolder := path.Append("../processing") + err = wfutils.CreateFolder(ctx, processingFolder) if err != nil { return err } @@ -47,26 +52,30 @@ func WatchFolderTranscode(ctx workflow.Context, params WatchFolderTranscodeInput if err != nil { return err } - outFolder, err := utils.GetSiblingFolder(path, "out") + outFolder := path.Append("../out") + err = wfutils.CreateFolder(ctx, outFolder) if err != nil { return err } - tmpFolder, err := utils.GetSiblingFolder(path, "tmp") + tmpFolder := path.Append("../tmp") + err = wfutils.CreateFolder(ctx, tmpFolder) if err != nil { return err } - errorFolder, err := utils.GetSiblingFolder(path, "error") + errorFolder := path.Append("../error") + err = wfutils.CreateFolder(ctx, errorFolder) if err != nil { return err } - processedFolder, err := utils.GetSiblingFolder(path, "processed") + processedFolder := path.Append("../processed") + err = wfutils.CreateFolder(ctx, processedFolder) if err != nil { return err } var transcodeOutput *activities.EncodeResult var transcribeOutput *activities.TranscribeResponse - ctx = workflow.WithTaskQueue(ctx, utils.GetTranscodeQueue()) + ctx = workflow.WithTaskQueue(ctx, environment.GetTranscodeQueue()) switch params.FolderName { case common.FolderProRes422HQHD: err = workflow.ExecuteActivity(ctx, activities.TranscodeToProResActivity, activities.EncodeParams{ @@ -110,7 +119,7 @@ func WatchFolderTranscode(ctx workflow.Context, params WatchFolderTranscodeInput Bitrate: "60M", }).Get(ctx, &transcodeOutput) case common.FolderTranscribe: - ctx = workflow.WithTaskQueue(ctx, utils.GetWorkerQueue()) + ctx = workflow.WithTaskQueue(ctx, environment.GetWorkerQueue()) err = workflow.ExecuteActivity(ctx, activities.Transcribe, activities.TranscribeParams{ Language: "no", File: path, @@ -120,7 +129,7 @@ func WatchFolderTranscode(ctx workflow.Context, params WatchFolderTranscodeInput err = fmt.Errorf("codec not supported: %s", params.FolderName) } - ctx = workflow.WithTaskQueue(ctx, utils.GetWorkerQueue()) + ctx = workflow.WithTaskQueue(ctx, environment.GetWorkerQueue()) if err != nil { path, _ = wfutils.MoveToFolder(ctx, path, errorFolder) From 98aec92cd8c8c66b536d24688b1a08f2225f5c52 Mon Sep 17 00:00:00 2001 From: Fredrik Vedvik Date: Thu, 9 Nov 2023 09:54:28 +0100 Subject: [PATCH 2/7] feat(paths): simplify names --- activities/analyze.go | 2 +- activities/files.go | 26 +++++++++++++------------- activities/normalize.go | 2 +- activities/preview.go | 4 ++-- activities/subtrans.go | 4 ++-- activities/transcode.go | 12 ++++++------ activities/transcribe.go | 4 ++-- activities/vidispine/files.go | 4 ++-- paths/paths.go | 8 ++++---- paths/paths_test.go | 2 +- services/baton/tasks.go | 2 +- services/ingest/types.go | 2 +- services/transcode/audio.go | 18 +++++++++--------- services/transcode/linear_normalize.go | 6 +++--- services/transcode/merge.go | 22 +++++++++++----------- services/transcode/mux.go | 10 +++++----- services/transcode/playout_mux.go | 10 +++++----- services/transcode/video.go | 10 +++++----- utils/wfutils/files.go | 2 +- workflows/export/mux_files.go | 10 +++++----- workflows/export/vx_export_bmm.go | 4 ++-- workflows/export/vx_export_playout.go | 2 +- workflows/export/vx_export_vod.go | 2 +- workflows/ingest/asset_ingest.go | 4 ++-- workflows/ingest/masters.go | 2 +- workflows/ingest/raw_material.go | 6 +++--- workflows/normalize_audio.go | 2 +- workflows/transcode_preview-file.go | 2 +- workflows/transcribe-vx.go | 2 +- 29 files changed, 93 insertions(+), 93 deletions(-) diff --git a/activities/analyze.go b/activities/analyze.go index 76671463..7280c747 100644 --- a/activities/analyze.go +++ b/activities/analyze.go @@ -20,7 +20,7 @@ func AnalyzeFile(ctx context.Context, input AnalyzeFileParams) (*AnalyzeFileResu logger := activity.GetLogger(ctx) logger.Info("Starting AnalyzeFileActivity") - info, err := ffmpeg.GetStreamInfo(input.FilePath.LocalPath()) + info, err := ffmpeg.GetStreamInfo(input.FilePath.Local()) if err != nil { return nil, err } diff --git a/activities/files.go b/activities/files.go index 4ef04a4b..011d524a 100644 --- a/activities/files.go +++ b/activities/files.go @@ -27,15 +27,15 @@ func MoveFile(ctx context.Context, input MoveFileInput) (*FileResult, error) { activity.RecordHeartbeat(ctx, "MoveFile") log.Info("Starting MoveFileActivity") - err := os.MkdirAll(filepath.Dir(input.Destination.LocalPath()), os.ModePerm) + err := os.MkdirAll(filepath.Dir(input.Destination.Local()), os.ModePerm) if err != nil { return nil, err } - err = os.Rename(input.Source.LocalPath(), input.Destination.LocalPath()) + err = os.Rename(input.Source.Local(), input.Destination.Local()) if err != nil { return nil, err } - _ = os.Chmod(input.Destination.LocalPath(), os.ModePerm) + _ = os.Chmod(input.Destination.Local(), os.ModePerm) return &FileResult{ Path: input.Destination, }, nil @@ -46,8 +46,8 @@ func StandardizeFileName(ctx context.Context, input FileInput) (*FileResult, err activity.RecordHeartbeat(ctx, "StandardizeFileName") log.Info("Starting StandardizeFileNameActivity") - path := paths.FixFilename(input.Path.LocalPath()) - err := os.Rename(input.Path.LocalPath(), path) + path := paths.FixFilename(input.Path.Local()) + err := os.Rename(input.Path.Local(), path) if err != nil { return nil, err } @@ -66,11 +66,11 @@ func CreateFolder(ctx context.Context, input CreateFolderInput) error { activity.RecordHeartbeat(ctx, "CreateFolder") log.Info("Starting CreateFolderActivity") - err := os.MkdirAll(input.Destination.LocalPath(), os.ModePerm) + err := os.MkdirAll(input.Destination.Local(), os.ModePerm) if err != nil { return err } - return os.Chmod(input.Destination.LocalPath(), os.ModePerm) + return os.Chmod(input.Destination.Local(), os.ModePerm) } type WriteFileInput struct { @@ -83,15 +83,15 @@ func WriteFile(ctx context.Context, input WriteFileInput) error { activity.RecordHeartbeat(ctx, "WriteFile") log.Info("Starting WriteFileActivity") - err := os.MkdirAll(filepath.Dir(input.Path.LocalPath()), os.ModePerm) + err := os.MkdirAll(filepath.Dir(input.Path.Local()), os.ModePerm) if err != nil { return err } - err = os.WriteFile(input.Path.LocalPath(), input.Data, os.ModePerm) + err = os.WriteFile(input.Path.Local(), input.Data, os.ModePerm) if err != nil { return err } - _ = os.Chmod(input.Path.LocalPath(), os.ModePerm) + _ = os.Chmod(input.Path.Local(), os.ModePerm) return nil } @@ -100,7 +100,7 @@ func ReadFile(ctx context.Context, input FileInput) ([]byte, error) { activity.RecordHeartbeat(ctx, "ReadFile") log.Info("Starting ReadFileActivity") - return os.ReadFile(input.Path.LocalPath()) + return os.ReadFile(input.Path.Local()) } func ListFiles(ctx context.Context, input FileInput) ([]paths.Path, error) { @@ -108,7 +108,7 @@ func ListFiles(ctx context.Context, input FileInput) ([]paths.Path, error) { activity.RecordHeartbeat(ctx, "ListFiles") log.Info("Starting ListFilesActivity") - files, err := filepath.Glob(filepath.Join(input.Path.LocalPath(), "*")) + files, err := filepath.Glob(filepath.Join(input.Path.Local(), "*")) if err != nil { return nil, err } @@ -122,5 +122,5 @@ func DeletePath(ctx context.Context, input FileInput) error { activity.RecordHeartbeat(ctx, "DeletePath") log.Info("Starting DeletePathActivity") - return os.RemoveAll(input.Path.LocalPath()) + return os.RemoveAll(input.Path.Local()) } diff --git a/activities/normalize.go b/activities/normalize.go index 0aa1714c..da347808 100644 --- a/activities/normalize.go +++ b/activities/normalize.go @@ -22,7 +22,7 @@ func AnalyzeEBUR128Activity(ctx context.Context, input AnalyzeEBUR128Params) (*c stop, progressCallback := registerProgressCallback(ctx) defer close(stop) - analyzeResult, err := ffmpeg.AnalyzeEBUR128(input.FilePath.LocalPath(), progressCallback) + analyzeResult, err := ffmpeg.AnalyzeEBUR128(input.FilePath.Local(), progressCallback) if err != nil { return nil, err } diff --git a/activities/preview.go b/activities/preview.go index 201ddb57..86e9d5f1 100644 --- a/activities/preview.go +++ b/activities/preview.go @@ -27,8 +27,8 @@ func TranscodePreview(ctx context.Context, input TranscodePreviewParams) (*Trans defer close(stop) result, err := transcode.Preview(transcode.PreviewInput{ - OutputDir: input.DestinationDirPath.LocalPath(), - FilePath: input.FilePath.LocalPath(), + OutputDir: input.DestinationDirPath.Local(), + FilePath: input.FilePath.Local(), }, progressCallback) if err != nil { fmt.Println(err.Error()) diff --git a/activities/subtrans.go b/activities/subtrans.go index 31d81289..5f60f2ea 100644 --- a/activities/subtrans.go +++ b/activities/subtrans.go @@ -97,7 +97,7 @@ func GetSubtransIDActivity(ctx context.Context, input *GetSubtransIDInput) (*Get func GetSubtitlesActivity(ctx context.Context, params GetSubtitlesInput) (map[string]paths.Path, error) { client := subtrans.NewClient(os.Getenv("SUBTRANS_BASE_URL"), os.Getenv("SUBTRANS_API_KEY")) - info, err := os.Stat(params.DestinationFolder.LocalPath()) + info, err := os.Stat(params.DestinationFolder.Local()) if os.IsNotExist(err) { return nil, err } @@ -119,7 +119,7 @@ func GetSubtitlesActivity(ctx context.Context, params GetSubtitlesInput) (map[st out := map[string]paths.Path{} for lang, sub := range subs { - path := path.Join(params.DestinationFolder.LocalPath(), params.FilePrefix+lang+"."+params.Format) + path := path.Join(params.DestinationFolder.Local(), params.FilePrefix+lang+"."+params.Format) err := os.WriteFile(path, []byte(sub), 0644) if err != nil { return nil, err diff --git a/activities/transcode.go b/activities/transcode.go index da81176b..4f979b1f 100644 --- a/activities/transcode.go +++ b/activities/transcode.go @@ -31,8 +31,8 @@ func TranscodeToProResActivity(ctx context.Context, input EncodeParams) (*Encode defer close(stop) transcodeResult, err := transcode.ProRes(transcode.ProResInput{ - FilePath: input.FilePath.LocalPath(), - OutputDir: input.OutputDir.LocalPath(), + FilePath: input.FilePath.Local(), + OutputDir: input.OutputDir.Local(), FrameRate: input.FrameRate, Resolution: input.Resolution, }, progressCallback) @@ -55,8 +55,8 @@ func TranscodeToH264Activity(ctx context.Context, input EncodeParams) (*EncodeRe defer close(stop) transcodeResult, err := transcode.H264(transcode.EncodeInput{ - FilePath: input.FilePath.LocalPath(), - OutputDir: input.OutputDir.LocalPath(), + FilePath: input.FilePath.Local(), + OutputDir: input.OutputDir.Local(), FrameRate: input.FrameRate, Resolution: input.Resolution, Bitrate: input.Bitrate, @@ -79,8 +79,8 @@ func TranscodeToXDCAMActivity(ctx context.Context, input EncodeParams) (*EncodeR defer close(stop) transcodeResult, err := transcode.XDCAM(transcode.EncodeInput{ - FilePath: input.FilePath.LocalPath(), - OutputDir: input.OutputDir.LocalPath(), + FilePath: input.FilePath.Local(), + OutputDir: input.OutputDir.Local(), FrameRate: input.FrameRate, Resolution: input.Resolution, Bitrate: input.Bitrate, diff --git a/activities/transcribe.go b/activities/transcribe.go index 492bca4f..64576ee0 100644 --- a/activities/transcribe.go +++ b/activities/transcribe.go @@ -33,14 +33,14 @@ func Transcribe( time.Sleep(time.Second * 10) - jobData, err := transcribe.DoTranscribe(ctx, input.File.LocalPath(), input.DestinationPath.LocalPath(), input.Language) + jobData, err := transcribe.DoTranscribe(ctx, input.File.Local(), input.DestinationPath.Local(), input.Language) if err != nil { return nil, err } log.Info("Finished Transcribe") - fileName := filepath.Base(input.File.LocalPath()) + fileName := filepath.Base(input.File.Local()) return &TranscribeResponse{ JSONPath: paths.MustParsePath(filepath.Join(jobData.OutputPath, fileName+".json")), SRTPath: paths.MustParsePath(filepath.Join(jobData.OutputPath, fileName+".srt")), diff --git a/activities/vidispine/files.go b/activities/vidispine/files.go index fa4fcabd..26c400fc 100644 --- a/activities/vidispine/files.go +++ b/activities/vidispine/files.go @@ -19,7 +19,7 @@ func ImportFileAsShapeActivity(ctx context.Context, params *ImportFileAsShapePar vsClient := GetClient() - fileID, err := vsClient.RegisterFile(params.FilePath.LocalPath(), vsapi.FileStateClosed) + fileID, err := vsClient.RegisterFile(params.FilePath.Local(), vsapi.FileStateClosed) if err != nil { return nil, err } @@ -46,7 +46,7 @@ func ImportFileAsSidecarActivity(ctx context.Context, params *ImportSubtitleAsSi vsClient := GetClient() - jobID, err := vsClient.AddSidecarToItem(params.AssetID, params.FilePath.LocalPath(), params.Language) + jobID, err := vsClient.AddSidecarToItem(params.AssetID, params.FilePath.Local(), params.Language) return &ImportFileAsSidecarResult{ JobID: jobID, }, err diff --git a/paths/paths.go b/paths/paths.go index d8290766..89d6900f 100644 --- a/paths/paths.go +++ b/paths/paths.go @@ -60,7 +60,7 @@ type Path struct { Path string } -func (p Path) LocalPath() string { +func (p Path) Local() string { return filepath.Join(drivePrefixes[p.Drive].Client, p.Path) } @@ -75,11 +75,11 @@ func (p Path) RcloneFsRemote() (string, string) { return "", "" } -func (p Path) RclonePath() string { +func (p Path) Rclone() string { return filepath.Join(drivePrefixes[p.Drive].Rclone, p.Path) } -func (p Path) BatonPath() string { +func (p Path) Baton() string { switch p.Drive { case IsilonDrive: return filepath.Join("\\\\10.12.130.61\\isilon", strings.ReplaceAll(p.Path, "/", "\\")) @@ -87,7 +87,7 @@ func (p Path) BatonPath() string { return "" } -func (p Path) FileName() string { +func (p Path) Base() string { return filepath.Base(p.Path) } diff --git a/paths/paths_test.go b/paths/paths_test.go index 9515ec57..bd316729 100644 --- a/paths/paths_test.go +++ b/paths/paths_test.go @@ -26,5 +26,5 @@ func Test_ParsePath(t *testing.T) { assert.Equal(t, IsilonDrive, path.Drive) assert.Equal(t, "test.xml", path.Path) - assert.Equal(t, "isilon:isilon/test.xml", path.RclonePath()) + assert.Equal(t, "isilon:isilon/test.xml", path.Rclone()) } diff --git a/services/baton/tasks.go b/services/baton/tasks.go index 8bb717f2..4fe21f56 100644 --- a/services/baton/tasks.go +++ b/services/baton/tasks.go @@ -12,7 +12,7 @@ type StartTaskResult struct { func StartTask(client *Client, filePath paths.Path, testPlan TestPlan) (*StartTaskResult, error) { req := client.restyClient.R() - req.SetQueryParam("mediaFilePath", filePath.BatonPath()) + req.SetQueryParam("mediaFilePath", filePath.Baton()) req.SetQueryParam("testPlan", testPlan.Value) req.SetResult(&StartTaskResult{}) res, err := req.Post("tasks") diff --git a/services/ingest/types.go b/services/ingest/types.go index 6289bcd3..9514ee90 100644 --- a/services/ingest/types.go +++ b/services/ingest/types.go @@ -44,7 +44,7 @@ type FileList struct { } type File struct { - FileName string `xml:"FileName"` + FileName string `xml:"Base"` IsFolder bool `xml:"isFolder"` FileSize int `xml:"fileSize"` FilePath string `xml:"filePath"` diff --git a/services/transcode/audio.go b/services/transcode/audio.go index 419a6c3b..7ae0c8a8 100644 --- a/services/transcode/audio.go +++ b/services/transcode/audio.go @@ -16,17 +16,17 @@ func AudioAac(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi params := []string{ "-progress", "pipe:1", "-hide_banner", - "-i", input.Path.LocalPath(), + "-i", input.Path.Local(), "-c:a", "aac", "-b:a", input.Bitrate, } - outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), filepath.Base(input.Path.LocalPath())) + outputFilePath := filepath.Join(input.DestinationPath.Local(), filepath.Base(input.Path.Local())) outputFilePath = fmt.Sprintf("%s-%s.aac", outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))], input.Bitrate) params = append(params, "-y", outputFilePath) - info, err := ffmpeg.GetStreamInfo(input.Path.LocalPath()) + info, err := ffmpeg.GetStreamInfo(input.Path.Local()) if err != nil { return nil, err } @@ -57,15 +57,15 @@ func AudioWav(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi params := []string{ "-progress", "pipe:1", "-hide_banner", - "-i", input.Path.LocalPath(), + "-i", input.Path.Local(), } - outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), filepath.Base(input.Path.LocalPath())) + outputFilePath := filepath.Join(input.DestinationPath.Local(), filepath.Base(input.Path.Local())) outputFilePath = fmt.Sprintf("%s-%s.wav", outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))], input.Bitrate) params = append(params, "-y", outputFilePath) - info, err := ffmpeg.GetStreamInfo(input.Path.LocalPath()) + info, err := ffmpeg.GetStreamInfo(input.Path.Local()) if err != nil { return nil, err } @@ -124,17 +124,17 @@ func AudioMP3(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi params := []string{ "-progress", "pipe:1", "-hide_banner", - "-i", input.Path.LocalPath(), + "-i", input.Path.Local(), "-c:a", "libmp3lame", "-q:a", fmt.Sprint(getQfactorFromBitrate(input.Bitrate)), } - outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), filepath.Base(input.Path.LocalPath())) + outputFilePath := filepath.Join(input.DestinationPath.Local(), filepath.Base(input.Path.Local())) outputFilePath = fmt.Sprintf("%s-%s.mp3", outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))], input.Bitrate) params = append(params, "-y", outputFilePath) - info, err := ffmpeg.GetStreamInfo(input.Path.LocalPath()) + info, err := ffmpeg.GetStreamInfo(input.Path.Local()) if err != nil { return nil, err } diff --git a/services/transcode/linear_normalize.go b/services/transcode/linear_normalize.go index 8914f922..d72d0b17 100644 --- a/services/transcode/linear_normalize.go +++ b/services/transcode/linear_normalize.go @@ -13,11 +13,11 @@ import ( // AdjustAudioLevel adjusts the audio level of the input file by the given adjustment in dB // without changing the dynamic range. This function does not protect against clipping! func AdjustAudioLevel(input common.AudioInput, adjustment float64, cb ffmpeg.ProgressCallback) (*common.AudioResult, error) { - outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), filepath.Base(input.Path.LocalPath())) + outputFilePath := filepath.Join(input.DestinationPath.Local(), filepath.Base(input.Path.Local())) outputFilePath = outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))] + "_normalized" + filepath.Ext(outputFilePath) params := []string{ - "-i", input.Path.LocalPath(), + "-i", input.Path.Local(), "-c:v", "copy", "-af", fmt.Sprintf("volume=%.2fdB", adjustment), outputFilePath, @@ -25,7 +25,7 @@ func AdjustAudioLevel(input common.AudioInput, adjustment float64, cb ffmpeg.Pro params = append(params, "-y", outputFilePath) - info, err := ffmpeg.GetStreamInfo(input.Path.LocalPath()) + info, err := ffmpeg.GetStreamInfo(input.Path.Local()) if err != nil { return nil, err } diff --git a/services/transcode/merge.go b/services/transcode/merge.go index 2837797a..cd2198c9 100644 --- a/services/transcode/merge.go +++ b/services/transcode/merge.go @@ -22,7 +22,7 @@ func getFramerate(input common.MergeInput) (int, error) { return a.End-a.Start > b.End-b.Start }) - info, err := ffmpeg.GetStreamInfo(longestItem.Path.LocalPath()) + info, err := ffmpeg.GetStreamInfo(longestItem.Path.Local()) if err != nil { return 0, err } @@ -40,7 +40,7 @@ func MergeVideo(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac var params []string for _, i := range input.Items { - params = append(params, "-i", i.Path.LocalPath()) + params = append(params, "-i", i.Path.Local()) } var filterComplex string @@ -63,7 +63,7 @@ func MergeVideo(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac // Concatenate the video streams. filterComplex += fmt.Sprintf("concat=n=%d:v=1:a=0[v]", len(input.Items)) - outputFilePath := filepath.Join(input.OutputDir.LocalPath(), filepath.Clean(input.Title)+".mxf") + outputFilePath := filepath.Join(input.OutputDir.Local(), filepath.Clean(input.Title)+".mxf") params = append(params, "-progress", "pipe:1", @@ -108,7 +108,7 @@ func MergeVideo(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac // mergeItemToStereoStream takes a merge input item and returns a string that can be used in a filter_complex to merge the audio streams. func mergeItemToStereoStream(index int, tag string, item common.MergeInputItem) (string, error) { - path := item.Path.LocalPath() + path := item.Path.Local() info, _ := ffmpeg.ProbeFile(path) if info == nil || len(info.Streams) == 0 { @@ -164,7 +164,7 @@ func MergeAudio(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac } for _, i := range input.Items { - params = append(params, "-i", i.Path.LocalPath()) + params = append(params, "-i", i.Path.Local()) } params = append(params, @@ -190,7 +190,7 @@ func MergeAudio(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac filterComplex += fmt.Sprintf("concat=n=%d:v=0:a=1 [a]", len(input.Items)) - outputFilePath := filepath.Join(input.OutputDir.LocalPath(), filepath.Clean(input.Title)+".wav") + outputFilePath := filepath.Join(input.OutputDir.Local(), filepath.Clean(input.Title)+".wav") params = append(params, "-filter_complex", filterComplex, "-map", "[a]", "-y", outputFilePath) @@ -229,9 +229,9 @@ func MergeSubtitles(input common.MergeInput, progressCallback ffmpeg.ProgressCal startAt := 0.0 for index, item := range input.Items { - file := filepath.Join(input.WorkDir.LocalPath(), fmt.Sprintf("%s-%d.srt", input.Title, index)) - fileOut := filepath.Join(input.WorkDir.LocalPath(), fmt.Sprintf("%s-%d-out.srt", input.Title, index)) - path := item.Path.LocalPath() + file := filepath.Join(input.WorkDir.Local(), fmt.Sprintf("%s-%d.srt", input.Title, index)) + fileOut := filepath.Join(input.WorkDir.Local(), fmt.Sprintf("%s-%d-out.srt", input.Title, index)) + path := item.Path.Local() cmd := exec.Command("ffmpeg", "-i", path, "-ss", fmt.Sprintf("%f", item.Start), "-to", fmt.Sprintf("%f", item.End), "-y", file) @@ -269,7 +269,7 @@ func MergeSubtitles(input common.MergeInput, progressCallback ffmpeg.ProgressCal content += fmt.Sprintf("file '%s'\n", f) } - subtitlesFile := filepath.Join(input.WorkDir.LocalPath(), input.Title+"-subtitles.txt") + subtitlesFile := filepath.Join(input.WorkDir.Local(), input.Title+"-subtitles.txt") err := os.WriteFile(subtitlesFile, []byte(content), os.ModePerm) if err != nil { @@ -278,7 +278,7 @@ func MergeSubtitles(input common.MergeInput, progressCallback ffmpeg.ProgressCal concatStr := fmt.Sprintf("concat:%s", strings.Join(files, "|")) - outputFilePath := filepath.Join(input.OutputDir.LocalPath(), filepath.Clean(input.Title)+".srt") + outputFilePath := filepath.Join(input.OutputDir.Local(), filepath.Clean(input.Title)+".srt") params := []string{ "-progress", "pipe:1", "-hide_banner", diff --git a/services/transcode/mux.go b/services/transcode/mux.go index 8f72aa80..208f3d6d 100644 --- a/services/transcode/mux.go +++ b/services/transcode/mux.go @@ -35,17 +35,17 @@ func languageFilesForPaths(paths map[string]paths.Path) []languageFile { // Mux multiplexes specified video, audio and subtitle tracks. func Mux(input common.MuxInput, progressCallback ffmpeg.ProgressCallback) (*common.MuxResult, error) { //Use ffmpeg to mux the video - info, err := ffmpeg.GetStreamInfo(input.VideoFilePath.LocalPath()) + info, err := ffmpeg.GetStreamInfo(input.VideoFilePath.Local()) if err != nil { return nil, err } - outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), input.FileName+".mp4") + outputFilePath := filepath.Join(input.DestinationPath.Local(), input.FileName+".mp4") params := []string{ "-progress", "pipe:1", "-hide_banner", - "-i", input.VideoFilePath.LocalPath(), + "-i", input.VideoFilePath.Local(), } audioFiles := languageFilesForPaths(input.AudioFilePaths) @@ -53,13 +53,13 @@ func Mux(input common.MuxInput, progressCallback ffmpeg.ProgressCallback) (*comm for _, f := range audioFiles { params = append(params, - "-i", f.Path.LocalPath(), + "-i", f.Path.Local(), ) } for _, f := range subtitleFiles { params = append(params, - "-i", f.Path.LocalPath(), + "-i", f.Path.Local(), ) } diff --git a/services/transcode/playout_mux.go b/services/transcode/playout_mux.go index c4f4c477..fdfc0f12 100644 --- a/services/transcode/playout_mux.go +++ b/services/transcode/playout_mux.go @@ -106,7 +106,7 @@ func generateFFmpegParamsForPlayoutMux(input common.PlayoutMuxInput, outputPath // Inputs ffmpegInputCount := 0 addInput := func(path paths.Path) { - params = append(params, "-i", path.LocalPath()) + params = append(params, "-i", path.Local()) ffmpegInputCount++ } addInput(input.VideoFilePath) @@ -131,7 +131,7 @@ func generateFFmpegParamsForPlayoutMux(input common.PlayoutMuxInput, outputPath audioLanguages[i] = &PlayoutLanguageState{ Code: lang, - FilePath: filePath.LocalPath(), + FilePath: filePath.Local(), CopyFrom: copyFrom, InputIndex: inputIndex, Stereo: i < 4, @@ -221,16 +221,16 @@ func generateFFmpegParamsForPlayoutMux(input common.PlayoutMuxInput, outputPath } func PlayoutMux(input common.PlayoutMuxInput, progressCallback ffmpeg.ProgressCallback) (*common.PlayoutMuxResult, error) { - base := filepath.Base(input.VideoFilePath.LocalPath()) + base := filepath.Base(input.VideoFilePath.Local()) fileNameWithoutExtension := base[:len(base)-len(filepath.Ext(base))] - outputFilePath := filepath.Join(input.OutputDir.LocalPath(), fileNameWithoutExtension+".mxf") + outputFilePath := filepath.Join(input.OutputDir.Local(), fileNameWithoutExtension+".mxf") params, err := generateFFmpegParamsForPlayoutMux(input, outputFilePath) if err != nil { return nil, err } - info, err := ffmpeg.GetStreamInfo(input.VideoFilePath.LocalPath()) + info, err := ffmpeg.GetStreamInfo(input.VideoFilePath.Local()) if err != nil { return nil, err } diff --git a/services/transcode/video.go b/services/transcode/video.go index 6b88a309..e33d9fe2 100644 --- a/services/transcode/video.go +++ b/services/transcode/video.go @@ -16,12 +16,12 @@ func VideoH264(input common.VideoInput, cb ffmpeg.ProgressCallback) (*common.Vid params := []string{ "-hide_banner", "-progress", "pipe:1", - "-i", input.Path.LocalPath(), + "-i", input.Path.Local(), } if input.WatermarkPath != nil { params = append(params, - "-i", input.WatermarkPath.LocalPath(), + "-i", input.WatermarkPath.Local(), ) } @@ -66,7 +66,7 @@ func VideoH264(input common.VideoInput, cb ffmpeg.ProgressCallback) (*common.Vid input.Width, input.Height) - info, err := ffmpeg.GetStreamInfo(input.Path.LocalPath()) + info, err := ffmpeg.GetStreamInfo(input.Path.Local()) if err != nil { return nil, err } @@ -86,11 +86,11 @@ func VideoH264(input common.VideoInput, cb ffmpeg.ProgressCallback) (*common.Vid "-map", "[out]", ) - filename := filepath.Base(input.Path.LocalPath()) + filename := filepath.Base(input.Path.Local()) filename = filename[:len(filename)-len(filepath.Ext(filename))] + fmt.Sprintf("_%dx%d.mp4", input.Width, input.Height) - outputFilePath := filepath.Join(input.DestinationPath.LocalPath(), filename) + outputFilePath := filepath.Join(input.DestinationPath.Local(), filename) params = append(params, "-y", outputFilePath, diff --git a/utils/wfutils/files.go b/utils/wfutils/files.go index a1a157c4..498369ad 100644 --- a/utils/wfutils/files.go +++ b/utils/wfutils/files.go @@ -34,7 +34,7 @@ func MoveFile(ctx workflow.Context, source, destination paths.Path) error { } func MoveToFolder(ctx workflow.Context, file, folder paths.Path) (paths.Path, error) { - newPath := folder.Append(filepath.Base(file.LocalPath())) + newPath := folder.Append(filepath.Base(file.Local())) err := MoveFile(ctx, file, newPath) return newPath, err } diff --git a/workflows/export/mux_files.go b/workflows/export/mux_files.go index d6ae0030..135ee9e5 100644 --- a/workflows/export/mux_files.go +++ b/workflows/export/mux_files.go @@ -78,7 +78,7 @@ func getSubtitlesResult(params MuxFilesParams) []smil.TextStream { for _, language := range subtitleLanguages { path := params.SubtitleFiles[language.ISO6391] subtitles = append(subtitles, smil.TextStream{ - Src: filepath.Base(path.LocalPath()), + Src: filepath.Base(path.Local()), SystemLanguage: language.ISO6391, SubtitleName: language.LanguageNameSystem, }) @@ -95,7 +95,7 @@ func startFileTasks(ctx workflow.Context, params MuxFilesParams, languages []bcc key := lang.ISO6391 fileTasks[key] = map[string]workflow.Future{} for _, q := range fileQualities { - base := filepath.Base(params.VideoFiles[q].LocalPath()) + base := filepath.Base(params.VideoFiles[q].Local()) fileName := base[:len(base)-len(filepath.Ext(base))] + "-" + key fileTasks[key][q] = wfutils.ExecuteWithQueue(ctx, activities.TranscodeMux, common.MuxInput{ FileName: fileName, @@ -130,7 +130,7 @@ func waitForFileTasks(ctx workflow.Context, params MuxFilesParams, tasks map[str Resolution: q, AudioLanguage: code, Mime: "video/mp4", - Path: filepath.Base(result.Path.LocalPath()), + Path: filepath.Base(result.Path.Local()), }) } } @@ -165,7 +165,7 @@ func startStreamTasks(ctx workflow.Context, params MuxFilesParams, languages map audioFilePaths[key] = params.AudioFiles[key] } - base := filepath.Base(path.LocalPath()) + base := filepath.Base(path.Local()) fileName := base[:len(base)-len(filepath.Ext(base))] tasks[q] = workflow.ExecuteActivity(ctx, activities.TranscodeMux, common.MuxInput{ @@ -191,7 +191,7 @@ func waitForStreamTasks(ctx workflow.Context, tasks map[string]workflow.Future, fileLanguages := languages[q] streams = append(streams, smil.Video{ - Src: filepath.Base(result.Path.LocalPath()), + Src: filepath.Base(result.Path.Local()), IncludeAudio: fmt.Sprintf("%t", len(fileLanguages) > 0), SystemLanguage: strings.Join(lo.Map(fileLanguages, func(i bccmflows.Language, _ int) string { return i.ISO6391 diff --git a/workflows/export/vx_export_bmm.go b/workflows/export/vx_export_bmm.go index d2d647cd..3974c6e5 100644 --- a/workflows/export/vx_export_bmm.go +++ b/workflows/export/vx_export_bmm.go @@ -141,7 +141,7 @@ func VXExportToBMM(ctx workflow.Context, params VXExportChildWorkflowParams) (*V ingestFolder := params.ExportData.SafeTitle + "_" + workflow.GetInfo(ctx).OriginalRunID err = workflow.ExecuteActivity(ctx, activities.RcloneCopyDir, activities.RcloneCopyDirInput{ - Source: outputFolder.RclonePath(), + Source: outputFolder.Rclone(), Destination: fmt.Sprintf("bmms3:/int-bmm-mediabanken/" + ingestFolder), }).Get(ctx, nil) if err != nil { @@ -209,7 +209,7 @@ func prepareBMMData(audioFiles map[string][]common.AudioResult, analysis map[str Bitrate: bitrate, VariableBitrate: true, ChannelCount: 2, - Path: path.Base(file.OutputPath.LocalPath()), // This needs to be relative to the resultintg JSON file + Path: path.Base(file.OutputPath.Local()), // This needs to be relative to the resultintg JSON file Lufs: analysis[lang].OutputAnalysis.IntegratedLoudness, DynamicRange: analysis[lang].OutputAnalysis.LoudnessRange, Language: lang, diff --git a/workflows/export/vx_export_playout.go b/workflows/export/vx_export_playout.go index 55fcd1a7..ebe57ae8 100644 --- a/workflows/export/vx_export_playout.go +++ b/workflows/export/vx_export_playout.go @@ -56,7 +56,7 @@ func VXExportToPlayout(ctx workflow.Context, params VXExportChildWorkflowParams) // Rclone to playout destination := "playout:/dropbox" err = workflow.ExecuteActivity(ctx, activities.RcloneCopyDir, activities.RcloneCopyDirInput{ - Source: params.OutputDir.RclonePath(), + Source: params.OutputDir.Rclone(), Destination: destination, }).Get(ctx, nil) if err != nil { diff --git a/workflows/export/vx_export_vod.go b/workflows/export/vx_export_vod.go index 73c5200b..46e7c1c0 100644 --- a/workflows/export/vx_export_vod.go +++ b/workflows/export/vx_export_vod.go @@ -129,7 +129,7 @@ func VXExportToVOD(ctx workflow.Context, params VXExportChildWorkflowParams) (*V outputPath := params.OutputDir err = workflow.ExecuteActivity(ctx, activities.RcloneCopyDir, activities.RcloneCopyDirInput{ - Source: outputPath.RclonePath(), + Source: outputPath.Rclone(), Destination: fmt.Sprintf("s3prod:vod-asset-ingest-prod/" + ingestFolder), }).Get(ctx, nil) if err != nil { diff --git a/workflows/ingest/asset_ingest.go b/workflows/ingest/asset_ingest.go index 023057a4..c0917047 100644 --- a/workflows/ingest/asset_ingest.go +++ b/workflows/ingest/asset_ingest.go @@ -114,8 +114,8 @@ func copyToDir(ctx workflow.Context, dest paths.Path, files []ingest.File) error } err = workflow.ExecuteActivity(ctx, activities.RcloneCopyDir, activities.RcloneCopyDirInput{ - Source: dir.RclonePath(), - Destination: dest.RclonePath(), + Source: dir.Rclone(), + Destination: dest.Rclone(), }).Get(ctx, nil) if err != nil { return err diff --git a/workflows/ingest/masters.go b/workflows/ingest/masters.go index fe9ba708..d0c09517 100644 --- a/workflows/ingest/masters.go +++ b/workflows/ingest/masters.go @@ -113,7 +113,7 @@ func uploadMaster(ctx workflow.Context, params MasterParams) (*MasterResult, err } plan := baton.TestPlanMXF - if filepath.Ext(file.FileName()) == ".mov" { + if filepath.Ext(file.Base()) == ".mov" { plan = baton.TestPlanMOV } diff --git a/workflows/ingest/raw_material.go b/workflows/ingest/raw_material.go index 91f2df30..7f62519e 100644 --- a/workflows/ingest/raw_material.go +++ b/workflows/ingest/raw_material.go @@ -34,10 +34,10 @@ func RawMaterial(ctx workflow.Context, params RawMaterialParams) error { var files []paths.Path for _, f := range originalFiles { - if !utils.ValidRawFilename(f.LocalPath()) { + if !utils.ValidRawFilename(f.Local()) { return fmt.Errorf("invalid filename: %s", f) } - newPath := outputFolder.Append(f.FileName()) + newPath := outputFolder.Append(f.Base()) err = wfutils.MoveFile(ctx, f, newPath) if err != nil { return err @@ -50,7 +50,7 @@ func RawMaterial(ctx workflow.Context, params RawMaterialParams) error { for _, file := range files { var result *importTagResult - result, err = importFileAsTag(ctx, "original", file, file.FileName()) + result, err = importFileAsTag(ctx, "original", file, file.Base()) if err != nil { return err } diff --git a/workflows/normalize_audio.go b/workflows/normalize_audio.go index 7d4774da..2e4c8e34 100644 --- a/workflows/normalize_audio.go +++ b/workflows/normalize_audio.go @@ -77,7 +77,7 @@ func NormalizeAudioLevelWorkflow( return nil, err } - out.FilePath = adjustResult.OutputPath.LocalPath() + out.FilePath = adjustResult.OutputPath.Local() if params.PerformOutputAnalysis { r128Result := &common.AnalyzeEBUR128Result{} diff --git a/workflows/transcode_preview-file.go b/workflows/transcode_preview-file.go index 7ed04c05..f9feb445 100644 --- a/workflows/transcode_preview-file.go +++ b/workflows/transcode_preview-file.go @@ -48,7 +48,7 @@ func TranscodePreviewFile( previewResponse := &activities.TranscodePreviewResponse{} err = workflow.ExecuteActivity(ctx, activities.TranscodePreview, activities.TranscodePreviewParams{ FilePath: filePath, - DestinationDirPath: paths.MustParsePath(filepath.Dir(filePath.LocalPath())), + DestinationDirPath: paths.MustParsePath(filepath.Dir(filePath.Local())), }).Get(ctx, previewResponse) if err != nil { diff --git a/workflows/transcribe-vx.go b/workflows/transcribe-vx.go index 3e5076ab..6b832686 100644 --- a/workflows/transcribe-vx.go +++ b/workflows/transcribe-vx.go @@ -134,7 +134,7 @@ func TranscribeVX( return err } - txtValue, err := os.ReadFile(transcriptionJob.TXTPath.LocalPath()) + txtValue, err := os.ReadFile(transcriptionJob.TXTPath.Local()) if err != nil { return err } From bd41f7ce8de5a3710b86d328c635a854b695e6c9 Mon Sep 17 00:00:00 2001 From: Fredrik Vedvik Date: Thu, 9 Nov 2023 09:58:45 +0100 Subject: [PATCH 3/7] feat(paths): replace filepath.Base with .Base() --- activities/transcribe.go | 2 +- services/transcode/audio.go | 6 +++--- services/transcode/linear_normalize.go | 2 +- services/transcode/playout_mux.go | 2 +- services/transcode/video.go | 2 +- utils/wfutils/files.go | 2 +- workflows/export/mux_files.go | 10 +++++----- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/activities/transcribe.go b/activities/transcribe.go index 64576ee0..fe8fc413 100644 --- a/activities/transcribe.go +++ b/activities/transcribe.go @@ -40,7 +40,7 @@ func Transcribe( log.Info("Finished Transcribe") - fileName := filepath.Base(input.File.Local()) + fileName := input.File.Base() return &TranscribeResponse{ JSONPath: paths.MustParsePath(filepath.Join(jobData.OutputPath, fileName+".json")), SRTPath: paths.MustParsePath(filepath.Join(jobData.OutputPath, fileName+".srt")), diff --git a/services/transcode/audio.go b/services/transcode/audio.go index 7ae0c8a8..caa19df1 100644 --- a/services/transcode/audio.go +++ b/services/transcode/audio.go @@ -21,7 +21,7 @@ func AudioAac(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi "-b:a", input.Bitrate, } - outputFilePath := filepath.Join(input.DestinationPath.Local(), filepath.Base(input.Path.Local())) + outputFilePath := filepath.Join(input.DestinationPath.Local(), input.Path.Base()) outputFilePath = fmt.Sprintf("%s-%s.aac", outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))], input.Bitrate) params = append(params, "-y", outputFilePath) @@ -60,7 +60,7 @@ func AudioWav(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi "-i", input.Path.Local(), } - outputFilePath := filepath.Join(input.DestinationPath.Local(), filepath.Base(input.Path.Local())) + outputFilePath := filepath.Join(input.DestinationPath.Local(), input.Path.Base()) outputFilePath = fmt.Sprintf("%s-%s.wav", outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))], input.Bitrate) params = append(params, "-y", outputFilePath) @@ -129,7 +129,7 @@ func AudioMP3(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi "-q:a", fmt.Sprint(getQfactorFromBitrate(input.Bitrate)), } - outputFilePath := filepath.Join(input.DestinationPath.Local(), filepath.Base(input.Path.Local())) + outputFilePath := filepath.Join(input.DestinationPath.Local(), input.Path.Base()) outputFilePath = fmt.Sprintf("%s-%s.mp3", outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))], input.Bitrate) params = append(params, "-y", outputFilePath) diff --git a/services/transcode/linear_normalize.go b/services/transcode/linear_normalize.go index d72d0b17..8107040e 100644 --- a/services/transcode/linear_normalize.go +++ b/services/transcode/linear_normalize.go @@ -13,7 +13,7 @@ import ( // AdjustAudioLevel adjusts the audio level of the input file by the given adjustment in dB // without changing the dynamic range. This function does not protect against clipping! func AdjustAudioLevel(input common.AudioInput, adjustment float64, cb ffmpeg.ProgressCallback) (*common.AudioResult, error) { - outputFilePath := filepath.Join(input.DestinationPath.Local(), filepath.Base(input.Path.Local())) + outputFilePath := filepath.Join(input.DestinationPath.Local(), input.Path.Base()) outputFilePath = outputFilePath[:len(outputFilePath)-len(filepath.Ext(outputFilePath))] + "_normalized" + filepath.Ext(outputFilePath) params := []string{ diff --git a/services/transcode/playout_mux.go b/services/transcode/playout_mux.go index fdfc0f12..b729725c 100644 --- a/services/transcode/playout_mux.go +++ b/services/transcode/playout_mux.go @@ -221,7 +221,7 @@ func generateFFmpegParamsForPlayoutMux(input common.PlayoutMuxInput, outputPath } func PlayoutMux(input common.PlayoutMuxInput, progressCallback ffmpeg.ProgressCallback) (*common.PlayoutMuxResult, error) { - base := filepath.Base(input.VideoFilePath.Local()) + base := input.VideoFilePath.Base() fileNameWithoutExtension := base[:len(base)-len(filepath.Ext(base))] outputFilePath := filepath.Join(input.OutputDir.Local(), fileNameWithoutExtension+".mxf") diff --git a/services/transcode/video.go b/services/transcode/video.go index e33d9fe2..aaa1220e 100644 --- a/services/transcode/video.go +++ b/services/transcode/video.go @@ -86,7 +86,7 @@ func VideoH264(input common.VideoInput, cb ffmpeg.ProgressCallback) (*common.Vid "-map", "[out]", ) - filename := filepath.Base(input.Path.Local()) + filename := input.Path.Base() filename = filename[:len(filename)-len(filepath.Ext(filename))] + fmt.Sprintf("_%dx%d.mp4", input.Width, input.Height) diff --git a/utils/wfutils/files.go b/utils/wfutils/files.go index 498369ad..98ea371d 100644 --- a/utils/wfutils/files.go +++ b/utils/wfutils/files.go @@ -34,7 +34,7 @@ func MoveFile(ctx workflow.Context, source, destination paths.Path) error { } func MoveToFolder(ctx workflow.Context, file, folder paths.Path) (paths.Path, error) { - newPath := folder.Append(filepath.Base(file.Local())) + newPath := folder.Append(file.Base()) err := MoveFile(ctx, file, newPath) return newPath, err } diff --git a/workflows/export/mux_files.go b/workflows/export/mux_files.go index 135ee9e5..0516500d 100644 --- a/workflows/export/mux_files.go +++ b/workflows/export/mux_files.go @@ -78,7 +78,7 @@ func getSubtitlesResult(params MuxFilesParams) []smil.TextStream { for _, language := range subtitleLanguages { path := params.SubtitleFiles[language.ISO6391] subtitles = append(subtitles, smil.TextStream{ - Src: filepath.Base(path.Local()), + Src: path.Base(), SystemLanguage: language.ISO6391, SubtitleName: language.LanguageNameSystem, }) @@ -95,7 +95,7 @@ func startFileTasks(ctx workflow.Context, params MuxFilesParams, languages []bcc key := lang.ISO6391 fileTasks[key] = map[string]workflow.Future{} for _, q := range fileQualities { - base := filepath.Base(params.VideoFiles[q].Local()) + base := params.VideoFiles[q].Base() fileName := base[:len(base)-len(filepath.Ext(base))] + "-" + key fileTasks[key][q] = wfutils.ExecuteWithQueue(ctx, activities.TranscodeMux, common.MuxInput{ FileName: fileName, @@ -130,7 +130,7 @@ func waitForFileTasks(ctx workflow.Context, params MuxFilesParams, tasks map[str Resolution: q, AudioLanguage: code, Mime: "video/mp4", - Path: filepath.Base(result.Path.Local()), + Path: result.Path.Base(), }) } } @@ -165,7 +165,7 @@ func startStreamTasks(ctx workflow.Context, params MuxFilesParams, languages map audioFilePaths[key] = params.AudioFiles[key] } - base := filepath.Base(path.Local()) + base := path.Base() fileName := base[:len(base)-len(filepath.Ext(base))] tasks[q] = workflow.ExecuteActivity(ctx, activities.TranscodeMux, common.MuxInput{ @@ -191,7 +191,7 @@ func waitForStreamTasks(ctx workflow.Context, tasks map[string]workflow.Future, fileLanguages := languages[q] streams = append(streams, smil.Video{ - Src: filepath.Base(result.Path.Local()), + Src: result.Path.Base(), IncludeAudio: fmt.Sprintf("%t", len(fileLanguages) > 0), SystemLanguage: strings.Join(lo.Map(fileLanguages, func(i bccmflows.Language, _ int) string { return i.ISO6391 From db40d4fff57a508c10d2aebe2120e41ca94beb3d Mon Sep 17 00:00:00 2001 From: Fredrik Vedvik Date: Thu, 9 Nov 2023 10:19:20 +0100 Subject: [PATCH 4/7] feat(paths): implement custom serializer for Drive and rename parse functions --- activities/files.go | 4 ++-- activities/preview.go | 2 +- activities/subtrans.go | 2 +- activities/transcode.go | 6 +++--- activities/transcribe.go | 6 +++--- activities/vidispine/meta.go | 2 +- paths/paths.go | 29 +++++++++++++++++++++++--- paths/paths_test.go | 2 +- services/transcode/audio.go | 6 +++--- services/transcode/linear_normalize.go | 2 +- services/transcode/merge.go | 6 +++--- services/transcode/mux.go | 2 +- services/transcode/playout_mux.go | 2 +- services/transcode/video.go | 2 +- services/transcode/video_test.go | 6 +++--- utils/wfutils/files.go | 4 ++-- workflows/export/merge_export_data.go | 6 +++--- workflows/export/vx_export_vod.go | 2 +- workflows/ingest/asset_ingest.go | 6 +++--- workflows/normalize_audio.go | 2 +- workflows/transcode_preview-file.go | 4 ++-- workflows/transcribe-file.go | 4 ++-- workflows/watch_folder_transcode.go | 2 +- 23 files changed, 66 insertions(+), 43 deletions(-) diff --git a/activities/files.go b/activities/files.go index 011d524a..ccbbc83d 100644 --- a/activities/files.go +++ b/activities/files.go @@ -53,7 +53,7 @@ func StandardizeFileName(ctx context.Context, input FileInput) (*FileResult, err } _ = os.Chmod(path, os.ModePerm) return &FileResult{ - Path: paths.MustParsePath(path), + Path: paths.MustParse(path), }, nil } @@ -113,7 +113,7 @@ func ListFiles(ctx context.Context, input FileInput) ([]paths.Path, error) { return nil, err } return lo.Map(files, func(i string, _ int) paths.Path { - return paths.MustParsePath(i) + return paths.MustParse(i) }), err } diff --git a/activities/preview.go b/activities/preview.go index 86e9d5f1..7db8ccf3 100644 --- a/activities/preview.go +++ b/activities/preview.go @@ -36,7 +36,7 @@ func TranscodePreview(ctx context.Context, input TranscodePreviewParams) (*Trans } return &TranscodePreviewResponse{ - PreviewFilePath: paths.MustParsePath(result.LowResolutionPath), + PreviewFilePath: paths.MustParse(result.LowResolutionPath), AudioOnly: result.AudioOnly, }, nil } diff --git a/activities/subtrans.go b/activities/subtrans.go index 5f60f2ea..fb2fc289 100644 --- a/activities/subtrans.go +++ b/activities/subtrans.go @@ -124,7 +124,7 @@ func GetSubtitlesActivity(ctx context.Context, params GetSubtitlesInput) (map[st if err != nil { return nil, err } - out[lang] = paths.MustParsePath(path) + out[lang] = paths.MustParse(path) } return out, nil diff --git a/activities/transcode.go b/activities/transcode.go index 4f979b1f..8a53a835 100644 --- a/activities/transcode.go +++ b/activities/transcode.go @@ -42,7 +42,7 @@ func TranscodeToProResActivity(ctx context.Context, input EncodeParams) (*Encode } return &EncodeResult{ - OutputPath: paths.MustParsePath(transcodeResult.OutputPath), + OutputPath: paths.MustParse(transcodeResult.OutputPath), }, nil } @@ -66,7 +66,7 @@ func TranscodeToH264Activity(ctx context.Context, input EncodeParams) (*EncodeRe } return &EncodeResult{ - OutputPath: paths.MustParsePath(transcodeResult.Path), + OutputPath: paths.MustParse(transcodeResult.Path), }, nil } @@ -90,7 +90,7 @@ func TranscodeToXDCAMActivity(ctx context.Context, input EncodeParams) (*EncodeR } return &EncodeResult{ - OutputPath: paths.MustParsePath(transcodeResult.Path), + OutputPath: paths.MustParse(transcodeResult.Path), }, nil } diff --git a/activities/transcribe.go b/activities/transcribe.go index fe8fc413..4f837d0e 100644 --- a/activities/transcribe.go +++ b/activities/transcribe.go @@ -42,8 +42,8 @@ func Transcribe( fileName := input.File.Base() return &TranscribeResponse{ - JSONPath: paths.MustParsePath(filepath.Join(jobData.OutputPath, fileName+".json")), - SRTPath: paths.MustParsePath(filepath.Join(jobData.OutputPath, fileName+".srt")), - TXTPath: paths.MustParsePath(filepath.Join(jobData.OutputPath, fileName+".txt")), + JSONPath: paths.MustParse(filepath.Join(jobData.OutputPath, fileName+".json")), + SRTPath: paths.MustParse(filepath.Join(jobData.OutputPath, fileName+".srt")), + TXTPath: paths.MustParse(filepath.Join(jobData.OutputPath, fileName+".txt")), }, nil } diff --git a/activities/vidispine/meta.go b/activities/vidispine/meta.go index a5f2325b..0e5ef8e3 100644 --- a/activities/vidispine/meta.go +++ b/activities/vidispine/meta.go @@ -36,7 +36,7 @@ func GetFileFromVXActivity(ctx context.Context, params GetFileFromVXParams) (*Ge } return &GetFileFromVXResult{ - FilePath: paths.MustParsePath(shape.GetPath()), + FilePath: paths.MustParse(shape.GetPath()), ShapeTag: tag, }, nil } diff --git a/paths/paths.go b/paths/paths.go index 89d6900f..4c6ee038 100644 --- a/paths/paths.go +++ b/paths/paths.go @@ -1,6 +1,7 @@ package paths import ( + "encoding/json" "github.com/ansel1/merry/v2" "github.com/bcc-code/bccm-flows/environment" "github.com/orsinium-labs/enum" @@ -26,6 +27,26 @@ func FixFilename(path string) string { type Drive enum.Member[string] +//goland:noinspection GoMixedReceiverTypes +func (d Drive) MarshalJSON() ([]byte, error) { + return json.Marshal(d.Value) +} + +//goland:noinspection GoMixedReceiverTypes +func (d *Drive) UnmarshalJSON(value []byte) error { + var stringValue string + err := json.Unmarshal(value, &stringValue) + if err != nil { + return err + } + drive := Drives.Parse(stringValue) + if drive == nil { + return ErrDriveNotFound + } + *d = *drive + return nil +} + var ( IsilonDrive = Drive{Value: "isilon"} TempDrive = Drive{Value: "temp"} @@ -35,6 +56,7 @@ var ( ErrPathNotValid = merry.Sentinel("path not valid") ) +//goland:noinspection GoMixedReceiverTypes func (d Drive) RcloneName() string { switch d { case IsilonDrive: @@ -45,6 +67,7 @@ func (d Drive) RcloneName() string { return "" } +//goland:noinspection GoMixedReceiverTypes func (d Drive) RclonePath() string { switch d { case IsilonDrive: @@ -108,7 +131,7 @@ var drivePrefixes = map[Drive]prefix{ TempDrive: {"/mnt/temp/", environment.GetTempMountPrefix(), "isilon:temp/"}, } -func ParsePath(path string) (Path, error) { +func Parse(path string) (Path, error) { for drive, ps := range drivePrefixes { prefixes := []string{ps.Linux, ps.Client, ps.Rclone} for _, p := range prefixes { @@ -123,8 +146,8 @@ func ParsePath(path string) (Path, error) { return Path{}, ErrPathNotValid } -func MustParsePath(path string) Path { - p, err := ParsePath(path) +func MustParse(path string) Path { + p, err := Parse(path) if err != nil { panic(err) } diff --git a/paths/paths_test.go b/paths/paths_test.go index bd316729..9a2ad3e9 100644 --- a/paths/paths_test.go +++ b/paths/paths_test.go @@ -19,7 +19,7 @@ func Test_GetSiblingFolder(t *testing.T) { func Test_ParsePath(t *testing.T) { pathString := "/mnt/isilon/test.xml" - path, err := ParsePath(pathString) + path, err := Parse(pathString) assert.Nil(t, err) diff --git a/services/transcode/audio.go b/services/transcode/audio.go index caa19df1..a9338fe7 100644 --- a/services/transcode/audio.go +++ b/services/transcode/audio.go @@ -41,7 +41,7 @@ func AudioAac(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi return nil, err } - outputPath, err := paths.ParsePath(outputFilePath) + outputPath, err := paths.Parse(outputFilePath) if err != nil { return nil, err } @@ -80,7 +80,7 @@ func AudioWav(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi return nil, err } - outputPath, err := paths.ParsePath(outputFilePath) + outputPath, err := paths.Parse(outputFilePath) if err != nil { return nil, err } @@ -149,7 +149,7 @@ func AudioMP3(input common.AudioInput, cb ffmpeg.ProgressCallback) (*common.Audi return nil, err } - outputPath, err := paths.ParsePath(outputFilePath) + outputPath, err := paths.Parse(outputFilePath) if err != nil { return nil, err } diff --git a/services/transcode/linear_normalize.go b/services/transcode/linear_normalize.go index 8107040e..c289325c 100644 --- a/services/transcode/linear_normalize.go +++ b/services/transcode/linear_normalize.go @@ -40,7 +40,7 @@ func AdjustAudioLevel(input common.AudioInput, adjustment float64, cb ffmpeg.Pro return nil, err } - outputPath, err := paths.ParsePath(outputFilePath) + outputPath, err := paths.Parse(outputFilePath) if err != nil { return nil, err } diff --git a/services/transcode/merge.go b/services/transcode/merge.go index cd2198c9..4b3ed1aa 100644 --- a/services/transcode/merge.go +++ b/services/transcode/merge.go @@ -96,7 +96,7 @@ func MergeVideo(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac return nil, err } - outputPath, err := paths.ParsePath(outputFilePath) + outputPath, err := paths.Parse(outputFilePath) if err != nil { return nil, err } @@ -200,7 +200,7 @@ func MergeAudio(input common.MergeInput, progressCallback ffmpeg.ProgressCallbac return nil, err } - outputPath, err := paths.ParsePath(outputFilePath) + outputPath, err := paths.Parse(outputFilePath) if err != nil { return nil, err } @@ -293,7 +293,7 @@ func MergeSubtitles(input common.MergeInput, progressCallback ffmpeg.ProgressCal return nil, err } - outputPath, err := paths.ParsePath(outputFilePath) + outputPath, err := paths.Parse(outputFilePath) if err != nil { return nil, err } diff --git a/services/transcode/mux.go b/services/transcode/mux.go index 208f3d6d..40885c47 100644 --- a/services/transcode/mux.go +++ b/services/transcode/mux.go @@ -106,7 +106,7 @@ func Mux(input common.MuxInput, progressCallback ffmpeg.ProgressCallback) (*comm return nil, err } - outputPath, err := paths.ParsePath(outputFilePath) + outputPath, err := paths.Parse(outputFilePath) if err != nil { return nil, err } diff --git a/services/transcode/playout_mux.go b/services/transcode/playout_mux.go index b729725c..09bb23a5 100644 --- a/services/transcode/playout_mux.go +++ b/services/transcode/playout_mux.go @@ -244,7 +244,7 @@ func PlayoutMux(input common.PlayoutMuxInput, progressCallback ffmpeg.ProgressCa return nil, err } - outputPath, err := paths.ParsePath(outputFilePath) + outputPath, err := paths.Parse(outputFilePath) if err != nil { return nil, err } diff --git a/services/transcode/video.go b/services/transcode/video.go index aaa1220e..8e7e6d9c 100644 --- a/services/transcode/video.go +++ b/services/transcode/video.go @@ -106,7 +106,7 @@ func VideoH264(input common.VideoInput, cb ffmpeg.ProgressCallback) (*common.Vid return nil, err } - outputPath, err := paths.ParsePath(outputFilePath) + outputPath, err := paths.Parse(outputFilePath) if err != nil { return nil, err } diff --git a/services/transcode/video_test.go b/services/transcode/video_test.go index 1a5d3d4b..74e3e533 100644 --- a/services/transcode/video_test.go +++ b/services/transcode/video_test.go @@ -11,10 +11,10 @@ func Test_Video(t *testing.T) { p, stop := printProgress() defer close(stop) - wm := paths.MustParsePath("/mnt/isilon/system/assets/BTV_LOGO_WATERMARK_BUG_GFX_1080.png") + wm := paths.MustParse("/mnt/isilon/system/assets/BTV_LOGO_WATERMARK_BUG_GFX_1080.png") _, err := VideoH264(common.VideoInput{ - Path: paths.MustParsePath("/mnt/isilon/system/tmp/workflows/07c4a523-ee62-481b-b473-80cc82fffb0a/SOTM_7v2123_SEQ.mxf"), - DestinationPath: paths.MustParsePath("/mnt/isilon/system/tmp/workflows/07c4a523-ee62-481b-b473-80cc82fffb0a"), + Path: paths.MustParse("/mnt/isilon/system/tmp/workflows/07c4a523-ee62-481b-b473-80cc82fffb0a/SOTM_7v2123_SEQ.mxf"), + DestinationPath: paths.MustParse("/mnt/isilon/system/tmp/workflows/07c4a523-ee62-481b-b473-80cc82fffb0a"), Bitrate: "1M", Width: 1280, Height: 720, diff --git a/utils/wfutils/files.go b/utils/wfutils/files.go index 98ea371d..d663efa6 100644 --- a/utils/wfutils/files.go +++ b/utils/wfutils/files.go @@ -85,7 +85,7 @@ func GetWorkflowIsilonOutputFolder(ctx workflow.Context, root string) (paths.Pat path := filepath.Join(environment.GetIsilonPrefix(), "Production", root, fmt.Sprintf("%d/%d/%d", date.Year(), date.Month(), date.Day()), info.OriginalRunID) - return paths.MustParsePath(path), CreateFolder(ctx, paths.MustParsePath(path)) + return paths.MustParse(path), CreateFolder(ctx, paths.MustParse(path)) } func GetWorkflowMastersOutputFolder(ctx workflow.Context) (paths.Path, error) { @@ -105,7 +105,7 @@ func GetWorkflowTempFolder(ctx workflow.Context) (paths.Path, error) { path := filepath.Join(environment.GetTempMountPrefix(), "workflows", info.OriginalRunID) - return paths.MustParsePath(path), CreateFolder(ctx, paths.MustParsePath(path)) + return paths.MustParse(path), CreateFolder(ctx, paths.MustParse(path)) } // GetMapKeysSafely makes sure that the order of the keys returned are identical to other workflow executions. diff --git a/workflows/export/merge_export_data.go b/workflows/export/merge_export_data.go index 4e4939b8..c03b153e 100644 --- a/workflows/export/merge_export_data.go +++ b/workflows/export/merge_export_data.go @@ -137,7 +137,7 @@ func exportDataToMergeInputs(data *vidispine.ExportData, tempDir, subtitlesDir p for _, clip := range data.Clips { mergeInput.Duration += clip.OutSeconds - clip.InSeconds mergeInput.Items = append(mergeInput.Items, common.MergeInputItem{ - Path: paths.MustParsePath(clip.VideoFile), + Path: paths.MustParse(clip.VideoFile), Start: clip.InSeconds, End: clip.OutSeconds, }) @@ -153,7 +153,7 @@ func exportDataToMergeInputs(data *vidispine.ExportData, tempDir, subtitlesDir p audioMergeInputs[lan].Duration += clip.OutSeconds - clip.InSeconds audioMergeInputs[lan].Items = append(audioMergeInputs[lan].Items, common.MergeInputItem{ - Path: paths.MustParsePath(af.File), + Path: paths.MustParse(af.File), Start: clip.InSeconds, End: clip.OutSeconds, Streams: af.Streams, @@ -171,7 +171,7 @@ func exportDataToMergeInputs(data *vidispine.ExportData, tempDir, subtitlesDir p subtitleMergeInputs[lan].Duration += clip.OutSeconds - clip.InSeconds subtitleMergeInputs[lan].Items = append(subtitleMergeInputs[lan].Items, common.MergeInputItem{ - Path: paths.MustParsePath(sf), + Path: paths.MustParse(sf), Start: clip.InSeconds, End: clip.OutSeconds, }) diff --git a/workflows/export/vx_export_vod.go b/workflows/export/vx_export_vod.go index 46e7c1c0..b137ba24 100644 --- a/workflows/export/vx_export_vod.go +++ b/workflows/export/vx_export_vod.go @@ -42,7 +42,7 @@ func VXExportToVOD(ctx workflow.Context, params VXExportChildWorkflowParams) (*V var result PrepareFilesResult var wm *paths.Path if params.ParentParams.WatermarkPath != "" { - path, err := paths.ParsePath(params.ParentParams.WatermarkPath) + path, err := paths.Parse(params.ParentParams.WatermarkPath) if err != nil { return nil, err } diff --git a/workflows/ingest/asset_ingest.go b/workflows/ingest/asset_ingest.go index c0917047..a98b6fd6 100644 --- a/workflows/ingest/asset_ingest.go +++ b/workflows/ingest/asset_ingest.go @@ -38,7 +38,7 @@ func Asset(ctx workflow.Context, params AssetParams) (*AssetResult, error) { options := wfutils.GetDefaultActivityOptions() ctx = workflow.WithActivityOptions(ctx, options) - xmlPath, err := paths.ParsePath(params.XMLPath) + xmlPath, err := paths.Parse(params.XMLPath) if err != nil { return nil, err } @@ -108,7 +108,7 @@ func copyToDir(ctx workflow.Context, dest paths.Path, files []ingest.File) error return fmt.Errorf("multiple directories not supported: %s", dirs) } - dir, err := paths.ParsePath(filepath.Join("/mnt/dmzshare", "workflow", dirs[0])) + dir, err := paths.Parse(filepath.Join("/mnt/dmzshare", "workflow", dirs[0])) if err != nil { return err } @@ -124,7 +124,7 @@ func copyToDir(ctx workflow.Context, dest paths.Path, files []ingest.File) error for _, file := range files { err = wfutils.DeletePath( ctx, - paths.MustParsePath(filepath.Join("/mnt/dmzshare", "workflow", file.FilePath, file.FileName)), + paths.MustParse(filepath.Join("/mnt/dmzshare", "workflow", file.FilePath, file.FileName)), ) if err != nil { return err diff --git a/workflows/normalize_audio.go b/workflows/normalize_audio.go index 2e4c8e34..9238887c 100644 --- a/workflows/normalize_audio.go +++ b/workflows/normalize_audio.go @@ -47,7 +47,7 @@ func NormalizeAudioLevelWorkflow( logger.Info("Starting NormalizeAudio workflow") - filePath, err := paths.ParsePath(params.FilePath) + filePath, err := paths.Parse(params.FilePath) if err != nil { return nil, err } diff --git a/workflows/transcode_preview-file.go b/workflows/transcode_preview-file.go index f9feb445..5ccb0159 100644 --- a/workflows/transcode_preview-file.go +++ b/workflows/transcode_preview-file.go @@ -40,7 +40,7 @@ func TranscodePreviewFile( logger.Info("Starting TranscodePreviewFile") - filePath, err := paths.ParsePath(params.FilePath) + filePath, err := paths.Parse(params.FilePath) if err != nil { return err } @@ -48,7 +48,7 @@ func TranscodePreviewFile( previewResponse := &activities.TranscodePreviewResponse{} err = workflow.ExecuteActivity(ctx, activities.TranscodePreview, activities.TranscodePreviewParams{ FilePath: filePath, - DestinationDirPath: paths.MustParsePath(filepath.Dir(filePath.Local())), + DestinationDirPath: paths.MustParse(filepath.Dir(filePath.Local())), }).Get(ctx, previewResponse) if err != nil { diff --git a/workflows/transcribe-file.go b/workflows/transcribe-file.go index 6b0cd82b..15c19b6e 100644 --- a/workflows/transcribe-file.go +++ b/workflows/transcribe-file.go @@ -50,7 +50,7 @@ func TranscribeFile( return err } - file, err := paths.ParsePath(params.File) + file, err := paths.Parse(params.File) if err != nil { return err } @@ -61,7 +61,7 @@ func TranscribeFile( DestinationPath: tempFolder, }).Get(ctx, &wavFile) - destination, err := paths.ParsePath(params.DestinationPath) + destination, err := paths.Parse(params.DestinationPath) if err != nil { return err } diff --git a/workflows/watch_folder_transcode.go b/workflows/watch_folder_transcode.go index 45505e36..9c106d05 100644 --- a/workflows/watch_folder_transcode.go +++ b/workflows/watch_folder_transcode.go @@ -35,7 +35,7 @@ func WatchFolderTranscode(ctx workflow.Context, params WatchFolderTranscodeInput logger.Info("Starting WatchFolderTranscode") - path, err := paths.ParsePath(params.Path) + path, err := paths.Parse(params.Path) if err != nil { return err } From 32e6921c31a636066f342187417c747e6edc50e2 Mon Sep 17 00:00:00 2001 From: Fredrik Vedvik Date: Thu, 9 Nov 2023 10:22:55 +0100 Subject: [PATCH 5/7] fix(subtrans): use correct path package --- activities/subtrans.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/activities/subtrans.go b/activities/subtrans.go index fb2fc289..fe8c3dfb 100644 --- a/activities/subtrans.go +++ b/activities/subtrans.go @@ -6,7 +6,7 @@ import ( "github.com/bcc-code/bccm-flows/paths" "net/url" "os" - "path" + "path/filepath" "strings" "github.com/bcc-code/bccm-flows/activities/vidispine" @@ -58,7 +58,7 @@ func GetSubtransIDActivity(ctx context.Context, input *GetSubtransIDInput) (*Get } // Extract file name - fileName := path.Base(parsedUri.Path) + fileName := filepath.Base(parsedUri.Path) // Split by dot fileNameSplit := strings.Split(fileName, ".") @@ -119,13 +119,12 @@ func GetSubtitlesActivity(ctx context.Context, params GetSubtitlesInput) (map[st out := map[string]paths.Path{} for lang, sub := range subs { - path := path.Join(params.DestinationFolder.Local(), params.FilePrefix+lang+"."+params.Format) + path := filepath.Join(params.DestinationFolder.Local(), params.FilePrefix+lang+"."+params.Format) err := os.WriteFile(path, []byte(sub), 0644) if err != nil { return nil, err } out[lang] = paths.MustParse(path) - } return out, nil } From fa2db18a39cf792c8dade6650314c4f56dbe66ce Mon Sep 17 00:00:00 2001 From: Fredrik Vedvik Date: Thu, 9 Nov 2023 10:23:23 +0100 Subject: [PATCH 6/7] fix(paths-test): use mustparse --- services/transcode/mux_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/services/transcode/mux_test.go b/services/transcode/mux_test.go index fcfd8e7f..fa11ea91 100644 --- a/services/transcode/mux_test.go +++ b/services/transcode/mux_test.go @@ -3,6 +3,7 @@ package transcode import ( "fmt" "github.com/bcc-code/bccm-flows/common" + "github.com/bcc-code/bccm-flows/paths" "github.com/bcc-code/bccm-flows/services/ffmpeg" "github.com/stretchr/testify/assert" "testing" @@ -39,12 +40,12 @@ func Test_MuxVideo(t *testing.T) { printer, stop := printProgress() defer close(stop) _, err := Mux(common.MuxInput{ - DestinationPath: "/Users/fredrikvedvik/Desktop/Transcoding/test/", - VideoFilePath: root + "SOTM_7v2123_SEQ.mxf", - AudioFilePaths: map[string]string{ - "nor": root + "SOTM_7v2123_SEQ-nor.wav", - "eng": root + "SOTM_7v2123_SEQ-eng.wav", - "nld": root + "SOTM_7v2123_SEQ-nld.wav", + DestinationPath: paths.MustParse("/Users/fredrikvedvik/Desktop/Transcoding/test/"), + VideoFilePath: paths.MustParse(root + "SOTM_7v2123_SEQ.mxf"), + AudioFilePaths: map[string]paths.Path{ + "nor": paths.MustParse(root + "SOTM_7v2123_SEQ-nor.wav"), + "eng": paths.MustParse(root + "SOTM_7v2123_SEQ-eng.wav"), + "nld": paths.MustParse(root + "SOTM_7v2123_SEQ-nld.wav"), }, }, printer) From 162e9f0a9833576e27b02065a64aa3b701ecb981 Mon Sep 17 00:00:00 2001 From: Fredrik Vedvik Date: Thu, 9 Nov 2023 10:36:09 +0100 Subject: [PATCH 7/7] feat(paths): tweak usages of mustparse --- paths/paths.go | 7 +++++++ utils/wfutils/files.go | 8 ++++---- workflows/transcode_preview-file.go | 3 +-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/paths/paths.go b/paths/paths.go index 4c6ee038..3df39328 100644 --- a/paths/paths.go +++ b/paths/paths.go @@ -83,6 +83,13 @@ type Path struct { Path string } +func (p Path) Dir() Path { + return Path{ + Drive: p.Drive, + Path: filepath.Dir(p.Path), + } +} + func (p Path) Local() string { return filepath.Join(drivePrefixes[p.Drive].Client, p.Path) } diff --git a/utils/wfutils/files.go b/utils/wfutils/files.go index d663efa6..49612b08 100644 --- a/utils/wfutils/files.go +++ b/utils/wfutils/files.go @@ -83,9 +83,9 @@ func GetWorkflowIsilonOutputFolder(ctx workflow.Context, root string) (paths.Pat date := time.Now() - path := filepath.Join(environment.GetIsilonPrefix(), "Production", root, fmt.Sprintf("%d/%d/%d", date.Year(), date.Month(), date.Day()), info.OriginalRunID) + path := paths.MustParse(filepath.Join(environment.GetIsilonPrefix(), "Production", root, fmt.Sprintf("%d/%d/%d", date.Year(), date.Month(), date.Day()), info.OriginalRunID)) - return paths.MustParse(path), CreateFolder(ctx, paths.MustParse(path)) + return path, CreateFolder(ctx, path) } func GetWorkflowMastersOutputFolder(ctx workflow.Context) (paths.Path, error) { @@ -103,9 +103,9 @@ func GetWorkflowAuxOutputFolder(ctx workflow.Context) (paths.Path, error) { func GetWorkflowTempFolder(ctx workflow.Context) (paths.Path, error) { info := workflow.GetInfo(ctx) - path := filepath.Join(environment.GetTempMountPrefix(), "workflows", info.OriginalRunID) + path := paths.MustParse(filepath.Join(environment.GetTempMountPrefix(), "workflows", info.OriginalRunID)) - return paths.MustParse(path), CreateFolder(ctx, paths.MustParse(path)) + return path, CreateFolder(ctx, path) } // GetMapKeysSafely makes sure that the order of the keys returned are identical to other workflow executions. diff --git a/workflows/transcode_preview-file.go b/workflows/transcode_preview-file.go index 5ccb0159..edddcf82 100644 --- a/workflows/transcode_preview-file.go +++ b/workflows/transcode_preview-file.go @@ -4,7 +4,6 @@ import ( "github.com/bcc-code/bccm-flows/activities" "github.com/bcc-code/bccm-flows/environment" "github.com/bcc-code/bccm-flows/paths" - "path/filepath" "time" "go.temporal.io/sdk/temporal" @@ -48,7 +47,7 @@ func TranscodePreviewFile( previewResponse := &activities.TranscodePreviewResponse{} err = workflow.ExecuteActivity(ctx, activities.TranscodePreview, activities.TranscodePreviewParams{ FilePath: filePath, - DestinationDirPath: paths.MustParse(filepath.Dir(filePath.Local())), + DestinationDirPath: filePath.Dir(), }).Get(ctx, previewResponse) if err != nil {