Merge pull request #121 from bcc-code/feat/led-material-import

feat(import): implement led material import

paths/paths.go

@@ -98,6 +98,10 @@ func (p Path) Local() string {
 	return filepath.Join(drivePrefixes[p.Drive].Client, p.Path)
 }
 
+func (p Path) Ext() string {
+	return filepath.Ext(p.Path)
+}
+
 // RcloneFsRemote returns (fs, remote) for rclone usage
 func (p Path) RcloneFsRemote() (string, string) {
 	switch p.Drive {

utils/files.go

@@ -1,25 +1,41 @@
 package utils
 
 import (
-	"github.com/samber/lo"
 	"path/filepath"
 	"regexp"
+
+	"github.com/samber/lo"
 )
 
 var alphanumericalRegex = regexp.MustCompile("^[a-zA-Z0-9_]+$")
 
-var supportedExtensions = []string{
-	".mxf",
-	".mov",
+var imageExtensions = []string{
 	".png",
 	".jpg",
 	".jpeg",
+}
+
+var presentationExtensions = []string{
+	".pptx",
+	".key",
+}
+
+var mediaExtensions = []string{
+	".mxf",
+	".mov",
 	".wav",
 }
 
+var supportedExtensions = append(imageExtensions, append(presentationExtensions, mediaExtensions...)...)
+
 func ValidRawFilename(filename string) bool {
 	extension := filepath.Ext(filename)
 	base := filepath.Base(filename)
 	name := base[:len(base)-len(extension)]
 	return alphanumericalRegex.MatchString(name) && lo.Contains(supportedExtensions, extension)
 }
+
+func IsMedia(filename string) bool {
+	extension := filepath.Ext(filename)
+	return lo.Contains(mediaExtensions, extension)
+}

workflows/ingest/asset_ingest.go

@@ -19,7 +19,8 @@ var (
 	OrderFormRawMaterial  = OrderForm{Value: "Rawmaterial"}
 	OrderFormVBMaster     = OrderForm{Value: "VB"}
 	OrderFormSeriesMaster = OrderForm{Value: "Series_Masters"}
-	OrderFormOtherMaster  = OrderForm{Value: "Other_Masters"}
+	OrderFormOtherMaster  = OrderForm{Value: "Other_Masters"} // TODO: set correct value
+	OrderFormLEDMaterial  = OrderForm{Value: "LED-Material"}
 	OrderFormPodcast      = OrderForm{Value: "Podcast"}
 	OrderForms            = enum.New(
 		OrderFormRawMaterial,
@@ -81,14 +82,21 @@ func Asset(ctx workflow.Context, params AssetParams) (*AssetResult, error) {
 	switch *orderForm {
 	case OrderFormRawMaterial:
 		err = workflow.ExecuteChildWorkflow(ctx, RawMaterial, RawMaterialParams{
+			OrderForm: *orderForm,
 			Metadata:  metadata,
 			Directory: fcOutputDir,
 		}).Get(ctx, nil)
-	case OrderFormSeriesMaster, OrderFormOtherMaster, OrderFormVBMaster, OrderFormPodcast:
+	case OrderFormSeriesMaster, OrderFormOtherMaster, OrderFormVBMaster, OrderFormLEDMaterial, OrderFormPodcast:
+		var outputDir paths.Path
+		outputDir, err = wfutils.GetWorkflowMastersOutputFolder(ctx)
+		if err != nil {
+			return nil, err
+		}
 		err = workflow.ExecuteChildWorkflow(ctx, Masters, MasterParams{
 			Metadata:  metadata,
 			OrderForm: *orderForm,
 			Directory: fcOutputDir,
+			OutputDir: outputDir,
 		}).Get(ctx, nil)
 	}
 	if err != nil {

workflows/ingest/common.go

@@ -1,8 +1,10 @@
 package ingestworkflows
 
 import (
+	"github.com/ansel1/merry/v2"
 	vsactivity "github.com/bcc-code/bccm-flows/activities/vidispine"
 	"github.com/bcc-code/bccm-flows/paths"
+	"github.com/bcc-code/bccm-flows/services/ingest"
 	"github.com/bcc-code/bccm-flows/workflows"
 	"go.temporal.io/sdk/workflow"
 )
@@ -35,7 +37,7 @@ func importFileAsTag(ctx workflow.Context, tag string, path paths.Path, title st
 	}, nil
 }
 
-func postImportActions(ctx workflow.Context, assetIDs []string, language string) error {
+func transcodeAndTranscribe(ctx workflow.Context, assetIDs []string, language string) error {
 	var wfFutures []workflow.ChildWorkflowFuture
 	for _, id := range assetIDs {
 		wfFutures = append(wfFutures, workflow.ExecuteChildWorkflow(ctx, workflows.TranscodePreviewVX, workflows.TranscodePreviewVXInput{
@@ -69,3 +71,16 @@ func postImportActions(ctx workflow.Context, assetIDs []string, language string)
 	}
 	return nil
 }
+
+func getOrderFormFilename(orderForm OrderForm, file paths.Path, props ingest.JobProperty) (string, error) {
+	switch orderForm {
+	case OrderFormRawMaterial:
+		// return filename without extension
+		base := file.Base()
+		ext := file.Ext()
+		return base[:len(base)-len(ext)], nil
+	case OrderFormLEDMaterial, OrderFormVBMaster, OrderFormSeriesMaster, OrderFormOtherMaster:
+		return masterFilename(props)
+	}
+	return "", merry.New("Unsupported order form")
+}

workflows/ingest/masters.go

@@ -14,6 +14,7 @@ import (
 	"github.com/bcc-code/bccm-flows/services/baton"
 	"github.com/bcc-code/bccm-flows/services/ingest"
 	"github.com/bcc-code/bccm-flows/services/vidispine/vscommon"
+	"github.com/bcc-code/bccm-flows/utils"
 	"github.com/bcc-code/bccm-flows/utils/workflows"
 	"go.temporal.io/sdk/workflow"
 )
@@ -23,10 +24,11 @@ type MasterParams struct {
 
 	OrderForm OrderForm
 	Directory paths.Path
+	OutputDir paths.Path
 }
 
 type MasterResult struct {
-	Report        baton.QCReport
+	Report        *baton.QCReport
 	AssetID       string
 	AnalyzeResult *common.AnalyzeEBUR128Result
 	Path          paths.Path
@@ -46,17 +48,19 @@ func Masters(ctx workflow.Context, params MasterParams) (*MasterResult, error) {
 		return nil, err
 	}
 
-	// This isn't run on VB masters in old system, but see no reason to not run it here.
-	result.AnalyzeResult, err = analyzeAudioAndSetMetadata(ctx, result.AssetID, result.Path)
-	if err != nil {
-		return nil, err
-	}
-
-	if result.Report.TopLevelInfo.Error == 0 {
-		err = postImportActions(ctx, []string{result.AssetID}, params.Metadata.JobProperty.Language)
+	if utils.IsMedia(result.Path.Local()) {
+		// This isn't run on VB masters in old system, but see no reason to not run it here.
+		result.AnalyzeResult, err = analyzeAudioAndSetMetadata(ctx, result.AssetID, result.Path)
 		if err != nil {
 			return nil, err
 		}
+
+		if result.Report.TopLevelInfo.Error == 0 {
+			err = transcodeAndTranscribe(ctx, []string{result.AssetID}, params.Metadata.JobProperty.Language)
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	return result, nil
@@ -66,7 +70,7 @@ func uploadMaster(ctx workflow.Context, params MasterParams) (*MasterResult, err
 	var filename string
 	var err error
 	switch params.OrderForm {
-	case OrderFormOtherMaster, OrderFormVBMaster, OrderFormSeriesMaster, OrderFormPodcast:
+	case OrderFormOtherMaster, OrderFormVBMaster, OrderFormSeriesMaster, OrderFormLEDMaterial, OrderFormPodcast:
 		filename, err = masterFilename(params.Metadata.JobProperty)
 	default:
 		return nil, fmt.Errorf("unsupported order form: %s", params.OrderForm)
@@ -87,12 +91,7 @@ func uploadMaster(ctx workflow.Context, params MasterParams) (*MasterResult, err
 		return nil, fmt.Errorf("too many files in directory: %s", params.Directory)
 	}
 
-	outputDir, err := wfutils.GetWorkflowMastersOutputFolder(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	file := outputDir.Append(filename)
+	file := params.OutputDir.Append(filename)
 	err = wfutils.MoveFile(ctx, files[0], file)
 	if err != nil {
 		return nil, err
@@ -113,18 +112,19 @@ func uploadMaster(ctx workflow.Context, params MasterParams) (*MasterResult, err
 		return nil, err
 	}
 
-	plan := baton.TestPlanMXF
-	if filepath.Ext(file.Base()) == ".mov" {
-		plan = baton.TestPlanMOV
-	}
-
-	var report baton.QCReport
-	err = wfutils.ExecuteWithQueue(ctx, batonactivities.QC, batonactivities.QCParams{
-		Path: file,
-		Plan: plan,
-	}).Get(ctx, &report)
-	if err != nil {
-		return nil, err
+	var report *baton.QCReport
+	if utils.IsMedia(file.Local()) {
+		plan := baton.TestPlanMXF
+		if filepath.Ext(file.Base()) == ".mov" {
+			plan = baton.TestPlanMOV
+		}
+		err = wfutils.ExecuteWithQueue(ctx, batonactivities.QC, batonactivities.QCParams{
+			Path: file,
+			Plan: plan,
+		}).Get(ctx, &report)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return &MasterResult{

workflows/ingest/raw_material.go

@@ -2,19 +2,18 @@ package ingestworkflows
 
 import (
 	"fmt"
-	"strconv"
 
 	"github.com/bcc-code/bccm-flows/activities"
 	vsactivity "github.com/bcc-code/bccm-flows/activities/vidispine"
 	"github.com/bcc-code/bccm-flows/paths"
 	"github.com/bcc-code/bccm-flows/services/ingest"
-	"github.com/bcc-code/bccm-flows/services/vidispine/vscommon"
 	"github.com/bcc-code/bccm-flows/utils"
 	"github.com/bcc-code/bccm-flows/utils/workflows"
 	"go.temporal.io/sdk/workflow"
 )
 
 type RawMaterialParams struct {
+	OrderForm OrderForm
 	Metadata  *ingest.Metadata
 	Directory paths.Path
 }
@@ -38,15 +37,21 @@ func RawMaterial(ctx workflow.Context, params RawMaterialParams) error {
 		if !utils.ValidRawFilename(f.Local()) {
 			return fmt.Errorf("invalid filename: %s", f)
 		}
-		newPath := outputFolder.Append(f.Base())
+
+		filename, err := getOrderFormFilename(params.OrderForm, f, params.Metadata.JobProperty)
+		if err != nil {
+			return err
+		}
+		newPath := outputFolder.Append(filename + f.Ext())
+
 		err = wfutils.MoveFile(ctx, f, newPath)
 		if err != nil {
 			return err
 		}
 		files = append(files, newPath)
 	}
 
-	var assetAnalyzeTasks = map[string]workflow.Future{}
+	var mediaAnalyzeTasks = map[string]workflow.Future{}
 	var vidispineJobIDs = map[string]string{}
 
 	for _, file := range files {
@@ -56,34 +61,31 @@ func RawMaterial(ctx workflow.Context, params RawMaterialParams) error {
 			return err
 		}
 		vidispineJobIDs[result.AssetID] = result.ImportJobID
-		assetAnalyzeTasks[result.AssetID] = wfutils.ExecuteWithQueue(ctx, activities.AnalyzeFile, activities.AnalyzeFileParams{
-			FilePath: file,
-		})
+
+		err = addMetaTags(ctx, result.AssetID, params.Metadata)
+		if err != nil {
+			return err
+		}
+		if utils.IsMedia(file.Local()) {
+			mediaAnalyzeTasks[result.AssetID] = wfutils.ExecuteWithQueue(ctx, activities.AnalyzeFile, activities.AnalyzeFileParams{
+				FilePath: file,
+			})
+		}
 	}
 
-	assetIDs, err := wfutils.GetMapKeysSafely(ctx, assetAnalyzeTasks)
+	mediaAssetIDs, err := wfutils.GetMapKeysSafely(ctx, mediaAnalyzeTasks)
 	if err != nil {
 		return err
 	}
 
-	for _, id := range assetIDs {
-		task := assetAnalyzeTasks[id]
+	for _, id := range mediaAssetIDs {
+		task := mediaAnalyzeTasks[id]
 		var result activities.AnalyzeFileResult
 		err = task.Get(ctx, &result)
 		if err != nil {
 			return err
 		}
 
-		err = wfutils.SetVidispineMeta(ctx, id, vscommon.FieldUploadedBy.Value, params.Metadata.JobProperty.SenderEmail)
-		if err != nil {
-			return err
-		}
-
-		err = wfutils.SetVidispineMeta(ctx, id, vscommon.FieldUploadJob.Value, strconv.Itoa(params.Metadata.JobProperty.JobID))
-		if err != nil {
-			return err
-		}
-
 		// need to wait for vidispine to import the file before we can create thumbnails
 		err = wfutils.WaitForVidispineJob(ctx, vidispineJobIDs[id])
 		if err != nil {
@@ -100,7 +102,7 @@ func RawMaterial(ctx workflow.Context, params RawMaterialParams) error {
 		}
 	}
 
-	err = postImportActions(ctx, assetIDs, params.Metadata.JobProperty.Language)
+	err = transcodeAndTranscribe(ctx, mediaAssetIDs, params.Metadata.JobProperty.Language)
 	if err != nil {
 		return err
 	}