Skip to content

Commit

Permalink
fix an issue where trying to upload large transcoded videos to the cl…
Browse files Browse the repository at this point in the history 
…oud would fail
  • Loading branch information
@mszulik
mszulik committed Jul 25, 2024
1 parent 98f0cbd commit 888c3d7
Showing 1 changed file with 45 additions and 30 deletions.
75 changes: 45 additions & 30 deletions app/Jobs/TranscodeVideo.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
use App\Models\UploadSlot;
use App\Models\Version;
use CdnHelper;
use CloudStorage;
use FFMpeg\Format\Video\X264;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Filesystem\Filesystem;
Expand Down Expand Up @@ -48,11 +47,6 @@ class TranscodeVideo implements ShouldQueue

protected string $originalFilePath;
protected string $uploadToken;

// Derivatives are saved to a temporary folder first, else race conditions could cause newer versions to be overwritten.
protected string $tempDerivativesDirectoryPath;
// Mp4 needs to be saved locally after transcoding, before being streamed to the configured disk.
protected string $tempMp4Filename;
// Videos stored in the cloud have to be downloaded for transcoding.
protected string $tempOriginalFilename;
protected string $derivativesDestinationPath;
Expand Down Expand Up @@ -88,7 +82,6 @@ public function handle(): void
$this->originalsDisk = MediaStorage::ORIGINALS->getDisk();
$this->derivativesDisk = MediaStorage::VIDEO_DERIVATIVES->getDisk();
$this->localDisk = Storage::disk('local');
$this->tempMp4Filename = $this->getTempMp4Filename();
$this->tempOriginalFilename = $this->getTempOriginalFilename();

$this->transcodeVideo();
Expand All @@ -112,13 +105,14 @@ public function failed(?Throwable $exception): void
{
// All properties have not yet been initialized, because failed jobs use a new instance.

$tempDerivativesDirectoryPath = $this->getTempVideoDerivativesDirectoryPath();
$localDisk = Storage::disk('local');

MediaStorage::VIDEO_DERIVATIVES->getDisk()->deleteDirectory($tempDerivativesDirectoryPath);
MediaStorage::VIDEO_DERIVATIVES->getDisk()->deleteDirectory($this->getTempDerivativesDirectoryPath());
$localDisk->delete($this->getTempMp4Filename());
$localDisk->delete($this->getTempOriginalFilename());
$localDisk->deleteDirectory($this->getFfmpegTempDirectory());
// This directory stores local temp derivatives in case cloud storage is used.
$localDisk->deleteDirectory($this->version->Media->User->name);

if (!$this->oldVersionNumber) {
// A failed upload must not create a version.
Expand Down Expand Up @@ -159,6 +153,7 @@ protected function transcodeVideo(): void

$this->localDisk->delete($this->tempOriginalFilename);
$this->localDisk->deleteDirectory($this->getFfmpegTempDirectory());
$this->localDisk->deleteDirectory($this->version->Media->User->name);

// Derivatives are generated at this point of time and located in the temporary folder.
$this->moveDerivativesToDestinationPath();
Expand Down Expand Up @@ -211,7 +206,6 @@ protected function openFromCloud(FFMpeg $ffmpeg): StreamingMedia
protected function setFilePaths(): void
{
$this->derivativesDestinationPath = $this->version->Media->baseDirectory();
$this->tempDerivativesDirectoryPath = $this->getTempVideoDerivativesDirectoryPath();
}

/**
Expand All @@ -224,14 +218,26 @@ protected function setFilePaths(): void
*/
protected function saveVideo(Streaming $video, string $format): void
{
$tempDerivativeFilePath = $this->getTempDerivativeFilePath($format);

// Save to temporary folder first, to prevent race conditions when multiple versions are uploaded simultaneously.
$this->isLocalFilesystem($this->derivativesDisk) ?
$video->save($this->derivativesDisk->path($this->getTempVideoDerivativeFilePath($format)))
: $video->save(null,
CloudStorage::getSaveConfiguration(
sprintf('%s/%s', $this->derivativesDisk->path($this->tempDerivativesDirectoryPath), $format), 'video'
)
);
if ($this->isLocalFilesystem($this->derivativesDisk)) {
$video->save($this->derivativesDisk->path($tempDerivativeFilePath));
} else {
// When using cloud storage, we save to local storage first and then upload manually,
// because the php-ffmpeg-video-streaming package direct upload functionality led to errors.
$video->save($this->localDisk->path($tempDerivativeFilePath));

$tempDerivativesFormatDirectoryPath = $this->getTempDerivativesFormatDirectoryPath($format);

foreach ($this->localDisk->allFiles($tempDerivativesFormatDirectoryPath) as $filePath) {
$this->derivativesDisk->writeStream(
$filePath,
$this->localDisk->readStream($filePath));
}

$this->localDisk->deleteDirectory($tempDerivativesFormatDirectoryPath);
}
}

/**
Expand All @@ -245,15 +251,16 @@ protected function saveVideo(Streaming $video, string $format): void
*/
protected function generateMp4(StreamingMedia $video): void
{
$video->save((new X264())->setAdditionalParameters(config('transmorpher.additional_transcoding_parameters')), $this->localDisk->path($this->tempMp4Filename));
$tempMp4Filename = $this->getTempMp4Filename();

$video->save((new X264())->setAdditionalParameters(config('transmorpher.additional_transcoding_parameters')), $this->localDisk->path($tempMp4Filename));

$derivativePath = $this->getTempVideoDerivativeFilePath('mp4');
$this->derivativesDisk->writeStream(
sprintf('%s.%s', $derivativePath, 'mp4'),
$this->localDisk->readStream($this->tempMp4Filename)
sprintf('%s.%s', $this->getTempDerivativeFilePath('mp4'), 'mp4'),
$this->localDisk->readStream($tempMp4Filename)
);

$this->localDisk->delete($this->tempMp4Filename);
$this->localDisk->delete($tempMp4Filename);
}

/**
Expand Down Expand Up @@ -290,7 +297,7 @@ protected function moveDerivativesToDestinationPath(): void
protected function moveFromTempDirectory(): void
{
if ($this->isLocalFilesystem($this->derivativesDisk)) {
$this->derivativesDisk->move($this->tempDerivativesDirectoryPath, $this->derivativesDestinationPath);
$this->derivativesDisk->move($this->getTempDerivativesDirectoryPath(), $this->derivativesDestinationPath);
} else {
$this->moveFromCloudTempDirectory();
}
Expand All @@ -304,8 +311,8 @@ protected function moveFromTempDirectory(): void
*/
protected function moveFromCloudTempDirectory(): void
{
$hlsFiles = $this->derivativesDisk->allFiles(sprintf('%s/%s/', $this->tempDerivativesDirectoryPath, StreamingFormat::HLS->value));
$dashFiles = $this->derivativesDisk->allFiles(sprintf('%s/%s/', $this->tempDerivativesDirectoryPath, StreamingFormat::DASH->value));
$hlsFiles = $this->derivativesDisk->allFiles($this->getTempDerivativesFormatDirectoryPath(StreamingFormat::HLS->value));
$dashFiles = $this->derivativesDisk->allFiles($this->getTempDerivativesFormatDirectoryPath(StreamingFormat::DASH->value));

foreach ($hlsFiles as $file) {
$this->derivativesDisk->move($file, $this->version->Media->videoDerivativeFilePath(StreamingFormat::HLS->value, basename($file)));
Expand All @@ -315,10 +322,9 @@ protected function moveFromCloudTempDirectory(): void
$this->derivativesDisk->move($file, $this->version->Media->videoDerivativeFilePath(StreamingFormat::DASH->value, basename($file)));
}

$tempDerivativePath = $this->getTempVideoDerivativeFilePath('mp4');
// Move MP4 file.
$this->derivativesDisk->move(
sprintf('%s.mp4', $tempDerivativePath),
sprintf('%s.mp4', $this->getTempDerivativeFilePath('mp4')),
sprintf('%s.mp4', $this->version->Media->videoDerivativeFilePath('mp4'))
);
}
Expand Down Expand Up @@ -376,9 +382,9 @@ protected function isMostRecentVersion(): bool
* @param string $format
* @return string
*/
protected function getTempVideoDerivativeFilePath(string $format): string
protected function getTempDerivativeFilePath(string $format): string
{
return sprintf('%s/%s/%s', $this->getTempVideoDerivativesDirectoryPath(), $format, 'video');
return sprintf('%s/%s', $this->getTempDerivativesFormatDirectoryPath($format), 'video');
}

/**
Expand All @@ -387,8 +393,17 @@ protected function getTempVideoDerivativeFilePath(string $format): string
*
* @return string
*/
protected function getTempVideoDerivativesDirectoryPath(): string
protected function getTempDerivativesDirectoryPath(): string
{
return sprintf('%s-%s-temp', $this->version->Media->baseDirectory(), $this->version->getKey());
}

/**
* Get the path to the temporary video derivatives directory for a format.
* Path structure: {username}/{identifier}-{versionKey}-temp/{format}
*/
protected function getTempDerivativesFormatDirectoryPath(string $format): string
{
return sprintf('%s/%s', $this->getTempDerivativesDirectoryPath(), $format);
}
}

0 comments on commit 888c3d7

Please sign in to comment.