diff --git a/TwitchDownloaderCore/ChatDownloader.cs b/TwitchDownloaderCore/ChatDownloader.cs index f0d560db..40fb718f 100644 --- a/TwitchDownloaderCore/ChatDownloader.cs +++ b/TwitchDownloaderCore/ChatDownloader.cs @@ -263,10 +263,16 @@ public async Task DownloadAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } throw; diff --git a/TwitchDownloaderCore/ChatRenderer.cs b/TwitchDownloaderCore/ChatRenderer.cs index d9373a07..293ceee2 100644 --- a/TwitchDownloaderCore/ChatRenderer.cs +++ b/TwitchDownloaderCore/ChatRenderer.cs @@ -85,10 +85,16 @@ public async Task RenderVideoAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } if (maskFileInfo is not null) @@ -96,7 +102,11 @@ public async Task RenderVideoAsync(CancellationToken cancellationToken) maskFileInfo.Refresh(); if (maskFileInfo.Exists && maskFileInfo.Length == 0) { - maskFileInfo.Delete(); + try + { + maskFileInfo.Delete(); + } + catch { } } } diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index 0eb70329..5370ea48 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -44,10 +44,16 @@ public async Task UpdateAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } throw; diff --git a/TwitchDownloaderCore/ClipDownloader.cs b/TwitchDownloaderCore/ClipDownloader.cs index bf6a91f6..5d14ad49 100644 --- a/TwitchDownloaderCore/ClipDownloader.cs +++ b/TwitchDownloaderCore/ClipDownloader.cs @@ -43,10 +43,16 @@ public async Task DownloadAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } throw; @@ -101,6 +107,8 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF } finally { + await Task.Delay(100, CancellationToken.None); + File.Delete(tempFile); } diff --git a/TwitchDownloaderCore/Tools/FfmpegConcatList.cs b/TwitchDownloaderCore/Tools/FfmpegConcatList.cs new file mode 100644 index 00000000..374c6dea --- /dev/null +++ b/TwitchDownloaderCore/Tools/FfmpegConcatList.cs @@ -0,0 +1,35 @@ +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace TwitchDownloaderCore.Tools +{ + // https://www.ffmpeg.org/ffmpeg-formats.html#toc-concat-1 + public static class FfmpegConcatList + { + private const string LINE_FEED = "\u000A"; + + public static async Task SerializeAsync(string filePath, M3U8 playlist, Range videoListCrop, CancellationToken cancellationToken = default) + { + await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); + await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; + + await sw.WriteLineAsync("ffconcat version 1.0"); + + foreach (var stream in playlist.Streams.Take(videoListCrop)) + { + cancellationToken.ThrowIfCancellationRequested(); + + await sw.WriteAsync("file '"); + await sw.WriteAsync(stream.Path); + await sw.WriteLineAsync('\''); + + await sw.WriteAsync("duration "); + await sw.WriteLineAsync(stream.PartInfo.Duration.ToString(CultureInfo.InvariantCulture)); + } + } + } +} \ No newline at end of file diff --git a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs index 023f288e..d5065c3f 100644 --- a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs +++ b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs @@ -14,7 +14,7 @@ public static class FfmpegMetadata private const string LINE_FEED = "\u000A"; public static async Task SerializeAsync(string filePath, string streamerName, string videoId, string videoTitle, DateTime videoCreation, int viewCount, string videoDescription = null, - TimeSpan startOffset = default, IEnumerable videoMomentEdges = null, CancellationToken cancellationToken = default) + TimeSpan startOffset = default, TimeSpan videoLength = default, IEnumerable videoMomentEdges = null, CancellationToken cancellationToken = default) { await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; @@ -22,7 +22,7 @@ public static async Task SerializeAsync(string filePath, string streamerName, st await SerializeGlobalMetadata(sw, streamerName, videoId, videoTitle, videoCreation, viewCount, videoDescription); await fs.FlushAsync(cancellationToken); - await SerializeChapters(sw, videoMomentEdges, startOffset); + await SerializeChapters(sw, videoMomentEdges, startOffset, videoLength); await fs.FlushAsync(cancellationToken); } @@ -44,14 +44,13 @@ private static async Task SerializeGlobalMetadata(StreamWriter sw, string stream await sw.WriteLineAsync(@$"Views: {viewCount}"); } - private static async Task SerializeChapters(StreamWriter sw, IEnumerable videoMomentEdges, TimeSpan startOffset) + private static async Task SerializeChapters(StreamWriter sw, IEnumerable videoMomentEdges, TimeSpan startOffset, TimeSpan videoLength) { if (videoMomentEdges is null) { return; } - // Note: FFmpeg automatically handles out of range chapters for us var startOffsetMillis = (int)startOffset.TotalMilliseconds; foreach (var momentEdge in videoMomentEdges) { @@ -64,6 +63,22 @@ private static async Task SerializeChapters(StreamWriter sw, IEnumerable TimeSpan.Zero) + { + var chapterStart = TimeSpan.FromMilliseconds(startMillis); + if (chapterStart >= videoLength) + { + continue; + } + + var chapterEnd = chapterStart + TimeSpan.FromMilliseconds(lengthMillis); + if (chapterEnd > videoLength) + { + lengthMillis = (int)(videoLength - chapterStart).TotalMilliseconds; + } + } + await sw.WriteLineAsync("[CHAPTER]"); await sw.WriteLineAsync("TIMEBASE=1/1000"); await sw.WriteLineAsync($"START={startMillis}"); diff --git a/TwitchDownloaderCore/Tools/M3U8.cs b/TwitchDownloaderCore/Tools/M3U8.cs index 3bc64ad5..15a35831 100644 --- a/TwitchDownloaderCore/Tools/M3U8.cs +++ b/TwitchDownloaderCore/Tools/M3U8.cs @@ -224,7 +224,7 @@ public enum PlaylistType // Twitch specific public uint TwitchLiveSequence { get; private set; } public decimal TwitchElapsedSeconds { get; private set; } - public decimal TwitchTotalSeconds { get; private set; } + public decimal TwitchTotalSeconds { get; internal set; } // Other headers that we don't have dedicated properties for. Useful for debugging. private readonly List> _unparsedValues = new(); diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs index b119eb81..b53295ff 100644 --- a/TwitchDownloaderCore/TsMerger.cs +++ b/TwitchDownloaderCore/TsMerger.cs @@ -38,10 +38,16 @@ public async Task MergeAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } throw; diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 042dfcd9..acc733db 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -51,10 +51,16 @@ public async Task DownloadAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } throw; @@ -69,7 +75,7 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF downloadOptions.TempFolder, $"{downloadOptions.Id}_{DateTimeOffset.UtcNow.Ticks}"); - _progress.SetStatus("Fetching Video Info [1/5]"); + _progress.SetStatus("Fetching Video Info [1/4]"); try { @@ -86,41 +92,34 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF var playlistUrl = qualityPlaylist.Path; var baseUrl = new Uri(playlistUrl[..(playlistUrl.LastIndexOf('/') + 1)], UriKind.Absolute); - var videoLength = TimeSpan.FromSeconds(videoInfoResponse.data.video.lengthSeconds); - CheckAvailableStorageSpace(qualityPlaylist.StreamInfo.Bandwidth, videoLength); + var videoInfo = videoInfoResponse.data.video; + var (playlist, airDate) = await GetVideoPlaylist(playlistUrl, cancellationToken); + + var videoListCrop = GetStreamListTrim(playlist.Streams, videoInfo, out var videoLength, out var startOffset, out var endOffset); - var (playlist, videoListCrop, airDate) = await GetVideoPlaylist(playlistUrl, cancellationToken); + CheckAvailableStorageSpace(qualityPlaylist.StreamInfo.Bandwidth, videoLength); if (Directory.Exists(downloadFolder)) Directory.Delete(downloadFolder, true); TwitchHelper.CreateDirectory(downloadFolder); - _progress.SetTemplateStatus("Downloading {0}% [2/5]", 0); + _progress.SetTemplateStatus("Downloading {0}% [2/4]", 0); await DownloadVideoPartsAsync(playlist.Streams, videoListCrop, baseUrl, downloadFolder, airDate, cancellationToken); - _progress.SetTemplateStatus("Verifying Parts {0}% [3/5]", 0); + _progress.SetTemplateStatus("Verifying Parts {0}% [3/4]", 0); await VerifyDownloadedParts(playlist.Streams, videoListCrop, baseUrl, downloadFolder, airDate, cancellationToken); - _progress.SetTemplateStatus("Combining Parts {0}% [4/5]", 0); - - await CombineVideoParts(downloadFolder, playlist.Streams, videoListCrop, cancellationToken); - - _progress.SetTemplateStatus("Finalizing Video {0}% [5/5]", 0); - - var startOffset = TimeSpan.FromSeconds((double)playlist.Streams - .Take(videoListCrop.Start.Value) - .Sum(x => x.PartInfo.Duration)); - - startOffset = downloadOptions.TrimBeginningTime - startOffset; - var seekDuration = downloadOptions.TrimEndingTime - downloadOptions.TrimBeginningTime; + _progress.SetTemplateStatus("Finalizing Video {0}% [4/4]", 0); string metadataPath = Path.Combine(downloadFolder, "metadata.txt"); - VideoInfo videoInfo = videoInfoResponse.data.video; await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, downloadOptions.Id.ToString(), videoInfo.title, videoInfo.createdAt, videoInfo.viewCount, videoInfo.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(), downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero, - videoChapterResponse.data.video.moments.edges, cancellationToken); + videoLength, videoChapterResponse.data.video.moments.edges, cancellationToken); + + var concatListPath = Path.Combine(downloadFolder, "concat.txt"); + await FfmpegConcatList.SerializeAsync(concatListPath, playlist, videoListCrop, cancellationToken); outputFs.Close(); @@ -128,7 +127,7 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d var ffmpegRetries = 0; do { - ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, outputFileInfo, metadataPath, startOffset, seekDuration > TimeSpan.Zero ? seekDuration : videoLength), cancellationToken); + ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, outputFileInfo, concatListPath, metadataPath, startOffset, endOffset, videoLength), cancellationToken); if (ffmpegExitCode != 0) { _progress.LogError($"Failed to finalize video (code {ffmpegExitCode}), retrying in 10 seconds..."); @@ -147,6 +146,8 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d } finally { + await Task.Delay(100, CancellationToken.None); + if (_shouldClearCache) { Cleanup(downloadFolder); @@ -348,24 +349,55 @@ private static bool VerifyVideoPart(string filePath) return true; } - private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string metadataPath, TimeSpan startOffset, TimeSpan seekDuration) + private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string concatListPath, string metadataPath, decimal startOffset, decimal endOffset, TimeSpan videoLength) { - var process = new Process + using var process = new Process { StartInfo = { FileName = downloadOptions.FfmpegPath, - Arguments = string.Format( - "-hide_banner -stats -y -avoid_negative_ts make_zero " + (downloadOptions.TrimBeginning ? "-ss {2} " : "") + "-i \"{0}\" -i \"{1}\" -map_metadata 1 -analyzeduration {3} -probesize {3} " + (downloadOptions.TrimEnding ? "-t {4} " : "") + "-c:v copy \"{5}\"", - Path.Combine(tempFolder, "output.ts"), metadataPath, startOffset.TotalSeconds.ToString(CultureInfo.InvariantCulture), int.MaxValue, seekDuration.TotalSeconds.ToString(CultureInfo.InvariantCulture), outputFile.FullName), UseShellExecute = false, CreateNoWindow = true, RedirectStandardInput = false, RedirectStandardOutput = true, - RedirectStandardError = true + RedirectStandardError = true, + WorkingDirectory = tempFolder } }; + var args = new List + { + "-stats", + "-y", + "-avoid_negative_ts", "make_zero", + "-analyzeduration", $"{int.MaxValue}", + "-probesize", $"{int.MaxValue}", + "-f", "concat", + "-i", concatListPath, + "-i", metadataPath, + "-map_metadata", "1", + "-c", "copy", + outputFile.FullName + }; + + // TODO: Make this optional - "Safe" and "Exact" trimming methods + if (endOffset > 0) + { + args.Insert(0, "-t"); + args.Insert(1, videoLength.TotalSeconds.ToString(CultureInfo.InvariantCulture)); + } + + if (startOffset > 0) + { + args.Insert(0, "-ss"); + args.Insert(1, startOffset.ToString(CultureInfo.InvariantCulture)); + } + + foreach (var arg in args) + { + process.StartInfo.ArgumentList.Add(arg); + } + var encodingTimeRegex = new Regex(@"(?<=time=)(\d\d):(\d\d):(\d\d)\.(\d\d)", RegexOptions.Compiled); var logQueue = new ConcurrentQueue(); @@ -376,7 +408,7 @@ private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string me logQueue.Enqueue(e.Data); // We cannot use -report ffmpeg arg because it redirects stderr - HandleFfmpegOutput(e.Data, encodingTimeRegex, seekDuration); + HandleFfmpegOutput(e.Data, encodingTimeRegex, videoLength); }; process.Start(); @@ -413,7 +445,7 @@ private void HandleFfmpegOutput(string output, Regex encodingTimeRegex, TimeSpan _progress.ReportProgress(Math.Clamp(percent, 0, 100)); } - private async Task<(M3U8 playlist, Range cropRange, DateTimeOffset airDate)> GetVideoPlaylist(string playlistUrl, CancellationToken cancellationToken) + private async Task<(M3U8 playlist, DateTimeOffset airDate)> GetVideoPlaylist(string playlistUrl, CancellationToken cancellationToken) { var playlistString = await _httpClient.GetStringAsync(playlistUrl, cancellationToken); var playlist = M3U8.Parse(playlistString); @@ -425,22 +457,26 @@ private void HandleFfmpegOutput(string output, Regex encodingTimeRegex, TimeSpan airDate = vodAirDate; } - var videoListCrop = GetStreamListCrop(playlist.Streams, downloadOptions); - - return (playlist, videoListCrop, airDate); + return (playlist, airDate); } - private static Range GetStreamListCrop(IList streamList, VideoDownloadOptions downloadOptions) + private Range GetStreamListTrim(IList streamList, VideoInfo videoInfo, out TimeSpan videoLength, out decimal startOffset, out decimal endOffset) { + startOffset = 0; + endOffset = 0; + var startIndex = 0; if (downloadOptions.TrimBeginning) { var startTime = 0m; - var cropTotalSeconds = (decimal)downloadOptions.TrimBeginningTime.TotalSeconds; + var trimTotalSeconds = (decimal)downloadOptions.TrimBeginningTime.TotalSeconds; foreach (var videoPart in streamList) { - if (startTime + videoPart.PartInfo.Duration > cropTotalSeconds) + if (startTime + videoPart.PartInfo.Duration > trimTotalSeconds) + { + startOffset = trimTotalSeconds - startTime; break; + } startIndex++; startTime += videoPart.PartInfo.Duration; @@ -451,17 +487,27 @@ private static Range GetStreamListCrop(IList streamList, VideoDownl if (downloadOptions.TrimEnding) { var endTime = streamList.Sum(x => x.PartInfo.Duration); - var cropTotalSeconds = (decimal)downloadOptions.TrimEndingTime.TotalSeconds; + var trimTotalSeconds = (decimal)downloadOptions.TrimEndingTime.TotalSeconds; for (var i = streamList.Count - 1; i >= 0; i--) { - if (endTime - streamList[i].PartInfo.Duration < cropTotalSeconds) + var videoPart = streamList[i]; + if (endTime - videoPart.PartInfo.Duration < trimTotalSeconds) + { + var offset = endTime - trimTotalSeconds; + if (offset > 0) endOffset = videoPart.PartInfo.Duration - offset; + break; + } endIndex--; - endTime -= streamList[i].PartInfo.Duration; + endTime -= videoPart.PartInfo.Duration; } } + videoLength = + (downloadOptions.TrimEnding ? downloadOptions.TrimEndingTime : TimeSpan.FromSeconds(videoInfo.lengthSeconds)) + - (downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero); + return new Range(startIndex, endIndex); } @@ -485,43 +531,7 @@ private static Range GetStreamListCrop(IList streamList, VideoDownl return m3u8.GetStreamOfQuality(downloadOptions.Quality); } - private async Task CombineVideoParts(string downloadFolder, IEnumerable playlist, Range videoListCrop, CancellationToken cancellationToken) - { - DriveInfo outputDrive = DriveHelper.GetOutputDrive(downloadFolder); - string outputFile = Path.Combine(downloadFolder, "output.ts"); - - int partCount = videoListCrop.End.Value - videoListCrop.Start.Value; - int doneCount = 0; - - await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.Read); - foreach (var part in playlist.Take(videoListCrop)) - { - await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); - - string partFile = Path.Combine(downloadFolder, DownloadTools.RemoveQueryString(part.Path)); - if (File.Exists(partFile)) - { - await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - await fs.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false); - } - - try - { - File.Delete(partFile); - } - catch { /* If we can't delete, oh well. It should get cleanup up later anyways */ } - } - - doneCount++; - int percent = (int)(doneCount / (double)partCount * 100); - _progress.ReportProgress(percent); - - cancellationToken.ThrowIfCancellationRequested(); - } - } - - private static void Cleanup(string downloadFolder) + private void Cleanup(string downloadFolder) { try { @@ -530,7 +540,10 @@ private static void Cleanup(string downloadFolder) Directory.Delete(downloadFolder, true); } } - catch (IOException) { } // Directory is probably being used by another process + catch (IOException e) + { + _progress.LogWarning($"Failed to delete download cache: {e.Message}"); + } } } } \ No newline at end of file