From 69de42358d2a13177b12f839cdf9765331389a61 Mon Sep 17 00:00:00 2001 From: bezzad Date: Sat, 27 Aug 2022 00:32:50 +0430 Subject: [PATCH 01/19] Added pause token to suspend tasks when pause method called --- .../UnitTests/PauseTokenTest.cs | 49 +++++++++++++++++++ src/Downloader/PauseToken.cs | 22 +++++++++ src/Downloader/PauseTokenSource.cs | 46 +++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 src/Downloader.Test/UnitTests/PauseTokenTest.cs create mode 100644 src/Downloader/PauseToken.cs create mode 100644 src/Downloader/PauseTokenSource.cs diff --git a/src/Downloader.Test/UnitTests/PauseTokenTest.cs b/src/Downloader.Test/UnitTests/PauseTokenTest.cs new file mode 100644 index 00000000..88f4047d --- /dev/null +++ b/src/Downloader.Test/UnitTests/PauseTokenTest.cs @@ -0,0 +1,49 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Threading; +using System.Threading.Tasks; + +namespace Downloader.Test.UnitTests +{ + [TestClass] + public class PauseTokenTest + { + private volatile int Counter; + + [TestMethod] + public void TestPauseTaskWithPauseToken() + { + // arrange + var cts = new CancellationTokenSource(); + var pts = new PauseTokenSource(); + Counter = 0; + var expectedCount = 0; + + // act + pts.Pause(); + Task.Run(() => IncreaseAsync(pts.Token, cts.Token)); + for (var i = 0; i < 100; i++) + { + Assert.IsTrue(expectedCount >= Counter); + pts.Resume(); + while (pts.IsPaused || expectedCount == Counter) + ; + pts.Pause(); + while (pts.IsPaused == false) + ; + Interlocked.Exchange(ref expectedCount, Counter+1); + Thread.Sleep(1); + } + cts.Cancel(); + } + + private async Task IncreaseAsync(PauseToken pause, CancellationToken cancel) + { + while (cancel.IsCancellationRequested == false) + { + await pause.WaitWhilePausedAsync(); + Interlocked.Increment(ref Counter); + await Task.Delay(1); + } + } + } +} diff --git a/src/Downloader/PauseToken.cs b/src/Downloader/PauseToken.cs new file mode 100644 index 00000000..882ff5b2 --- /dev/null +++ b/src/Downloader/PauseToken.cs @@ -0,0 +1,22 @@ +using System.Threading.Tasks; + +namespace Downloader +{ + internal struct PauseToken + { + private readonly PauseTokenSource tokenSource; + public bool IsPaused => tokenSource?.IsPaused == true; + + internal PauseToken(PauseTokenSource source) + { + tokenSource = source; + } + + public Task WaitWhilePausedAsync() + { + return IsPaused + ? tokenSource.WaitWhilePausedAsync() + : PauseTokenSource.CompletedTask; + } + } +} diff --git a/src/Downloader/PauseTokenSource.cs b/src/Downloader/PauseTokenSource.cs new file mode 100644 index 00000000..f5b48e5b --- /dev/null +++ b/src/Downloader/PauseTokenSource.cs @@ -0,0 +1,46 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Downloader +{ + internal class PauseTokenSource + { + private volatile TaskCompletionSource tcsPaused; + internal static readonly Task CompletedTask = Task.FromResult(true); + + public PauseToken Token => new PauseToken(this); + public bool IsPaused => tcsPaused != null; + + public void Pause() + { + // if (tcsPause == new TaskCompletionSource()) tcsPause = null; + Interlocked.CompareExchange(ref tcsPaused, new TaskCompletionSource(), null); + } + + public void Resume() + { + // we need to do this in a standard compare-exchange loop: + // grab the current value, do the compare exchange assuming that value, + // and if the value actually changed between the time we grabbed it + // and the time we did the compare-exchange, repeat. + while (true) + { + var tcs = tcsPaused; + + if (tcs == null) + return; + + if (Interlocked.CompareExchange(ref tcsPaused, null, tcs) == tcs) + { + tcs.SetResult(true); + break; + } + } + } + + internal Task WaitWhilePausedAsync() + { + return tcsPaused?.Task ?? CompletedTask; + } + } +} From cf466d53703de220e71e7650a3032701ba873457 Mon Sep 17 00:00:00 2001 From: bezzad Date: Sat, 27 Aug 2022 00:35:20 +0430 Subject: [PATCH 02/19] Fixed pause token unit test --- src/Downloader.Test/UnitTests/PauseTokenTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Downloader.Test/UnitTests/PauseTokenTest.cs b/src/Downloader.Test/UnitTests/PauseTokenTest.cs index 88f4047d..f45223b0 100644 --- a/src/Downloader.Test/UnitTests/PauseTokenTest.cs +++ b/src/Downloader.Test/UnitTests/PauseTokenTest.cs @@ -21,7 +21,7 @@ public void TestPauseTaskWithPauseToken() // act pts.Pause(); Task.Run(() => IncreaseAsync(pts.Token, cts.Token)); - for (var i = 0; i < 100; i++) + for (var i = 0; i < 10; i++) { Assert.IsTrue(expectedCount >= Counter); pts.Resume(); @@ -31,7 +31,7 @@ public void TestPauseTaskWithPauseToken() while (pts.IsPaused == false) ; Interlocked.Exchange(ref expectedCount, Counter+1); - Thread.Sleep(1); + Thread.Sleep(10); } cts.Cancel(); } From 0bc190e9c8a86e5e62a6ed899abcd7cd33bbe820 Mon Sep 17 00:00:00 2001 From: bezzad Date: Sat, 27 Aug 2022 00:37:53 +0430 Subject: [PATCH 03/19] check pause token is in pause state or not --- src/Downloader.Test/UnitTests/PauseTokenTest.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Downloader.Test/UnitTests/PauseTokenTest.cs b/src/Downloader.Test/UnitTests/PauseTokenTest.cs index f45223b0..f895818f 100644 --- a/src/Downloader.Test/UnitTests/PauseTokenTest.cs +++ b/src/Downloader.Test/UnitTests/PauseTokenTest.cs @@ -34,6 +34,10 @@ public void TestPauseTaskWithPauseToken() Thread.Sleep(10); } cts.Cancel(); + + // assert + Assert.IsTrue(expectedCount >= Counter); + Assert.IsTrue(pts.IsPaused); } private async Task IncreaseAsync(PauseToken pause, CancellationToken cancel) From 270c7b3e3424ab9debcf842a03c7cf1f9e546a5e Mon Sep 17 00:00:00 2001 From: bezzad Date: Sat, 27 Aug 2022 01:33:42 +0430 Subject: [PATCH 04/19] Added pause token and tests. #98, #99 --- src/Downloader.Sample/Program.cs | 13 ++++-- .../DownloadIntegrationTest.cs | 41 ++++++++++++++++++ .../UnitTests/ChunkDownloaderTest.cs | 13 +++--- .../UnitTests/PauseTokenTest.cs | 12 ++++-- src/Downloader/ChunkDownloader.cs | 26 ++++++------ src/Downloader/Download.cs | 12 ++++++ src/Downloader/DownloadService.cs | 42 +++++++++++++------ .../{DowloadStatus.cs => DownloadStatus.cs} | 1 + src/Downloader/IDownload.cs | 28 +++++++------ src/Downloader/IDownloadService.cs | 2 + 10 files changed, 138 insertions(+), 52 deletions(-) rename src/Downloader/{DowloadStatus.cs => DownloadStatus.cs} (88%) diff --git a/src/Downloader.Sample/Program.cs b/src/Downloader.Sample/Program.cs index 176694d1..1decf724 100644 --- a/src/Downloader.Sample/Program.cs +++ b/src/Downloader.Sample/Program.cs @@ -67,6 +67,7 @@ private static void Initial() private static void AddKeyboardHandler() { Console.WriteLine("\nPress Esc to Stop current file download"); + Console.WriteLine("\nPress P to Pause and R to Resume downloading"); Console.WriteLine("\nPress Up Arrow to Increase download speed 2X"); Console.WriteLine("\nPress Down Arrow to Decrease download speed 2X"); Console.WriteLine(); @@ -78,13 +79,19 @@ private static void AddKeyboardHandler() Thread.Sleep(50); } - if (Console.ReadKey(true).Key == ConsoleKey.Escape) + if (Console.ReadKey(true).Key == ConsoleKey.P) + CurrentDownloadService?.Pause(); + + if (Console.ReadKey(true).Key == ConsoleKey.R) + CurrentDownloadService?.Resume(); + + else if (Console.ReadKey(true).Key == ConsoleKey.Escape) CurrentDownloadService?.CancelAsync(); - if (Console.ReadKey(true).Key == ConsoleKey.UpArrow) + else if (Console.ReadKey(true).Key == ConsoleKey.UpArrow) CurrentDownloadConfiguration.MaximumBytesPerSecond *= 2; - if (Console.ReadKey(true).Key == ConsoleKey.DownArrow) + else if (Console.ReadKey(true).Key == ConsoleKey.DownArrow) CurrentDownloadConfiguration.MaximumBytesPerSecond /= 2; } } diff --git a/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs b/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs index ad192f4d..8295d649 100644 --- a/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs +++ b/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs @@ -1,9 +1,11 @@ using Downloader.DummyHttpServer; +using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.IO; using System.Linq; using System.Threading; +using System.Threading.Tasks; namespace Downloader.Test.IntegrationTests { @@ -145,6 +147,7 @@ public void StopResumeDownloadTest() { downloader.DownloadFileTaskAsync(downloader.Package).Wait(); // resume download from stopped point. } + var stream = File.ReadAllBytes(downloader.Package.FileName); // assert Assert.IsTrue(File.Exists(downloader.Package.FileName)); @@ -152,6 +155,44 @@ public void StopResumeDownloadTest() Assert.AreEqual(expectedStopCount, stopCount); Assert.AreEqual(expectedStopCount, cancellationsOccurrenceCount); Assert.IsTrue(downloadCompletedSuccessfully); + Assert.IsTrue(DummyFileHelper.File16Kb.SequenceEqual(stream.ToArray())); + + File.Delete(downloader.Package.FileName); + } + + [TestMethod] + public void PauseResumeDownloadTest() + { + // arrange + var expectedPauseCount = 2; + var pauseCount = 0; + var downloadCompletedSuccessfully = false; + var downloader = new DownloadService(Config); + downloader.DownloadFileCompleted += (s, e) => { + if (e.Cancelled == false && e.Error is null) + downloadCompletedSuccessfully = true; + }; + downloader.DownloadProgressChanged += delegate { + if (expectedPauseCount > pauseCount) + { + // Stopping after start of downloading + downloader.Pause(); + pauseCount++; + downloader.Resume(); + } + }; + + // act + downloader.DownloadFileTaskAsync(DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb), Path.GetTempFileName()).Wait(); + var stream = File.ReadAllBytes(downloader.Package.FileName); + + // assert + Assert.IsFalse(downloader.IsPause); + Assert.IsTrue(File.Exists(downloader.Package.FileName)); + Assert.AreEqual(DummyFileHelper.FileSize16Kb, downloader.Package.TotalFileSize); + Assert.AreEqual(expectedPauseCount, pauseCount); + Assert.IsTrue(downloadCompletedSuccessfully); + Assert.IsTrue(DummyFileHelper.File16Kb.SequenceEqual(stream.ToArray())); File.Delete(downloader.Package.FileName); } diff --git a/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs b/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs index e4a3843f..fca3f4c4 100644 --- a/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs +++ b/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs @@ -1,12 +1,11 @@ -using System; +using Downloader.DummyHttpServer; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; -using Downloader.DummyHttpServer; -using Downloader.Test.Helper; -using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Downloader.Test.UnitTests { @@ -50,7 +49,7 @@ private void ReadStreamTest(IStorage storage) using var memoryStream = new MemoryStream(randomlyBytes); // act - chunkDownloader.ReadStream(memoryStream, new CancellationToken()).Wait(); + chunkDownloader.ReadStream(memoryStream, new PauseTokenSource().Token, new CancellationToken()).Wait(); // assert Assert.AreEqual(memoryStream.Length, chunkDownloader.Chunk.Storage.GetLength()); @@ -91,7 +90,7 @@ private void ReadStreamProgressEventsTest(IStorage storage) }; // act - chunkDownloader.ReadStream(sourceMemoryStream, new CancellationToken()).Wait(); + chunkDownloader.ReadStream(sourceMemoryStream, new PauseTokenSource().Token, new CancellationToken()).Wait(); // assert Assert.AreEqual(streamSize/_configuration.BufferBlockSize, eventCount); @@ -113,7 +112,7 @@ public void ReadStreamTimeoutExceptionTest() var canceledToken = new CancellationToken(true); // act - async Task CallReadStream() => await chunkDownloader.ReadStream(new MemoryStream(), canceledToken).ConfigureAwait(false); + async Task CallReadStream() => await chunkDownloader.ReadStream(new MemoryStream(), new PauseTokenSource().Token, canceledToken).ConfigureAwait(false); // assert Assert.ThrowsExceptionAsync(CallReadStream); diff --git a/src/Downloader.Test/UnitTests/PauseTokenTest.cs b/src/Downloader.Test/UnitTests/PauseTokenTest.cs index f895818f..c4b4b60b 100644 --- a/src/Downloader.Test/UnitTests/PauseTokenTest.cs +++ b/src/Downloader.Test/UnitTests/PauseTokenTest.cs @@ -1,4 +1,5 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.TestTools.UnitTesting; using System.Threading; using System.Threading.Tasks; @@ -21,22 +22,25 @@ public void TestPauseTaskWithPauseToken() // act pts.Pause(); Task.Run(() => IncreaseAsync(pts.Token, cts.Token)); + Task.Run(() => IncreaseAsync(pts.Token, cts.Token)); + Task.Run(() => IncreaseAsync(pts.Token, cts.Token)); + Task.Run(() => IncreaseAsync(pts.Token, cts.Token)); for (var i = 0; i < 10; i++) { - Assert.IsTrue(expectedCount >= Counter); + Assert.IsTrue(expectedCount >= Counter, $"Expected: {expectedCount}, Actual: {Counter}"); pts.Resume(); while (pts.IsPaused || expectedCount == Counter) ; pts.Pause(); while (pts.IsPaused == false) ; - Interlocked.Exchange(ref expectedCount, Counter+1); + Interlocked.Exchange(ref expectedCount, Counter+4); Thread.Sleep(10); } cts.Cancel(); // assert - Assert.IsTrue(expectedCount >= Counter); + Assert.IsTrue(expectedCount >= Counter, $"Expected: {expectedCount}, Actual: {Counter}"); Assert.IsTrue(pts.IsPaused); } diff --git a/src/Downloader/ChunkDownloader.cs b/src/Downloader/ChunkDownloader.cs index 6dcac6ed..5a756ad7 100644 --- a/src/Downloader/ChunkDownloader.cs +++ b/src/Downloader/ChunkDownloader.cs @@ -33,24 +33,25 @@ private void ConfigurationPropertyChanged(object sender, PropertyChangedEventArg } } - public async Task Download(Request downloadRequest, CancellationToken cancellationToken) + public async Task Download(Request downloadRequest, + PauseToken pause, CancellationToken cancellationToken) { try { - await DownloadChunk(downloadRequest, cancellationToken).ConfigureAwait(false); + await DownloadChunk(downloadRequest, pause, cancellationToken).ConfigureAwait(false); return Chunk; } catch (TaskCanceledException) // when stream reader timeout occurred { // re-request and continue downloading... - return await Download(downloadRequest, cancellationToken).ConfigureAwait(false); + return await Download(downloadRequest, pause, cancellationToken).ConfigureAwait(false); } catch (WebException) when (Chunk.CanTryAgainOnFailover()) { // when the host forcibly closed the connection. await Task.Delay(Chunk.Timeout, cancellationToken).ConfigureAwait(false); // re-request and continue downloading... - return await Download(downloadRequest, cancellationToken).ConfigureAwait(false); + return await Download(downloadRequest, pause, cancellationToken).ConfigureAwait(false); } catch (Exception error) when (Chunk.CanTryAgainOnFailover() && (error.HasSource("System.Net.Http") || @@ -61,7 +62,7 @@ public async Task Download(Request downloadRequest, CancellationToken can Chunk.Timeout += TimeoutIncrement; // decrease download speed to down pressure on host await Task.Delay(Chunk.Timeout, cancellationToken).ConfigureAwait(false); // re-request and continue downloading... - return await Download(downloadRequest, cancellationToken).ConfigureAwait(false); + return await Download(downloadRequest, pause, cancellationToken).ConfigureAwait(false); } finally { @@ -69,9 +70,9 @@ public async Task Download(Request downloadRequest, CancellationToken can } } - private async Task DownloadChunk(Request downloadRequest, CancellationToken token) + private async Task DownloadChunk(Request downloadRequest, PauseToken pauseToken, CancellationToken cancelToken) { - token.ThrowIfCancellationRequested(); + cancelToken.ThrowIfCancellationRequested(); if (Chunk.IsDownloadCompleted() == false) { HttpWebRequest request = downloadRequest.GetRequest(); @@ -79,9 +80,9 @@ private async Task DownloadChunk(Request downloadRequest, CancellationToken toke using HttpWebResponse downloadResponse = request.GetResponse() as HttpWebResponse; if (downloadResponse.StatusCode == HttpStatusCode.OK || downloadResponse.StatusCode == HttpStatusCode.PartialContent || - downloadResponse.StatusCode == HttpStatusCode.Created || - downloadResponse.StatusCode == HttpStatusCode.Accepted || - downloadResponse.StatusCode == HttpStatusCode.ResetContent) + downloadResponse.StatusCode == HttpStatusCode.Created || + downloadResponse.StatusCode == HttpStatusCode.Accepted || + downloadResponse.StatusCode == HttpStatusCode.ResetContent) { Configuration.RequestConfiguration.CookieContainer = request.CookieContainer; using Stream responseStream = downloadResponse?.GetResponseStream(); @@ -89,7 +90,7 @@ private async Task DownloadChunk(Request downloadRequest, CancellationToken toke { using (destinationStream = new ThrottledStream(responseStream, Configuration.MaximumSpeedPerChunk)) { - await ReadStream(destinationStream, token).ConfigureAwait(false); + await ReadStream(destinationStream, pauseToken, cancelToken).ConfigureAwait(false); } } } @@ -110,12 +111,13 @@ private void SetRequestRange(HttpWebRequest request) } } - internal async Task ReadStream(Stream stream, CancellationToken token) + internal async Task ReadStream(Stream stream, PauseToken pauseToken, CancellationToken token) { int readSize = 1; while (CanReadStream() && readSize > 0) { token.ThrowIfCancellationRequested(); + await pauseToken.WaitWhilePausedAsync().ConfigureAwait(false); using var innerCts = new CancellationTokenSource(Chunk.Timeout); byte[] buffer = new byte[Configuration.BufferBlockSize]; diff --git a/src/Downloader/Download.cs b/src/Downloader/Download.cs index 9e9e9016..b98efee7 100644 --- a/src/Downloader/Download.cs +++ b/src/Downloader/Download.cs @@ -93,6 +93,18 @@ public void Stop() Status = DownloadStatus.Stopped; } + public void Pause() + { + downloadService.Pause(); + Status = DownloadStatus.Paused; + } + + public void Resume() + { + downloadService.Resume(); + Status = DownloadStatus.Running; + } + public void Clear() { Stop(); diff --git a/src/Downloader/DownloadService.cs b/src/Downloader/DownloadService.cs index 505ce4e4..3ef8a7c4 100644 --- a/src/Downloader/DownloadService.cs +++ b/src/Downloader/DownloadService.cs @@ -15,6 +15,7 @@ public class DownloadService : IDownloadService, IDisposable { private ChunkHub _chunkHub; private CancellationTokenSource _globalCancellationTokenSource; + private PauseTokenSource _pauseTokenSource; private Request _requestInstance; private Stream _destinationStream; private readonly Bandwidth _bandwidth; @@ -22,6 +23,7 @@ public class DownloadService : IDownloadService, IDisposable protected DownloadConfiguration Options { get; set; } public bool IsBusy { get; private set; } public bool IsCancelled => _globalCancellationTokenSource?.IsCancellationRequested == true; + public bool IsPause => _pauseTokenSource.IsPaused; public DownloadPackage Package { get; set; } public event EventHandler DownloadFileCompleted; public event EventHandler DownloadProgressChanged; @@ -31,6 +33,7 @@ public class DownloadService : IDownloadService, IDisposable // ReSharper disable once MemberCanBePrivate.Global public DownloadService() { + _pauseTokenSource = new PauseTokenSource(); _bandwidth = new Bandwidth(); Options = new DownloadConfiguration(); Package = new DownloadPackage(); @@ -149,7 +152,6 @@ private async Task GetFilename() if (string.IsNullOrWhiteSpace(filename)) { filename = _requestInstance.GetFileName(); - if (string.IsNullOrWhiteSpace(filename)) { filename = Guid.NewGuid().ToString("N"); @@ -162,6 +164,17 @@ private async Task GetFilename() public void CancelAsync() { _globalCancellationTokenSource?.Cancel(false); + Resume(); + } + + public void Resume() + { + _pauseTokenSource.Resume(); + } + + public void Pause() + { + _pauseTokenSource.Pause(); } public void Clear() @@ -172,7 +185,8 @@ public void Clear() _bandwidth.Reset(); _requestInstance = null; IsBusy = false; - // Note: don't clear package from `DownloadService.Dispose()`. Because maybe it will use in another time. + // Note: don't clear package from `DownloadService.Dispose()`. + // Because maybe it will use in another time. } private void InitialDownloader(string address) @@ -182,7 +196,7 @@ private void InitialDownloader(string address) _requestInstance = new Request(address, Options.RequestConfiguration); Package.Address = _requestInstance.Address.OriginalString; _chunkHub = new ChunkHub(Options); - _parallelSemaphore = new SemaphoreSlim(Options.ParallelCount); + _parallelSemaphore = new SemaphoreSlim(Options.ParallelCount); // TODO: Add Options.ParallelCount to MaxCount } private async Task StartDownload(string fileName) @@ -197,18 +211,20 @@ private async Task StartDownload() { Package.TotalFileSize = await _requestInstance.GetFileSize().ConfigureAwait(false); Package.IsSupportDownloadInRange = await _requestInstance.IsSupportDownloadInRange().ConfigureAwait(false); - OnDownloadStarted(new DownloadStartedEventArgs(Package.FileName, Package.TotalFileSize)); ValidateBeforeChunking(); Package.Chunks ??= _chunkHub.ChunkFile(Package.TotalFileSize, Options.ChunkCount, Options.RangeLow); Package.Validate(); + // firing the start event after creating chunks + OnDownloadStarted(new DownloadStartedEventArgs(Package.FileName, Package.TotalFileSize)); + if (Options.ParallelDownload) { - await ParallelDownload(_globalCancellationTokenSource.Token).ConfigureAwait(false); + await ParallelDownload(_pauseTokenSource.Token, _globalCancellationTokenSource.Token).ConfigureAwait(false); } else { - await SerialDownload(_globalCancellationTokenSource.Token).ConfigureAwait(false); + await SerialDownload(_pauseTokenSource.Token, _globalCancellationTokenSource.Token).ConfigureAwait(false); } await StoreDownloadedFile(_globalCancellationTokenSource.Token).ConfigureAwait(false); @@ -269,7 +285,7 @@ private void SetRangedSizes() { if (Options.RangeDownload) { - if(!Package.IsSupportDownloadInRange) + if (!Package.IsSupportDownloadInRange) { throw new NotSupportedException("The server of your desired address does not support download in a specific range"); } @@ -334,28 +350,28 @@ private void CheckSupportDownloadInRange() } } - private async Task ParallelDownload(CancellationToken cancellationToken) + private async Task ParallelDownload(PauseToken pauseToken, CancellationToken cancellationToken) { - var tasks = Package.Chunks.Select(chunk => DownloadChunk(chunk, cancellationToken)); + var tasks = Package.Chunks.Select(chunk => DownloadChunk(chunk, pauseToken, cancellationToken)); await Task.WhenAll(tasks).ConfigureAwait(false); } - private async Task SerialDownload(CancellationToken cancellationToken) + private async Task SerialDownload(PauseToken pauseToken, CancellationToken cancellationToken) { foreach (var chunk in Package.Chunks) { - await DownloadChunk(chunk, cancellationToken).ConfigureAwait(false); + await DownloadChunk(chunk, pauseToken, cancellationToken).ConfigureAwait(false); } } - private async Task DownloadChunk(Chunk chunk, CancellationToken cancellationToken) + private async Task DownloadChunk(Chunk chunk, PauseToken pause, CancellationToken cancellationToken) { ChunkDownloader chunkDownloader = new ChunkDownloader(chunk, Options); chunkDownloader.DownloadProgressChanged += OnChunkDownloadProgressChanged; await _parallelSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); try { - return await chunkDownloader.Download(_requestInstance, cancellationToken).ConfigureAwait(false); + return await chunkDownloader.Download(_requestInstance, pause, cancellationToken).ConfigureAwait(false); } finally { diff --git a/src/Downloader/DowloadStatus.cs b/src/Downloader/DownloadStatus.cs similarity index 88% rename from src/Downloader/DowloadStatus.cs rename to src/Downloader/DownloadStatus.cs index 6dd41a04..24a5c964 100644 --- a/src/Downloader/DowloadStatus.cs +++ b/src/Downloader/DownloadStatus.cs @@ -6,5 +6,6 @@ public enum DownloadStatus Created = 1, Running = 2, Stopped = 3, + Paused = 4 } } diff --git a/src/Downloader/IDownload.cs b/src/Downloader/IDownload.cs index 7646e7c3..166b72b5 100644 --- a/src/Downloader/IDownload.cs +++ b/src/Downloader/IDownload.cs @@ -6,20 +6,22 @@ namespace Downloader { public interface IDownload { - string Url { get; } - string Folder { get; } - string Filename { get; } - long DownloadedFileSize { get; } - long TotalFileSize { get; } - DownloadStatus Status { get; } + public string Url { get; } + public string Folder { get; } + public string Filename { get; } + public long DownloadedFileSize { get; } + public long TotalFileSize { get; } + public DownloadStatus Status { get; } - event EventHandler ChunkDownloadProgressChanged; - event EventHandler DownloadFileCompleted; - event EventHandler DownloadProgressChanged; - event EventHandler DownloadStarted; + public event EventHandler ChunkDownloadProgressChanged; + public event EventHandler DownloadFileCompleted; + public event EventHandler DownloadProgressChanged; + public event EventHandler DownloadStarted; - void Clear(); - Task StartAsync(); - void Stop(); + public void Clear(); + public Task StartAsync(); + public void Stop(); + public void Pause(); + public void Resume(); } } diff --git a/src/Downloader/IDownloadService.cs b/src/Downloader/IDownloadService.cs index 0125113a..49c4af1a 100644 --- a/src/Downloader/IDownloadService.cs +++ b/src/Downloader/IDownloadService.cs @@ -20,6 +20,8 @@ public interface IDownloadService Task DownloadFileTaskAsync(string address, string fileName); Task DownloadFileTaskAsync(string address, DirectoryInfo folder); void CancelAsync(); + void Pause(); + void Resume(); void Clear(); } } \ No newline at end of file From d63edfc7b8492e7d446cd8c5d0183adf75d5aa04 Mon Sep 17 00:00:00 2001 From: bezzad Date: Sat, 27 Aug 2022 02:17:52 +0430 Subject: [PATCH 05/19] Update sample repository --- src/Downloader.Sample/Program.cs | 37 ++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/src/Downloader.Sample/Program.cs b/src/Downloader.Sample/Program.cs index 1decf724..373ae8f1 100644 --- a/src/Downloader.Sample/Program.cs +++ b/src/Downloader.Sample/Program.cs @@ -16,12 +16,14 @@ namespace Downloader.Sample internal static class Program { private const string DownloadListFile = "DownloadList.json"; + private static List DownloadList; private static ProgressBar ConsoleProgress; private static ConcurrentDictionary ChildConsoleProgresses; private static ProgressBarOptions ChildOption; private static ProgressBarOptions ProcessBarOption; private static DownloadService CurrentDownloadService; private static DownloadConfiguration CurrentDownloadConfiguration; + private static CancellationTokenSource CancelAllTokenSource; private static async Task Main() { @@ -30,10 +32,10 @@ private static async Task Main() DummyHttpServer.HttpServer.Run(3333); await Task.Delay(1000); Console.Clear(); - new Thread(AddKeyboardHandler) { IsBackground = true }.Start(); Initial(); - List downloadList = GetDownloadItems(); - await DownloadAll(downloadList).ConfigureAwait(false); + Console.CancelKeyPress += CancelAll; + _=AddKeyboardHandler().ConfigureAwait(false); + await DownloadAll(DownloadList, CancelAllTokenSource.Token).ConfigureAwait(false); } catch (Exception e) { @@ -41,12 +43,15 @@ private static async Task Main() Debugger.Break(); } - Console.WriteLine("END"); + Console.WriteLine("\n\n END \n\n"); Console.Read(); } private static void Initial() { + CancelAllTokenSource = new CancellationTokenSource(); + DownloadList = GetDownloadItems(); + ProcessBarOption = new ProgressBarOptions { ForegroundColor = ConsoleColor.Green, ForegroundColorDone = ConsoleColor.DarkGreen, @@ -64,7 +69,7 @@ private static void Initial() }; } - private static void AddKeyboardHandler() + private static async Task AddKeyboardHandler() { Console.WriteLine("\nPress Esc to Stop current file download"); Console.WriteLine("\nPress P to Pause and R to Resume downloading"); @@ -76,13 +81,18 @@ private static void AddKeyboardHandler() { while (!Console.KeyAvailable) { - Thread.Sleep(50); + await Task.Delay(10).ConfigureAwait(false); } if (Console.ReadKey(true).Key == ConsoleKey.P) + { CurrentDownloadService?.Pause(); + Console.Beep(); + if (Console.Title.EndsWith("Paused") == false) + Console.Title += " - Paused"; + } - if (Console.ReadKey(true).Key == ConsoleKey.R) + else if (Console.ReadKey(true).Key == ConsoleKey.R) CurrentDownloadService?.Resume(); else if (Console.ReadKey(true).Key == ConsoleKey.Escape) @@ -96,6 +106,12 @@ private static void AddKeyboardHandler() } } + private static void CancelAll(object sender, ConsoleCancelEventArgs e) + { + CancelAllTokenSource.Cancel(); + CurrentDownloadService?.CancelAsync(); + } + private static DownloadConfiguration GetDownloadConfiguration() { var cookies = new CookieContainer(); @@ -114,7 +130,7 @@ private static DownloadConfiguration GetDownloadConfiguration() RangeDownload = false, // set true if you want to download just a specific range of bytes of a large file RangeLow = 0, // floor offset of download range of a large file RangeHigh = 0, // ceiling offset of download range of a large file - RequestConfiguration = + RequestConfiguration = { // config and customize request headers Accept = "*/*", @@ -147,10 +163,13 @@ private static List GetDownloadItems() return downloadList; } - private static async Task DownloadAll(IEnumerable downloadList) + private static async Task DownloadAll(IEnumerable downloadList, CancellationToken cancelToken) { foreach (DownloadItem downloadItem in downloadList) { + if (cancelToken.IsCancellationRequested) + return; + // begin download from url DownloadService ds = await DownloadFile(downloadItem).ConfigureAwait(false); From c2045f32124f57dab3f75fffd0d0f1e5a6b7dba1 Mon Sep 17 00:00:00 2001 From: bezzad Date: Fri, 2 Sep 2022 17:35:34 +0430 Subject: [PATCH 06/19] Fixed Pause and Resume downloads as fast as without interrupt --- src/Downloader.Sample/Helper.cs | 5 +- src/Downloader.Sample/Program.cs | 57 +++++++++---------- .../DownloadIntegrationTest.cs | 2 +- src/Downloader/DownloadService.cs | 2 +- 4 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/Downloader.Sample/Helper.cs b/src/Downloader.Sample/Helper.cs index cef6338b..79fce9c4 100644 --- a/src/Downloader.Sample/Helper.cs +++ b/src/Downloader.Sample/Helper.cs @@ -27,7 +27,7 @@ public static string CalcMemoryMensurableUnit(this double bytes) return result; } - public static void UpdateTitleInfo(this DownloadProgressChangedEventArgs e) + public static void UpdateTitleInfo(this DownloadProgressChangedEventArgs e, bool isPaused) { double nonZeroSpeed = e.BytesPerSecondSpeed + 0.0001; int estimateTime = (int)((e.TotalBytesToReceive - e.ReceivedBytesSize) / nonZeroSpeed); @@ -55,7 +55,8 @@ public static void UpdateTitleInfo(this DownloadProgressChangedEventArgs e) Console.Title = $"{progressPercentage}% - " + $"{speed}/s (avg: {avgSpeed}/s) - " + $"{estimateTime} {timeLeftUnit} left - " + - $"[{bytesReceived} of {totalBytesToReceive}]"; + $"[{bytesReceived} of {totalBytesToReceive}]" + + (isPaused ? " - Paused" : ""); } } } diff --git a/src/Downloader.Sample/Program.cs b/src/Downloader.Sample/Program.cs index 373ae8f1..eced3ff5 100644 --- a/src/Downloader.Sample/Program.cs +++ b/src/Downloader.Sample/Program.cs @@ -33,8 +33,7 @@ private static async Task Main() await Task.Delay(1000); Console.Clear(); Initial(); - Console.CancelKeyPress += CancelAll; - _=AddKeyboardHandler().ConfigureAwait(false); + new Task(KeyboardHandler).Start(); await DownloadAll(DownloadList, CancelAllTokenSource.Token).ConfigureAwait(false); } catch (Exception e) @@ -46,7 +45,6 @@ private static async Task Main() Console.WriteLine("\n\n END \n\n"); Console.Read(); } - private static void Initial() { CancelAllTokenSource = new CancellationTokenSource(); @@ -68,44 +66,42 @@ private static void Initial() ProgressBarOnBottom = true }; } - - private static async Task AddKeyboardHandler() + private static void KeyboardHandler() { + ConsoleKeyInfo cki; + Console.CancelKeyPress += CancelAll; Console.WriteLine("\nPress Esc to Stop current file download"); Console.WriteLine("\nPress P to Pause and R to Resume downloading"); Console.WriteLine("\nPress Up Arrow to Increase download speed 2X"); Console.WriteLine("\nPress Down Arrow to Decrease download speed 2X"); Console.WriteLine(); - while (true) // continue download other files of the list + while (true) { - while (!Console.KeyAvailable) - { - await Task.Delay(10).ConfigureAwait(false); - } - - if (Console.ReadKey(true).Key == ConsoleKey.P) + cki = Console.ReadKey(true); + switch (cki.Key) { - CurrentDownloadService?.Pause(); - Console.Beep(); - if (Console.Title.EndsWith("Paused") == false) - Console.Title += " - Paused"; + case ConsoleKey.P: + CurrentDownloadService?.Pause(); + Console.Beep(); + break; + case ConsoleKey.R: + CurrentDownloadService?.Resume(); + break; + case ConsoleKey.Escape: + CurrentDownloadService?.CancelAsync(); + break; + + case ConsoleKey.UpArrow: + CurrentDownloadConfiguration.MaximumBytesPerSecond *= 2; + break; + + case ConsoleKey.DownArrow: + CurrentDownloadConfiguration.MaximumBytesPerSecond /= 2; + break; } - - else if (Console.ReadKey(true).Key == ConsoleKey.R) - CurrentDownloadService?.Resume(); - - else if (Console.ReadKey(true).Key == ConsoleKey.Escape) - CurrentDownloadService?.CancelAsync(); - - else if (Console.ReadKey(true).Key == ConsoleKey.UpArrow) - CurrentDownloadConfiguration.MaximumBytesPerSecond *= 2; - - else if (Console.ReadKey(true).Key == ConsoleKey.DownArrow) - CurrentDownloadConfiguration.MaximumBytesPerSecond /= 2; } } - private static void CancelAll(object sender, ConsoleCancelEventArgs e) { CancelAllTokenSource.Cancel(); @@ -234,7 +230,8 @@ private static void OnChunkDownloadProgressChanged(object sender, DownloadProgre private static void OnDownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e) { ConsoleProgress.Tick((int)(e.ProgressPercentage * 100)); - e.UpdateTitleInfo(); + if (sender is DownloadService ds) + e.UpdateTitleInfo(ds.IsPaused); } } } \ No newline at end of file diff --git a/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs b/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs index 8295d649..8e915507 100644 --- a/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs +++ b/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs @@ -187,7 +187,7 @@ public void PauseResumeDownloadTest() var stream = File.ReadAllBytes(downloader.Package.FileName); // assert - Assert.IsFalse(downloader.IsPause); + Assert.IsFalse(downloader.IsPaused); Assert.IsTrue(File.Exists(downloader.Package.FileName)); Assert.AreEqual(DummyFileHelper.FileSize16Kb, downloader.Package.TotalFileSize); Assert.AreEqual(expectedPauseCount, pauseCount); diff --git a/src/Downloader/DownloadService.cs b/src/Downloader/DownloadService.cs index 3ef8a7c4..d1b5407d 100644 --- a/src/Downloader/DownloadService.cs +++ b/src/Downloader/DownloadService.cs @@ -23,7 +23,7 @@ public class DownloadService : IDownloadService, IDisposable protected DownloadConfiguration Options { get; set; } public bool IsBusy { get; private set; } public bool IsCancelled => _globalCancellationTokenSource?.IsCancellationRequested == true; - public bool IsPause => _pauseTokenSource.IsPaused; + public bool IsPaused => _pauseTokenSource.IsPaused; public DownloadPackage Package { get; set; } public event EventHandler DownloadFileCompleted; public event EventHandler DownloadProgressChanged; From c74b7e6d31a7ce244bf89b1f8b61a28f74ef1130 Mon Sep 17 00:00:00 2001 From: bezzad Date: Fri, 2 Sep 2022 18:27:52 +0430 Subject: [PATCH 07/19] Resume download from beginning when a server does not support download in range --- src/Downloader.Sample/DownloadList.json | 4 ++++ .../UnitTests/DownloadPackageTest.cs | 14 ++++++++++++++ src/Downloader/DownloadPackage.cs | 6 ++++++ 3 files changed, 24 insertions(+) diff --git a/src/Downloader.Sample/DownloadList.json b/src/Downloader.Sample/DownloadList.json index 5c8d2a5c..3771fa5f 100644 --- a/src/Downloader.Sample/DownloadList.json +++ b/src/Downloader.Sample/DownloadList.json @@ -11,6 +11,10 @@ // "FileName": "D:\\TestDownload\\Hello - 81606.mp4", // "Url": "https://cdn.pixabay.com/vimeo/576083058/Hello%20-%2081605.mp4?width=1920&hash=e6f56273dcd2f28fd1a9fe6e77f66d7e157b33f6&download=1" //}, + //{ + // "FileName": "D:\\TestDownload\\VS.exe", + // "Url": "https://c2rsetup.officeapps.live.com/c2r/downloadVS.aspx?sku=community&channel=Release&version=VS2022&source=VSLandingPage&includeRecommended=true&cid=2030:9bf2104738684908988ca7dcd5dafed1" + //}, { "FileName": "D:\\TestDownload\\LocalFile100MB_Raw.dat", "Url": "http://localhost:3333/dummyfile/file/size/104857600" diff --git a/src/Downloader.Test/UnitTests/DownloadPackageTest.cs b/src/Downloader.Test/UnitTests/DownloadPackageTest.cs index 0952f809..421da6ed 100644 --- a/src/Downloader.Test/UnitTests/DownloadPackageTest.cs +++ b/src/Downloader.Test/UnitTests/DownloadPackageTest.cs @@ -119,5 +119,19 @@ public void PackageValidateTest() // assert Assert.AreEqual(actualPosition, _package.Chunks[0].Position); } + + [TestMethod] + public void TestPackageValidateWhenDoesNotSupportDownloadInRange() + { + // arrange + _package.Chunks[0].Position = 1000; + _package.IsSupportDownloadInRange = false; + + // act + _package.Validate(); + + // assert + Assert.AreEqual(0, _package.Chunks[0].Position); + } } } diff --git a/src/Downloader/DownloadPackage.cs b/src/Downloader/DownloadPackage.cs index 7e62f381..832bc61b 100644 --- a/src/Downloader/DownloadPackage.cs +++ b/src/Downloader/DownloadPackage.cs @@ -52,6 +52,12 @@ public void Validate() } chunk.SetValidPosition(); } + + if (IsSupportDownloadInRange == false) + { + // reset chunk to download from zero byte + chunk.Clear(); + } } } } From ab3c66020e213533c0cc4454a7cfe23072030fd7 Mon Sep 17 00:00:00 2001 From: bezzad Date: Sun, 4 Sep 2022 13:55:05 +0430 Subject: [PATCH 08/19] Refactor sample program --- README.md | 83 +++++++++++++++++++++----------- src/Downloader.Sample/Program.cs | 33 ++++++++++--- 2 files changed, 81 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 3889488f..c3a761a3 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ Downloader is compatible with .NET Standard 2.0 and above, running on Windows, L - Store download package object to resume the download when you want. - Get download speed or progress percentage in each progress event. - Get download progress events per chunk downloads. -- Pause and Resume your downloads with package object. +- Fast Pause and Resume downloads asynchronously. +- Stop and Resume downloads whenever you want with the package object. - Supports large file download. - Set a dynamic speed limit on downloads (changeable speed limitation on the go). - Download files without storing on disk and get a memory stream for each downloaded file. @@ -72,37 +73,53 @@ var downloadOpt = new DownloadConfiguration() ### Complex Configuration + +> **Note**: *Do not use all of the below options in your applications, just add which one you need.* + ```csharp var downloadOpt = new DownloadConfiguration() { - BufferBlockSize = 10240, // usually, hosts support max to 8000 bytes, default values is 8000 - ChunkCount = 8, // file parts to download, default value is 1 - MaximumBytesPerSecond = 1024 * 1024 * 2, // download speed limited to 2MB/s, default values is zero or unlimited - MaxTryAgainOnFailover = 5, // the maximum number of times to fail - OnTheFlyDownload = false, // caching in-memory or not? default values is true - ParallelDownload = true, // download parts of file as parallel or not. Default value is false - ParallelCount = 4, // number of parallel downloads. The default value is the same as the chunk count - TempDirectory = @"C:\temp", // Set the temp path for buffering chunk files, the default path is Path.GetTempPath() - Timeout = 1000, // timeout (millisecond) per stream block reader, default values is 1000 - RangeDownload = false, // set true if you want to download just a specific range of bytes of a large file - RangeLow = 0, // floor offset of download range of a large file - RangeHigh = 0, // ceiling offset of download range of a large file + // usually, hosts support max to 8000 bytes, default values is 8000 + BufferBlockSize = 10240, + // file parts to download, default value is 1 + ChunkCount = 8, + // download speed limited to 2MB/s, default values is zero or unlimited + MaximumBytesPerSecond = 1024*1024*2, + // the maximum number of times to fail + MaxTryAgainOnFailover = 5, + // caching in-memory or not? default values is true + OnTheFlyDownload = false, + // download parts of file as parallel or not. Default value is false + ParallelDownload = true, + // number of parallel downloads. The default value is the same as the chunk count + ParallelCount = 4, + // Set the temp path for buffering chunk files, the default path is Path.GetTempPath() + TempDirectory = @"C:\temp", + // timeout (millisecond) per stream block reader, default values is 1000 + Timeout = 1000, + // set true if you want to download just a specific range of bytes of a large file + RangeDownload = false, + // floor offset of download range of a large file + RangeLow = 0, + // ceiling offset of download range of a large file + RangeHigh = 0, + // config and customize request headers RequestConfiguration = - { - // config and customize request headers + { Accept = "*/*", CookieContainer = cookies, - Headers = new WebHeaderCollection(), // { Add your custom headers } - KeepAlive = true, // default value is false + Headers = new WebHeaderCollection(), // { your custom headers } + KeepAlive = true, // default value is false ProtocolVersion = HttpVersion.Version11, // default value is HTTP 1.1 UseDefaultCredentials = false, - UserAgent = $"DownloaderSample/{Assembly.GetExecutingAssembly().GetName().Version?.ToString(3)}" - // Proxy = new WebProxy() { - // Address = new Uri("http://YourProxyServer/proxy.pac"), - // UseDefaultCredentials = false, - // Credentials = System.Net.CredentialCache.DefaultNetworkCredentials, - // BypassProxyOnLocal = true - // } + // your custom user agent or your_app_name/app_version. + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + Proxy = new WebProxy() { + Address = new Uri("http://YourProxyServer/proxy.pac"), + UseDefaultCredentials = false, + Credentials = System.Net.CredentialCache.DefaultNetworkCredentials, + BypassProxyOnLocal = true + } } }; ``` @@ -119,13 +136,19 @@ var downloader = new DownloadService(downloadOpt); // Provide `FileName` and `TotalBytesToReceive` at the start of each downloads downloader.DownloadStarted += OnDownloadStarted; -// Provide any information about chunker downloads, like progress percentage per chunk, speed, total received bytes and received bytes array to live streaming. +// Provide any information about chunker downloads, +// like progress percentage per chunk, speed, +// total received bytes and received bytes array to live streaming. downloader.ChunkDownloadProgressChanged += OnChunkDownloadProgressChanged; -// Provide any information about download progress, like progress percentage of sum of chunks, total speed, average speed, total received bytes and received bytes array to live streaming. +// Provide any information about download progress, +// like progress percentage of sum of chunks, total speed, +// average speed, total received bytes and received bytes array +// to live streaming. downloader.DownloadProgressChanged += OnDownloadProgressChanged; -// Download completed event that can include occurred errors or cancelled or download completed successfully. +// Download completed event that can include occurred errors or +// cancelled or download completed successfully. downloader.DownloadFileCompleted += OnDownloadFileCompleted; ``` @@ -142,13 +165,15 @@ await downloader.DownloadFileTaskAsync(url, file); ```csharp DirectoryInfo path = new DirectoryInfo("Your_Path"); string url = @"https://file-examples.com/fileName.zip"; -await downloader.DownloadFileTaskAsync(url, path); // download into "Your_Path\fileName.zip" +// download into "Your_Path\fileName.zip" +await downloader.DownloadFileTaskAsync(url, path); ``` ## **Step 4c**: Download in MemoryStream ```csharp -Stream destinationStream = await downloader.DownloadFileTaskAsync(url); +// After download completion, it gets a MemoryStream +Stream destinationStream = await downloader.DownloadFileTaskAsync(url); ``` --- diff --git a/src/Downloader.Sample/Program.cs b/src/Downloader.Sample/Program.cs index eced3ff5..2ec23b2a 100644 --- a/src/Downloader.Sample/Program.cs +++ b/src/Downloader.Sample/Program.cs @@ -131,10 +131,11 @@ private static DownloadConfiguration GetDownloadConfiguration() // config and customize request headers Accept = "*/*", CookieContainer = cookies, - Headers = new WebHeaderCollection(), // { Add your custom headers } + Headers = new WebHeaderCollection(), // { your custom headers } KeepAlive = true, // default value is false ProtocolVersion = HttpVersion.Version11, // default value is HTTP 1.1 UseDefaultCredentials = false, + // your custom user agent or your_app_name/app_version. UserAgent = $"DownloaderSample/{Assembly.GetExecutingAssembly().GetName().Version?.ToString(3)}" // Proxy = new WebProxy() { // Address = new Uri("http://YourProxyServer/proxy.pac"), @@ -176,11 +177,7 @@ private static async Task DownloadAll(IEnumerable downloadList, Ca private static async Task DownloadFile(DownloadItem downloadItem) { CurrentDownloadConfiguration = GetDownloadConfiguration(); - CurrentDownloadService = new DownloadService(CurrentDownloadConfiguration); - CurrentDownloadService.ChunkDownloadProgressChanged += OnChunkDownloadProgressChanged; - CurrentDownloadService.DownloadProgressChanged += OnDownloadProgressChanged; - CurrentDownloadService.DownloadFileCompleted += OnDownloadFileCompleted; - CurrentDownloadService.DownloadStarted += OnDownloadStarted; + CurrentDownloadService = CreateDownloadService(CurrentDownloadConfiguration); if (string.IsNullOrWhiteSpace(downloadItem.FileName)) { @@ -193,7 +190,31 @@ private static async Task DownloadFile(DownloadItem downloadIte return CurrentDownloadService; } + private static DownloadService CreateDownloadService(DownloadConfiguration config) + { + var downloadService = new DownloadService(config); + + // Provide `FileName` and `TotalBytesToReceive` at the start of each downloads + downloadService.DownloadStarted += OnDownloadStarted; + + // Provide any information about chunker downloads, + // like progress percentage per chunk, speed, + // total received bytes and received bytes array to live streaming. + downloadService.ChunkDownloadProgressChanged += OnChunkDownloadProgressChanged; + // Provide any information about download progress, + // like progress percentage of sum of chunks, total speed, + // average speed, total received bytes and received bytes array + // to live streaming. + downloadService.DownloadProgressChanged += OnDownloadProgressChanged; + + // Download completed event that can include occurred errors or + // cancelled or download completed successfully. + downloadService.DownloadFileCompleted += OnDownloadFileCompleted; + + return downloadService; + } + private static void OnDownloadStarted(object sender, DownloadStartedEventArgs e) { ConsoleProgress = new ProgressBar(10000, From c7ec7a7dd258853226265e235dc0daddba9a0e85 Mon Sep 17 00:00:00 2001 From: bezzad Date: Sun, 4 Sep 2022 15:22:07 +0430 Subject: [PATCH 09/19] Added pause and resume test to fluent builder --- .../UnitTests/DownloadBuilderTest.cs | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/Downloader.Test/UnitTests/DownloadBuilderTest.cs b/src/Downloader.Test/UnitTests/DownloadBuilderTest.cs index 35d53b81..e93986cb 100644 --- a/src/Downloader.Test/UnitTests/DownloadBuilderTest.cs +++ b/src/Downloader.Test/UnitTests/DownloadBuilderTest.cs @@ -1,4 +1,5 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Downloader.DummyHttpServer; +using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.IO; @@ -16,7 +17,7 @@ public class DownloadBuilderTest public DownloadBuilderTest() { // arrange - url = "http://host.com/file2.txt"; + url = DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb); filename = "test.txt"; folder = Path.GetTempPath().TrimEnd('\\', '/'); path = Path.Combine(folder, filename); @@ -105,5 +106,35 @@ public void TestPathless() // assert Assert.ThrowsException(act); } + + [TestMethod] + public void TestPauseAndResume() + { + // arrange + var pauseCount = 0; + var downloader = DownloadBuilder.New() + .WithUrl(url) + .WithFileLocation(path) + .Build(); + + downloader.DownloadProgressChanged += (s, e) => { + if (pauseCount < 10) + { + downloader.Pause(); + pauseCount++; + downloader.Resume(); + } + }; + + // act + downloader.StartAsync().Wait(); + + // assert + Assert.AreEqual(10, pauseCount); + Assert.IsTrue(File.Exists(path)); + + // clean up + File.Delete(path); + } } } From eee3bce983ad490cfde4b7c52587099c6a49f9da Mon Sep 17 00:00:00 2001 From: bezzad Date: Sun, 4 Sep 2022 15:22:38 +0430 Subject: [PATCH 10/19] Update readme to explain pause and resume functionality --- README.md | 58 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index c3a761a3..c564ac07 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,9 @@ Downloader is a modern, fluent, asynchronous, testable and portable library for .NET. This is a multipart downloader with asynchronous progress events. This library can added in your `.Net Core v3.1` and later or `.Net Framework v4.5` or later projects. -Downloader is compatible with .NET Standard 2.0 and above, running on Windows, Linux, and macOS, in full .NET Framework or .NET Core. +Downloader is compatible with .NET Standard 2.0 and above, running on Windows, Linux, and macOS, in full .NET Framework or .NET Core. + +> For a complete example see [Downloader.Sample](https://github.com/bezzad/Downloader/blob/master/src/Downloader.Sample/Program.cs) project from this repository. ## Sample Console Application @@ -177,34 +179,46 @@ Stream destinationStream = await downloader.DownloadFileTaskAsync(url); ``` --- +## How to **pause** and **resume** downloads quickly -## How to stop and resume downloads +When you want to resume a download quickly after pausing a few seconds. You can call the `Pause` function of the downloader service. This way, streams stay alive and are only suspended by a locker to be released and resumed whenever you want. -The ‍`DownloadService` class has a property called `Package` that stores each step of the download. To stopping or pause the download you must call the `CancelAsync` method, and if you want to continue again, you must call the same `DownloadFileTaskAsync` function with the `Package` parameter to resume your download! -For example: +```csharp +// Pause the download +DownloadService.Pause(); -Keep `Package` file to resume from last download positions: +// Resume the download +DownloadService.Resume(); +``` + +--- +## How to **stop** and **resume** downloads other time + +The ‍`DownloadService` class has a property called `Package` that stores each step of the download. To stopping the download you must call the `CancelAsync` method. Now, if you want to continue again, you must call the same `DownloadFileTaskAsync` function with the `Package` parameter to resume your download. For example: ```csharp +// At first, keep and store the Package file to resume +// your download from the last download position: DownloadPackage pack = downloader.Package; ``` -**Stop or Pause Download:** +**Stop or cancel download:** ```csharp +// This function breaks your stream and cancels progress. downloader.CancelAsync(); ``` -**Resume Download:** +**Resuming download after cancelation:** ```csharp await downloader.DownloadFileTaskAsync(pack); ``` -So that you can even save your large downloads with a very small amount in the Package and after restarting the program, restore it again and start continuing your download. In fact, the packages are your instant download snapshots. If your download config has OnTheFlyDownload, the downloaded bytes ​​will be stored in the package itself, but otherwise, only the downloaded file address will be included and you can resume it whenever you like. -For more detail see [StopResumeDownloadTest](https://github.com/bezzad/Downloader/blob/master/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs#L114) method +So that you can even save your large downloads with a very small amount in the Package and after restarting the program, restore it again and start continuing your download. The packages are your snapshot of the download instance. If your download config has `OnTheFlyDownload`, the downloaded bytes ​​will be stored in the package itself. But otherwise, if you download on temp, only the downloaded temp files' addresses will be included in the package and you can resume it whenever you want. +For more detail see [StopResumeDownloadTest](https://github.com/bezzad/Downloader/blob/master/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs#L115) method -> Note: for complete sample see `Downloader.Sample` project from this repository. +> Note: Sometimes a server does not support downloading in a specific range. That time, we can't resume downloads after canceling. So, the downloader starts from the beginning. --- @@ -238,18 +252,38 @@ download.DownloadStarted += DownloadStarted; download.ChunkDownloadProgressChanged += ChunkDownloadProgressChanged; await download.StartAsync(); + +download.Stop(); // cancel current download ``` Resume the existing download package: ```csharp -await DownloadBuilder.Build(package).StartAsync(); +await DownloadBuilder.New() + .Build(package) + .StartAsync(); ``` Resume the existing download package with a new configuration: ```csharp -await DownloadBuilder.Build(package, new DownloadConfiguration()).StartAsync(); +await DownloadBuilder.New() + .Build(package) + .StartAsync(); +``` + +[Pause and Resume quickly](https://github.com/bezzad/Downloader/blob/master/src/Downloader.Test/UnitTests/DownloadBuilderTest.cs#L110): + +```csharp +var download = DownloadBuilder.New() + .Build() + .WithUrl(url) + .WithFileLocation(path); +await download.StartAsync(); + +download.Pause(); // pause current download quickly + +download.Resume(); // continue current download quickly ``` --- From 58f93d5548bfadbae0a33dede2a9676d48278f72 Mon Sep 17 00:00:00 2001 From: bezzad Date: Sun, 4 Sep 2022 15:59:52 +0430 Subject: [PATCH 11/19] Fixed test on file and memory storage for ChunkDownloader class --- .../DownloadIntegrationTest.cs | 2 - .../UnitTests/ChunkDownloaderTest.cs | 90 ++++++++++--------- 2 files changed, 48 insertions(+), 44 deletions(-) diff --git a/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs b/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs index 8e915507..8d2bece3 100644 --- a/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs +++ b/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs @@ -1,11 +1,9 @@ using Downloader.DummyHttpServer; -using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.IO; using System.Linq; using System.Threading; -using System.Threading.Tasks; namespace Downloader.Test.IntegrationTests { diff --git a/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs b/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs index fca3f4c4..a1e3b435 100644 --- a/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs +++ b/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs @@ -9,42 +9,21 @@ namespace Downloader.Test.UnitTests { - [TestClass] - public class ChunkDownloaderTest + public abstract class ChunkDownloaderTest { - private DownloadConfiguration _configuration; + protected DownloadConfiguration _configuration; + protected IStorage _storage; [TestInitialize] - public void Initial() - { - _configuration = new DownloadConfiguration { - BufferBlockSize = 1024, - ChunkCount = 16, - ParallelDownload = true, - MaxTryAgainOnFailover = 100, - Timeout = 100, - OnTheFlyDownload = true - }; - } - - [TestMethod] - public void ReadStreamWhenFileStorageTest() - { - ReadStreamTest(new FileStorage("")); - } + public abstract void InitialTest(); [TestMethod] - public void ReadStreamWhenMemoryStorageTest() - { - ReadStreamTest(new MemoryStorage()); - } - - private void ReadStreamTest(IStorage storage) + public void ReadStreamTest() { // arrange var streamSize = 20480; var randomlyBytes = DummyData.GenerateRandomBytes(streamSize); - var chunk = new Chunk(0, streamSize - 1) { Timeout = 100, Storage = storage }; + var chunk = new Chunk(0, streamSize - 1) { Timeout = 100, Storage = _storage }; var chunkDownloader = new ChunkDownloader(chunk, _configuration); using var memoryStream = new MemoryStream(randomlyBytes); @@ -63,18 +42,7 @@ private void ReadStreamTest(IStorage storage) } [TestMethod] - public void ReadStreamProgressEventsWhenMemoryStorageTest() - { - ReadStreamProgressEventsTest(new MemoryStorage()); - } - - [TestMethod] - public void ReadStreamProgressEventsWhenFileStorageTest() - { - ReadStreamProgressEventsTest(new FileStorage("")); - } - - private void ReadStreamProgressEventsTest(IStorage storage) + public void ReadStreamProgressEventsTest() { // arrange var eventCount = 0; @@ -82,7 +50,7 @@ private void ReadStreamProgressEventsTest(IStorage storage) var streamSize = 9 * _configuration.BufferBlockSize; var source = DummyData.GenerateRandomBytes(streamSize); using var sourceMemoryStream = new MemoryStream(source); - var chunk = new Chunk(0, streamSize - 1) { Timeout = 100, Storage = storage }; + var chunk = new Chunk(0, streamSize - 1) { Timeout = 100, Storage = _storage }; var chunkDownloader = new ChunkDownloader(chunk, _configuration); chunkDownloader.DownloadProgressChanged += (s, e) => { eventCount++; @@ -106,16 +74,54 @@ public void ReadStreamTimeoutExceptionTest() // arrange var streamSize = 20480; var randomlyBytes = DummyData.GenerateRandomBytes(streamSize); - var chunk = new Chunk(0, streamSize - 1) { Timeout = 100 }; + var chunk = new Chunk(0, streamSize - 1) { Timeout = 100, Storage = _storage }; var chunkDownloader = new ChunkDownloader(chunk, _configuration); using var memoryStream = new MemoryStream(randomlyBytes); var canceledToken = new CancellationToken(true); // act - async Task CallReadStream() => await chunkDownloader.ReadStream(new MemoryStream(), new PauseTokenSource().Token, canceledToken).ConfigureAwait(false); + async Task CallReadStream() => await chunkDownloader + .ReadStream(new MemoryStream(), new PauseTokenSource().Token, canceledToken) + .ConfigureAwait(false); // assert Assert.ThrowsExceptionAsync(CallReadStream); } } + + [TestClass] + public class ChunkDownloaderOnMemoryTest : ChunkDownloaderTest + { + [TestInitialize] + public override void InitialTest() + { + _configuration = new DownloadConfiguration { + BufferBlockSize = 1024, + ChunkCount = 16, + ParallelDownload = true, + MaxTryAgainOnFailover = 100, + Timeout = 100, + OnTheFlyDownload = true + }; + _storage = new MemoryStorage(); + } + } + + [TestClass] + public class ChunkDownloaderOnFileTest : ChunkDownloaderTest + { + [TestInitialize] + public override void InitialTest() + { + _configuration = new DownloadConfiguration { + BufferBlockSize = 1024, + ChunkCount = 16, + ParallelDownload = true, + MaxTryAgainOnFailover = 100, + Timeout = 100, + OnTheFlyDownload = true + }; + _storage = new FileStorage(); + } + } } \ No newline at end of file From 0d5c9d87a370fcdd1a17b59f0fdab23db7ef9617 Mon Sep 17 00:00:00 2001 From: bezzad Date: Sun, 4 Sep 2022 16:06:21 +0430 Subject: [PATCH 12/19] Added PauseResumeReadStream test method --- .../UnitTests/ChunkDownloaderTest.cs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs b/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs index a1e3b435..71d25da4 100644 --- a/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs +++ b/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs @@ -41,6 +41,39 @@ public void ReadStreamTest() chunkDownloader.Chunk.Clear(); } + [TestMethod] + public void PauseResumeReadStreamTest() + { + // arrange + var streamSize = 16*1024; + var randomlyBytes = DummyData.GenerateRandomBytes(streamSize); + var chunk = new Chunk(0, streamSize - 1) { Timeout = 100, Storage = _storage }; + var chunkDownloader = new ChunkDownloader(chunk, _configuration); + using var memoryStream = new MemoryStream(randomlyBytes); + var pauseToken = new PauseTokenSource(); + var pauseCount = 0; + + // act + chunkDownloader.DownloadProgressChanged += (sender, e) => { + if (pauseCount < 10) + { + pauseToken.Pause(); + pauseCount++; + pauseToken.Resume(); + } + }; + chunkDownloader.ReadStream(memoryStream, pauseToken.Token, new CancellationToken()).Wait(); + + // assert + Assert.AreEqual(memoryStream.Length, chunkDownloader.Chunk.Storage.GetLength()); + Assert.AreEqual(10, pauseCount); + var chunkStream = chunkDownloader.Chunk.Storage.OpenRead(); + for (int i = 0; i < streamSize; i++) + Assert.AreEqual(randomlyBytes[i], chunkStream.ReadByte()); + + chunkDownloader.Chunk.Clear(); + } + [TestMethod] public void ReadStreamProgressEventsTest() { From cd8ec0577aad71b76f005b57fa635d931e3b27d9 Mon Sep 17 00:00:00 2001 From: bezzad Date: Mon, 5 Sep 2022 12:02:05 +0430 Subject: [PATCH 13/19] Added GetFileWithNoAcceptRange method to dummy server to test no accept range servers --- .../Controllers/DummyFileController.cs | 19 +++++++- .../DummyFileHelper.cs | 5 +++ .../HelperTests/DummyFileControllerTest.cs | 44 ++++++++++++++++++- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/Downloader.DummyHttpServer/Controllers/DummyFileController.cs b/src/Downloader.DummyHttpServer/Controllers/DummyFileController.cs index 9cd0ada0..4dfbc109 100644 --- a/src/Downloader.DummyHttpServer/Controllers/DummyFileController.cs +++ b/src/Downloader.DummyHttpServer/Controllers/DummyFileController.cs @@ -24,6 +24,7 @@ public DummyFileController(ILogger logger) [Route("file/size/{size}")] public IActionResult GetFile(int size) { + _logger.Log(LogLevel.Information, $"file/size/{size}"); var data = DummyData.GenerateOrderedBytes(size); return File(data, "application/octet-stream", true); } @@ -35,8 +36,9 @@ public IActionResult GetFile(int size) /// Query param of the file size /// [Route("file/{fileName}")] - public IActionResult GetFileWithName(string fileName, int size, bool noheader) + public IActionResult GetFileWithName(string fileName, [FromQuery]int size, [FromQuery]bool noheader) { + _logger.Log(LogLevel.Information, $"file/{fileName}?size={size}&noheader={noheader}"); if (noheader) { var stream = new MemoryStream(DummyData.GenerateOrderedBytes(size)); @@ -58,8 +60,23 @@ public IActionResult GetFileWithName(string fileName, int size, bool noheader) [Route("file/{fileName}/size/{size}")] public IActionResult GetFileWithContentDisposition(string fileName, int size) { + _logger.Log(LogLevel.Information, $"file/{fileName}/size/{size}"); byte[] fileData = DummyData.GenerateOrderedBytes(size); return File(fileData, "application/octet-stream", fileName, true); } + + /// + /// Return the file stream with header content-length and filename. + /// + /// The file name + /// Size of the File + /// + [Route("file/{fileName}/size/{size}/norange")] + public IActionResult GetFileWithNoAcceptRange(string fileName, int size) + { + _logger.Log(LogLevel.Information, $"file/{fileName}/size/{size}/norange"); + byte[] fileData = DummyData.GenerateOrderedBytes(size); + return File(fileData, "application/octet-stream", fileName, false); + } } } diff --git a/src/Downloader.DummyHttpServer/DummyFileHelper.cs b/src/Downloader.DummyHttpServer/DummyFileHelper.cs index 8be54c2d..a23e92cb 100644 --- a/src/Downloader.DummyHttpServer/DummyFileHelper.cs +++ b/src/Downloader.DummyHttpServer/DummyFileHelper.cs @@ -40,6 +40,11 @@ public static string GetFileWithContentDispositionUrl(string filename, int size) return $"http://localhost:{Port}/dummyfile/file/{filename}/size/{size}"; } + public static string GetFileWithNoAcceptRangeUrl(string filename, int size) + { + return $"http://localhost:{Port}/dummyfile/file/{filename}/size/{size}/norange"; + } + public static bool AreEqual(this byte[] expected, Stream actual) { using (actual) diff --git a/src/Downloader.Test/HelperTests/DummyFileControllerTest.cs b/src/Downloader.Test/HelperTests/DummyFileControllerTest.cs index 3f7501a0..e4a67bbd 100644 --- a/src/Downloader.Test/HelperTests/DummyFileControllerTest.cs +++ b/src/Downloader.Test/HelperTests/DummyFileControllerTest.cs @@ -86,9 +86,51 @@ public void GetFileWithContentDispositionTest() Assert.IsTrue(headers["Content-Disposition"].Contains($"filename={filename};")); } - private WebHeaderCollection ReadAndGetHeaders(string url, byte[] bytes) + [TestMethod] + public void GetFileWithRangeTest() + { + // arrange + int size = 1024; + byte[] bytes = new byte[size]; + string url = DummyFileHelper.GetFileUrl(size); + var dummyData = DummyData.GenerateOrderedBytes(size); + + // act + var headers = ReadAndGetHeaders(url, bytes, justFirst512Bytes: true); + + // assert + Assert.IsTrue(dummyData.Take(512).SequenceEqual(bytes.Take(512))); + Assert.AreEqual(contentType, headers["Content-Type"]); + Assert.AreEqual("512", headers["Content-Length"]); + Assert.AreEqual("bytes 0-511/1024", headers["Content-Range"]); + Assert.AreEqual("bytes", headers["Accept-Ranges"]); + } + + [TestMethod] + public void GetFileWithNoAcceptRangeTest() + { + // arrange + int size = 1024; + byte[] bytes = new byte[size]; + string filename = "testfilename.dat"; + string url = DummyFileHelper.GetFileWithNoAcceptRangeUrl(filename, size); + var dummyData = DummyData.GenerateOrderedBytes(size); + + // act + var headers = ReadAndGetHeaders(url, bytes, justFirst512Bytes: true); + + // assert + Assert.IsTrue(dummyData.SequenceEqual(bytes)); + Assert.AreEqual(size.ToString(), headers["Content-Length"]); + Assert.AreEqual(contentType, headers["Content-Type"]); + Assert.IsNull(headers["Accept-Ranges"]); + } + + private WebHeaderCollection ReadAndGetHeaders(string url, byte[] bytes, bool justFirst512Bytes = false) { HttpWebRequest request = WebRequest.CreateHttp(url); + if (justFirst512Bytes) + request.AddRange(0, 511); using HttpWebResponse downloadResponse = request.GetResponse() as HttpWebResponse; var respStream = downloadResponse.GetResponseStream(); respStream.Read(bytes); From 53aea6f37c31057e751208f26ab52101eba0db9d Mon Sep 17 00:00:00 2001 From: bezzad Date: Mon, 5 Sep 2022 12:54:32 +0430 Subject: [PATCH 14/19] Update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c564ac07..131c11c8 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,10 @@ # Downloader -:rocket: Fast and reliable multipart downloader with **.Net Core 3.1+** supporting :rocket: +:rocket: Fast, cross-platform and reliable multipart downloader with **.Net Core** supporting :rocket: Downloader is a modern, fluent, asynchronous, testable and portable library for .NET. This is a multipart downloader with asynchronous progress events. -This library can added in your `.Net Core v3.1` and later or `.Net Framework v4.5` or later projects. +This library can added in your `.Net Core v2` and later or `.Net Framework v4.5` or later projects. Downloader is compatible with .NET Standard 2.0 and above, running on Windows, Linux, and macOS, in full .NET Framework or .NET Core. From 1abe01b7459605b3aa61698829636cb90f460683 Mon Sep 17 00:00:00 2001 From: bezzad Date: Mon, 5 Sep 2022 14:19:54 +0430 Subject: [PATCH 15/19] Updated downloader info and version --- src/Downloader/Downloader.csproj | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Downloader/Downloader.csproj b/src/Downloader/Downloader.csproj index 315294e2..7021f452 100644 --- a/src/Downloader/Downloader.csproj +++ b/src/Downloader/Downloader.csproj @@ -3,7 +3,9 @@ netstandard2.0;netstandard2.1;netcoreapp3.1;net452;net6.0 latestMajor - 2.3.7 + 2.3.8 + 2.3.8 + 2.3.8 Downloader Behzad Khosravifar bezzad @@ -11,7 +13,7 @@ https://github.com/bezzad/Downloader https://github.com/bezzad/Downloader download-manager, downloader, download-file, stream-downloader, multipart-download - Fixed parallel downloading when a server not support download in rang #98 #99 + Add pause and resume quickly feature true Downloader.snk Copyright (C) 2019-2022 Behzad Khosravifar @@ -20,8 +22,7 @@ LICENSE downloader.png git - 2.3.7 - 2.3.7 + README.md embedded From 4e0329c0bc669407594ac9ceb34d4a8cfd31df09 Mon Sep 17 00:00:00 2001 From: bezzad Date: Mon, 5 Sep 2022 14:20:38 +0430 Subject: [PATCH 16/19] Fixed active chunks count issue and add it tests --- .../IntegrationTests/DownloadServiceTest.cs | 264 +++++++++++++++--- src/Downloader/DownloadService.cs | 8 +- 2 files changed, 236 insertions(+), 36 deletions(-) diff --git a/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs b/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs index 565ce73b..801eed12 100644 --- a/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs +++ b/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs @@ -1,6 +1,7 @@ using Downloader.DummyHttpServer; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; +using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.IO; @@ -12,19 +13,25 @@ namespace Downloader.Test.IntegrationTests [TestClass] public class DownloadServiceTest : DownloadService { - [TestMethod] - public void CancelAsyncTest() + private DownloadConfiguration GetDefaultConfig() { - // arrange - AsyncCompletedEventArgs eventArgs = null; - string address = DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb); - Options = new DownloadConfiguration { + return new DownloadConfiguration { BufferBlockSize = 1024, ChunkCount = 8, + ParallelCount = 4, ParallelDownload = true, MaxTryAgainOnFailover = 100, OnTheFlyDownload = true }; + } + + [TestMethod] + public void CancelAsyncTest() + { + // arrange + AsyncCompletedEventArgs eventArgs = null; + string address = DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb); + Options = GetDefaultConfig(); DownloadStarted += (s, e) => CancelAsync(); DownloadFileCompleted += (s, e) => eventArgs = e; @@ -47,13 +54,8 @@ public void CompletesWithErrorWhenBadUrlTest() Exception onCompletionException = null; string address = "https://nofile"; FileInfo file = new FileInfo(Path.GetTempFileName()); - Options = new DownloadConfiguration { - BufferBlockSize = 1024, - ChunkCount = 8, - ParallelDownload = true, - MaxTryAgainOnFailover = 0, - OnTheFlyDownload = true - }; + Options = GetDefaultConfig(); + Options.MaxTryAgainOnFailover = 0; DownloadFileCompleted += delegate (object sender, AsyncCompletedEventArgs e) { onCompletionException = e.Error; }; @@ -136,13 +138,7 @@ public void CancelPerformanceTest() AsyncCompletedEventArgs eventArgs = null; var watch = new Stopwatch(); string address = DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb); - Options = new DownloadConfiguration { - BufferBlockSize = 1024, - ChunkCount = 8, - ParallelDownload = true, - MaxTryAgainOnFailover = 100, - OnTheFlyDownload = true - }; + Options = GetDefaultConfig(); DownloadStarted += (s, e) => { watch.Start(); CancelAsync(); @@ -156,6 +152,8 @@ public void CancelPerformanceTest() // assert Assert.IsTrue(eventArgs?.Cancelled); Assert.IsTrue(watch.ElapsedMilliseconds < 1000); + Assert.AreEqual(4, Options.ParallelCount); + Assert.AreEqual(8, Options.ChunkCount); Clear(); } @@ -168,20 +166,19 @@ public void ResumePerformanceTest() var watch = new Stopwatch(); var isCancelled = false; string address = DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb); - Options = new DownloadConfiguration { - BufferBlockSize = 1024, - ChunkCount = 8, - ParallelDownload = true, - MaxTryAgainOnFailover = 100, - OnTheFlyDownload = true - }; - DownloadStarted += (s, e) => { - if (isCancelled == false) - CancelAsync(); - isCancelled=true; - }; + Options = GetDefaultConfig(); DownloadFileCompleted += (s, e) => eventArgs = e; - DownloadProgressChanged += (s, e) => watch.Stop(); + DownloadProgressChanged += (s, e) => { + if (isCancelled == false) + { + CancelAsync(); + isCancelled=true; + } + else + { + watch.Stop(); + } + }; // act DownloadFileTaskAsync(address).Wait(); @@ -191,7 +188,208 @@ public void ResumePerformanceTest() // assert Assert.IsFalse(eventArgs?.Cancelled); Assert.IsTrue(watch.ElapsedMilliseconds < 1000); + Assert.AreEqual(4, Options.ParallelCount); + Assert.AreEqual(8, Options.ChunkCount); + + Clear(); + } + + [TestMethod] + public void PauseResumeTest() + { + // arrange + AsyncCompletedEventArgs eventArgs = null; + var paused = false; + var cancelled = false; + string address = DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb); + Options = GetDefaultConfig(); + + DownloadFileCompleted += (s, e) => eventArgs = e; + + // act + DownloadProgressChanged += (s, e) => { + Pause(); + cancelled = IsCancelled; + paused = IsPaused; + Resume(); + }; + DownloadFileTaskAsync(address).Wait(); + + // assert + Assert.IsTrue(paused); + Assert.IsFalse(cancelled); + Assert.AreEqual(4, Options.ParallelCount); + Assert.AreEqual(8, Options.ChunkCount); + + // clean up + Clear(); + } + + [TestMethod] + public void DownloadParallelNotSupportedUrlTest() + { + // arrange + var actualChunksCount = 0; + AsyncCompletedEventArgs eventArgs = null; + string address = DummyFileHelper.GetFileWithNoAcceptRangeUrl("test.dat", DummyFileHelper.FileSize16Kb); + Options = GetDefaultConfig(); + DownloadFileCompleted += (s, e) => eventArgs = e; + DownloadStarted += (s, e) => { + actualChunksCount = Package.Chunks.Length; + }; + + // act + DownloadFileTaskAsync(address).Wait(); + + // assert + Assert.IsFalse(Package.IsSupportDownloadInRange); + Assert.AreEqual(1, Options.ParallelCount); + Assert.AreEqual(1, Options.ChunkCount); + Assert.IsFalse(eventArgs?.Cancelled); + Assert.IsTrue(Package.IsSaveComplete); + Assert.IsNull(eventArgs?.Error); + Assert.AreEqual(1, actualChunksCount); + + // clean up + Clear(); + } + + [TestMethod] + public void ResumeNotSupportedUrlTest() + { + // arrange + AsyncCompletedEventArgs eventArgs = null; + var isCancelled = false; + var actualChunksCount = 0; + var progressCount = 0; + var cancelOnProgressNo = 6; + var maxProgressPercentage = 0d; + var address = DummyFileHelper.GetFileWithNoAcceptRangeUrl("test.dat", DummyFileHelper.FileSize16Kb); + Options = GetDefaultConfig(); + DownloadFileCompleted += (s, e) => eventArgs = e; + DownloadProgressChanged += (s, e) => { + if (cancelOnProgressNo == progressCount++) + { + CancelAsync(); + isCancelled=true; + } + else if (isCancelled) + { + actualChunksCount = Package.Chunks.Length; + } + maxProgressPercentage = Math.Max(e.ProgressPercentage, maxProgressPercentage); + }; + + // act + DownloadFileTaskAsync(address).Wait(); // start the download + DownloadFileTaskAsync(Package).Wait(); // resume the downlaod after canceling + + // assert + Assert.IsTrue(isCancelled); + Assert.IsFalse(Package.IsSupportDownloadInRange); + Assert.AreEqual(1, Options.ParallelCount); + Assert.AreEqual(1, Options.ChunkCount); + Assert.IsFalse(eventArgs?.Cancelled); + Assert.IsTrue(Package.IsSaveComplete); + Assert.IsNull(eventArgs?.Error); + Assert.AreEqual(1, actualChunksCount); + Assert.AreEqual(100, maxProgressPercentage); + + // clean up + Clear(); + } + + [TestMethod] + public void ActiveChunksTest() + { + // arrange + var allActiveChunksCount = new List(20); + string address = DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb); + Options = GetDefaultConfig(); + + // act + DownloadProgressChanged += (s, e) => { + allActiveChunksCount.Add(e.ActiveChunks); + }; + DownloadFileTaskAsync(address).Wait(); + + // assert + Assert.AreEqual(4, Options.ParallelCount); + Assert.AreEqual(8, Options.ChunkCount); + Assert.IsTrue(Package.IsSupportDownloadInRange); + Assert.IsTrue(Package.IsSaveComplete); + foreach (var activeChunks in allActiveChunksCount) + Assert.IsTrue(1 <= activeChunks && activeChunks <= 4); + + // clean up + Clear(); + } + + [TestMethod] + public void ActiveChunksWithRangeNotSupportedUrlTest() + { + // arrange + var allActiveChunksCount = new List(20); + string address = DummyFileHelper.GetFileWithNoAcceptRangeUrl("test.dat", DummyFileHelper.FileSize16Kb); + Options = GetDefaultConfig(); + + // act + DownloadProgressChanged += (s, e) => { + allActiveChunksCount.Add(e.ActiveChunks); + }; + DownloadFileTaskAsync(address).Wait(); + + // assert + Assert.AreEqual(1, Options.ParallelCount); + Assert.AreEqual(1, Options.ChunkCount); + Assert.IsFalse(Package.IsSupportDownloadInRange); + Assert.IsTrue(Package.IsSaveComplete); + foreach (var activeChunks in allActiveChunksCount) + Assert.IsTrue(1 <= activeChunks && activeChunks <= 4); + + // clean up + Clear(); + } + + [TestMethod] + public void ActiveChunksAfterCancelResumeWithNotSupportedUrlTest() + { + // arrange + var allActiveChunksCount = new List(20); + var isCancelled = false; + var actualChunksCount = 0; + var progressCount = 0; + var cancelOnProgressNo = 6; + var address = DummyFileHelper.GetFileWithNoAcceptRangeUrl("test.dat", DummyFileHelper.FileSize16Kb); + Options = GetDefaultConfig(); + DownloadProgressChanged += (s, e) => { + allActiveChunksCount.Add(e.ActiveChunks); + if (cancelOnProgressNo == progressCount++) + { + CancelAsync(); + isCancelled=true; + } + else if (isCancelled) + { + actualChunksCount = Package.Chunks.Length; + } + }; + + // act + DownloadFileTaskAsync(address).Wait(); // start the download + DownloadFileTaskAsync(Package).Wait(); // resume the downlaod after canceling + + // assert + Assert.IsTrue(isCancelled); + Assert.IsFalse(Package.IsSupportDownloadInRange); + Assert.IsTrue(Package.IsSaveComplete); + Assert.AreEqual(1, actualChunksCount); + Assert.AreEqual(1, Options.ParallelCount); + Assert.AreEqual(1, Options.ChunkCount); + foreach (var activeChunks in allActiveChunksCount) + Assert.IsTrue(1 <= activeChunks && activeChunks <= 4); + // clean up Clear(); } } diff --git a/src/Downloader/DownloadService.cs b/src/Downloader/DownloadService.cs index d1b5407d..83bfe4db 100644 --- a/src/Downloader/DownloadService.cs +++ b/src/Downloader/DownloadService.cs @@ -196,7 +196,7 @@ private void InitialDownloader(string address) _requestInstance = new Request(address, Options.RequestConfiguration); Package.Address = _requestInstance.Address.OriginalString; _chunkHub = new ChunkHub(Options); - _parallelSemaphore = new SemaphoreSlim(Options.ParallelCount); // TODO: Add Options.ParallelCount to MaxCount + _parallelSemaphore = new SemaphoreSlim(Options.ParallelCount, Options.ParallelCount); } private async Task StartDownload(string fileName) @@ -275,9 +275,9 @@ private async Task StoreDownloadedFile(CancellationToken cancellationToken) private void ValidateBeforeChunking() { + CheckUnlimitedDownload(); CheckSupportDownloadInRange(); SetRangedSizes(); - CheckUnlimitedDownload(); CheckSizes(); } @@ -337,8 +337,8 @@ private void CheckUnlimitedDownload() { if (Package.TotalFileSize <= 1) { + Package.IsSupportDownloadInRange = false; Package.TotalFileSize = 0; - Options.ChunkCount = 1; } } @@ -347,6 +347,8 @@ private void CheckSupportDownloadInRange() if (Package.IsSupportDownloadInRange == false) { Options.ChunkCount = 1; + Options.ParallelCount = 1; + _parallelSemaphore = new SemaphoreSlim(1, 1); } } From 00b489229ddca4615d472ffe6c828eaef2bfc4a2 Mon Sep 17 00:00:00 2001 From: bezzad Date: Mon, 5 Sep 2022 14:29:34 +0430 Subject: [PATCH 17/19] Added cancel after pause test --- .../IntegrationTests/DownloadServiceTest.cs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs b/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs index 801eed12..b2aba4c8 100644 --- a/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs +++ b/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs @@ -225,6 +225,47 @@ public void PauseResumeTest() Clear(); } + [TestMethod] + public void CancelAfterPauseTest() + { + // arrange + AsyncCompletedEventArgs eventArgs = null; + var pauseStateBeforeCancel = false; + var cancelStateBeforeCancel = false; + var pauseStateAfterCancel = false; + var cancelStateAfterCancel = false; + string address = DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb); + Options = GetDefaultConfig(); + + DownloadFileCompleted += (s, e) => eventArgs = e; + + // act + DownloadProgressChanged += (s, e) => { + Pause(); + cancelStateBeforeCancel = IsCancelled; + pauseStateBeforeCancel = IsPaused; + CancelAsync(); + pauseStateAfterCancel = IsPaused; + cancelStateAfterCancel = IsCancelled; + + }; + DownloadFileTaskAsync(address).Wait(); + + // assert + Assert.IsTrue(pauseStateBeforeCancel); + Assert.IsFalse(cancelStateBeforeCancel); + Assert.IsFalse(pauseStateAfterCancel); + Assert.IsTrue(cancelStateAfterCancel); + Assert.AreEqual(4, Options.ParallelCount); + Assert.AreEqual(8, Options.ChunkCount); + Assert.AreEqual(8, Options.ChunkCount); + Assert.IsFalse(Package.IsSaveComplete); + Assert.IsTrue(eventArgs.Cancelled); + + // clean up + Clear(); + } + [TestMethod] public void DownloadParallelNotSupportedUrlTest() { From 6bd28744a430468a30c0d395fdfda16715f16074 Mon Sep 17 00:00:00 2001 From: bezzad Date: Mon, 5 Sep 2022 14:45:16 +0430 Subject: [PATCH 18/19] Fixed codefactor issues --- .../IntegrationTests/DownloadServiceTest.cs | 7 ++-- .../UnitTests/ChunkDownloaderOnFileTest.cs | 22 ++++++++++++ .../UnitTests/ChunkDownloaderOnMemoryTest.cs | 22 ++++++++++++ .../UnitTests/ChunkDownloaderTest.cs | 36 ------------------- .../UnitTests/PauseTokenTest.cs | 6 ++-- 5 files changed, 49 insertions(+), 44 deletions(-) create mode 100644 src/Downloader.Test/UnitTests/ChunkDownloaderOnFileTest.cs create mode 100644 src/Downloader.Test/UnitTests/ChunkDownloaderOnMemoryTest.cs diff --git a/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs b/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs index b2aba4c8..3c22fce6 100644 --- a/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs +++ b/src/Downloader.Test/IntegrationTests/DownloadServiceTest.cs @@ -247,7 +247,6 @@ public void CancelAfterPauseTest() CancelAsync(); pauseStateAfterCancel = IsPaused; cancelStateAfterCancel = IsCancelled; - }; DownloadFileTaskAsync(address).Wait(); @@ -360,7 +359,7 @@ public void ActiveChunksTest() Assert.IsTrue(Package.IsSupportDownloadInRange); Assert.IsTrue(Package.IsSaveComplete); foreach (var activeChunks in allActiveChunksCount) - Assert.IsTrue(1 <= activeChunks && activeChunks <= 4); + Assert.IsTrue(activeChunks >= 1 && activeChunks <= 4); // clean up Clear(); @@ -386,7 +385,7 @@ public void ActiveChunksWithRangeNotSupportedUrlTest() Assert.IsFalse(Package.IsSupportDownloadInRange); Assert.IsTrue(Package.IsSaveComplete); foreach (var activeChunks in allActiveChunksCount) - Assert.IsTrue(1 <= activeChunks && activeChunks <= 4); + Assert.IsTrue(activeChunks >= 1 && activeChunks <= 4); // clean up Clear(); @@ -428,7 +427,7 @@ public void ActiveChunksAfterCancelResumeWithNotSupportedUrlTest() Assert.AreEqual(1, Options.ParallelCount); Assert.AreEqual(1, Options.ChunkCount); foreach (var activeChunks in allActiveChunksCount) - Assert.IsTrue(1 <= activeChunks && activeChunks <= 4); + Assert.IsTrue(activeChunks >= 1 && activeChunks <= 4); // clean up Clear(); diff --git a/src/Downloader.Test/UnitTests/ChunkDownloaderOnFileTest.cs b/src/Downloader.Test/UnitTests/ChunkDownloaderOnFileTest.cs new file mode 100644 index 00000000..d10460fc --- /dev/null +++ b/src/Downloader.Test/UnitTests/ChunkDownloaderOnFileTest.cs @@ -0,0 +1,22 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Downloader.Test.UnitTests +{ + [TestClass] + public class ChunkDownloaderOnFileTest : ChunkDownloaderTest + { + [TestInitialize] + public override void InitialTest() + { + _configuration = new DownloadConfiguration { + BufferBlockSize = 1024, + ChunkCount = 16, + ParallelDownload = true, + MaxTryAgainOnFailover = 100, + Timeout = 100, + OnTheFlyDownload = true + }; + _storage = new FileStorage(); + } + } +} diff --git a/src/Downloader.Test/UnitTests/ChunkDownloaderOnMemoryTest.cs b/src/Downloader.Test/UnitTests/ChunkDownloaderOnMemoryTest.cs new file mode 100644 index 00000000..97f5c5f9 --- /dev/null +++ b/src/Downloader.Test/UnitTests/ChunkDownloaderOnMemoryTest.cs @@ -0,0 +1,22 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Downloader.Test.UnitTests +{ + [TestClass] + public class ChunkDownloaderOnMemoryTest : ChunkDownloaderTest + { + [TestInitialize] + public override void InitialTest() + { + _configuration = new DownloadConfiguration { + BufferBlockSize = 1024, + ChunkCount = 16, + ParallelDownload = true, + MaxTryAgainOnFailover = 100, + Timeout = 100, + OnTheFlyDownload = true + }; + _storage = new MemoryStorage(); + } + } +} diff --git a/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs b/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs index 71d25da4..bb9ed4ab 100644 --- a/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs +++ b/src/Downloader.Test/UnitTests/ChunkDownloaderTest.cs @@ -121,40 +121,4 @@ async Task CallReadStream() => await chunkDownloader Assert.ThrowsExceptionAsync(CallReadStream); } } - - [TestClass] - public class ChunkDownloaderOnMemoryTest : ChunkDownloaderTest - { - [TestInitialize] - public override void InitialTest() - { - _configuration = new DownloadConfiguration { - BufferBlockSize = 1024, - ChunkCount = 16, - ParallelDownload = true, - MaxTryAgainOnFailover = 100, - Timeout = 100, - OnTheFlyDownload = true - }; - _storage = new MemoryStorage(); - } - } - - [TestClass] - public class ChunkDownloaderOnFileTest : ChunkDownloaderTest - { - [TestInitialize] - public override void InitialTest() - { - _configuration = new DownloadConfiguration { - BufferBlockSize = 1024, - ChunkCount = 16, - ParallelDownload = true, - MaxTryAgainOnFailover = 100, - Timeout = 100, - OnTheFlyDownload = true - }; - _storage = new FileStorage(); - } - } } \ No newline at end of file diff --git a/src/Downloader.Test/UnitTests/PauseTokenTest.cs b/src/Downloader.Test/UnitTests/PauseTokenTest.cs index c4b4b60b..bdc27249 100644 --- a/src/Downloader.Test/UnitTests/PauseTokenTest.cs +++ b/src/Downloader.Test/UnitTests/PauseTokenTest.cs @@ -29,11 +29,9 @@ public void TestPauseTaskWithPauseToken() { Assert.IsTrue(expectedCount >= Counter, $"Expected: {expectedCount}, Actual: {Counter}"); pts.Resume(); - while (pts.IsPaused || expectedCount == Counter) - ; + while (pts.IsPaused || expectedCount == Counter); pts.Pause(); - while (pts.IsPaused == false) - ; + while (pts.IsPaused == false); Interlocked.Exchange(ref expectedCount, Counter+4); Thread.Sleep(10); } From 1eb044e1cd9de4ea4a584c593d0f7648430e02f3 Mon Sep 17 00:00:00 2001 From: bezzad Date: Mon, 5 Sep 2022 15:25:24 +0430 Subject: [PATCH 19/19] Fixed sample links --- src/Downloader.Sample/DownloadList.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Downloader.Sample/DownloadList.json b/src/Downloader.Sample/DownloadList.json index 3771fa5f..c43c5d47 100644 --- a/src/Downloader.Sample/DownloadList.json +++ b/src/Downloader.Sample/DownloadList.json @@ -1,12 +1,4 @@ [ - //{ - // "FileName": "D:\\TestDownload\\file_example_MP4_1920_18MG.mp4", - // "Url": "https://file-examples.com/storage/fef3ae9ac162ce030988192/2017/04/file_example_MP4_1920_18MG.mp4" - //}, - //{ - // "FileName": "D:\\TestDownload\\Minions.The.Rise.of.Gru.2022.mkv", - // "Url": "https://5f8u2z8mn5qjqvfdxs59z5g6aw8djtnew.kinguploadf2m15.xyz/Film/2022/Minions.The.Rise.of.Gru.2022.1080p.WEB-DL.GalaxyRG.Farsi.Sub.Film2Media.mkv" - //}, //{ // "FileName": "D:\\TestDownload\\Hello - 81606.mp4", // "Url": "https://cdn.pixabay.com/vimeo/576083058/Hello%20-%2081605.mp4?width=1920&hash=e6f56273dcd2f28fd1a9fe6e77f66d7e157b33f6&download=1"