Skip to content

Commit

Permalink
Merge branch 'release/v2.3.7'
Browse files Browse the repository at this point in the history
  • Loading branch information
bezzad committed Aug 23, 2022
2 parents cdab6db + 3fe6881 commit 92f2a6e
Show file tree
Hide file tree
Showing 9 changed files with 256 additions and 63 deletions.
66 changes: 44 additions & 22 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,28 +75,34 @@ var downloadOpt = new DownloadConfiguration()
```csharp
var downloadOpt = new DownloadConfiguration()
{
BufferBlockSize = 10240, // usually, hosts support max to 8000 bytes, default values is 8000
ChunkCount = 8, // file parts to download, default value is 1
MaximumBytesPerSecond = 1024 * 1024, // download speed limited to 1MB/s, default values is zero or unlimited
MaxTryAgainOnFailover = int.MaxValue, // the maximum number of times to fail
OnTheFlyDownload = false, // caching in-memory or not? default values is true
ParallelDownload = true, // download parts of file as parallel or not. Default value is false
ParallelCount = 4, // number of parallel downloads
TempDirectory = "C:\\temp", // Set the temp path for buffering chunk files, the default path is Path.GetTempPath()
Timeout = 1000, // timeout (millisecond) per stream block reader, default values is 1000
RangeDownload = true, // set true if you want to download just a certain range of bytes of a large file
RangeLow = 273618157, // floor offset of download range of a large file
RangeHigh = 305178560, // ceiling offset of download range of a large file
RequestConfiguration = // config and customize request headers
BufferBlockSize = 10240, // usually, hosts support max to 8000 bytes, default values is 8000
ChunkCount = 8, // file parts to download, default value is 1
MaximumBytesPerSecond = 1024 * 1024 * 2, // download speed limited to 2MB/s, default values is zero or unlimited
MaxTryAgainOnFailover = 5, // the maximum number of times to fail
OnTheFlyDownload = false, // caching in-memory or not? default values is true
ParallelDownload = true, // download parts of file as parallel or not. Default value is false
ParallelCount = 4, // number of parallel downloads. The default value is the same as the chunk count
TempDirectory = @"C:\temp", // Set the temp path for buffering chunk files, the default path is Path.GetTempPath()
Timeout = 1000, // timeout (millisecond) per stream block reader, default values is 1000
RangeDownload = false, // set true if you want to download just a specific range of bytes of a large file
RangeLow = 0, // floor offset of download range of a large file
RangeHigh = 0, // ceiling offset of download range of a large file
RequestConfiguration =
{
// config and customize request headers
Accept = "*/*",
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
CookieContainer = new CookieContainer(), // Add your cookies
Headers = new WebHeaderCollection(), // Add your custom headers
KeepAlive = false, // default value is false
ProtocolVersion = HttpVersion.Version11, // Default value is HTTP 1.1
CookieContainer = cookies,
Headers = new WebHeaderCollection(), // { Add your custom headers }
KeepAlive = true, // default value is false
ProtocolVersion = HttpVersion.Version11, // default value is HTTP 1.1
UseDefaultCredentials = false,
UserAgent = $"DownloaderSample/{Assembly.GetExecutingAssembly().GetName().Version.ToString(3)}"
UserAgent = $"DownloaderSample/{Assembly.GetExecutingAssembly().GetName().Version?.ToString(3)}"
// Proxy = new WebProxy() {
// Address = new Uri("http://YourProxyServer/proxy.pac"),
// UseDefaultCredentials = false,
// Credentials = System.Net.CredentialCache.DefaultNetworkCredentials,
// BypassProxyOnLocal = true
// }
}
};
```
Expand Down Expand Up @@ -212,18 +218,34 @@ await download.StartAsync();
Resume the existing download package:

```csharp
await DownloadBuilder.Build(package);
await DownloadBuilder.Build(package).StartAsync();
```

Resume the existing download package with a new configuration:

```csharp
await DownloadBuilder.Build(package, new DownloadConfiguration())
.StartAsync();
await DownloadBuilder.Build(package, new DownloadConfiguration()).StartAsync();
```

---

## When does Downloader fail to download in multiple chunks?

### Content-Length:
If your URL server does not provide the file size in the response header (`Content-Length`).
The Downloader cannot split the file into multiple parts and continues its work with one chunk.

### Accept-Ranges:
If the server return `Accept-Ranges: none` in the responses header then that means the server does not support download in range and
the Downloader cannot use multiple chunking and continues its work with one chunk.

### Content-Range:
At first, the Downloader sends a GET request to the server to fetch the file's size in the range.
If the server does not provide `Content-Range` in the header then that means the server does not support download in range.
Therefore, the Downloader has to continue its work with one chunk.

---

## How to serialize and deserialize downloader package

### **What is Serialization?**
Expand Down
4 changes: 4 additions & 0 deletions src/Downloader.Sample/DownloadList.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@
// "FileName": "D:\\TestDownload\\Minions.The.Rise.of.Gru.2022.mkv",
// "Url": "https://5f8u2z8mn5qjqvfdxs59z5g6aw8djtnew.kinguploadf2m15.xyz/Film/2022/Minions.The.Rise.of.Gru.2022.1080p.WEB-DL.GalaxyRG.Farsi.Sub.Film2Media.mkv"
//},
//{
// "FileName": "D:\\TestDownload\\Hello - 81606.mp4",
// "Url": "https://cdn.pixabay.com/vimeo/576083058/Hello%20-%2081605.mp4?width=1920&hash=e6f56273dcd2f28fd1a9fe6e77f66d7e157b33f6&download=1"
//},
{
"FileName": "D:\\TestDownload\\LocalFile100MB_Raw.dat",
"Url": "http://localhost:3333/dummyfile/file/size/104857600"
Expand Down
38 changes: 19 additions & 19 deletions src/Downloader.Sample/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -91,38 +91,38 @@ private static void AddKeyboardHandler()

private static DownloadConfiguration GetDownloadConfiguration()
{
string version = Assembly.GetExecutingAssembly().GetName().Version?.ToString(3) ?? "1";
var cookies = new CookieContainer();
cookies.Add(new Cookie("download-type", "test") { Domain = "domain.com" });

return new DownloadConfiguration {
BufferBlockSize = 10240, // usually, hosts support max to 8000 bytes, default values is 8000
ChunkCount = 8, // file parts to download, default value is 1
BufferBlockSize = 10240, // usually, hosts support max to 8000 bytes, default values is 8000
ChunkCount = 8, // file parts to download, default value is 1
MaximumBytesPerSecond = 1024 * 1024 * 2, // download speed limited to 2MB/s, default values is zero or unlimited
MaxTryAgainOnFailover = 5, // the maximum number of times to fail
OnTheFlyDownload = false, // caching in-memory or not? default values is true
ParallelDownload = true, // download parts of file as parallel or not. Default value is false
ParallelCount = 4, // number of parallel downloads
TempDirectory = "C:\\temp", // Set the temp path for buffering chunk files, the default path is Path.GetTempPath()
Timeout = 1000, // timeout (millisecond) per stream block reader, default values is 1000
RangeDownload = false,
RangeLow = 0,
RangeHigh = 0,
RequestConfiguration = {
MaxTryAgainOnFailover = 5, // the maximum number of times to fail
OnTheFlyDownload = false, // caching in-memory or not? default values is true
ParallelDownload = true, // download parts of file as parallel or not. Default value is false
ParallelCount = 4, // number of parallel downloads. The default value is the same as the chunk count
TempDirectory = @"C:\temp", // Set the temp path for buffering chunk files, the default path is Path.GetTempPath()
Timeout = 1000, // timeout (millisecond) per stream block reader, default values is 1000
RangeDownload = false, // set true if you want to download just a specific range of bytes of a large file
RangeLow = 0, // floor offset of download range of a large file
RangeHigh = 0, // ceiling offset of download range of a large file
RequestConfiguration =
{
// config and customize request headers
Accept = "*/*",
CookieContainer = cookies,
Headers = new WebHeaderCollection(), // { Add your custom headers }
KeepAlive = true, // default value is false
ProtocolVersion = HttpVersion.Version11, // Default value is HTTP 1.1
Headers = new WebHeaderCollection(), // { Add your custom headers }
KeepAlive = true, // default value is false
ProtocolVersion = HttpVersion.Version11, // default value is HTTP 1.1
UseDefaultCredentials = false,
UserAgent = $"DownloaderSample/{version}"
//Proxy = new WebProxy() {
UserAgent = $"DownloaderSample/{Assembly.GetExecutingAssembly().GetName().Version?.ToString(3)}"
// Proxy = new WebProxy() {
// Address = new Uri("http://YourProxyServer/proxy.pac"),
// UseDefaultCredentials = false,
// Credentials = System.Net.CredentialCache.DefaultNetworkCredentials,
// BypassProxyOnLocal = true
//}
// }
}
};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -391,10 +391,10 @@ public void TestContentWhenDownloadOnMemoryStream()
var downloader = new DownloadService(Config);

// act
using var stream = (MemoryStream)downloader.DownloadFileTaskAsync(DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize1Kb)).Result;
using var stream = (MemoryStream)downloader.DownloadFileTaskAsync(DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb)).Result;

// assert
Assert.IsTrue(DummyFileHelper.File1Kb.SequenceEqual(stream.ToArray()));
Assert.IsTrue(DummyFileHelper.File16Kb.SequenceEqual(stream.ToArray()));
}

[TestMethod]
Expand Down
95 changes: 92 additions & 3 deletions src/Downloader.Test/UnitTests/RequestTest.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
using System.Net;
using System.Text;
using Downloader.DummyHttpServer;
using Downloader.DummyHttpServer;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using System.Collections.Generic;
using System.Net;
using System.Text;

namespace Downloader.Test.UnitTests
{
Expand Down Expand Up @@ -357,6 +358,94 @@ public void GetFileSizeTest()
Assert.AreEqual(expectedSize, actualSize);
}

[TestMethod]
public void IsSupportDownloadInRangeTest()
{
// arrange
var url = DummyFileHelper.GetFileUrl(DummyFileHelper.FileSize16Kb);

// act
var actualCan = new Request(url).IsSupportDownloadInRange().Result;

// assert
Assert.IsTrue(actualCan);
}

[TestMethod]
public void GetTotalSizeFromContentLengthTest()
{
// arrange
var length = 23432L;
var headers = new Dictionary<string, string>() { { "Content-Length", length.ToString() } };
var request = new Request("www.example.com");

// act
var actualLength = request.GetTotalSizeFromContentLength(headers);

// assert
Assert.AreEqual(length, actualLength);
}

[TestMethod]
public void GetTotalSizeFromContentLengthWhenNoHeaderTest()
{
// arrange
var length = -1;
var headers = new Dictionary<string, string>();
var request = new Request("www.example.com");

// act
var actualLength = request.GetTotalSizeFromContentLength(headers);

// assert
Assert.AreEqual(length, actualLength);
}

[TestMethod]
public void GetTotalSizeFromContentRangeTest()
{
TestGetTotalSizeFromContentRange(23432, "bytes 0-0/23432");
}

[TestMethod]
public void GetTotalSizeFromContentRangeWhenUnknownSizeTest()
{
TestGetTotalSizeFromContentRange(-1, "bytes 0-1000/*");
}

[TestMethod]
public void GetTotalSizeFromContentRangeWhenUnknownRangeTest()
{
TestGetTotalSizeFromContentRange(23529, "bytes */23529");
}

[TestMethod]
public void GetTotalSizeFromContentRangeWhenIncorrectTest()
{
TestGetTotalSizeFromContentRange(23589, "bytes -0/23589");
}

[TestMethod]
public void GetTotalSizeFromContentRangeWhenNoHeaderTest()
{
TestGetTotalSizeFromContentRange(-1, null);
}

private void TestGetTotalSizeFromContentRange(long expectedLength, string contentRange)
{
// arrange
var request = new Request("www.example.com");
var headers = new Dictionary<string, string>();
if (string.IsNullOrEmpty(contentRange) == false)
headers["Content-Range"] = contentRange;

// act
var actualLength = request.GetTotalSizeFromContentRange(headers);

// assert
Assert.AreEqual(expectedLength, actualLength);
}

[TestMethod]
public void ToUnicodeFromEnglishToEnglishTest()
{
Expand Down
1 change: 1 addition & 0 deletions src/Downloader/DownloadPackage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ public class DownloadPackage
public string FileName { get; set; }
public Chunk[] Chunks { get; set; }
public long ReceivedBytesSize => Chunks?.Sum(chunk => chunk.Position) ?? 0;
public bool IsSupportDownloadInRange { get; set; } = true;

public void Clear()
{
Expand Down
17 changes: 16 additions & 1 deletion src/Downloader/DownloadService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ private async Task<Stream> StartDownload()
try
{
Package.TotalFileSize = await _requestInstance.GetFileSize().ConfigureAwait(false);
Package.IsSupportDownloadInRange = await _requestInstance.IsSupportDownloadInRange().ConfigureAwait(false);
OnDownloadStarted(new DownloadStartedEventArgs(Package.FileName, Package.TotalFileSize));
ValidateBeforeChunking();
Package.Chunks ??= _chunkHub.ChunkFile(Package.TotalFileSize, Options.ChunkCount, Options.RangeLow);
Expand Down Expand Up @@ -258,6 +259,7 @@ private async Task StoreDownloadedFile(CancellationToken cancellationToken)

private void ValidateBeforeChunking()
{
CheckSupportDownloadInRange();
SetRangedSizes();
CheckUnlimitedDownload();
CheckSizes();
Expand All @@ -267,6 +269,11 @@ private void SetRangedSizes()
{
if (Options.RangeDownload)
{
if(!Package.IsSupportDownloadInRange)
{
throw new NotSupportedException("The server of your desired address does not support download in a specific range");
}

if (Options.RangeHigh < Options.RangeLow)
{
Options.RangeLow = Options.RangeHigh - 1;
Expand Down Expand Up @@ -312,13 +319,21 @@ private void CheckSizes()

private void CheckUnlimitedDownload()
{
if (Package.TotalFileSize <= 0)
if (Package.TotalFileSize <= 1)
{
Package.TotalFileSize = 0;
Options.ChunkCount = 1;
}
}

private void CheckSupportDownloadInRange()
{
if (Package.IsSupportDownloadInRange == false)
{
Options.ChunkCount = 1;
}
}

private async Task ParallelDownload(CancellationToken cancellationToken)
{
var tasks = Package.Chunks.Select(chunk => DownloadChunk(chunk, cancellationToken));
Expand Down
9 changes: 4 additions & 5 deletions src/Downloader/Downloader.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,15 @@
<PropertyGroup>
<TargetFrameworks>netstandard2.0;netstandard2.1;netcoreapp3.1;net452;net6.0</TargetFrameworks>
<LangVersion>latestMajor</LangVersion>
<Version>2.3.6</Version>
<Version>2.3.7</Version>
<Title>Downloader</Title>
<Authors>Behzad Khosravifar</Authors>
<Company>bezzad</Company>
<Description>Fast and reliable multipart downloader with asynchronous progress events for .NET</Description>
<PackageProjectUrl>https://github.com/bezzad/Downloader</PackageProjectUrl>
<RepositoryUrl>https://github.com/bezzad/Downloader</RepositoryUrl>
<PackageTags>download-manager, downloader, download-file, stream-downloader, multipart-download</PackageTags>
<PackageReleaseNotes>* Merge PR #97
* Fix issue #93</PackageReleaseNotes>
<PackageReleaseNotes>Fixed parallel downloading when a server not support download in rang #98 #99</PackageReleaseNotes>
<SignAssembly>true</SignAssembly>
<AssemblyOriginatorKeyFile>Downloader.snk</AssemblyOriginatorKeyFile>
<Copyright>Copyright (C) 2019-2022 Behzad Khosravifar</Copyright>
Expand All @@ -21,8 +20,8 @@
<PackageLicenseFile>LICENSE</PackageLicenseFile>
<PackageIcon>downloader.png</PackageIcon>
<RepositoryType>git</RepositoryType>
<AssemblyVersion>2.3.6</AssemblyVersion>
<FileVersion>2.3.6</FileVersion>
<AssemblyVersion>2.3.7</AssemblyVersion>
<FileVersion>2.3.7</FileVersion>
<PackageReadmeFile>README.md</PackageReadmeFile>
<DebugType>embedded</DebugType>
</PropertyGroup>
Expand Down
Loading

0 comments on commit 92f2a6e

Please sign in to comment.