Skip to content

Commit

Permalink
Merge pull request #102 from bezzad/develop
Browse files Browse the repository at this point in the history 
Fixed no supported servers issue #98
  • Loading branch information
@bezzad
bezzad authored Sep 5, 2022
2 parents 92f2a6e + 1eb044e commit 42e87ac
Show file tree
Hide file tree
Showing 25 changed files with 904 additions and 205 deletions.
145 changes: 102 additions & 43 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@

# Downloader

:rocket: Fast and reliable multipart downloader with **.Net Core 3.1+** supporting :rocket:
:rocket: Fast, cross-platform and reliable multipart downloader with **.Net Core** supporting :rocket:

Downloader is a modern, fluent, asynchronous, testable and portable library for .NET. This is a multipart downloader with asynchronous progress events.
This library can added in your `.Net Core v3.1` and later or `.Net Framework v4.5` or later projects.
This library can added in your `.Net Core v2` and later or `.Net Framework v4.5` or later projects.

Downloader is compatible with .NET Standard 2.0 and above, running on Windows, Linux, and macOS, in full .NET Framework or .NET Core.
Downloader is compatible with .NET Standard 2.0 and above, running on Windows, Linux, and macOS, in full .NET Framework or .NET Core.

> For a complete example see [Downloader.Sample](https://github.com/bezzad/Downloader/blob/master/src/Downloader.Sample/Program.cs) project from this repository.
## Sample Console Application

Expand All @@ -37,7 +39,8 @@ Downloader is compatible with .NET Standard 2.0 and above, running on Windows, L
- Store download package object to resume the download when you want.
- Get download speed or progress percentage in each progress event.
- Get download progress events per chunk downloads.
- Pause and Resume your downloads with package object.
- Fast Pause and Resume downloads asynchronously.
- Stop and Resume downloads whenever you want with the package object.
- Supports large file download.
- Set a dynamic speed limit on downloads (changeable speed limitation on the go).
- Download files without storing on disk and get a memory stream for each downloaded file.
Expand Down Expand Up @@ -72,37 +75,53 @@ var downloadOpt = new DownloadConfiguration()

### Complex Configuration


> **Note**: *Do not use all of the below options in your applications, just add which one you need.*
```csharp
var downloadOpt = new DownloadConfiguration()
{
BufferBlockSize = 10240, // usually, hosts support max to 8000 bytes, default values is 8000
ChunkCount = 8, // file parts to download, default value is 1
MaximumBytesPerSecond = 1024 * 1024 * 2, // download speed limited to 2MB/s, default values is zero or unlimited
MaxTryAgainOnFailover = 5, // the maximum number of times to fail
OnTheFlyDownload = false, // caching in-memory or not? default values is true
ParallelDownload = true, // download parts of file as parallel or not. Default value is false
ParallelCount = 4, // number of parallel downloads. The default value is the same as the chunk count
TempDirectory = @"C:\temp", // Set the temp path for buffering chunk files, the default path is Path.GetTempPath()
Timeout = 1000, // timeout (millisecond) per stream block reader, default values is 1000
RangeDownload = false, // set true if you want to download just a specific range of bytes of a large file
RangeLow = 0, // floor offset of download range of a large file
RangeHigh = 0, // ceiling offset of download range of a large file
// usually, hosts support max to 8000 bytes, default values is 8000
BufferBlockSize = 10240,
// file parts to download, default value is 1
ChunkCount = 8,
// download speed limited to 2MB/s, default values is zero or unlimited
MaximumBytesPerSecond = 1024*1024*2,
// the maximum number of times to fail
MaxTryAgainOnFailover = 5,
// caching in-memory or not? default values is true
OnTheFlyDownload = false,
// download parts of file as parallel or not. Default value is false
ParallelDownload = true,
// number of parallel downloads. The default value is the same as the chunk count
ParallelCount = 4,
// Set the temp path for buffering chunk files, the default path is Path.GetTempPath()
TempDirectory = @"C:\temp",
// timeout (millisecond) per stream block reader, default values is 1000
Timeout = 1000,
// set true if you want to download just a specific range of bytes of a large file
RangeDownload = false,
// floor offset of download range of a large file
RangeLow = 0,
// ceiling offset of download range of a large file
RangeHigh = 0,
// config and customize request headers
RequestConfiguration =
{
// config and customize request headers
{
Accept = "*/*",
CookieContainer = cookies,
Headers = new WebHeaderCollection(), // { Add your custom headers }
KeepAlive = true, // default value is false
Headers = new WebHeaderCollection(), // { your custom headers }
KeepAlive = true, // default value is false
ProtocolVersion = HttpVersion.Version11, // default value is HTTP 1.1
UseDefaultCredentials = false,
UserAgent = $"DownloaderSample/{Assembly.GetExecutingAssembly().GetName().Version?.ToString(3)}"
// Proxy = new WebProxy() {
// Address = new Uri("http://YourProxyServer/proxy.pac"),
// UseDefaultCredentials = false,
// Credentials = System.Net.CredentialCache.DefaultNetworkCredentials,
// BypassProxyOnLocal = true
// }
// your custom user agent or your_app_name/app_version.
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
Proxy = new WebProxy() {
Address = new Uri("http://YourProxyServer/proxy.pac"),
UseDefaultCredentials = false,
Credentials = System.Net.CredentialCache.DefaultNetworkCredentials,
BypassProxyOnLocal = true
}
}
};
```
Expand All @@ -119,13 +138,19 @@ var downloader = new DownloadService(downloadOpt);
// Provide `FileName` and `TotalBytesToReceive` at the start of each downloads
downloader.DownloadStarted += OnDownloadStarted;

// Provide any information about chunker downloads, like progress percentage per chunk, speed, total received bytes and received bytes array to live streaming.
// Provide any information about chunker downloads,
// like progress percentage per chunk, speed,
// total received bytes and received bytes array to live streaming.
downloader.ChunkDownloadProgressChanged += OnChunkDownloadProgressChanged;

// Provide any information about download progress, like progress percentage of sum of chunks, total speed, average speed, total received bytes and received bytes array to live streaming.
// Provide any information about download progress,
// like progress percentage of sum of chunks, total speed,
// average speed, total received bytes and received bytes array
// to live streaming.
downloader.DownloadProgressChanged += OnDownloadProgressChanged;

// Download completed event that can include occurred errors or cancelled or download completed successfully.
// Download completed event that can include occurred errors or
// cancelled or download completed successfully.
downloader.DownloadFileCompleted += OnDownloadFileCompleted;
```

Expand All @@ -142,44 +167,58 @@ await downloader.DownloadFileTaskAsync(url, file);
```csharp
DirectoryInfo path = new DirectoryInfo("Your_Path");
string url = @"https://file-examples.com/fileName.zip";
await downloader.DownloadFileTaskAsync(url, path); // download into "Your_Path\fileName.zip"
// download into "Your_Path\fileName.zip"
await downloader.DownloadFileTaskAsync(url, path);
```

## **Step 4c**: Download in MemoryStream

```csharp
Stream destinationStream = await downloader.DownloadFileTaskAsync(url);
// After download completion, it gets a MemoryStream
Stream destinationStream = await downloader.DownloadFileTaskAsync(url);
```

---
## How to **pause** and **resume** downloads quickly

## How to stop and resume downloads
When you want to resume a download quickly after pausing a few seconds. You can call the `Pause` function of the downloader service. This way, streams stay alive and are only suspended by a locker to be released and resumed whenever you want.

The ‍`DownloadService` class has a property called `Package` that stores each step of the download. To stopping or pause the download you must call the `CancelAsync` method, and if you want to continue again, you must call the same `DownloadFileTaskAsync` function with the `Package` parameter to resume your download!
For example:
```csharp
// Pause the download
DownloadService.Pause();

Keep `Package` file to resume from last download positions:
// Resume the download
DownloadService.Resume();
```

---
## How to **stop** and **resume** downloads other time

The ‍`DownloadService` class has a property called `Package` that stores each step of the download. To stopping the download you must call the `CancelAsync` method. Now, if you want to continue again, you must call the same `DownloadFileTaskAsync` function with the `Package` parameter to resume your download. For example:

```csharp
// At first, keep and store the Package file to resume
// your download from the last download position:
DownloadPackage pack = downloader.Package;
```

**Stop or Pause Download:**
**Stop or cancel download:**

```csharp
// This function breaks your stream and cancels progress.
downloader.CancelAsync();
```

**Resume Download:**
**Resuming download after cancelation:**

```csharp
await downloader.DownloadFileTaskAsync(pack);
```

So that you can even save your large downloads with a very small amount in the Package and after restarting the program, restore it again and start continuing your download. In fact, the packages are your instant download snapshots. If your download config has OnTheFlyDownload, the downloaded bytes ​​will be stored in the package itself, but otherwise, only the downloaded file address will be included and you can resume it whenever you like.
For more detail see [StopResumeDownloadTest](https://github.com/bezzad/Downloader/blob/master/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs#L114) method
So that you can even save your large downloads with a very small amount in the Package and after restarting the program, restore it again and start continuing your download. The packages are your snapshot of the download instance. If your download config has `OnTheFlyDownload`, the downloaded bytes ​​will be stored in the package itself. But otherwise, if you download on temp, only the downloaded temp files' addresses will be included in the package and you can resume it whenever you want.
For more detail see [StopResumeDownloadTest](https://github.com/bezzad/Downloader/blob/master/src/Downloader.Test/IntegrationTests/DownloadIntegrationTest.cs#L115) method

> Note: for complete sample see `Downloader.Sample` project from this repository.
> Note: Sometimes a server does not support downloading in a specific range. That time, we can't resume downloads after canceling. So, the downloader starts from the beginning.
---

Expand Down Expand Up @@ -213,18 +252,38 @@ download.DownloadStarted += DownloadStarted;
download.ChunkDownloadProgressChanged += ChunkDownloadProgressChanged;

await download.StartAsync();

download.Stop(); // cancel current download
```

Resume the existing download package:

```csharp
await DownloadBuilder.Build(package).StartAsync();
await DownloadBuilder.New()
.Build(package)
.StartAsync();
```

Resume the existing download package with a new configuration:

```csharp
await DownloadBuilder.Build(package, new DownloadConfiguration()).StartAsync();
await DownloadBuilder.New()
.Build(package)
.StartAsync();
```

[Pause and Resume quickly](https://github.com/bezzad/Downloader/blob/master/src/Downloader.Test/UnitTests/DownloadBuilderTest.cs#L110):

```csharp
var download = DownloadBuilder.New()
.Build()
.WithUrl(url)
.WithFileLocation(path);
await download.StartAsync();

download.Pause(); // pause current download quickly
download.Resume(); // continue current download quickly
```

---
Expand Down
19 changes: 18 additions & 1 deletion src/Downloader.DummyHttpServer/Controllers/DummyFileController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ public DummyFileController(ILogger<DummyFileController> logger)
[Route("file/size/{size}")]
public IActionResult GetFile(int size)
{
_logger.Log(LogLevel.Information, $"file/size/{size}");
var data = DummyData.GenerateOrderedBytes(size);
return File(data, "application/octet-stream", true);
}
Expand All @@ -35,8 +36,9 @@ public IActionResult GetFile(int size)
/// <param name="size">Query param of the file size</param>
/// <returns></returns>
[Route("file/{fileName}")]
public IActionResult GetFileWithName(string fileName, int size, bool noheader)
public IActionResult GetFileWithName(string fileName, [FromQuery]int size, [FromQuery]bool noheader)
{
_logger.Log(LogLevel.Information, $"file/{fileName}?size={size}&noheader={noheader}");
if (noheader)
{
var stream = new MemoryStream(DummyData.GenerateOrderedBytes(size));
Expand All @@ -58,8 +60,23 @@ public IActionResult GetFileWithName(string fileName, int size, bool noheader)
[Route("file/{fileName}/size/{size}")]
public IActionResult GetFileWithContentDisposition(string fileName, int size)
{
_logger.Log(LogLevel.Information, $"file/{fileName}/size/{size}");
byte[] fileData = DummyData.GenerateOrderedBytes(size);
return File(fileData, "application/octet-stream", fileName, true);
}

/// <summary>
/// Return the file stream with header content-length and filename.
/// </summary>
/// <param name="fileName">The file name</param>
/// <param name="size">Size of the File</param>
/// <returns></returns>
[Route("file/{fileName}/size/{size}/norange")]
public IActionResult GetFileWithNoAcceptRange(string fileName, int size)
{
_logger.Log(LogLevel.Information, $"file/{fileName}/size/{size}/norange");
byte[] fileData = DummyData.GenerateOrderedBytes(size);
return File(fileData, "application/octet-stream", fileName, false);
}
}
}
5 changes: 5 additions & 0 deletions src/Downloader.DummyHttpServer/DummyFileHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ public static string GetFileWithContentDispositionUrl(string filename, int size)
return $"http://localhost:{Port}/dummyfile/file/{filename}/size/{size}";
}

public static string GetFileWithNoAcceptRangeUrl(string filename, int size)
{
return $"http://localhost:{Port}/dummyfile/file/{filename}/size/{size}/norange";
}

public static bool AreEqual(this byte[] expected, Stream actual)
{
using (actual)
Expand Down
12 changes: 4 additions & 8 deletions src/Downloader.Sample/DownloadList.json
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
[
//{
// "FileName": "D:\\TestDownload\\file_example_MP4_1920_18MG.mp4",
// "Url": "https://file-examples.com/storage/fef3ae9ac162ce030988192/2017/04/file_example_MP4_1920_18MG.mp4"
//},
//{
// "FileName": "D:\\TestDownload\\Minions.The.Rise.of.Gru.2022.mkv",
// "Url": "https://5f8u2z8mn5qjqvfdxs59z5g6aw8djtnew.kinguploadf2m15.xyz/Film/2022/Minions.The.Rise.of.Gru.2022.1080p.WEB-DL.GalaxyRG.Farsi.Sub.Film2Media.mkv"
//},
//{
// "FileName": "D:\\TestDownload\\Hello - 81606.mp4",
// "Url": "https://cdn.pixabay.com/vimeo/576083058/Hello%20-%2081605.mp4?width=1920&hash=e6f56273dcd2f28fd1a9fe6e77f66d7e157b33f6&download=1"
//},
//{
// "FileName": "D:\\TestDownload\\VS.exe",
// "Url": "https://c2rsetup.officeapps.live.com/c2r/downloadVS.aspx?sku=community&channel=Release&version=VS2022&source=VSLandingPage&includeRecommended=true&cid=2030:9bf2104738684908988ca7dcd5dafed1"
//},
{
"FileName": "D:\\TestDownload\\LocalFile100MB_Raw.dat",
"Url": "http://localhost:3333/dummyfile/file/size/104857600"
Expand Down
5 changes: 3 additions & 2 deletions src/Downloader.Sample/Helper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public static string CalcMemoryMensurableUnit(this double bytes)
return result;
}

public static void UpdateTitleInfo(this DownloadProgressChangedEventArgs e)
public static void UpdateTitleInfo(this DownloadProgressChangedEventArgs e, bool isPaused)
{
double nonZeroSpeed = e.BytesPerSecondSpeed + 0.0001;
int estimateTime = (int)((e.TotalBytesToReceive - e.ReceivedBytesSize) / nonZeroSpeed);
Expand Down Expand Up @@ -55,7 +55,8 @@ public static void UpdateTitleInfo(this DownloadProgressChangedEventArgs e)
Console.Title = $"{progressPercentage}% - " +
$"{speed}/s (avg: {avgSpeed}/s) - " +
$"{estimateTime} {timeLeftUnit} left - " +
$"[{bytesReceived} of {totalBytesToReceive}]";
$"[{bytesReceived} of {totalBytesToReceive}]" +
(isPaused ? " - Paused" : "");
}
}
}
Loading

0 comments on commit 42e87ac

Please sign in to comment.