Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
@bezzad
bezzad committed Jun 16, 2020
2 parents c53c333 + 86b455b commit 70ad0bf
Show file tree
Hide file tree
Showing 6 changed files with 79 additions and 78 deletions.
13 changes: 11 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,17 @@ var downloadOpt = new DownloadConfiguration()
BufferBlockSize = 10240, // usually, hosts support max to 8000 bytes
ChunkCount = 8, // file parts to download
MaxTryAgainOnFailover = int.MaxValue, // the maximum number of times to fail.
OnTheFlyDownload = true, // caching in-memory mode
Timeout = 1000 // timeout (millisecond) per stream block reader
OnTheFlyDownload = false, // caching in-memory or not?
Timeout = 1000, // timeout (millisecond) per stream block reader
RequestConfiguration = // config and customize request headers
{
Accept = "*/*",
UserAgent = $"DownloaderSample/{Assembly.GetExecutingAssembly().GetName().Version.ToString(3)}",
ProtocolVersion = HttpVersion.Version11,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
KeepAlive = true,
UseDefaultCredentials = false
}
};
```

Expand Down
2 changes: 2 additions & 0 deletions src/Downloader.Sample/DownloadList.json
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
[
//{ "FileName": "D:\\TestDownload\\H2017.06.06.0000.0001a.patch", "Url": "http://patch-dl.ffxiv.com/game/4e9a232b/H2017.06.06.0000.0001a.patch" },
//{ "FileName": "D:\\TestDownload\\2020.rar", "Url": "http://resources.downloads.pokecity.club/2020.rar" },
{ "FileName": "D:\\TestDownload\\test.xls", "Url": "https://file-examples.com/wp-content/uploads/2017/02/file_example_XLS_5000.xls" },
{ "FileName": "D:\\TestDownload\\100MB_0.zip", "Url": "http://ipv4.download.thinkbroadband.com/100MB.zip" },
{ "FileName": "D:\\TestDownload\\100MB_1.zip", "Url": "http://ipv4.download.thinkbroadband.com/100MB.zip" },
Expand Down
18 changes: 8 additions & 10 deletions src/Downloader.Sample/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ class Program

static async Task Main(string[] args)
{
var chunkCount = 8;
DownloadList = File.Exists(DownloadListFile)
? JsonConvert.DeserializeObject<List<DownloadItem>>(File.ReadAllText(DownloadListFile))
: null;
Expand Down Expand Up @@ -53,18 +52,17 @@ static async Task Main(string[] args)
{
ParallelDownload = true, // download parts of file as parallel or not
BufferBlockSize = 10240, // usually, hosts support max to 8000 bytes
ChunkCount = chunkCount, // file parts to download
ChunkCount = 8, // file parts to download
MaxTryAgainOnFailover = int.MaxValue, // the maximum number of times to fail.
OnTheFlyDownload = false, // caching in-memory mode or not?
OnTheFlyDownload = false, // caching in-memory or not?
Timeout = 1000, // timeout (millisecond) per stream block reader
RequestConfiguration =
RequestConfiguration = // config and customize request headers
{
Accept = "*/*",
UserAgent = $"DownloaderSample/{Assembly.GetExecutingAssembly().GetName().Version.ToString(3)}",
ProtocolVersion = HttpVersion.Version11,
// AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
KeepAlive = true,
UseDefaultCredentials = false
Accept = "*/*",
UserAgent = $"DownloaderSample/{Assembly.GetExecutingAssembly().GetName().Version.ToString(3)}",
ProtocolVersion = HttpVersion.Version11,
KeepAlive = true,
UseDefaultCredentials = false
}
};

Expand Down
1 change: 0 additions & 1 deletion src/Downloader/DownloadService.Chunk.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ public Chunk(long start, long end)
Start = start;
End = end;
Position = 0;
Data = new byte[Length];
}

public long Id => Start.PairingFunction(End);
Expand Down
117 changes: 53 additions & 64 deletions src/Downloader/DownloadService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,11 +129,9 @@ protected HttpWebRequest GetRequest(string method, Uri address)
protected long GetFileSize(Uri address)
{
var request = GetRequest("HEAD", address);
using (var response = request.GetResponse())
{
// if (long.TryParse(webResponse.Headers.Get("Content-Length"), out var respLength))
return response.ContentLength;
}
using var response = request.GetResponse();
// if (long.TryParse(webResponse.Headers.Get("Content-Length"), out var respLength))
return response.ContentLength;
}
protected Chunk[] ChunkFile(long fileSize, int parts)
{
Expand Down Expand Up @@ -208,25 +206,23 @@ protected async Task<Chunk> DownloadChunk(Uri address, Chunk chunk, Cancellation
var request = GetRequest("GET", address);
request.AddRange(chunk.Start + chunk.Position, chunk.End);

using (var httpWebResponse = request.GetResponse() as HttpWebResponse)
using var httpWebResponse = request.GetResponse() as HttpWebResponse;
if (httpWebResponse == null)
return chunk;

var stream = httpWebResponse.GetResponseStream();
using (stream)
{
if (httpWebResponse == null)
if (stream == null)
return chunk;

var stream = httpWebResponse.GetResponseStream();
using (stream)
{
if (stream == null)
return chunk;

if (Package.Options.OnTheFlyDownload)
await ReadStreamOnTheFly(stream, chunk, token);
else
await ReadStreamOnTheFile(stream, chunk, token);
}

return chunk;
if (Package.Options.OnTheFlyDownload)
await ReadStreamOnTheFly(stream, chunk, token);
else
await ReadStreamOnTheFile(stream, chunk, token);
}

return chunk;
}
catch (TaskCanceledException) // when stream reader timeout occured
{
Expand Down Expand Up @@ -268,24 +264,23 @@ protected async Task<Chunk> DownloadChunk(Uri address, Chunk chunk, Cancellation
protected async Task ReadStreamOnTheFly(Stream stream, Chunk chunk, CancellationToken token)
{
var bytesToReceiveCount = chunk.Length - chunk.Position;
chunk.Data ??= new byte[chunk.Length];
while (bytesToReceiveCount > 0)
{
if (token.IsCancellationRequested)
return;

using (var innerCts = new CancellationTokenSource(Package.Options.Timeout))
{
var count = bytesToReceiveCount > Package.Options.BufferBlockSize
? Package.Options.BufferBlockSize
: (int)bytesToReceiveCount;
var readSize = await stream.ReadAsync(chunk.Data, chunk.Position, count, innerCts.Token);
Package.BytesReceived += readSize;
chunk.Position += readSize;
bytesToReceiveCount = chunk.Length - chunk.Position;

OnChunkDownloadProgressChanged(new DownloadProgressChangedEventArgs(chunk.Id.ToString(), chunk.Length, chunk.Position, DownloadSpeed));
OnDownloadProgressChanged(new DownloadProgressChangedEventArgs(MainProgressName, Package.TotalFileSize, Package.BytesReceived, DownloadSpeed));
}
using var innerCts = new CancellationTokenSource(Package.Options.Timeout);
var count = bytesToReceiveCount > Package.Options.BufferBlockSize
? Package.Options.BufferBlockSize
: (int)bytesToReceiveCount;
var readSize = await stream.ReadAsync(chunk.Data, chunk.Position, count, innerCts.Token);
Package.BytesReceived += readSize;
chunk.Position += readSize;
bytesToReceiveCount = chunk.Length - chunk.Position;

OnChunkDownloadProgressChanged(new DownloadProgressChangedEventArgs(chunk.Id.ToString(), chunk.Length, chunk.Position, DownloadSpeed));
OnDownloadProgressChanged(new DownloadProgressChangedEventArgs(MainProgressName, Package.TotalFileSize, Package.BytesReceived, DownloadSpeed));
}
}
protected async Task ReadStreamOnTheFile(Stream stream, Chunk chunk, CancellationToken token)
Expand All @@ -294,29 +289,25 @@ protected async Task ReadStreamOnTheFile(Stream stream, Chunk chunk, Cancellatio
if (string.IsNullOrWhiteSpace(chunk.FileName) || File.Exists(chunk.FileName) == false)
chunk.FileName = Path.GetTempFileName();

using (var writer = new FileStream(chunk.FileName, FileMode.Append, FileAccess.Write, FileShare.Delete))
using var writer = new FileStream(chunk.FileName, FileMode.Append, FileAccess.Write, FileShare.Delete);
while (bytesToReceiveCount > 0)
{
while (bytesToReceiveCount > 0)
{
if (token.IsCancellationRequested)
return;

using (var innerCts = new CancellationTokenSource(Package.Options.Timeout))
{
var count = bytesToReceiveCount > Package.Options.BufferBlockSize
? Package.Options.BufferBlockSize
: (int)bytesToReceiveCount;
var buffer = new byte[count];
var readSize = await stream.ReadAsync(buffer, 0, count, innerCts.Token);
await writer.WriteAsync(buffer, 0, readSize, token);
Package.BytesReceived += readSize;
chunk.Position += readSize;
bytesToReceiveCount = chunk.Length - chunk.Position;

OnChunkDownloadProgressChanged(new DownloadProgressChangedEventArgs(chunk.Id.ToString(), chunk.Length, chunk.Position, DownloadSpeed));
OnDownloadProgressChanged(new DownloadProgressChangedEventArgs(MainProgressName, Package.TotalFileSize, Package.BytesReceived, DownloadSpeed));
}
}
if (token.IsCancellationRequested)
return;

using var innerCts = new CancellationTokenSource(Package.Options.Timeout);
var count = bytesToReceiveCount > Package.Options.BufferBlockSize
? Package.Options.BufferBlockSize
: (int)bytesToReceiveCount;
var buffer = new byte[count];
var readSize = await stream.ReadAsync(buffer, 0, count, innerCts.Token);
await writer.WriteAsync(buffer, 0, readSize, token);
Package.BytesReceived += readSize;
chunk.Position += readSize;
bytesToReceiveCount = chunk.Length - chunk.Position;

OnChunkDownloadProgressChanged(new DownloadProgressChangedEventArgs(chunk.Id.ToString(), chunk.Length, chunk.Position, DownloadSpeed));
OnDownloadProgressChanged(new DownloadProgressChangedEventArgs(MainProgressName, Package.TotalFileSize, Package.BytesReceived, DownloadSpeed));
}
}
protected async Task MergeChunks(Chunk[] chunks)
Expand All @@ -328,17 +319,15 @@ protected async Task MergeChunks(Chunk[] chunks)
if (Directory.Exists(directory) == false)
Directory.CreateDirectory(directory);

using (var destinationStream = new FileStream(Package.FileName, FileMode.Append, FileAccess.Write))
using var destinationStream = new FileStream(Package.FileName, FileMode.Append, FileAccess.Write);
foreach (var chunk in chunks.OrderBy(c => c.Start))
{
foreach (var chunk in chunks.OrderBy(c => c.Start))
if (Package.Options.OnTheFlyDownload)
await destinationStream.WriteAsync(chunk.Data, 0, (int)chunk.Length);
else if (File.Exists(chunk.FileName))
{
if (Package.Options.OnTheFlyDownload)
await destinationStream.WriteAsync(chunk.Data, 0, (int)chunk.Length);
else if (File.Exists(chunk.FileName))
{
using (var reader = File.OpenRead(chunk.FileName))
await reader.CopyToAsync(destinationStream);
}
using var reader = File.OpenRead(chunk.FileName);
await reader.CopyToAsync(destinationStream);
}
}
}
Expand Down
6 changes: 5 additions & 1 deletion src/Downloader/Downloader.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@
<PackageProjectUrl>https://github.com/bezzad/Downloader</PackageProjectUrl>
<RepositoryUrl>https://github.com/bezzad/Downloader.git</RepositoryUrl>
<PackageTags>download-manager, downloader, download-file, stream-downloader, multipart-download</PackageTags>
<PackageReleaseNotes>Fixed KeepAlive and support old systems</PackageReleaseNotes>
<PackageReleaseNotes>Fixed KeepAlive and support old systems
Added RequestConfiguration to customize requests by user
Fixed memory lake problem in `OnTheFlyDownload=false` mode
</PackageReleaseNotes>
<SignAssembly>true</SignAssembly>
<AssemblyOriginatorKeyFile>Downloader.snk</AssemblyOriginatorKeyFile>
<Copyright>Copyright (C) 2020 Behzad Khosravifar</Copyright>
Expand All @@ -20,6 +23,7 @@
<PackageIcon>downloader.png</PackageIcon>
<RepositoryType>git</RepositoryType>
<AssemblyVersion>1.2.0.0</AssemblyVersion>
<LangVersion>8</LangVersion>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
Expand Down

0 comments on commit 70ad0bf

Please sign in to comment.