Skip to content

Commit

Permalink
Update ver 1.3.3
Browse files Browse the repository at this point in the history 
+ Support large file upload
  • Loading branch information
@trannamtrung1st
trannamtrung1st committed Jul 25, 2021
1 parent 9effb3c commit af1c7e5
Show file tree
Hide file tree
Showing 46 changed files with 1,623 additions and 320 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,3 +358,4 @@ elFinder.Net.Core/Demos/elFinder.Net.AdvancedDemo/wwwroot/lib/
elFinder.Net.Core/Demos/elFinder.Net.AdvancedDemo/wwwroot/upload/
elFinder.Net.Core/Demos/elFinder.Net.AdvancedDemo/thumb/
elFinder.Net.Core/Demos/elFinder.Net.AdvancedDemo/.storage/
elFinder.Net.Core/Demos/elFinder.Net.AdvancedDemo/.temp/
14 changes: 10 additions & 4 deletions elFinder.Net.Core/Demos/elFinder.Net.AdvancedDemo/Controllers/FilesController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,17 @@ private void CustomizeResponse(ConnectorResult connectorResult, IVolume volume,
_connector.PluginManager.Features[typeof(QuotaOptions)] = quotaOptions;

// Volume initialization
var volume = new Volume(_driver, Startup.MapStoragePath($"./upload/{volumePath}"),
$"/api/files/storage/upload/{volumePath}/", $"/api/files/thumb/")
var volume = new Volume(_driver,
Startup.MapStoragePath($"./upload/{volumePath}"),
Startup.TempPath,
$"/api/files/storage/upload/{volumePath}/",
$"/api/files/thumb/",
thumbnailDirectory: Startup.MapStoragePath($"./thumb/{volumePath}"))
{
Name = "My volume",
ThumbnailDirectory = Startup.MapStoragePath($"./thumb/{volumePath}")
MaxUploadFiles = 20,
MaxUploadSizeInMb = 10,
MaxUploadConnections = 3 // 3 upload requests at a time
};

_connector.AddVolume(volume);
Expand Down Expand Up @@ -215,7 +221,7 @@ private void CustomizeResponse(ConnectorResult connectorResult, IVolume volume,
{
Expression = $"file.ext = 'exe'", // Example only
FileFilter = (file) => file.Extension == ".exe",
Locked = true, Write = false, ShowOnly = true, Read = false
Write = false, ShowOnly = true, Read = false
},
new FilteredObjectAttribute()
{
Expand Down
1 change: 1 addition & 0 deletions ...nder.Net.Core/Demos/elFinder.Net.AdvancedDemo/Models/Responses/ApplicationInitResponse.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ public ApplicationInitResponse(InitResponse initResp)
cwd = initResp.cwd;
files = initResp.files;
options = initResp.options;
uplMaxFile = initResp.uplMaxFile;
}

public long usage { get; set; }
Expand Down
1 change: 1 addition & 0 deletions ...nder.Net.Core/Demos/elFinder.Net.AdvancedDemo/Models/Responses/ApplicationOpenResponse.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ public ApplicationOpenResponse(OpenResponse openResp)
cwd = openResp.cwd;
files = openResp.files;
options = openResp.options;
uplMaxFile = openResp.uplMaxFile;
}

public long usage { get; set; }
Expand Down
1 change: 1 addition & 0 deletions elFinder.Net.Core/Demos/elFinder.Net.AdvancedDemo/Pages/Index.cshtml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
contextmenu: contextMenu,
lang: 'vi',
requestType: 'post',
uploadMaxChunkSize: 1024 * 1024 * 10,
//onlyMimes: ["image", "text/plain"] // Get files of requested mime types only
};
Expand Down
2 changes: 1 addition & 1 deletion elFinder.Net.Core/Demos/elFinder.Net.AdvancedDemo/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public static void Main(string[] args)
Id = 2,
UserName = "msdiana",
VolumePath = "msdiana",
QuotaInBytes = 20 * (long)Math.Pow(1024, 2),
QuotaInBytes = (long)Math.Pow(1024, 3),
});

dbContext.SaveChanges();
Expand Down
9 changes: 8 additions & 1 deletion elFinder.Net.Core/Demos/elFinder.Net.AdvancedDemo/Startup.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using elFinder.Net.Core.Services.Drawing;
using elFinder.Net.Drivers.FileSystem.Extensions;
using elFinder.Net.Drivers.FileSystem.Helpers;
using elFinder.Net.Drivers.FileSystem.Services;
using elFinder.Net.Plugins.FileSystemQuotaManagement.Extensions;
using elFinder.Net.Plugins.LoggingExample.Extensions;
using Microsoft.AspNetCore.Authentication.Cookies;
Expand All @@ -24,16 +25,19 @@ namespace elFinder.Net.AdvancedDemo
public class Startup
{
public const string StorageFolder = ".storage";
public const string TempFileFolder = ".temp";

public Startup(IConfiguration configuration, IWebHostEnvironment env)
{
Configuration = configuration;
WebRootPath = env.WebRootPath;
StoragePath = new DirectoryInfo(StorageFolder).FullName;
TempPath = new DirectoryInfo(TempFileFolder).FullName;
}

public static string WebRootPath { get; private set; }
public static string StoragePath { get; private set; }
public static string TempPath { get; private set; }

public static string MapStoragePath(string path)
{
Expand All @@ -49,7 +53,10 @@ public void ConfigureServices(IServiceCollection services)
var pluginCollection = new PluginCollection();

services.AddElFinderAspNetCore()
.AddFileSystemDriver()
.AddFileSystemDriver(tempFileCleanerConfig: (opt) =>
{
opt.ScanFolders.Add(TempPath, TempFileCleanerOptions.DefaultUnmanagedLifeTime);
})
.AddFileSystemQuotaManagement(pluginCollection)
.AddElFinderLoggingExample(pluginCollection)
.AddElFinderPlugins(pluginCollection);
Expand Down
8 changes: 6 additions & 2 deletions elFinder.Net.Core/Demos/elFinder.Net.Demo31/Controllers/FilesController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,15 @@ private async Task SetupConnectorAsync()
for (var i = 0; i < 5; i++)
{
var volume = new Volume(_driver,
Startup.MapPath($"~/upload/volume-{i}"), $"/upload/volume-{i}", $"/api/files/thumb/")
Startup.MapPath($"~/upload/volume-{i}"),
Startup.TempPath,
$"/upload/volume-{i}",
$"/api/files/thumb/",
thumbnailDirectory: PathHelper.GetFullPath("./thumb"))
{
StartDirectory = Startup.MapPath($"~/upload/volume-{i}/start"),
Name = $"Volume {i}",
ThumbnailDirectory = PathHelper.GetFullPath("./thumb")
MaxUploadConnections = 3
};

_connector.AddVolume(volume);
Expand Down
7 changes: 6 additions & 1 deletion elFinder.Net.Core/Demos/elFinder.Net.Demo31/Startup.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using elFinder.Net.Core;
using elFinder.Net.Drivers.FileSystem.Extensions;
using elFinder.Net.Drivers.FileSystem.Helpers;
using elFinder.Net.Drivers.FileSystem.Services;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.ResponseCompression;
Expand All @@ -22,6 +23,7 @@ public Startup(IConfiguration configuration, IWebHostEnvironment env)
}

public static string WebRootPath { get; private set; }
public static string TempPath { get; } = Path.GetTempPath();

public static string MapPath(string path, string basePath = null)
{
Expand All @@ -41,7 +43,10 @@ public void ConfigureServices(IServiceCollection services)
{
#region elFinder
services.AddElFinderAspNetCore()
.AddFileSystemDriver();
.AddFileSystemDriver(tempFileCleanerConfig: (opt) =>
{
opt.ScanFolders.Add(TempPath, TempFileCleanerOptions.DefaultUnmanagedLifeTime);
});
#endregion

services.AddResponseCompression(options =>
Expand Down
4 changes: 2 additions & 2 deletions .../elFinder.Net.Plugins.FileSystemQuotaManagement/Extensions/ServiceCollectionExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ public static IServiceCollection AddFileSystemQuotaManagement(this IServiceColle
if (storageManagerOptionsConfig == null)
storageManagerOptionsConfig = (options) =>
{
options.StorageCachingMinutes = StorageManagerOptions.DefaultStorageCachingMinutes;
options.StorageCachingLifeTime = StorageManagerOptions.DefaultStorageCachingLifeTime;
options.MaximumItems = StorageManagerOptions.DefaultMaximumItems;
options.ReservationsAfterCleanUp = StorageManagerOptions.DefaultReservationsAfterCleanUp;
options.PollingIntervalInMinutes = StorageManagerOptions.DefaultPollingIntervalInMinutes;
options.PollingInterval = StorageManagerOptions.DefaultPollingInterval;
};

services.AddScoped<DriverInterceptor>()
Expand Down
156 changes: 131 additions & 25 deletions .../Plugins/elFinder.Net.Plugins.FileSystemQuotaManagement/Interceptors/DriverInterceptor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,29 +321,33 @@ protected virtual void InterceptRm(IInvocation invocation, QuotaOptions quotaOpt
{
rmLength = file.LengthAsync.Result;
}
else if (args is IDirectory dir)
{
rmLength = dir.GetSizeAndCountAsync(false, _ => true, _ => true, cancellationToken: cancellationToken).Result.Size;
}
};

driver.OnAfterRemove += (sender, args) =>
{
if (args is IFile file)
{
var volume = file.Volume;
var volumeDir = driver.CreateDirectory(volume.RootDirectory, volume);
Func<string, Task<long>> createFunc = (_) => volumeDir.GetPhysicalStorageUsageAsync(cancellationToken);
if (rmLength == 0) return;

var (storageCache, _) = storageManager.Lock(volume.RootDirectory, createFunc);
var volume = args.Volume;
var volumeDir = driver.CreateDirectory(volume.RootDirectory, volume);
Func<string, Task<long>> createFunc = (_) => volumeDir.GetPhysicalStorageUsageAsync(cancellationToken);

try
{
storageCache.Storage -= rmLength.Value;
rmLength = null;
proceededDirs.Add(volumeDir);
}
finally
{
if (storageCache != null)
storageManager.Unlock(storageCache);
}
var (storageCache, _) = storageManager.Lock(volume.RootDirectory, createFunc);

try
{
storageCache.Storage -= rmLength.Value;
rmLength = null;
proceededDirs.Add(volumeDir);
}
finally
{
if (storageCache != null)
storageManager.Unlock(storageCache);
storageCache = null;
}
};
}
Expand All @@ -370,15 +374,14 @@ protected virtual void InterceptUpload(IInvocation invocation, QuotaOptions quot
var cmd = invocation.Arguments[0] as UploadCommand;
var volume = cmd.TargetPath.Volume;
var volumeDir = driver.CreateDirectory(volume.RootDirectory, volume);
var proceededDirs = new HashSet<IDirectory>();
var cancellationToken = (CancellationToken)invocation.Arguments.Last();
double? maximum = null;
DirectoryStorageCache storageCache = null;

if (quotaOptions.Enabled && quotaOptions.Quotas.TryGetValue(volume.VolumeId, out var volumeQuota))
maximum = volumeQuota.MaxStorageSize;

DirectoryStorageCache storageCache = null;
bool proceeded = false;

if (!_registeredHandlers.Contains(nameof(InterceptUpload)))
{
_registeredHandlers.Add(nameof(InterceptUpload));
Expand All @@ -391,6 +394,30 @@ protected virtual void InterceptUpload(IInvocation invocation, QuotaOptions quot
{
(storageCache, _) = storageManager.Lock(volume.RootDirectory, createFunc);

if (args.IsChunking)
{
try
{
var totalUploadLength = cmd.RangeInfo.TotalBytes;

if (args.IsOverwrite)
{
var destLength = args.DestFile.LengthAsync.Result;
totalUploadLength -= destLength;
}

if (storageCache.Storage + totalUploadLength > maximum)
throw new QuotaException(maximum.Value, storageCache.Storage, quotaOptions);

return;
}
finally
{
storageManager.Unlock(storageCache);
storageCache = null;
}
}

uploadLength = args.FormFile.Length;
if (args.IsOverwrite)
{
Expand All @@ -400,13 +427,69 @@ protected virtual void InterceptUpload(IInvocation invocation, QuotaOptions quot
if (storageCache.Storage + uploadLength > maximum)
throw new QuotaException(maximum.Value, storageCache.Storage, quotaOptions);

proceeded = true;
proceededDirs.Add(volumeDir);
};

driver.OnAfterUpload += (sender, args) =>
{
if (storageCache == null) return;
if (storageCache == null && uploadLength == 0) return;
storageCache.Storage += uploadLength;
uploadLength = 0;
storageManager.Unlock(storageCache);
storageCache = null;
};

long transferLength = 0;
long originalLength = 0;
long tempTransferedLength = 0;

driver.OnBeforeChunkMerged += (sender, args) =>
{
originalLength = args.IsOverwrite ? args.File.LengthAsync.Result : 0;
};

driver.OnBeforeChunkTransfer += (sender, args) =>
{
(storageCache, _) = storageManager.Lock(volume.RootDirectory, createFunc);

if (originalLength > 0)
{
tempTransferedLength -= originalLength;
originalLength = 0;
}

transferLength = args.ChunkFile.LengthAsync.Result;
tempTransferedLength += transferLength;

if (storageCache.Storage + tempTransferedLength > maximum)
throw new QuotaException(maximum.Value, storageCache.Storage, quotaOptions);

proceededDirs.Add(volumeDir);
};

driver.OnAfterChunkTransfer += (sender, args) =>
{
if (storageCache != null && tempTransferedLength > 0)
{
storageCache.Storage += tempTransferedLength;
tempTransferedLength = 0;
}

storageManager.Unlock(storageCache);
storageCache = null;
};

driver.OnAfterChunkMerged += (sender, arge) =>
{
if (storageCache == null)
(storageCache, _) = storageManager.Lock(volume.RootDirectory, createFunc);

if (tempTransferedLength < 0)
{
storageCache.Storage += tempTransferedLength;
tempTransferedLength = 0;
}

storageManager.Unlock(storageCache);
storageCache = null;
};
Expand All @@ -415,9 +498,32 @@ protected virtual void InterceptUpload(IInvocation invocation, QuotaOptions quot
{
storageManager.Unlock(storageCache);
storageCache = null;

if (exception is QuotaException)
throw exception;
};

long rmLength = 0;

driver.OnBeforeRollbackChunk += (sender, args) =>
{
if (args is IFile file)
{
rmLength = file.LengthAsync.Result;
proceededDirs.Add(volumeDir);
}
};

driver.OnAfterRollbackChunk += (sender, args) =>
{
if (rmLength == 0 || args is IDirectory) return;

if (storageCache == null)
(storageCache, _) = storageManager.Lock(volume.RootDirectory, createFunc);

storageCache.Storage -= rmLength;
rmLength = 0;
};
}

try
Expand All @@ -429,8 +535,8 @@ protected virtual void InterceptUpload(IInvocation invocation, QuotaOptions quot
storageManager.Unlock(storageCache);
storageCache = null;

if (proceeded)
storageManager.StartSizeCalculationThread(volumeDir);
foreach (var dir in proceededDirs)
storageManager.StartSizeCalculationThread(dir);
}
}

Expand Down Expand Up @@ -602,7 +708,7 @@ protected virtual void InterceptArchive(IInvocation invocation, QuotaOptions quo
{
try
{
file.DeleteAsync().Wait();
file.DeleteAsync(cancellationToken: cancellationToken).Wait();
}
finally
{
Expand Down
Loading

0 comments on commit af1c7e5

Please sign in to comment.