Merge pull request #727 from Crypter-File-Transfer/feature/multipart-…

…upload-algorithm Simple algorithm to target 1 second multipart upload request times
Crypter-File-Transfer · Sep 18, 2024 · 0c757ce · 0c757ce
2 parents 2318cf0 + 5e3921a
commit 0c757ce
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 2 deletions.
diff --git a/Crypter.Common.Client/Transfer/Handlers/UploadFileHandler.cs b/Crypter.Common.Client/Transfer/Handlers/UploadFileHandler.cs
@@ -27,6 +27,7 @@
 using System;
 using System.Buffers;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Threading;
@@ -159,13 +160,16 @@ await Parallel.ForEachAsync(asyncEnumerable, parallelOptions, async (streamOpene
         async IAsyncEnumerable<Func<MemoryStream>> SplitEncryptionStreamAsync(EncryptionStream encryptionStream)
         {
             bool endOfStream = false;
+            Stopwatch loopStopwatch = new Stopwatch();
+            short blocksPerRequest = ClientTransferSettings.InitialMultipartReadBlocks;
             do
             {
-                int bufferSize = ClientTransferSettings.MaximumMultipartReadBlocks * encryptionStream.MinimumBufferSize;
+                loopStopwatch.Restart();
+                int bufferSize = blocksPerRequest * encryptionStream.MinimumBufferSize;
                 byte[] buffer = ArrayPool<byte>.Shared.Rent(bufferSize);
 
                 int totalBytesRead = 0;
-                for (int i = 0; i < ClientTransferSettings.MaximumMultipartReadBlocks; i++)
+                for (int i = 0; i < blocksPerRequest; i++)
                 {
                     int bytesRead = await encryptionStream.ReadAsync(buffer.AsMemory(totalBytesRead, encryptionStream.MinimumBufferSize));
                     totalBytesRead += bytesRead;
@@ -187,6 +191,15 @@ async IAsyncEnumerable<Func<MemoryStream>> SplitEncryptionStreamAsync(Encryption
                         ArrayPool<byte>.Shared.Return(buffer, clearArray: true);   
                     }
                 }
+
+                if (loopStopwatch.Elapsed < TimeSpan.FromSeconds(1) && blocksPerRequest < ClientTransferSettings.MaximumMultipartReadBlocks)
+                {
+                    blocksPerRequest += 5;
+                }
+                else if (loopStopwatch.Elapsed > TimeSpan.FromSeconds(1) && blocksPerRequest > ClientTransferSettings.InitialMultipartReadBlocks)
+                {
+                    blocksPerRequest -= 5;
+                }
             } while (!endOfStream);
         }
     }

diff --git a/Crypter.Common.Client/Transfer/Models/ClientTransferSettings.cs b/Crypter.Common.Client/Transfer/Models/ClientTransferSettings.cs
@@ -48,6 +48,11 @@ public class ClientTransferSettings
     /// </summary>
     public short MaximumMultipartReadBlocks { get; init; }
 
+    /// <summary>
+    /// Set the initial number of read blocks in a single multipart request body.
+    /// </summary>
+    public short InitialMultipartReadBlocks { get; init; }
+
     /// <summary>
     /// Set the maximum degrees of parallelism for multipart uploads.
     /// </summary>

diff --git a/Crypter.Web/wwwroot/appsettings.json b/Crypter.Web/wwwroot/appsettings.json
@@ -7,6 +7,7 @@
       "MaximumUploadStreamSizeMB": 250,
       "MaximumMultipartUploadSizeMB": 250,
       "MaximumMultipartReadBlocks": 120,
+      "InitialMultipartReadBlocks": 10,
       "MaximumMultipartParallelism": 1,
       "MaxReadSize": 32704,
       "PadSize": 64