Merge pull request #13 from DIG-Network/release/v0.0.1-alpha.13

Release/v0.0.1 alpha.13

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.0.1-alpha.13](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.10...v0.0.1-alpha.13) (2024-09-10)
+
+
+### Features
+
+* add plimit to directory add ([468cff2](https://github.com/DIG-Network/dig-chia-sdk/commit/468cff28397c993b22af7388a8472b1d2068aebd))
 
 ### [0.0.1-alpha.12](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.10...v0.0.1-alpha.12) (2024-09-10)
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.12",
+  "version": "0.0.1-alpha.13",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",

src/utils/directoryUtils.ts

@@ -1,23 +1,38 @@
-import * as fs from "fs";
 import * as path from "path";
-import { promisify } from "util";
+import * as fs from "fs";
 import ignore from "ignore";
-import pLimit from "p-limit";
 import { DataIntegrityTree } from "../DataIntegrityTree";
 
+// Custom concurrency handler
+const limitConcurrency = async (concurrencyLimit: number, tasks: (() => Promise<void>)[]) => {
+  const results = [];
+  const executing: Promise<void>[] = [];
+
+  for (const task of tasks) {
+    const p = task();
+    results.push(p);
+
+    // Once the limit is reached, wait for one to complete
+    if (executing.length >= concurrencyLimit) {
+      await Promise.race(executing);
+    }
+
+    // Add the new task to the executing array
+    executing.push(p);
 
-// Function to dynamically load p-limit since it's an ES module
-async function loadPLimit() {
-  const { default: pLimit } = await import('p-limit');
-  return pLimit;
-}
+    // When a task completes, remove it from the executing array
+    p.finally(() => executing.splice(executing.indexOf(p), 1));
+  }
+
+  // Wait for all remaining tasks to complete
+  return Promise.all(results);
+};
 
 export const addDirectory = async (
   datalayer: DataIntegrityTree,
   dirPath: string,
   baseDir: string = dirPath
 ): Promise<void> => {
-  const limit = await loadPLimit();  // Dynamically load p-limit and get the default export
   const ig = ignore();
   const gitignorePath = path.join(baseDir, ".gitignore");
 
@@ -28,59 +43,36 @@ export const addDirectory = async (
   }
 
   const files = fs.readdirSync(dirPath);
+  const tasks: (() => Promise<void>)[] = [];
 
-  await Promise.all(
-    files.map(async (file) => {
-      const filePath = path.join(dirPath, file);
-      const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/");
-
-      // Skip the .dig, .git folders and files or directories ignored by .gitignore
-      if (file === ".dig" || file === ".git" || ig.ignores(relativePath)) {
-        return;
-      }
-
-      const stat = fs.statSync(filePath);
-
-      if (stat.isDirectory()) {
-        await addDirectory(datalayer, filePath, baseDir);
-      } else {
-        // Use the dynamically loaded p-limit to limit concurrent file processing
-        return limit(10)(() =>
-          new Promise<void>((resolve, reject) => {
-            const stream = fs.createReadStream(filePath);
-            datalayer
-              .upsertKey(stream, Buffer.from(relativePath).toString("hex"))
-              .then(resolve)
-              .catch(reject);
-          })
-        );
-      }
-    })
-  );
-};
-
-
-
-/**
- * Calculate the total size of the DIG_FOLDER_PATH
- * @param folderPath - The path of the folder to calculate size.
- * @returns The total size of the folder in bytes.
- */
-export const calculateFolderSize = (folderPath: string): bigint => {
-  let totalSize = BigInt(0);
+  for (const file of files) {
+    const filePath = path.join(dirPath, file);
+    const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/");
 
-  const files = fs.readdirSync(folderPath);
+    // Skip the .dig, .git folders and files or directories ignored by .gitignore
+    if (file === ".dig" || file === ".git" || ig.ignores(relativePath)) {
+      continue;
+    }
 
-  for (const file of files) {
-    const filePath = path.join(folderPath, file);
     const stat = fs.statSync(filePath);
 
     if (stat.isDirectory()) {
-      totalSize += calculateFolderSize(filePath);
+      // Recursively process the directory
+      tasks.push(() => addDirectory(datalayer, filePath, baseDir));
     } else {
-      totalSize += BigInt(stat.size);
+      // Add a task for each file to be processed
+      tasks.push(() =>
+        new Promise<void>((resolve, reject) => {
+          const stream = fs.createReadStream(filePath);
+          datalayer
+            .upsertKey(stream, Buffer.from(relativePath).toString("hex"))
+            .then(resolve)
+            .catch(reject);
+        })
+      );
     }
   }
 
-  return totalSize;
+  // Run tasks with limited concurrency (set the concurrency limit as needed)
+  await limitConcurrency(10, tasks); // Adjust 10 based on your system limits
 };