Merge pull request #11 from DIG-Network/release/v0.0.1-alpha.11

Release/v0.0.1 alpha.11

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.0.1-alpha.11](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.10...v0.0.1-alpha.11) (2024-09-10)
+
+
+### Features
+
+* add plimit to directory add ([468cff2](https://github.com/DIG-Network/dig-chia-sdk/commit/468cff28397c993b22af7388a8472b1d2068aebd))
+
 ### [0.0.1-alpha.10](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.9...v0.0.1-alpha.10) (2024-09-10)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dignetwork/dig-sdk",
-  "version": "0.0.1-alpha.10",
+  "version": "0.0.1-alpha.11",
   "description": "",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -41,6 +41,7 @@
     "merkletreejs": "^0.4.0",
     "nanospinner": "^1.1.0",
     "nconf": "^0.12.1",
+    "p-limit": "^6.1.0",
     "superagent": "^10.0.0"
   },
   "devDependencies": {

src/utils/directoryUtils.ts

@@ -1,8 +1,18 @@
 import * as fs from "fs";
 import * as path from "path";
+import { promisify } from "util";
 import ignore from "ignore";
+import pLimit from "p-limit";
 import { DataIntegrityTree } from "../DataIntegrityTree";
 
+
+const limit = pLimit(10); // Limit the concurrency to 10 (adjust based on your system's file descriptor limit)
+
+// Promisify fs methods
+const readdir = promisify(fs.readdir);
+const stat = promisify(fs.stat);
+const readFile = promisify(fs.readFile);
+
 /**
  * Recursively add all files in a directory to the Merkle tree, skipping the .dig, .git folders, and files in .gitignore.
  * @param datalayer - The DataStoreManager instance.
@@ -19,37 +29,45 @@ export const addDirectory = async (
 
   // Load .gitignore rules if the file exists
   if (fs.existsSync(gitignorePath)) {
-    const gitignoreContent = fs.readFileSync(gitignorePath, "utf-8");
+    const gitignoreContent = await readFile(gitignorePath, "utf-8");
     ig.add(gitignoreContent);
   }
 
-  const files = fs.readdirSync(dirPath);
+  const files = await readdir(dirPath);
 
-  for (const file of files) {
-    const filePath = path.join(dirPath, file);
-    const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/");
+  // Process each file or directory
+  await Promise.all(
+    files.map(async (file) => {
+      const filePath = path.join(dirPath, file);
+      const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/");
 
-    // Skip the .dig, .git folders and files or directories ignored by .gitignore
-    if (file === ".dig" || file === ".git" || ig.ignores(relativePath)) {
-      continue;
-    }
+      // Skip the .dig, .git folders and files or directories ignored by .gitignore
+      if (file === ".dig" || file === ".git" || ig.ignores(relativePath)) {
+        return;
+      }
 
-    const stat = fs.statSync(filePath);
+      const fileStat = await stat(filePath);
 
-    if (stat.isDirectory()) {
-      await addDirectory(datalayer, filePath, baseDir);
-    } else {
-      await new Promise<void>((resolve, reject) => {
-        const stream = fs.createReadStream(filePath);
-        datalayer
-          .upsertKey(stream, Buffer.from(relativePath).toString("hex"))
-          .then(resolve)
-          .catch(reject);
-      });
-    }
-  }
+      if (fileStat.isDirectory()) {
+        // Recursively process the directory
+        return addDirectory(datalayer, filePath, baseDir);
+      } else {
+        // Process the file with limited concurrency
+        return limit(() =>
+          new Promise<void>((resolve, reject) => {
+            const stream = fs.createReadStream(filePath);
+            datalayer
+              .upsertKey(stream, Buffer.from(relativePath).toString("hex"))
+              .then(resolve)
+              .catch(reject);
+          })
+        );
+      }
+    })
+  );
 };
 
+
 /**
  * Calculate the total size of the DIG_FOLDER_PATH
  * @param folderPath - The path of the folder to calculate size.