diff --git a/CHANGELOG.md b/CHANGELOG.md index 675d649..e5903a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.11](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.10...v0.0.1-alpha.11) (2024-09-10) + + +### Features + +* add plimit to directory add ([468cff2](https://github.com/DIG-Network/dig-chia-sdk/commit/468cff28397c993b22af7388a8472b1d2068aebd)) + ### [0.0.1-alpha.10](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.9...v0.0.1-alpha.10) (2024-09-10) diff --git a/package-lock.json b/package-lock.json index d9fc7e1..91cd401 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.10", + "version": "0.0.1-alpha.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.10", + "version": "0.0.1-alpha.11", "license": "ISC", "dependencies": { "bip39": "^3.1.0", @@ -25,6 +25,7 @@ "merkletreejs": "^0.4.0", "nanospinner": "^1.1.0", "nconf": "^0.12.1", + "p-limit": "^6.1.0", "superagent": "^10.0.0" }, "devDependencies": { @@ -3684,15 +3685,14 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.1.0.tgz", + "integrity": "sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.1.1" }, "engines": { - "node": ">=10" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -3713,6 +3713,33 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -4964,12 +4991,11 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz", + "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" diff --git a/package.json b/package.json index ffc6542..1e5c0da 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.10", + "version": "0.0.1-alpha.11", "description": "", "type": "commonjs", "main": "./dist/index.js", @@ -41,6 +41,7 @@ "merkletreejs": "^0.4.0", "nanospinner": "^1.1.0", "nconf": "^0.12.1", + "p-limit": "^6.1.0", "superagent": "^10.0.0" }, "devDependencies": { diff --git a/src/utils/directoryUtils.ts b/src/utils/directoryUtils.ts index b508bdb..72b6ddd 100644 --- a/src/utils/directoryUtils.ts +++ b/src/utils/directoryUtils.ts @@ -1,8 +1,18 @@ import * as fs from "fs"; import * as path from "path"; +import { promisify } from "util"; import ignore from "ignore"; +import pLimit from "p-limit"; import { DataIntegrityTree } from "../DataIntegrityTree"; + +const limit = pLimit(10); // Limit the concurrency to 10 (adjust based on your system's file descriptor limit) + +// Promisify fs methods +const readdir = promisify(fs.readdir); +const stat = promisify(fs.stat); +const readFile = promisify(fs.readFile); + /** * Recursively add all files in a directory to the Merkle tree, skipping the .dig, .git folders, and files in .gitignore. * @param datalayer - The DataStoreManager instance. @@ -19,37 +29,45 @@ export const addDirectory = async ( // Load .gitignore rules if the file exists if (fs.existsSync(gitignorePath)) { - const gitignoreContent = fs.readFileSync(gitignorePath, "utf-8"); + const gitignoreContent = await readFile(gitignorePath, "utf-8"); ig.add(gitignoreContent); } - const files = fs.readdirSync(dirPath); + const files = await readdir(dirPath); - for (const file of files) { - const filePath = path.join(dirPath, file); - const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/"); + // Process each file or directory + await Promise.all( + files.map(async (file) => { + const filePath = path.join(dirPath, file); + const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/"); - // Skip the .dig, .git folders and files or directories ignored by .gitignore - if (file === ".dig" || file === ".git" || ig.ignores(relativePath)) { - continue; - } + // Skip the .dig, .git folders and files or directories ignored by .gitignore + if (file === ".dig" || file === ".git" || ig.ignores(relativePath)) { + return; + } - const stat = fs.statSync(filePath); + const fileStat = await stat(filePath); - if (stat.isDirectory()) { - await addDirectory(datalayer, filePath, baseDir); - } else { - await new Promise((resolve, reject) => { - const stream = fs.createReadStream(filePath); - datalayer - .upsertKey(stream, Buffer.from(relativePath).toString("hex")) - .then(resolve) - .catch(reject); - }); - } - } + if (fileStat.isDirectory()) { + // Recursively process the directory + return addDirectory(datalayer, filePath, baseDir); + } else { + // Process the file with limited concurrency + return limit(() => + new Promise((resolve, reject) => { + const stream = fs.createReadStream(filePath); + datalayer + .upsertKey(stream, Buffer.from(relativePath).toString("hex")) + .then(resolve) + .catch(reject); + }) + ); + } + }) + ); }; + /** * Calculate the total size of the DIG_FOLDER_PATH * @param folderPath - The path of the folder to calculate size.