From e6e1accd168ed34898ff91e89de2ad98fdece3a8 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Tue, 10 Sep 2024 16:52:43 -0400 Subject: [PATCH 1/2] chore(release): 0.0.1-alpha.12 --- CHANGELOG.md | 7 +++ package-lock.json | 4 +- package.json | 2 +- src/utils/directoryUtils.ts | 100 +++++++++++++++++------------------- 4 files changed, 56 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5903a7..bde2874 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.12](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.10...v0.0.1-alpha.12) (2024-09-10) + + +### Features + +* add plimit to directory add ([468cff2](https://github.com/DIG-Network/dig-chia-sdk/commit/468cff28397c993b22af7388a8472b1d2068aebd)) + ### [0.0.1-alpha.11](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.10...v0.0.1-alpha.11) (2024-09-10) diff --git a/package-lock.json b/package-lock.json index 91cd401..13908d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.11", + "version": "0.0.1-alpha.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.11", + "version": "0.0.1-alpha.12", "license": "ISC", "dependencies": { "bip39": "^3.1.0", diff --git a/package.json b/package.json index 1e5c0da..8251f61 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.11", + "version": "0.0.1-alpha.12", "description": "", "type": "commonjs", "main": "./dist/index.js", diff --git a/src/utils/directoryUtils.ts b/src/utils/directoryUtils.ts index 9f69c7b..4008332 100644 --- a/src/utils/directoryUtils.ts +++ b/src/utils/directoryUtils.ts @@ -1,23 +1,38 @@ -import * as fs from "fs"; import * as path from "path"; -import { promisify } from "util"; +import * as fs from "fs"; import ignore from "ignore"; -import pLimit from "p-limit"; import { DataIntegrityTree } from "../DataIntegrityTree"; +// Custom concurrency handler +const limitConcurrency = async (concurrencyLimit: number, tasks: (() => Promise)[]) => { + const results = []; + const executing: Promise[] = []; + + for (const task of tasks) { + const p = task(); + results.push(p); + + // Once the limit is reached, wait for one to complete + if (executing.length >= concurrencyLimit) { + await Promise.race(executing); + } + + // Add the new task to the executing array + executing.push(p); -// Function to dynamically load p-limit since it's an ES module -async function loadPLimit() { - const { default: pLimit } = await import('p-limit'); - return pLimit; -} + // When a task completes, remove it from the executing array + p.finally(() => executing.splice(executing.indexOf(p), 1)); + } + + // Wait for all remaining tasks to complete + return Promise.all(results); +}; export const addDirectory = async ( datalayer: DataIntegrityTree, dirPath: string, baseDir: string = dirPath ): Promise => { - const limit = await loadPLimit(); // Dynamically load p-limit and get the default export const ig = ignore(); const gitignorePath = path.join(baseDir, ".gitignore"); @@ -28,59 +43,36 @@ export const addDirectory = async ( } const files = fs.readdirSync(dirPath); + const tasks: (() => Promise)[] = []; - await Promise.all( - files.map(async (file) => { - const filePath = path.join(dirPath, file); - const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/"); - - // Skip the .dig, .git folders and files or directories ignored by .gitignore - if (file === ".dig" || file === ".git" || ig.ignores(relativePath)) { - return; - } - - const stat = fs.statSync(filePath); - - if (stat.isDirectory()) { - await addDirectory(datalayer, filePath, baseDir); - } else { - // Use the dynamically loaded p-limit to limit concurrent file processing - return limit(10)(() => - new Promise((resolve, reject) => { - const stream = fs.createReadStream(filePath); - datalayer - .upsertKey(stream, Buffer.from(relativePath).toString("hex")) - .then(resolve) - .catch(reject); - }) - ); - } - }) - ); -}; - - - -/** - * Calculate the total size of the DIG_FOLDER_PATH - * @param folderPath - The path of the folder to calculate size. - * @returns The total size of the folder in bytes. - */ -export const calculateFolderSize = (folderPath: string): bigint => { - let totalSize = BigInt(0); + for (const file of files) { + const filePath = path.join(dirPath, file); + const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/"); - const files = fs.readdirSync(folderPath); + // Skip the .dig, .git folders and files or directories ignored by .gitignore + if (file === ".dig" || file === ".git" || ig.ignores(relativePath)) { + continue; + } - for (const file of files) { - const filePath = path.join(folderPath, file); const stat = fs.statSync(filePath); if (stat.isDirectory()) { - totalSize += calculateFolderSize(filePath); + // Recursively process the directory + tasks.push(() => addDirectory(datalayer, filePath, baseDir)); } else { - totalSize += BigInt(stat.size); + // Add a task for each file to be processed + tasks.push(() => + new Promise((resolve, reject) => { + const stream = fs.createReadStream(filePath); + datalayer + .upsertKey(stream, Buffer.from(relativePath).toString("hex")) + .then(resolve) + .catch(reject); + }) + ); } } - return totalSize; + // Run tasks with limited concurrency (set the concurrency limit as needed) + await limitConcurrency(10, tasks); // Adjust 10 based on your system limits }; From 9cbe760987c8e0c14255a5f8c71621cc1a489eac Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Tue, 10 Sep 2024 17:04:10 -0400 Subject: [PATCH 2/2] chore(release): 0.0.1-alpha.13 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bde2874..ed611a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.13](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.10...v0.0.1-alpha.13) (2024-09-10) + + +### Features + +* add plimit to directory add ([468cff2](https://github.com/DIG-Network/dig-chia-sdk/commit/468cff28397c993b22af7388a8472b1d2068aebd)) + ### [0.0.1-alpha.12](https://github.com/DIG-Network/dig-chia-sdk/compare/v0.0.1-alpha.10...v0.0.1-alpha.12) (2024-09-10) diff --git a/package-lock.json b/package-lock.json index 13908d1..71efcfb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.12", + "version": "0.0.1-alpha.13", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.12", + "version": "0.0.1-alpha.13", "license": "ISC", "dependencies": { "bip39": "^3.1.0", diff --git a/package.json b/package.json index 8251f61..9ea6efb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dignetwork/dig-sdk", - "version": "0.0.1-alpha.12", + "version": "0.0.1-alpha.13", "description": "", "type": "commonjs", "main": "./dist/index.js",