diff --git a/@types/lib/util/devops.d.ts.map b/@types/lib/util/devops.d.ts.map index 91514d9d9..d8c1e2e2d 100644 --- a/@types/lib/util/devops.d.ts.map +++ b/@types/lib/util/devops.d.ts.map @@ -1 +1 @@ -{"version":3,"file":"devops.d.ts","sourceRoot":"","sources":["../../../lib/util/devops.js"],"names":[],"mappings":";yBAYa,OAAO,wBAAwB,EAAE,UAAU;uBAC3C,OAAO,wBAAwB,EAAE,QAAQ;oBACzC,OAAO,wBAAwB,EAAE,KAAK;0BACtC,OAAO,wBAAwB,EAAE,WAAW;8BAC5C,OAAO,wBAAwB,EAAE,eAAe;2BAChD,OAAO,wBAAwB,EAAE,YAAY;sBAC7C,OAAO,wBAAwB,EAAE,OAAO;+BACxC,OAAO,wBAAwB,EAAE,gBAAgB;mCACjD,OAAO,wBAAwB,EAAE,oBAAoB;kCACrD,OAAO,wBAAwB,EAAE,mBAAmB;8BACpD,OAAO,wBAAwB,EAAE,eAAe;iCAChD,OAAO,wBAAwB,EAAE,kBAAkB;oCACnD,OAAO,wBAAwB,EAAE,qBAAqB;mCACtD,OAAO,wBAAwB,EAAE,oBAAoB;gCACrD,OAAO,wBAAwB,EAAE,iBAAiB;0BAClD,OAAO,wBAAwB,EAAE,WAAW;2BAC5C,OAAO,wBAAwB,EAAE,YAAY;;IAOtD;;;;;;;;;;OAUG;IACH,kCAPW,OAAO,UACP,MAAM,oBACN,OAAO,mBACP,MAAM,kBACN,MAAM,GACJ,OAAO,CAAE,YAAY,EAAE,CAAC,CA2TpC;IAED;;;;;;;;OAQG;IACH,2CANW,OAAO,SACP,MAAM,YACN,YAAY,EAAE,kBACd,MAAM,GACJ,OAAO,CAAE,YAAY,EAAE,CAAC,CAqLpC;IAED;;;;;;OAMG;IACH,6BAJW,MAAM,cACN,MAAM,GACJ,IAAI,CA4BhB;IAED;;;;;;;;;OASG;IACH,sCANW,OAAO,YACP,QAAQ,gBACR,MAAM,UACN,MAAM,EAAE,GACN,OAAO,CAAE,MAAM,EAAE,CAAC,CAM9B"} \ No newline at end of file +{"version":3,"file":"devops.d.ts","sourceRoot":"","sources":["../../../lib/util/devops.js"],"names":[],"mappings":";yBAca,OAAO,wBAAwB,EAAE,UAAU;uBAC3C,OAAO,wBAAwB,EAAE,QAAQ;oBACzC,OAAO,wBAAwB,EAAE,KAAK;0BACtC,OAAO,wBAAwB,EAAE,WAAW;8BAC5C,OAAO,wBAAwB,EAAE,eAAe;2BAChD,OAAO,wBAAwB,EAAE,YAAY;sBAC7C,OAAO,wBAAwB,EAAE,OAAO;+BACxC,OAAO,wBAAwB,EAAE,gBAAgB;mCACjD,OAAO,wBAAwB,EAAE,oBAAoB;kCACrD,OAAO,wBAAwB,EAAE,mBAAmB;8BACpD,OAAO,wBAAwB,EAAE,eAAe;iCAChD,OAAO,wBAAwB,EAAE,kBAAkB;oCACnD,OAAO,wBAAwB,EAAE,qBAAqB;mCACtD,OAAO,wBAAwB,EAAE,oBAAoB;gCACrD,OAAO,wBAAwB,EAAE,iBAAiB;0BAClD,OAAO,wBAAwB,EAAE,WAAW;2BAC5C,OAAO,wBAAwB,EAAE,YAAY;;IAOtD;;;;;;;;;;OAUG;IACH,kCAPW,OAAO,UACP,MAAM,oBACN,OAAO,mBACP,MAAM,kBACN,MAAM,GACJ,OAAO,CAAE,YAAY,EAAE,CAAC,CAmVpC;IAED;;;;;;;;OAQG;IACH,2CANW,OAAO,SACP,MAAM,YACN,YAAY,EAAE,kBACd,MAAM,GACJ,OAAO,CAAE,YAAY,EAAE,CAAC,CAqLpC;IAED;;;;;;OAMG;IACH,6BAJW,MAAM,cACN,MAAM,GACJ,IAAI,CA4BhB;IAED;;;;;;;;;OASG;IACH,sCANW,OAAO,YACP,QAAQ,gBACR,MAAM,UACN,MAAM,EAAE,GACN,OAAO,CAAE,MAAM,EAAE,CAAC,CAM9B"} \ No newline at end of file diff --git a/lib/util/devops.js b/lib/util/devops.js index 70142788f..cac62f1ff 100644 --- a/lib/util/devops.js +++ b/lib/util/devops.js @@ -8,6 +8,8 @@ const git = simpleGit(); import Builder from '../Builder.js'; import MetadataType from '../MetadataTypeInfo.js'; import jsonToTable from 'json-to-table'; +import pLimit from 'p-limit'; +import cliProgress from 'cli-progress'; /** * @typedef {import('../../types/mcdev.d.js').AuthObject} AuthObject @@ -97,11 +99,13 @@ const DevOps = { 'add/update': 0, move: 0, }; + const diffSummary = await git.diffSummary([range]); + Util.logger.info(`Found ${diffSummary.files.length} changed files in the commit range`); /** * @type {DeltaPkgItem[]} */ // first, process everything that can be done synchronously here - const deltaSync = (await git.diffSummary([range])).files + const deltaSync = diffSummary.files // populate additional info for all changed files .map((/** @type {DeltaPkgItem} */ file) => { // If file was moved it's path needs to be parsed @@ -149,72 +153,94 @@ const DevOps = { return false; } }); + + Util.logger.info( + `Identified ${deltaSync.length} relevant file changes after filtering file types and folders. Processing related metadata now.` + ); // second, we need to do asynchronous file operations + + const extendedBar = new cliProgress.SingleBar( + { + format: ' Processing changes [{bar}] {percentage}% | {value}/{total}', + }, + cliProgress.Presets.shades_classic + ); + extendedBar.start(deltaSync.length, 0); + const rateLimit = pLimit(10); + /** * @type {DeltaPkgItem[]} */ const delta = await Promise.all( - deltaSync.map(async (/** @type {DeltaPkgItem} */ file) => { - // Gets external key based on file name und the assumption that filename = externalKey - if (file.type === 'folder') { - file.externalKey = null; - file.name = path.basename(file.file).split('.').shift(); - } else { - // find the key in paths like: - // - retrieve/cred/bu/asset/block/016aecc7-7063-4b78-93f4-aa119ea933c7.asset-block-meta.html - // - retrieve/cred/bu/asset/message/003c1ef5-f538-473a-91da-26942024a64a/blocks/views.html.slots.[bottom-8frq7iw2k99].asset-message-meta.html - // - retrieve/cred/bu/query/03efd5f1-ba1f-487a-9c9a-36aeb2ae5192.query-meta.sql - file.externalKey = - file.type === 'asset' - ? file.file.split('/')[5].split('.')[0] // assets have an additional folder level for their subtype - : file.file.split('/')[4].split('.')[0]; - file.name = null; - } - - // Check if file doesn't exist in reported path, that means it was a git deletion - // TODO: improve git action detection by switching from diffSummary to diff with --summary result parsing - if (!(await File.pathExists(file.file))) { - file.gitAction = 'delete'; - } else if (file.moved) { - file.gitAction = 'move'; - } else { - file.gitAction = 'add/update'; - } - gitActionsCounter[file.gitAction]++; - file._credential = file.file.split('/')[1]; - file._businessUnit = file.file.split('/')[2]; - - // Parse retrieve directory to also populate the name field (not possible for deleted files) - if (file.gitAction !== 'delete' && file.type !== 'folder') { - // folders are saved with their name as file-name, not with their key, hence this section can be skipped for folders - const buPath = `${properties.directories.retrieve}/${file._credential}/${file._businessUnit}/`; - if (!metadata[file._credential]) { - metadata[file._credential] = {}; + deltaSync.map((/** @type {DeltaPkgItem} */ file) => + rateLimit(async () => { + // Gets external key based on file name und the assumption that filename = externalKey + if (file.type === 'folder') { + file.externalKey = null; + file.name = path.basename(file.file).split('.').shift(); + } else { + // find the key in paths like: + // - retrieve/cred/bu/asset/block/016aecc7-7063-4b78-93f4-aa119ea933c7.asset-block-meta.html + // - retrieve/cred/bu/asset/message/003c1ef5-f538-473a-91da-26942024a64a/blocks/views.html.slots.[bottom-8frq7iw2k99].asset-message-meta.html + // - retrieve/cred/bu/query/03efd5f1-ba1f-487a-9c9a-36aeb2ae5192.query-meta.sql + file.externalKey = + file.type === 'asset' + ? file.file.split('/')[5].split('.')[0] // assets have an additional folder level for their subtype + : file.file.split('/')[4].split('.')[0]; + file.name = null; } - if (!metadata[file._credential][file._businessUnit]) { - metadata[file._credential][file._businessUnit] = {}; + + // Check if file doesn't exist in reported path, that means it was a git deletion + // TODO: improve git action detection by switching from diffSummary to diff with --summary result parsing + if (!(await File.pathExists(file.file))) { + file.gitAction = 'delete'; + } else if (file.moved) { + file.gitAction = 'move'; + } else { + file.gitAction = 'add/update'; } - if (!metadata[file._credential][file._businessUnit][file.type]) { - try { - await MetadataType[file.type].readBUMetadataForType( - buPath, - false, - metadata[file._credential][file._businessUnit] - ); - } catch (ex) { - // silently catch directory-not-found errors here - Util.logger.debug(ex.message); + gitActionsCounter[file.gitAction]++; + file._credential = file.file.split('/')[1]; + file._businessUnit = file.file.split('/')[2]; + + // Parse retrieve directory to also populate the name field (not possible for deleted files) + if (file.gitAction !== 'delete' && file.type !== 'folder') { + // folders are saved with their name as file-name, not with their key, hence this section can be skipped for folders + const buPath = `${properties.directories.retrieve}/${file._credential}/${file._businessUnit}/`; + if (!metadata[file._credential]) { + metadata[file._credential] = {}; + } + if (!metadata[file._credential][file._businessUnit]) { + metadata[file._credential][file._businessUnit] = {}; + } + if (!metadata[file._credential][file._businessUnit][file.type]) { + try { + await MetadataType[file.type].readBUMetadataForType( + buPath, + false, + metadata[file._credential][file._businessUnit] + ); + } catch (ex) { + // silently catch directory-not-found errors here + Util.logger.debug(ex.message); + } + } + const fileContent = + metadata[file._credential][file._businessUnit][file.type][ + file.externalKey + ]; + if (fileContent) { + file.name = fileContent[MetadataType[file.type].definition.nameField]; } } - const fileContent = - metadata[file._credential][file._businessUnit][file.type][file.externalKey]; - if (fileContent) { - file.name = fileContent[MetadataType[file.type].definition.nameField]; - } - } - return file; - }) + extendedBar.increment(); + return file; + }) + ) ); + // stop the progress bar + extendedBar.stop(); + if ( !gitActionsCounter['add/update'] && !gitActionsCounter.move &&