Skip to content

Commit

Permalink
#2011: throttle createDeltaPkg file parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@JoernBerkefeld
JoernBerkefeld committed Jan 27, 2025
1 parent 671ca48 commit aac7823
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 58 deletions.
2 changes: 1 addition & 1 deletion @types/lib/util/devops.d.ts.map
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

140 changes: 83 additions & 57 deletions lib/util/devops.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ const git = simpleGit();
import Builder from '../Builder.js';
import MetadataType from '../MetadataTypeInfo.js';
import jsonToTable from 'json-to-table';
import pLimit from 'p-limit';
import cliProgress from 'cli-progress';

/**
* @typedef {import('../../types/mcdev.d.js').AuthObject} AuthObject
Expand Down Expand Up @@ -97,11 +99,13 @@ const DevOps = {
'add/update': 0,
move: 0,
};
const diffSummary = await git.diffSummary([range]);
Util.logger.info(`Found ${diffSummary.files.length} changed files in the commit range`);
/**
* @type {DeltaPkgItem[]}
*/
// first, process everything that can be done synchronously here
const deltaSync = (await git.diffSummary([range])).files
const deltaSync = diffSummary.files
// populate additional info for all changed files
.map((/** @type {DeltaPkgItem} */ file) => {
// If file was moved it's path needs to be parsed
Expand Down Expand Up @@ -149,72 +153,94 @@ const DevOps = {
return false;
}
});

Util.logger.info(
`Identified ${deltaSync.length} relevant file changes after filtering file types and folders. Processing related metadata now.`
);
// second, we need to do asynchronous file operations

const extendedBar = new cliProgress.SingleBar(
{
format: ' Processing changes [{bar}] {percentage}% | {value}/{total}',
},
cliProgress.Presets.shades_classic
);
extendedBar.start(deltaSync.length, 0);
const rateLimit = pLimit(10);

/**
* @type {DeltaPkgItem[]}
*/
const delta = await Promise.all(
deltaSync.map(async (/** @type {DeltaPkgItem} */ file) => {
// Gets external key based on file name und the assumption that filename = externalKey
if (file.type === 'folder') {
file.externalKey = null;
file.name = path.basename(file.file).split('.').shift();
} else {
// find the key in paths like:
// - retrieve/cred/bu/asset/block/016aecc7-7063-4b78-93f4-aa119ea933c7.asset-block-meta.html
// - retrieve/cred/bu/asset/message/003c1ef5-f538-473a-91da-26942024a64a/blocks/views.html.slots.[bottom-8frq7iw2k99].asset-message-meta.html
// - retrieve/cred/bu/query/03efd5f1-ba1f-487a-9c9a-36aeb2ae5192.query-meta.sql
file.externalKey =
file.type === 'asset'
? file.file.split('/')[5].split('.')[0] // assets have an additional folder level for their subtype
: file.file.split('/')[4].split('.')[0];
file.name = null;
}

// Check if file doesn't exist in reported path, that means it was a git deletion
// TODO: improve git action detection by switching from diffSummary to diff with --summary result parsing
if (!(await File.pathExists(file.file))) {
file.gitAction = 'delete';
} else if (file.moved) {
file.gitAction = 'move';
} else {
file.gitAction = 'add/update';
}
gitActionsCounter[file.gitAction]++;
file._credential = file.file.split('/')[1];
file._businessUnit = file.file.split('/')[2];

// Parse retrieve directory to also populate the name field (not possible for deleted files)
if (file.gitAction !== 'delete' && file.type !== 'folder') {
// folders are saved with their name as file-name, not with their key, hence this section can be skipped for folders
const buPath = `${properties.directories.retrieve}/${file._credential}/${file._businessUnit}/`;
if (!metadata[file._credential]) {
metadata[file._credential] = {};
deltaSync.map((/** @type {DeltaPkgItem} */ file) =>
rateLimit(async () => {
// Gets external key based on file name und the assumption that filename = externalKey
if (file.type === 'folder') {
file.externalKey = null;
file.name = path.basename(file.file).split('.').shift();
} else {
// find the key in paths like:
// - retrieve/cred/bu/asset/block/016aecc7-7063-4b78-93f4-aa119ea933c7.asset-block-meta.html
// - retrieve/cred/bu/asset/message/003c1ef5-f538-473a-91da-26942024a64a/blocks/views.html.slots.[bottom-8frq7iw2k99].asset-message-meta.html
// - retrieve/cred/bu/query/03efd5f1-ba1f-487a-9c9a-36aeb2ae5192.query-meta.sql
file.externalKey =
file.type === 'asset'
? file.file.split('/')[5].split('.')[0] // assets have an additional folder level for their subtype
: file.file.split('/')[4].split('.')[0];
file.name = null;
}
if (!metadata[file._credential][file._businessUnit]) {
metadata[file._credential][file._businessUnit] = {};

// Check if file doesn't exist in reported path, that means it was a git deletion
// TODO: improve git action detection by switching from diffSummary to diff with --summary result parsing
if (!(await File.pathExists(file.file))) {
file.gitAction = 'delete';
} else if (file.moved) {
file.gitAction = 'move';
} else {
file.gitAction = 'add/update';
}
if (!metadata[file._credential][file._businessUnit][file.type]) {
try {
await MetadataType[file.type].readBUMetadataForType(
buPath,
false,
metadata[file._credential][file._businessUnit]
);
} catch (ex) {
// silently catch directory-not-found errors here
Util.logger.debug(ex.message);
gitActionsCounter[file.gitAction]++;
file._credential = file.file.split('/')[1];
file._businessUnit = file.file.split('/')[2];

// Parse retrieve directory to also populate the name field (not possible for deleted files)
if (file.gitAction !== 'delete' && file.type !== 'folder') {
// folders are saved with their name as file-name, not with their key, hence this section can be skipped for folders
const buPath = `${properties.directories.retrieve}/${file._credential}/${file._businessUnit}/`;
if (!metadata[file._credential]) {
metadata[file._credential] = {};
}
if (!metadata[file._credential][file._businessUnit]) {
metadata[file._credential][file._businessUnit] = {};
}
if (!metadata[file._credential][file._businessUnit][file.type]) {
try {
await MetadataType[file.type].readBUMetadataForType(
buPath,
false,
metadata[file._credential][file._businessUnit]
);
} catch (ex) {
// silently catch directory-not-found errors here
Util.logger.debug(ex.message);
}
}
const fileContent =
metadata[file._credential][file._businessUnit][file.type][
file.externalKey
];
if (fileContent) {
file.name = fileContent[MetadataType[file.type].definition.nameField];
}
}
const fileContent =
metadata[file._credential][file._businessUnit][file.type][file.externalKey];
if (fileContent) {
file.name = fileContent[MetadataType[file.type].definition.nameField];
}
}
return file;
})
extendedBar.increment();
return file;
})
)
);
// stop the progress bar
extendedBar.stop();

if (
!gitActionsCounter['add/update'] &&
!gitActionsCounter.move &&
Expand Down

0 comments on commit aac7823

Please sign in to comment.