From f4befd9b78fe35e7089bdf6393baa15970b9ee25 Mon Sep 17 00:00:00 2001 From: Tianchu Zhao Date: Tue, 10 Dec 2024 17:59:06 +0530 Subject: [PATCH 1/8] feat: export installed package to file --- bin/install.js | 6 ++++++ lib/install.js | 13 +++++++++++-- lib/utils.js | 15 +++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- tests/base.test.js | 35 +++++++++++++++++++++++++++++++++++ 6 files changed, 70 insertions(+), 5 deletions(-) diff --git a/bin/install.js b/bin/install.js index adf09d1..431c9fa 100755 --- a/bin/install.js +++ b/bin/install.js @@ -58,6 +58,11 @@ yargs "Print JSON-formatted dependency graph of packages to be installed without actually installing them", default: false, }) + .option("export-package-names", { + type: "string", + description: "export installed packages to a file", + default: "", + }) .option("azure", { alias: "az", type: "boolean", @@ -71,6 +76,7 @@ yargs timeZone: argv["tz"], preview: argv["p"], azure: argv["az"], + exportPackageNameFilePath: argv["export-package-names"], }; if (argv.global) { return installGlobal(argv.package, argv.registry, argv.namespace, argv.cluster, options).then( diff --git a/lib/install.js b/lib/install.js index 9be7259..679ffb4 100644 --- a/lib/install.js +++ b/lib/install.js @@ -6,6 +6,7 @@ const K8sInstaller = require("./k8s").K8sInstaller; const S3 = require("./s3").S3; const listDirs = require("./utils").listDirs; const deleteDir = require("./utils").deleteDir; +const appendToFileSync = require("./utils").appendToFileSync; const fs = require("fs"); const { DashboardInstaller } = require("./dashboard"); const { constants } = require("./constants"); @@ -84,7 +85,7 @@ const packageDetailsFromPath = function (path) { * @param {string} namespace * @param {boolean} save * @param {boolean} cluster - * @param {{force:boolean,cronString:string,timeZone,preview:boolean,azure:boolean}} options + * @param {{force:boolean,cronString:string,timeZone,preview:boolean,azure:boolean,exportPackageNameFilePath:string}} options * @param {string} dirPath */ const install = function (packageName, registry, namespace, save, cluster, options, dirPath = process.cwd()) { @@ -217,7 +218,7 @@ const install = function (packageName, registry, namespace, save, cluster, optio } if (options.preview) { - installed.push(packageDetailsFromPath(".")); + installed.push(packageDetailsFromPath(dirPath)); return Promise.resolve(); // Skip installation } @@ -248,6 +249,14 @@ const install = function (packageName, registry, namespace, save, cluster, optio if (options.preview) { console.log(JSON.stringify(installed, null, 2)); } + console.log(dirPath); + if (options.exportPackageNameFilePath !== "") { + var packageSet = new Set(); + installed.forEach(function (data) { + packageSet.add(data.name); + }); + appendToFileSync(options.exportPackageNameFilePath, Array.from(packageSet).join(" ")); + } const parsedPackage = npa(parentPackageName); return parsedPackage.name; }); diff --git a/lib/utils.js b/lib/utils.js index 86d7f4c..3176d54 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -2,6 +2,7 @@ const Promise = require("bluebird"); const fs = require("fs").promises; const path = require("path"); const { rimraf } = require("rimraf"); +const fsSync = require("fs"); /** * Recursively walk through the folder and return all file paths @@ -53,6 +54,20 @@ async function deleteDir(dir) { exports.deleteDir = deleteDir; +/** + * Append content to file + * @param {string} filePath + * @param {string} content + * @returns {Promise} + */ +function appendToFileSync(filePath, content) { + const dirPath = path.dirname(filePath); + fsSync.mkdirSync(dirPath, { recursive: true }); + fsSync.appendFileSync(filePath, content); +} + +exports.appendToFileSync = appendToFileSync; + /** * Generate arguments * @param {[string]} args diff --git a/package-lock.json b/package-lock.json index 85e1e29..53cec89 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "argopm", - "version": "0.10.19", + "version": "0.10.20", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "argopm", - "version": "0.10.19", + "version": "0.10.20", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.637.0", diff --git a/package.json b/package.json index 9df436f..a3513f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "argopm", - "version": "0.10.19", + "version": "0.10.20", "description": "Argo package manager", "main": "./lib/index.js", "scripts": { diff --git a/tests/base.test.js b/tests/base.test.js index f924ea8..49cc9d0 100644 --- a/tests/base.test.js +++ b/tests/base.test.js @@ -1,6 +1,10 @@ const { uninstall } = require("../lib/index"); const { install, installGlobal } = require("../lib/install.js"); const { getPackageName, MOCK_PACKAGE_PATH, REGISTRY } = require("./test-utils"); +const { k8s } = require("../lib/k8s.js"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); describe.skip("simulate package install", () => { const namespace = "default"; @@ -25,3 +29,34 @@ describe.skip("simulate package install", () => { expect(result).toBeTruthy(); }); }); + +describe("verify export-package-names", () => { + test("verify content", async () => { + const namespace = "default"; + const cluster = false; + + const currentTime = Date.now(); + const tempDir = os.tmpdir(); + const filePath = path.join(tempDir, `${currentTime}.txt`); + + const result = await install( + ".", + REGISTRY, + namespace, + false, + cluster, + { preview: true, exportPackageNameFilePath: filePath }, + MOCK_PACKAGE_PATH + ) + .then(() => { + return true; + }) + .catch((err) => { + console.error(err); + throw err; + }); + expect(result).toBeTruthy(); + const data = fs.readFileSync(filePath, "utf8"); + expect(data).toEqual("@atlan/mock-package-delete-me"); + }); +}); From ecb95cda6e56121d3deb8ca6a7493d6b21fecff1 Mon Sep 17 00:00:00 2001 From: Tianchu Zhao Date: Tue, 10 Dec 2024 18:00:43 +0530 Subject: [PATCH 2/8] fix: remove debug log --- lib/install.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/install.js b/lib/install.js index 679ffb4..1ba0291 100644 --- a/lib/install.js +++ b/lib/install.js @@ -249,7 +249,6 @@ const install = function (packageName, registry, namespace, save, cluster, optio if (options.preview) { console.log(JSON.stringify(installed, null, 2)); } - console.log(dirPath); if (options.exportPackageNameFilePath !== "") { var packageSet = new Set(); installed.forEach(function (data) { From 0615a8cab59b73fbde5ee82b23520ed36f0614c5 Mon Sep 17 00:00:00 2001 From: Tianchu Zhao Date: Tue, 10 Dec 2024 18:25:35 +0530 Subject: [PATCH 3/8] feat: allow skip packageInstall in localInstall --- bin/local-install.js | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/bin/local-install.js b/bin/local-install.js index b568ca1..ed71d79 100644 --- a/bin/local-install.js +++ b/bin/local-install.js @@ -230,18 +230,22 @@ async function run( extraArgs, channel, snapshotInstall, - skipVersionCheck + skipVersionCheck, + skipPackages ) { const packagesMap = getAllPackagesMap(); const installedPackages = await getInstalledPackages(); - const packagesToInstall = getPackagesToInstall( + const initPackagesToInstall = getPackagesToInstall( packageName, packagesMap, installedPackages, skipVersionCheck, snapshotInstall ); + const skipPackagesArray = JSON.parse("[" + skipPackages + "]"); + const packagesToInstall = initPackagesToInstall.removeAll(skipPackagesArray); + console.log("Packages skipped install: " + skipPackages); console.log( "Packages to install: " + Array.from(packagesToInstall) @@ -313,9 +317,19 @@ const channel = process.argv[7]; // It respects bypassSafetyCheck, and added a -snapshot suffix to the version const snapshotInstallString = process.argv[8]; const skipVersionCheckString = process.argv[9]; +const skipPackages = process.argv[10]; const bypassSafetyCheck = bypassSafetyCheckString === "true"; const snapshotInstall = snapshotInstallString === "true"; const skipVersionCheck = skipVersionCheckString === "true"; -run(packageName, azureArtifacts, bypassSafetyCheck, extraArgs, channel, snapshotInstall, skipVersionCheck); +run( + packageName, + azureArtifacts, + bypassSafetyCheck, + extraArgs, + channel, + snapshotInstall, + skipVersionCheck, + skipPackages +); From 46dc075e9e4fa6ee6b0374ab0e97970ab3c9854e Mon Sep 17 00:00:00 2001 From: Tianchu Zhao Date: Wed, 11 Dec 2024 16:12:27 +0530 Subject: [PATCH 4/8] fix --- lib/install.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/install.js b/lib/install.js index 1ba0291..fdd1a54 100644 --- a/lib/install.js +++ b/lib/install.js @@ -164,6 +164,7 @@ const install = function (packageName, registry, namespace, save, cluster, optio return k8sInstaller .install(cluster) .then((_) => { + installed.push(packageDetailsFromPath(innerDir)); return dashboardInstaller.install(); }) .then(() => { @@ -198,6 +199,9 @@ const install = function (packageName, registry, namespace, save, cluster, optio return k8sInstaller .install(cluster) + .then((_) => { + installed.push(packageDetailsFromPath(dir)); + }) .then((_) => { return dashboardInstaller.install(); }) @@ -234,6 +238,7 @@ const install = function (packageName, registry, namespace, save, cluster, optio return k8sInstaller .install(cluster) .then((_) => { + installed.push(packageDetailsFromPath(dirPath)); return dashboardInstaller.install(); }) .then(() => { @@ -254,7 +259,7 @@ const install = function (packageName, registry, namespace, save, cluster, optio installed.forEach(function (data) { packageSet.add(data.name); }); - appendToFileSync(options.exportPackageNameFilePath, Array.from(packageSet).join(" ")); + appendToFileSync(options.exportPackageNameFilePath, Array.from(packageSet).join(",")); } const parsedPackage = npa(parentPackageName); return parsedPackage.name; From bf2d6fee6b9d4b58d00c47657ec91084afbd9b3c Mon Sep 17 00:00:00 2001 From: Tianchu Zhao Date: Wed, 11 Dec 2024 16:35:14 +0530 Subject: [PATCH 5/8] fix: fix --- bin/local-install.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/local-install.js b/bin/local-install.js index ed71d79..7a52d11 100644 --- a/bin/local-install.js +++ b/bin/local-install.js @@ -243,7 +243,9 @@ async function run( skipVersionCheck, snapshotInstall ); - const skipPackagesArray = JSON.parse("[" + skipPackages + "]"); + const skipPackagesArray = JSON.parse(skipPackages) + .map((item) => item.split(",")) + .flat(); const packagesToInstall = initPackagesToInstall.removeAll(skipPackagesArray); console.log("Packages skipped install: " + skipPackages); console.log( From f2dd2e86e032508961e50cedf431e9041fa8d916 Mon Sep 17 00:00:00 2001 From: Tianchu Zhao Date: Wed, 11 Dec 2024 17:10:26 +0530 Subject: [PATCH 6/8] fix --- bin/local-install.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bin/local-install.js b/bin/local-install.js index 7a52d11..496c321 100644 --- a/bin/local-install.js +++ b/bin/local-install.js @@ -243,11 +243,15 @@ async function run( skipVersionCheck, snapshotInstall ); - const skipPackagesArray = JSON.parse(skipPackages) - .map((item) => item.split(",")) - .flat(); - const packagesToInstall = initPackagesToInstall.removeAll(skipPackagesArray); - console.log("Packages skipped install: " + skipPackages); + const skipPackagesArray = Array.from( + JSON.parse(skipPackages) + .map((item) => item.split(",")) + .flat() + ); + const packagesToInstall = Array.from(initPackagesToInstall).filter( + (item) => !skipPackagesArray.includes(item.name) + ); + console.log("Packages skipped install: " + skipPackagesArray); console.log( "Packages to install: " + Array.from(packagesToInstall) From c6ef339a209814ede4aa00290479a406301f1f2d Mon Sep 17 00:00:00 2001 From: Tianchu Zhao Date: Wed, 11 Dec 2024 19:49:38 +0530 Subject: [PATCH 7/8] put safety check to standalone step --- bin/local-install.js | 261 +++----------------------------------- bin/safety-check.js | 88 +++++++++++++ lib/local-install-util.js | 180 ++++++++++++++++++++++++++ 3 files changed, 283 insertions(+), 246 deletions(-) create mode 100644 bin/safety-check.js create mode 100644 lib/local-install-util.js diff --git a/bin/local-install.js b/bin/local-install.js index 496c321..d2524bf 100644 --- a/bin/local-install.js +++ b/bin/local-install.js @@ -1,204 +1,7 @@ const fs = require("fs"); const path = require("path"); const execSync = require("child_process").execSync; - -const k8s = require("@kubernetes/client-node"); -const { exit } = require("process"); - -// Kube config -const kc = new k8s.KubeConfig(); -kc.loadFromDefault(); - -function skipRunningPackagesCheck(packageName, bypassSafetyCheck) { - /** - * Check if the last safe release was more than 24 hours ago. If not prevent safety check install. - */ - if (bypassSafetyCheck) { - return true; - } - - const safetyCheckFile = `/tmp/atlan-update/${packageName.replace("/", "-")}-last-safe-run.txt`; - if (!fs.existsSync(safetyCheckFile)) { - return true; - } - - const lastSafeRelease = parseInt(fs.readFileSync(safetyCheckFile, "utf-8"), 10); - const lastSafeReleaseDate = new Date(lastSafeRelease); - const now = new Date(); - const diff = now - lastSafeReleaseDate; - const diffInHours = diff / (1000 * 60 * 60); - if (diffInHours < 24) { - return false; - } - return true; -} - -function getAllPackagesMap() { - /** - * Returns a map of all packages in the packages directory - */ - const packagesMap = {}; - console.log("Reading packages from " + marketplacePackagesPath); - - const packages = fs - .readdirSync(marketplacePackagesPath, { recursive: true, withFileTypes: false }) - .filter((file) => fs.lstatSync(path.join(marketplacePackagesPath, file)).isDirectory()); - - for (const packageName of packages) { - // Skip if packageName contains node_modules - if (packageName.includes("node_modules")) { - continue; - } - - const packagePath = path.join(marketplacePackagesPath, packageName); - console.log("Reading package " + packageName + " from " + packagePath); - - const packageJSONFileExists = fs.existsSync(path.join(packagePath, "package.json")); - - if (!packageJSONFileExists) { - continue; - } - - const packageJSON = JSON.parse(fs.readFileSync(path.join(packagePath, "package.json"), "utf-8")); - - const isNumaflowPackage = fs.existsSync(path.join(packagePath, "pipelines")); - - packagesMap[packageJSON.name] = { - name: packageJSON.name, - version: packageJSON.version, - dependencies: packageJSON.dependencies, - path: packagePath, - isNumaflowPackage: isNumaflowPackage, - }; - } - console.log("Found " + Object.keys(packagesMap).length + " packages"); - return packagesMap; -} - -async function getAllRunningPackages() { - /** - * Returns a list of all packages that are currently running - */ - - // Fetch all running workflows - const workflowClient = kc.makeApiClient(k8s.CustomObjectsApi); - const workflows = await workflowClient.listNamespacedCustomObject( - "argoproj.io", - "v1alpha1", - "default", - "workflows", - undefined, - undefined, - undefined, - undefined, - "workflows.argoproj.io/phase=Running" - ); - - // For every running workflow, check which package it belongs to - const runningPackages = []; - for (const workflow of workflows.body.items) { - const package = workflow.metadata.annotations["package.argoproj.io/name"]; - if (package) { - runningPackages.push(package); - } - } - return runningPackages; -} - -async function getInstalledPackages() { - /** - * Returns a list of all packages that are currently installed on the cluster - */ - const clusterWorkflowTemplateClient = kc.makeApiClient(k8s.CustomObjectsApi); - const clusterWorkflowTemplates = await clusterWorkflowTemplateClient.listClusterCustomObject( - "argoproj.io", - "v1alpha1", - "clusterworkflowtemplates" - ); - const installedPackages = {}; - for (const clusterWorkflowTemplate of clusterWorkflowTemplates.body.items) { - if (!clusterWorkflowTemplate.metadata.annotations || !clusterWorkflowTemplate.metadata.labels) { - continue; - } - const package = clusterWorkflowTemplate.metadata.annotations["package.argoproj.io/name"]; - const packageVersion = clusterWorkflowTemplate.metadata.labels["package.argoproj.io/version"]; - if (package && packageVersion) { - installedPackages[package] = packageVersion; - } - } - console.log("Installed packages: " + Object.keys(installedPackages).join(", ")); - return installedPackages; -} - -function getPackagesToInstall(packageName, packagesMap, installedPackages, skipVersionCheck, snapshotInstall) { - /** - * Returns a list of all packages that need to be installed - */ - var packagesToInstall = new Set(); - const package = packagesMap[packageName]; - if (!package) { - throw new Error(`Package ${packageName} not found`); - } - - const snapshotInstallSuffix = "-snapshot"; - - for (const dependency of Object.keys(package.dependencies)) { - let dependencyPackage = packagesMap[dependency]; - if (!dependencyPackage) { - throw new Error(`Dependency ${dependency} not found`); - } - - if (snapshotInstall) { - if (!dependencyPackage.version.endsWith(snapshotInstallSuffix)) { - dependencyPackage.version = dependencyPackage.version + snapshotInstallSuffix; - } - packagesToInstall.add(dependencyPackage); - } - - if (!installedPackages[dependencyPackage.name] || dependencyPackage.isNumaflowPackage) { - packagesToInstall.add(dependencyPackage); - } - - if (skipVersionCheck || installedPackages[dependencyPackage.name] !== dependencyPackage.version) { - packagesToInstall.add(dependencyPackage); - } - - if (dependencyPackage.dependencies) { - const dependencyPackagesToInstall = getPackagesToInstall( - dependency, - packagesMap, - installedPackages, - skipVersionCheck, - snapshotInstall - ); - packagesToInstall = new Set([...packagesToInstall, ...dependencyPackagesToInstall]); - } - } - return packagesToInstall; -} - -function getConnectorPackages() { - //All the connector packages don't have dependency to @atlan/connectors - //If changes for canary are present in canary deployment, and the crawler is running, then we have to stop installation of @atlan/connectors package - //Hence if any of these are running, we have to skip the installation of @atlan/connectors package. - - //Read all the packages - //Check for isVerified, isCertified - //Check for type miner, utility and return for custom, connectors etc. - const packages = fs - .readdirSync(marketplacePackagesPath, { recursive: true, withFileTypes: false }) - .filter((file) => file.endsWith("package.json")) - .map((file) => JSON.parse(fs.readFileSync(path.join(marketplacePackagesPath, file), "utf-8"))) - .filter((pkg) => pkg.config?.labels?.["orchestration.atlan.com/certified"] === "true") - .filter( - (pkg) => - pkg.config?.labels?.["orchestration.atlan.com/type"] !== "miner" && - pkg.config?.labels?.["orchestration.atlan.com/type"] !== "utility" - ) - .map((pkg) => pkg.name); - - return packages; -} +const { getPackagesToInstall, getInstalledPackages, getAllPackagesMap } = require("../lib/local-install-util"); function installPackages(packages, extraArgs, azureArtifacts) { // Install packages @@ -224,16 +27,16 @@ function installPackages(packages, extraArgs, azureArtifacts) { } async function run( + marketplacePackagesPath, packageName, azureArtifacts, - bypassSafetyCheck, extraArgs, channel, snapshotInstall, skipVersionCheck, skipPackages ) { - const packagesMap = getAllPackagesMap(); + const packagesMap = getAllPackagesMap(marketplacePackagesPath); const installedPackages = await getInstalledPackages(); const initPackagesToInstall = getPackagesToInstall( @@ -261,44 +64,11 @@ async function run( // Always install numaflow packages since delete-pipelines may have deleted them const numaflowPackages = [...packagesToInstall].filter((pkg) => pkg.isNumaflowPackage); - const connectorPackages = [...packagesToInstall].find((pkg) => "@atlan/connectors" === pkg.name) - ? getConnectorPackages() - : []; if (packageName != "@atlan/cloud-packages") { console.log("Numaflow packages to install: " + numaflowPackages.map((pkg) => pkg.name).join(", ")); installPackages(numaflowPackages, extraArgs, azureArtifacts); } - var safeToInstall = true; - if (!skipRunningPackagesCheck(packageName, bypassSafetyCheck)) { - // Check if running workflows have packages that need to be installed - const runningPackages = await getAllRunningPackages(); - console.log("Running packages: " + runningPackages.join(", ")); - const packagesToInstallNames = Array.from(packagesToInstall).map((pkg) => pkg.name); - for (const runningPackage of runningPackages) { - if (packagesToInstallNames.includes(runningPackage)) { - safeToInstall = false; - break; - } - if (connectorPackages.includes(runningPackage)) { - //If any of the connector packages are running, then we have to skip the installation of @atlan/connectors package. - safeToInstall = false; - console.log( - `Connector package ${runningPackage} is running. Skipping installation of @atlan/connectors package` - ); - break; - } - } - } - console.log("Safe to install: " + safeToInstall); - - if (!safeToInstall) { - console.warn("Not safe to install. Waiting for running workflows to complete before installing packages."); - // use custom exit code 100 to bypass workflow failure - // choose code 100 to avoid collision https://node.readthedocs.io/en/latest/api/process/ - exit(100); - } - // Install packages const argoPackages = [...packagesToInstall].filter((pkg) => !pkg.isNumaflowPackage); console.log("Argo packages to install: " + argoPackages.map((pkg) => pkg.name).join(", ")); @@ -306,33 +76,32 @@ async function run( installPackages(argoPackages, extraArgs, azureArtifacts, snapshotInstall); // Write last safe release - fs.writeFileSync( - `/tmp/atlan-update/${packageName.replace("/", "-")}-last-safe-run.txt`, - `${Math.floor(new Date().getTime())}` - ); + const filePath = `/tmp/atlan-update/${packageName.replace("/", "-")}-last-safe-run.txt`; + const dirPath = path.dirname(filePath); + fs.mkdirSync(dirPath, { recursive: true }); + fs.writeFileSync(filePath, `${Math.floor(new Date().getTime())}`); } // Take package name as input const marketplacePackagesPath = process.argv[2]; const packageName = process.argv[3]; const azureArtifacts = process.argv[4]; -const bypassSafetyCheckString = process.argv[5]; -const extraArgs = process.argv[6]; -const channel = process.argv[7]; +const extraArgs = process.argv[5]; +const channel = process.argv[6]; // snapshotInstall install package regardless of package version -// It respects bypassSafetyCheck, and added a -snapshot suffix to the version -const snapshotInstallString = process.argv[8]; -const skipVersionCheckString = process.argv[9]; -const skipPackages = process.argv[10]; +// It adds a -snapshot suffix to the version +const snapshotInstallString = process.argv[7]; +const skipVersionCheckString = process.argv[8]; +const skipPackagesString = process.argv[9]; -const bypassSafetyCheck = bypassSafetyCheckString === "true"; const snapshotInstall = snapshotInstallString === "true"; const skipVersionCheck = skipVersionCheckString === "true"; +const skipPackages = skipPackagesString.startsWith("[") ? skipPackagesString : "[]"; run( + marketplacePackagesPath, packageName, azureArtifacts, - bypassSafetyCheck, extraArgs, channel, snapshotInstall, diff --git a/bin/safety-check.js b/bin/safety-check.js new file mode 100644 index 0000000..9bc0f3d --- /dev/null +++ b/bin/safety-check.js @@ -0,0 +1,88 @@ +const { exit } = require("process"); +const fs = require("fs"); +const { + getAllRunningPackages, + getPackagesToInstall, + getInstalledPackages, + getAllPackagesMap, + getConnectorPackages, +} = require("../lib/local-install-util"); + +function skipRunningPackagesCheck(packageName) { + /** + * Check if the last safe release was more than 24 hours ago. If not prevent safety check install. + */ + + const safetyCheckFile = `/tmp/atlan-update/${packageName.replace("/", "-")}-last-safe-run.txt`; + if (!fs.existsSync(safetyCheckFile)) { + return true; + } + + const lastSafeRelease = parseInt(fs.readFileSync(safetyCheckFile, "utf-8"), 10); + const lastSafeReleaseDate = new Date(lastSafeRelease); + const now = new Date(); + const diff = now - lastSafeReleaseDate; + const diffInHours = diff / (1000 * 60 * 60); + if (diffInHours < 24) { + return false; + } + return true; +} + +async function run(marketplacePackagesPath, packageName, snapshotInstall, skipVersionCheck) { + if (snapshotInstall) { + console.log("snapshot install, safe to proceed"); + exit(0); + } + const packagesMap = getAllPackagesMap(marketplacePackagesPath); + const installedPackages = await getInstalledPackages(); + console.log(123); + const packagesToInstall = getPackagesToInstall( + packageName, + packagesMap, + installedPackages, + skipVersionCheck, + snapshotInstall + ); + var safeToInstall = true; + if (!skipRunningPackagesCheck(packageName)) { + // Check if running workflows have packages that need to be installed + const runningPackages = await getAllRunningPackages(); + console.log("Running packages: " + runningPackages.join(", ")); + const packagesToInstallNames = Array.from(packagesToInstall).map((pkg) => pkg.name); + const connectorPackages = [...packagesToInstall].find((pkg) => "@atlan/connectors" === pkg.name) + ? getConnectorPackages(marketplacePackagesPath) + : []; + for (const runningPackage of runningPackages) { + if (packagesToInstallNames.includes(runningPackage)) { + safeToInstall = false; + break; + } + if (connectorPackages.includes(runningPackage)) { + //If any of the connector packages are running, then we have to skip the installation of @atlan/connectors package. + safeToInstall = false; + console.log( + `Connector package ${runningPackage} is running. Skipping installation of @atlan/connectors package` + ); + break; + } + } + } + console.log("Safe to install: " + safeToInstall); + + if (!safeToInstall) { + console.warn("Not safe to install. Waiting for running workflows to complete before installing packages."); + // use custom exit code 100 to bypass workflow failure + // choose code 100 to avoid collision https://node.readthedocs.io/en/latest/api/process/ + exit(100); + } +} + +const marketplacePackagesPath = process.argv[2]; +const packageName = process.argv[3]; +const snapshotInstallString = process.argv[4]; +const skipVersionCheckString = process.argv[5]; +const snapshotInstall = snapshotInstallString === "true"; +const skipVersionCheck = skipVersionCheckString === "true"; + +run(marketplacePackagesPath, packageName, snapshotInstall, skipVersionCheck); diff --git a/lib/local-install-util.js b/lib/local-install-util.js new file mode 100644 index 0000000..a9d38d1 --- /dev/null +++ b/lib/local-install-util.js @@ -0,0 +1,180 @@ +const fs = require("fs"); +const path = require("path"); +const k8s = require("@kubernetes/client-node"); + +// Kube config +const kc = new k8s.KubeConfig(); +kc.loadFromDefault(); + +async function getAllRunningPackages() { + /** + * Returns a list of all packages that are currently running + */ + + // Fetch all running workflows + const workflowClient = kc.makeApiClient(k8s.CustomObjectsApi); + const workflows = await workflowClient.listNamespacedCustomObject( + "argoproj.io", + "v1alpha1", + "default", + "workflows", + undefined, + undefined, + undefined, + undefined, + "workflows.argoproj.io/phase=Running" + ); + + // For every running workflow, check which package it belongs to + const runningPackages = []; + for (const workflow of workflows.body.items) { + const package = workflow.metadata.annotations["package.argoproj.io/name"]; + if (package) { + runningPackages.push(package); + } + } + return runningPackages; +} + +async function getInstalledPackages() { + /** + * Returns a list of all packages that are currently installed on the cluster + */ + const clusterWorkflowTemplateClient = kc.makeApiClient(k8s.CustomObjectsApi); + const clusterWorkflowTemplates = await clusterWorkflowTemplateClient.listClusterCustomObject( + "argoproj.io", + "v1alpha1", + "clusterworkflowtemplates" + ); + const installedPackages = {}; + for (const clusterWorkflowTemplate of clusterWorkflowTemplates.body.items) { + if (!clusterWorkflowTemplate.metadata.annotations || !clusterWorkflowTemplate.metadata.labels) { + continue; + } + const package = clusterWorkflowTemplate.metadata.annotations["package.argoproj.io/name"]; + const packageVersion = clusterWorkflowTemplate.metadata.labels["package.argoproj.io/version"]; + if (package && packageVersion) { + installedPackages[package] = packageVersion; + } + } + console.log("Installed packages: " + Object.keys(installedPackages).join(", ")); + return installedPackages; +} + +function getPackagesToInstall(packageName, packagesMap, installedPackages, skipVersionCheck, snapshotInstall) { + /** + * Returns a list of all packages that need to be installed + */ + var packagesToInstall = new Set(); + const package = packagesMap[packageName]; + if (!package) { + throw new Error(`Package ${packageName} not found`); + } + + const snapshotInstallSuffix = "-snapshot"; + + for (const dependency of Object.keys(package.dependencies)) { + let dependencyPackage = packagesMap[dependency]; + if (!dependencyPackage) { + throw new Error(`Dependency ${dependency} not found`); + } + + if (snapshotInstall) { + if (!dependencyPackage.version.endsWith(snapshotInstallSuffix)) { + dependencyPackage.version = dependencyPackage.version + snapshotInstallSuffix; + } + packagesToInstall.add(dependencyPackage); + } + + if (!installedPackages[dependencyPackage.name] || dependencyPackage.isNumaflowPackage) { + packagesToInstall.add(dependencyPackage); + } + + if (skipVersionCheck || installedPackages[dependencyPackage.name] !== dependencyPackage.version) { + packagesToInstall.add(dependencyPackage); + } + + if (dependencyPackage.dependencies) { + const dependencyPackagesToInstall = getPackagesToInstall( + dependency, + packagesMap, + installedPackages, + skipVersionCheck, + snapshotInstall + ); + packagesToInstall = new Set([...packagesToInstall, ...dependencyPackagesToInstall]); + } + } + return packagesToInstall; +} + +function getAllPackagesMap(marketplacePackagesPath) { + /** + * Returns a map of all packages in the packages directory + */ + const packagesMap = {}; + console.log("Reading packages from " + marketplacePackagesPath); + + const packages = fs + .readdirSync(marketplacePackagesPath, { recursive: true, withFileTypes: false }) + .filter((file) => fs.lstatSync(path.join(marketplacePackagesPath, file)).isDirectory()); + + for (const packageName of packages) { + // Skip if packageName contains node_modules + if (packageName.includes("node_modules")) { + continue; + } + + const packagePath = path.join(marketplacePackagesPath, packageName); + console.log("Reading package " + packageName + " from " + packagePath); + + const packageJSONFileExists = fs.existsSync(path.join(packagePath, "package.json")); + + if (!packageJSONFileExists) { + continue; + } + + const packageJSON = JSON.parse(fs.readFileSync(path.join(packagePath, "package.json"), "utf-8")); + + const isNumaflowPackage = fs.existsSync(path.join(packagePath, "pipelines")); + + packagesMap[packageJSON.name] = { + name: packageJSON.name, + version: packageJSON.version, + dependencies: packageJSON.dependencies, + path: packagePath, + isNumaflowPackage: isNumaflowPackage, + }; + } + console.log("Found " + Object.keys(packagesMap).length + " packages"); + return packagesMap; +} + +function getConnectorPackages(marketplacePackagesPath) { + //All the connector packages don't have dependency to @atlan/connectors + //If changes for canary are present in canary deployment, and the crawler is running, then we have to stop installation of @atlan/connectors package + //Hence if any of these are running, we have to skip the installation of @atlan/connectors package. + + //Read all the packages + //Check for isVerified, isCertified + //Check for type miner, utility and return for custom, connectors etc. + const packages = fs + .readdirSync(marketplacePackagesPath, { recursive: true, withFileTypes: false }) + .filter((file) => file.endsWith("package.json")) + .map((file) => JSON.parse(fs.readFileSync(path.join(marketplacePackagesPath, file), "utf-8"))) + .filter((pkg) => pkg.config?.labels?.["orchestration.atlan.com/certified"] === "true") + .filter( + (pkg) => + pkg.config?.labels?.["orchestration.atlan.com/type"] !== "miner" && + pkg.config?.labels?.["orchestration.atlan.com/type"] !== "utility" + ) + .map((pkg) => pkg.name); + + return packages; +} + +exports.getAllRunningPackages = getAllRunningPackages; +exports.getPackagesToInstall = getPackagesToInstall; +exports.getInstalledPackages = getInstalledPackages; +exports.getAllPackagesMap = getAllPackagesMap; +exports.getConnectorPackages = getConnectorPackages; From 122516415020fd5566260a9b3153fec8a0c85f42 Mon Sep 17 00:00:00 2001 From: Tianchu Zhao Date: Thu, 12 Dec 2024 12:20:03 +0530 Subject: [PATCH 8/8] Release 0.10.20