diff --git a/.gitignore b/.gitignore index d0901c12..fb51602f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .nyc_output/ +artifacts configs coverage +local-storage.json node_modules package-lock.json third_party diff --git a/README.md b/README.md index 6cd747f5..b81a926b 100644 --- a/README.md +++ b/README.md @@ -482,7 +482,7 @@ git cherry-pick --continue git push # create pull request -e pr --backport 1234 +e pr open --backport 1234 ``` ## Common Usage diff --git a/package.json b/package.json index 94291a99..8518f2c8 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "@marshallofsound/chrome-cookies-secure": "^2.1.1", "@octokit/auth-oauth-device": "^3.1.1", "@octokit/rest": "^18.5.2", + "adm-zip": "^0.5.16", "ajv": "^8.11.0", "ajv-formats": "^2.1.1", "chalk": "^2.4.1", diff --git a/src/e b/src/e index 92ec439b..8aae46d3 100755 --- a/src/e +++ b/src/e @@ -169,7 +169,7 @@ program .command('backport [pr]', 'Assists with manual backport processes') .command('show ', 'Show info about the current build config') .command('test [specRunnerArgs...]', `Run Electron's spec runner`) - .command('pr [options]', 'Open a GitHub URL where you can PR your changes') + .command('pr [subcommand]', 'Work with PRs to electron/electron') .command('patches ', 'Refresh the patches in $root/src/electron/patches/$target') .command('open ', 'Open a GitHub URL for the given commit hash / pull # / issue #') .command('auto-update', 'Check for build-tools updates or enable/disable automatic updates') @@ -179,7 +179,8 @@ program 'Opens a PR to electron/electron that backport the given CL into our patches folder', ) .alias('auto-cherry-pick') - .command('gh-auth', 'Generates a device oauth token'); + .command('gh-auth', 'Generates a device oauth token') + .command('clean', 'Cleanup artifacts produced by build tools'); const ci = program.command('ci').executableDir(path.join(__dirname, 'ci')); ci.command('status', 'Show information about CI job statuses'); diff --git a/src/e-clean.js b/src/e-clean.js new file mode 100644 index 00000000..db76392c --- /dev/null +++ b/src/e-clean.js @@ -0,0 +1,29 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const { ARTIFACTS_DIR, getStaleArtifacts } = require('./utils/artifacts'); + +const program = require('commander'); +const { color, fatal } = require('./utils/logging'); + +function clean(options) { + try { + if (options.stale) { + const staleFiles = getStaleArtifacts(); + staleFiles.forEach((file) => { + const filePath = path.join(ARTIFACTS_DIR, file); + fs.rmSync(filePath, { recursive: true, force: true }); + }); + console.log(color.success, `${staleFiles.length} stale artifact(s) removed.`); + } else { + fs.rmSync(ARTIFACTS_DIR, { recursive: true, force: true }); + console.log(color.success, 'Artifacts directory cleaned successfully.'); + } + } catch (error) { + fatal(error); + } +} + +program.action(clean).option('--stale', 'Only clean stale artifacts'); + +program.parse(process.argv); diff --git a/src/e-pr.js b/src/e-pr.js index 0ceb756b..7a2a1f99 100644 --- a/src/e-pr.js +++ b/src/e-pr.js @@ -1,15 +1,23 @@ #!/usr/bin/env node const childProcess = require('child_process'); +const fs = require('fs'); const path = require('path'); + +const AdmZip = require('adm-zip'); const querystring = require('querystring'); const semver = require('semver'); - const open = require('open'); const program = require('commander'); +const { Octokit } = require('@octokit/rest'); +const inquirer = require('inquirer'); +const { getGitHubAuthToken } = require('./utils/github-auth'); const { current } = require('./evm-config'); const { color, fatal } = require('./utils/logging'); +const { ARTIFACTS_DIR, maybeCheckStaleArtifacts } = require('./utils/artifacts'); + +const d = require('debug')('build-tools:pr'); // Adapted from https://github.com/electron/clerk function findNoteInPRBody(body) { @@ -134,27 +142,23 @@ function pullRequestSource(source) { } program + .command('open', null, { isDefault: true }) .description('Open a GitHub URL where you can PR your changes') - .option( - '-s, --source ', - 'Where the changes are coming from', - guessPRSource(current()), - ) - .option( - '-t, --target ', - 'Where the changes are going to', - guessPRTarget(current()), - ) + .option('-s, --source [source_branch]', 'Where the changes are coming from') + .option('-t, --target [target_branch]', 'Where the changes are going to') .option('-b, --backport ', 'Pull request being backported') .action(async (options) => { - if (!options.source) { + const source = options.source || guessPRSource(current()); + const target = options.target || guessPRTarget(current()); + + if (!source) { fatal(`'source' is required to create a PR`); - } else if (!options.target) { + } else if (!target) { fatal(`'target' is required to create a PR`); } const repoBaseUrl = 'https://github.com/electron/electron'; - const comparePath = `${options.target}...${pullRequestSource(options.source)}`; + const comparePath = `${target}...${pullRequestSource(source)}`; const queryParams = { expand: 1 }; if (!options.backport) { @@ -188,5 +192,193 @@ program } return open(`${repoBaseUrl}/compare/${comparePath}?${querystring.stringify(queryParams)}`); - }) - .parse(process.argv); + }); + +program + .command('download-dist ') + .description('Download a pull request dist') + .option( + '--platform [platform]', + 'Platform to download dist for. Defaults to current platform.', + process.platform, + ) + .option( + '--arch [arch]', + 'Architecture to download dist for. Defaults to current arch.', + process.arch, + ) + .option( + '-o, --output ', + 'Specify the output directory for downloaded artifacts. ' + + 'Defaults to ~/.electron_build_tools/artifacts/pr_{number}_{platform}_{arch}', + ) + .option('-s, --skip-confirmation', 'Skip the confirmation prompt before downloading the dist.') + .action(async (pullRequestNumber, options) => { + if (!pullRequestNumber) { + fatal(`Pull request number is required to download a PR`); + } + + maybeCheckStaleArtifacts(); + + d('checking auth...'); + const octokit = new Octokit({ + auth: await getGitHubAuthToken(['repo']), + }); + + d('fetching pr info...'); + let pullRequest; + try { + const { data } = await octokit.pulls.get({ + owner: 'electron', + repo: 'electron', + pull_number: pullRequestNumber, + }); + pullRequest = data; + } catch (error) { + console.error(`Failed to get pull request: ${error}`); + return; + } + + if (!options.skipConfirmation) { + const isElectronRepo = pullRequest.head.repo.full_name !== 'electron/electron'; + const { proceed } = await inquirer.prompt([ + { + type: 'confirm', + name: 'proceed', + message: `You are about to download artifacts from: + +“${pullRequest.title} (#${pullRequest.number})” by ${pullRequest.user.login} +${pullRequest.head.repo.html_url}${isElectronRepo ? ' (fork)' : ''} + +Proceed?`, + }, + ]); + + if (!proceed) return; + } + + d('fetching workflow runs...'); + let workflowRuns; + try { + const { data } = await octokit.actions.listWorkflowRunsForRepo({ + owner: 'electron', + repo: 'electron', + branch: pullRequest.head.ref, + name: 'Build', + event: 'pull_request', + status: 'completed', + per_page: 10, + sort: 'created', + direction: 'desc', + }); + workflowRuns = data.workflow_runs; + } catch (error) { + console.error(`Failed to list workflow runs: ${error}`); + return; + } + + const latestBuildWorkflowRun = workflowRuns.find((run) => run.name === 'Build'); + if (!latestBuildWorkflowRun) { + fatal(`No 'Build' workflow runs found for pull request #${pullRequestNumber}`); + return; + } + + d('fetching artifacts...'); + let artifacts; + try { + const { data } = await octokit.actions.listWorkflowRunArtifacts({ + owner: 'electron', + repo: 'electron', + run_id: latestBuildWorkflowRun.id, + }); + artifacts = data.artifacts; + } catch (error) { + console.error(`Failed to list artifacts: ${error}`); + return; + } + + const artifactName = `generated_artifacts_${options.platform}_${options.arch}`; + const artifact = artifacts.find((artifact) => artifact.name === artifactName); + if (!artifact) { + console.error(`Failed to find artifact: ${artifactName}`); + return; + } + + let outputDir; + + if (options.output) { + outputDir = path.resolve(options.output); + + if (!(await fs.promises.stat(outputDir).catch(() => false))) { + fatal(`The output directory '${options.output}' does not exist`); + return; + } + } else { + const defaultDir = path.resolve( + ARTIFACTS_DIR, + `pr_${pullRequest.number}_${options.platform}_${options.arch}`, + ); + + // Clean up the directory if it exists + try { + await fs.promises.rm(defaultDir, { recursive: true, force: true }); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + // Create the directory + await fs.promises.mkdir(defaultDir, { recursive: true }); + + outputDir = defaultDir; + } + + console.log( + `Downloading artifact '${artifactName}' from pull request #${pullRequestNumber}...`, + ); + + const response = await octokit.actions.downloadArtifact({ + owner: 'electron', + repo: 'electron', + artifact_id: artifact.id, + archive_format: 'zip', + }); + + // Extract artifact zip in-memory + const artifactZip = new AdmZip(Buffer.from(response.data)); + + const distZipEntry = artifactZip.getEntry('dist.zip'); + if (!distZipEntry) { + fatal('dist.zip not found in build artifact.'); + return; + } + + // Extract dist.zip in-memory + const distZipContents = artifactZip.readFile(distZipEntry); + const distZip = new AdmZip(distZipContents); + + const platformExecutables = { + win32: 'electron.exe', + darwin: 'Electron.app/', + linux: 'electron', + }; + + const executableName = platformExecutables[options.platform]; + if (!executableName) { + fatal(`Unable to find executable for platform '${options.platform}'`); + return; + } + + if (!distZip.getEntry(executableName)) { + fatal(`${executableName} not found within dist.zip.`); + return; + } + + // Extract dist.zip to the output directory + await distZip.extractAllToAsync(outputDir); + + console.info(`Downloaded to ${outputDir}`); + }); + +program.parse(process.argv); diff --git a/src/utils/artifacts.js b/src/utils/artifacts.js new file mode 100644 index 00000000..df165c72 --- /dev/null +++ b/src/utils/artifacts.js @@ -0,0 +1,51 @@ +const path = require('path'); +const { localStorage } = require('./local-storage'); +const fs = require('fs'); + +const ARTIFACTS_DIR = path.join(__dirname, '..', '..', 'artifacts'); + +/** How often to check for stale files. */ +const STALE_CHECK_INTERVAL = 7 * 24 * 60 * 60 * 1000; // 1 week + +/** How old a file is to be considered stale. */ +const STALE_FILE_AGE = 30 * 24 * 60 * 60 * 1000; // 1 month + +function getStaleArtifacts() { + const now = Date.now(); + let files; + try { + files = fs.readdirSync(ARTIFACTS_DIR); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + return []; + } + const staleFiles = files.filter((file) => { + const filePath = path.join(ARTIFACTS_DIR, file); + const stats = fs.statSync(filePath); + return stats.mtimeMs < now - STALE_FILE_AGE; + }); + return staleFiles; +} + +function maybeCheckStaleArtifacts() { + const lastChecked = parseInt(localStorage.getItem('lastArtifactsCheck'), 10); + const now = Date.now(); + + if (!lastChecked || lastChecked < now - STALE_CHECK_INTERVAL) { + const staleArtifacts = getStaleArtifacts(); + if (staleArtifacts.length > 0) { + console.warn( + `Stale artifact(s) found:\n\t${staleArtifacts.join('\n\t')}\n\nRun 'e clean --stale' to cleanup artifacts.`, + ); + } + localStorage.setItem('lastArtifactsCheck', now); + } +} + +module.exports = { + ARTIFACTS_DIR, + getStaleArtifacts, + maybeCheckStaleArtifacts, +}; diff --git a/src/utils/local-storage.js b/src/utils/local-storage.js new file mode 100644 index 00000000..1e5635c4 --- /dev/null +++ b/src/utils/local-storage.js @@ -0,0 +1,65 @@ +const fs = require('fs'); +const path = require('path'); + +/** + * Simple localStorage implementation before it becomes widely available in NodeJS. + * https://github.com/nodejs/node/blob/main/doc/api/globals.md#localstorage + */ +class LocalStorage { + constructor() { + this.filePath = path.resolve(__dirname, '..', '..', 'local-storage.json'); + } + + #load() { + try { + return JSON.parse(fs.readFileSync(this.filePath, 'utf8')); + } catch { + return {}; + } + } + + #save() { + fs.writeFileSync(this.filePath, JSON.stringify(this.store, null, 2)); + } + + /** Lazy-load reading from disk. */ + get store() { + return (this.store = this.#load()); + } + set store(value) { + Object.defineProperty(this, 'store', { value }); + } + + getItem(key) { + return this.store[key]; + } + + setItem(key, value) { + this.store[key] = JSON.stringify(value); + this.#save(); + } + + removeItem(key) { + delete this.store[key]; + this.#save(); + } + + clear() { + this.store = {}; + this.#save(); + } + + key(index) { + return Object.keys(this.store)[index]; + } + + get length() { + return Object.keys(this.store).length; + } +} + +const localStorage = new LocalStorage(); + +module.exports = { + localStorage, +}; diff --git a/yarn.lock b/yarn.lock index 5f0edcf5..1e064907 100644 --- a/yarn.lock +++ b/yarn.lock @@ -864,6 +864,11 @@ acorn@^8.8.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== +adm-zip@^0.5.16: + version "0.5.16" + resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.16.tgz#0b5e4c779f07dedea5805cdccb1147071d94a909" + integrity sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ== + agent-base@6, agent-base@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" @@ -4698,16 +4703,7 @@ string-argv@~0.3.2: resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.2.tgz#2b6d0ef24b656274d957d54e0a4bbf6153dc02b6" integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -4782,14 +4778,7 @@ string_decoder@^1.1.1: dependencies: safe-buffer "~5.2.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -5333,7 +5322,7 @@ word-wrap@^1.2.3: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.4.tgz#cb4b50ec9aca570abd1f52f33cd45b6c61739a9f" integrity sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -5351,15 +5340,6 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"