diff --git a/evm-config.schema.json b/evm-config.schema.json index 7760c276..3f4cefd6 100644 --- a/evm-config.schema.json +++ b/evm-config.schema.json @@ -42,6 +42,14 @@ "msft" ] }, + "reclient": { + "description": "Whether to use the Electron RBE infrastructure", + "type": "string", + "enum": [ + "remote_exec", + "none" + ] + }, "root": { "description": "Path of the top directory. Home of the .gclient file", "type": "string", diff --git a/package.json b/package.json index 408091fe..60253810 100644 --- a/package.json +++ b/package.json @@ -18,12 +18,12 @@ "author": "Electron Authors", "license": "MIT", "dependencies": { + "@marshallofsound/chrome-cookies-secure": "^2.1.1", "@octokit/auth-oauth-device": "^3.1.1", "@octokit/rest": "^18.5.2", "ajv": "^8.11.0", "ajv-formats": "^2.1.1", "chalk": "^2.4.1", - "@marshallofsound/chrome-cookies-secure": "^2.1.1", "command-exists": "^1.2.8", "commander": "^9.0.0", "cross-zip": "^3.0.0", diff --git a/src/download.js b/src/download.js index d7b43749..9740382d 100644 --- a/src/download.js +++ b/src/download.js @@ -8,7 +8,7 @@ const pipeline = promisify(stream.pipeline); const { fatal } = require('./utils/logging'); -const MB_BYTES = 1025 * 1024; +const MB_BYTES = 1024 * 1024; const progressStream = function(tokens) { var pt = new stream.PassThrough(); @@ -16,10 +16,14 @@ const progressStream = function(tokens) { pt.on('pipe', function(stream) { stream.on('response', function(res) { const total = parseInt(res.headers['content-length'], 10); - const bar = new ProgressBar(tokens, { total: Math.round(total / MB_BYTES) }); + const bar = new ProgressBar(tokens, { total: Math.round(total) }); pt.on('data', function(chunk) { - bar.tick(chunk.length / MB_BYTES); + const elapsed = new Date() - bar.start; + const rate = bar.curr / (elapsed / 1000); + bar.tick(chunk.length, { + mbRate: (rate / MB_BYTES).toFixed(2), + }); }); }); }); @@ -27,7 +31,7 @@ const progressStream = function(tokens) { return pt; }; -const progress = progressStream('[:bar] :rateMB/s :percent :etas'); +const progress = progressStream('[:bar] :mbRateMB/s :percent :etas'); const write = fs.createWriteStream(process.argv[3]); function tryDownload(attemptsLeft = 3) { diff --git a/src/e-build.js b/src/e-build.js index 1876cf81..368a8567 100644 --- a/src/e-build.js +++ b/src/e-build.js @@ -9,6 +9,7 @@ const evmConfig = require('./evm-config'); const { color, fatal } = require('./utils/logging'); const depot = require('./utils/depot-tools'); const goma = require('./utils/goma'); +const reclient = require('./utils/reclient'); function runGNGen(config) { depot.ensure(); @@ -44,6 +45,14 @@ function runNinja(config, target, useGoma, ninjaArgs) { if (!ninjaArgs.includes('-j') && !ninjaArgs.find(arg => /^-j[0-9]+$/.test(arg.trim()))) { ninjaArgs.push('-j', 200); } + } else if (config.reclient !== 'none') { + reclient.downloadAndPrepare(config); + reclient.auth(config); + + // Autoninja sets this absurdly high, we take it down a notch + if (!ninjaArgs.includes('-j') && !ninjaArgs.find(arg => /^-j[0-9]+$/.test(arg.trim()))) { + ninjaArgs.push('-j', 200); + } } else { console.info(`${color.info} Building ${target} with Goma disabled`); } @@ -51,7 +60,11 @@ function runNinja(config, target, useGoma, ninjaArgs) { depot.ensure(config); ensureGNGen(config); - const exec = os.platform() === 'win32' ? 'ninja.bat' : 'ninja'; + // Using remoteexec means that we need autoninja so that reproxy is started + stopped + // correctly + const ninjaName = config.reclient !== 'none' ? 'autoninja' : 'ninja'; + + const exec = os.platform() === 'win32' ? `${ninjaName}.bat` : ninjaName; const args = [...ninjaArgs, target]; const opts = { cwd: evmConfig.outDir(config), diff --git a/src/e-depot-tools.js b/src/e-depot-tools.js index c2aaf3bc..d93b9f25 100644 --- a/src/e-depot-tools.js +++ b/src/e-depot-tools.js @@ -7,6 +7,7 @@ const evmConfig = require('./evm-config'); const { fatal } = require('./utils/logging'); const depot = require('./utils/depot-tools'); const goma = require('./utils/goma'); +const reclient = require('./utils/reclient'); program .command('depot-tools') @@ -39,6 +40,11 @@ program args.unshift('python3'); } + if (args[0] === 'rbe') { + reclient.downloadAndPrepare(evmConfig.current()); + args[0] = reclient.helperPath; + } + if (args[0] === '--') { args.shift(); } diff --git a/src/evm-config.js b/src/evm-config.js index 15319068..d9d46505 100644 --- a/src/evm-config.js +++ b/src/evm-config.js @@ -185,15 +185,6 @@ function sanitizeConfig(name, config, overwrite = false) { changes.push(`added missing property ${color.config('goma: cache-only')}`); } - if ( - config.goma !== 'none' && - (!config.gen || !config.gen.args || !config.gen.args.find(arg => arg.includes(goma.gnFilePath))) - ) { - const str = `import("${goma.gnFilePath}")`; - config.gen.args.push(str); - changes.push(`added ${color.cmd(str)} needed by goma`); - } - if (config.origin) { config.remotes = { electron: { @@ -218,6 +209,53 @@ function sanitizeConfig(name, config, overwrite = false) { changes.push(`removed ${color.config('cc_wrapper')} definition because goma is enabled`); } + if (!config.reclient) { + config.reclient = 'none'; + changes.push(`defined ${color.config('remoteexec')} to default value of none`); + } + + if (!['none', 'remote_exec'].includes(config.reclient)) { + config.reclient = 'none'; + changes.push(`fixed invalid property ${color.config('remoteexec: none')}`); + } + + if (config.reclient !== 'none' && config.goma !== 'none') { + config.goma = 'none'; + changes.push(`disabled ${color.config('goma')} as ${color.config('remoteexec')} is enabled`); + } + + const gomaGnArg = `import("${goma.gnFilePath}")`; + const hasGomaImport = !( + !config.gen || + !config.gen.args || + !config.gen.args.find(arg => arg.includes(goma.gnFilePath)) + ); + if (config.goma !== 'none' && !hasGomaImport) { + config.gen = config.gen || {}; + config.gen.args = config.gen.args || []; + config.gen.args.push(gomaGnArg); + changes.push(`added ${color.cmd(gomaGnArg)} needed by goma`); + } else if (config.goma === 'none' && hasGomaImport) { + config.gen.args = config.gen.args.filter(arg => !arg.includes(goma.gnFilePath)); + changes.push(`removed gn arg ${color.cmd(gomaGnArg)} as goma is disabled`); + } + + const remoteExecGnArg = 'use_remoteexec = true'; + const hasRemoteExecGN = !( + !config.gen || + !config.gen.args || + !config.gen.args.find(arg => /^use_remoteexec ?= ?true$/.test(arg)) + ); + if (config.reclient !== 'none' && !hasRemoteExecGN) { + config.gen = config.gen || {}; + config.gen.args = config.gen.args || []; + config.gen.args.push(remoteExecGnArg); + changes.push(`added gn arg ${color.cmd(remoteExecGnArg)} needed by remoteexec`); + } else if (config.reclient === 'none' && hasRemoteExecGN) { + config.gen.args = config.gen.args.filter(arg => !/^use_remoteexec ?= ?true$/.test(arg)); + changes.push(`removed gn arg ${color.cmd(remoteExecGnArg)} as remoteexec is disabled`); + } + if (!config.env) config.env = {}; if (!config.env.CHROMIUM_BUILDTOOLS_PATH) { diff --git a/src/utils/depot-tools.js b/src/utils/depot-tools.js index eaf11f1e..256c131a 100644 --- a/src/utils/depot-tools.js +++ b/src/utils/depot-tools.js @@ -96,6 +96,7 @@ function depotOpts(config, opts = {}) { ...opts.env, // Circular reference so we have to delay load ...require('./goma').env(config), + ...require('./reclient').env(config), }; // put depot tools at the front of the path diff --git a/src/utils/reclient.js b/src/utils/reclient.js new file mode 100644 index 00000000..ec059f91 --- /dev/null +++ b/src/utils/reclient.js @@ -0,0 +1,115 @@ +const childProcess = require('child_process'); +const fs = require('fs'); +const path = require('path'); +const rimraf = require('rimraf'); + +const { color, fatal } = require('./logging'); + +const reclientDir = path.resolve(__dirname, '..', '..', 'third_party', 'reclient'); +const reclientTagFile = path.resolve(reclientDir, '.tag'); +const reclientHelperPath = path.resolve( + reclientDir, + `electron-rbe-credential-helper${process.platform === 'win32' ? '.exe' : ''}`, +); + +const CREDENTIAL_HELPER_TAG = 'v0.0.6'; + +function downloadAndPrepareReclient(config) { + if (config.reclient === 'none') return; + + // Reclient itself comes down with a "gclient sync" + // run. We just need to ensure we have the cred helper + let targetPlatform = null; + switch (process.platform) { + case 'win32': { + targetPlatform = `windows-${process.arch === 'arm64' ? 'arm64' : 'amd64'}`; + break; + } + case 'darwin': { + targetPlatform = `darwin-${process.arch === 'arm64' ? 'arm64' : 'amd64'}`; + break; + } + case 'linux': { + targetPlatform = `linux-${process.arch === 'arm64' ? 'arm64' : 'amd64'}`; + break; + } + } + + // Not supported + if (!targetPlatform) return; + + if (!fs.existsSync(path.dirname(reclientDir))) { + fs.mkdirSync(path.dirname(reclientDir)); + } + + if ( + fs.existsSync(reclientTagFile) && + fs.readFileSync(reclientTagFile, 'utf8') === CREDENTIAL_HELPER_TAG + ) + return; + + const tmpDownload = path.resolve(reclientDir, '..', 'reclient.tar.gz'); + // Clean Up + rimraf.sync(reclientDir); + rimraf.sync(tmpDownload); + + const downloadURL = `https://dev-cdn.electronjs.org/reclient/credential-helper/${CREDENTIAL_HELPER_TAG}/electron-rbe-credential-helper-${targetPlatform}.tar.gz`; + console.log(`Downloading ${color.cmd(downloadURL)} into ${color.path(tmpDownload)}`); + const { status } = childProcess.spawnSync( + process.execPath, + [path.resolve(__dirname, '..', 'download.js'), downloadURL, tmpDownload], + { + stdio: 'inherit', + }, + ); + if (status !== 0) { + rimraf.sync(tmpDownload); + fatal(`Failure while downloading reclient`); + } + + const targetDir = path.resolve(tmpDownload, '..'); + + fs.mkdirSync(reclientDir); + const result = childProcess.spawnSync('tar', ['zxvf', 'reclient.tar.gz', '-C', reclientDir], { + cwd: targetDir, + }); + if (result.status !== 0) { + fatal('Failed to extract reclient'); + } + rimraf.sync(tmpDownload); + fs.writeFileSync(reclientTagFile, CREDENTIAL_HELPER_TAG); + return; +} + +function reclientEnv(config) { + if (config && config.reclient === 'none') { + return {}; + } + + return { + RBE_service: 'rbe.notgoma.com:443', + RBE_experimental_credentials_helper: reclientHelperPath, + }; +} + +function ensureHelperAuth(config) { + const result = childProcess.spawnSync(reclientHelperPath, ['status'], { + stdio: 'pipe', + }); + if (result.status !== 0) { + console.error(result.stdout.toString()); + console.error( + `${color.err} You do not have valid auth for Reclient, please run ${color.cmd( + 'e d rbe login', + )}`, + ); + process.exit(result.status || 1); + } +} + +module.exports = { + env: reclientEnv, + downloadAndPrepare: downloadAndPrepareReclient, + helperPath: reclientHelperPath, + auth: ensureHelperAuth, +}; diff --git a/tests/evm-config-spec.js b/tests/evm-config-spec.js index a09e575a..3308dc0e 100644 --- a/tests/evm-config-spec.js +++ b/tests/evm-config-spec.js @@ -14,6 +14,7 @@ const validConfig = { }, }, goma: 'none', + reclient: 'none', gen: { args: [], out: 'Testing',