Skip to content

Commit

Permalink
feat: add support for reclient
Browse files Browse the repository at this point in the history
  • Loading branch information
MarshallOfSound committed Dec 30, 2023
1 parent c9c02d0 commit 8a3ca6f
Show file tree
Hide file tree
Showing 9 changed files with 201 additions and 15 deletions.
8 changes: 8 additions & 0 deletions evm-config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,14 @@
"msft"
]
},
"reclient": {
"description": "Whether to use the Electron RBE infrastructure",
"type": "string",
"enum": [
"remote_exec",
"none"
]
},
"root": {
"description": "Path of the top directory. Home of the .gclient file",
"type": "string",
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@
"author": "Electron Authors",
"license": "MIT",
"dependencies": {
"@marshallofsound/chrome-cookies-secure": "^2.1.1",
"@octokit/auth-oauth-device": "^3.1.1",
"@octokit/rest": "^18.5.2",
"ajv": "^8.11.0",
"ajv-formats": "^2.1.1",
"chalk": "^2.4.1",
"@marshallofsound/chrome-cookies-secure": "^2.1.1",
"command-exists": "^1.2.8",
"commander": "^9.0.0",
"cross-zip": "^3.0.0",
Expand Down
12 changes: 8 additions & 4 deletions src/download.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,30 @@ const pipeline = promisify(stream.pipeline);

const { fatal } = require('./utils/logging');

const MB_BYTES = 1025 * 1024;
const MB_BYTES = 1024 * 1024;

const progressStream = function(tokens) {
var pt = new stream.PassThrough();

pt.on('pipe', function(stream) {
stream.on('response', function(res) {
const total = parseInt(res.headers['content-length'], 10);
const bar = new ProgressBar(tokens, { total: Math.round(total / MB_BYTES) });
const bar = new ProgressBar(tokens, { total: Math.round(total) });

pt.on('data', function(chunk) {
bar.tick(chunk.length / MB_BYTES);
const elapsed = new Date() - bar.start;
const rate = bar.curr / (elapsed / 1000);
bar.tick(chunk.length, {
mbRate: (rate / MB_BYTES).toFixed(2),
});
});
});
});

return pt;
};

const progress = progressStream('[:bar] :rateMB/s :percent :etas');
const progress = progressStream('[:bar] :mbRateMB/s :percent :etas');
const write = fs.createWriteStream(process.argv[3]);

function tryDownload(attemptsLeft = 3) {
Expand Down
15 changes: 14 additions & 1 deletion src/e-build.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ const evmConfig = require('./evm-config');
const { color, fatal } = require('./utils/logging');
const depot = require('./utils/depot-tools');
const goma = require('./utils/goma');
const reclient = require('./utils/reclient');

function runGNGen(config) {
depot.ensure();
Expand Down Expand Up @@ -44,14 +45,26 @@ function runNinja(config, target, useGoma, ninjaArgs) {
if (!ninjaArgs.includes('-j') && !ninjaArgs.find(arg => /^-j[0-9]+$/.test(arg.trim()))) {
ninjaArgs.push('-j', 200);
}
} else if (config.reclient !== 'none') {
reclient.downloadAndPrepare(config);
reclient.auth(config);

// Autoninja sets this absurdly high, we take it down a notch
if (!ninjaArgs.includes('-j') && !ninjaArgs.find(arg => /^-j[0-9]+$/.test(arg.trim()))) {
ninjaArgs.push('-j', 200);
}
} else {
console.info(`${color.info} Building ${target} with Goma disabled`);
}

depot.ensure(config);
ensureGNGen(config);

const exec = os.platform() === 'win32' ? 'ninja.bat' : 'ninja';
// Using remoteexec means that we need autoninja so that reproxy is started + stopped
// correctly
const ninjaName = config.reclient !== 'none' ? 'autoninja' : 'ninja';

const exec = os.platform() === 'win32' ? `${ninjaName}.bat` : ninjaName;
const args = [...ninjaArgs, target];
const opts = {
cwd: evmConfig.outDir(config),
Expand Down
6 changes: 6 additions & 0 deletions src/e-depot-tools.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ const evmConfig = require('./evm-config');
const { fatal } = require('./utils/logging');
const depot = require('./utils/depot-tools');
const goma = require('./utils/goma');
const reclient = require('./utils/reclient');

program
.command('depot-tools')
Expand Down Expand Up @@ -39,6 +40,11 @@ program
args.unshift('python3');
}

if (args[0] === 'rbe') {
reclient.downloadAndPrepare(evmConfig.current());
args[0] = reclient.helperPath;
}

if (args[0] === '--') {
args.shift();
}
Expand Down
56 changes: 47 additions & 9 deletions src/evm-config.js
Original file line number Diff line number Diff line change
Expand Up @@ -185,15 +185,6 @@ function sanitizeConfig(name, config, overwrite = false) {
changes.push(`added missing property ${color.config('goma: cache-only')}`);
}

if (
config.goma !== 'none' &&
(!config.gen || !config.gen.args || !config.gen.args.find(arg => arg.includes(goma.gnFilePath)))
) {
const str = `import("${goma.gnFilePath}")`;
config.gen.args.push(str);
changes.push(`added ${color.cmd(str)} needed by goma`);
}

if (config.origin) {
config.remotes = {
electron: {
Expand All @@ -218,6 +209,53 @@ function sanitizeConfig(name, config, overwrite = false) {
changes.push(`removed ${color.config('cc_wrapper')} definition because goma is enabled`);
}

if (!config.reclient) {
config.reclient = 'none';
changes.push(`defined ${color.config('remoteexec')} to default value of none`);
}

if (!['none', 'remote_exec'].includes(config.reclient)) {
config.reclient = 'none';
changes.push(`fixed invalid property ${color.config('remoteexec: none')}`);
}

if (config.reclient !== 'none' && config.goma !== 'none') {
config.goma = 'none';
changes.push(`disabled ${color.config('goma')} as ${color.config('remoteexec')} is enabled`);
}

const gomaGnArg = `import("${goma.gnFilePath}")`;
const hasGomaImport = !(
!config.gen ||
!config.gen.args ||
!config.gen.args.find(arg => arg.includes(goma.gnFilePath))
);
if (config.goma !== 'none' && !hasGomaImport) {
config.gen = config.gen || {};
config.gen.args = config.gen.args || [];
config.gen.args.push(gomaGnArg);
changes.push(`added ${color.cmd(gomaGnArg)} needed by goma`);
} else if (config.goma === 'none' && hasGomaImport) {
config.gen.args = config.gen.args.filter(arg => !arg.includes(goma.gnFilePath));
changes.push(`removed gn arg ${color.cmd(gomaGnArg)} as goma is disabled`);
}

const remoteExecGnArg = 'use_remoteexec = true';
const hasRemoteExecGN = !(
!config.gen ||
!config.gen.args ||
!config.gen.args.find(arg => /^use_remoteexec ?= ?true$/.test(arg))
);
if (config.reclient !== 'none' && !hasRemoteExecGN) {
config.gen = config.gen || {};
config.gen.args = config.gen.args || [];
config.gen.args.push(remoteExecGnArg);
changes.push(`added gn arg ${color.cmd(remoteExecGnArg)} needed by remoteexec`);
} else if (config.reclient === 'none' && hasRemoteExecGN) {
config.gen.args = config.gen.args.filter(arg => !/^use_remoteexec ?= ?true$/.test(arg));
changes.push(`removed gn arg ${color.cmd(remoteExecGnArg)} as remoteexec is disabled`);
}

if (!config.env) config.env = {};

if (!config.env.CHROMIUM_BUILDTOOLS_PATH) {
Expand Down
1 change: 1 addition & 0 deletions src/utils/depot-tools.js
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ function depotOpts(config, opts = {}) {
...opts.env,
// Circular reference so we have to delay load
...require('./goma').env(config),
...require('./reclient').env(config),
};

// put depot tools at the front of the path
Expand Down
115 changes: 115 additions & 0 deletions src/utils/reclient.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
const childProcess = require('child_process');
const fs = require('fs');
const path = require('path');
const rimraf = require('rimraf');

const { color, fatal } = require('./logging');

const reclientDir = path.resolve(__dirname, '..', '..', 'third_party', 'reclient');
const reclientTagFile = path.resolve(reclientDir, '.tag');
const reclientHelperPath = path.resolve(
reclientDir,
`electron-rbe-credential-helper${process.platform === 'win32' ? '.exe' : ''}`,
);

const CREDENTIAL_HELPER_TAG = 'v0.0.6';

function downloadAndPrepareReclient(config) {
if (config.reclient === 'none') return;

// Reclient itself comes down with a "gclient sync"
// run. We just need to ensure we have the cred helper
let targetPlatform = null;
switch (process.platform) {
case 'win32': {
targetPlatform = `windows-${process.arch === 'arm64' ? 'arm64' : 'amd64'}`;
break;
}
case 'darwin': {
targetPlatform = `darwin-${process.arch === 'arm64' ? 'arm64' : 'amd64'}`;
break;
}
case 'linux': {
targetPlatform = `linux-${process.arch === 'arm64' ? 'arm64' : 'amd64'}`;
break;
}
}

// Not supported
if (!targetPlatform) return;

if (!fs.existsSync(path.dirname(reclientDir))) {
fs.mkdirSync(path.dirname(reclientDir));
}

if (
fs.existsSync(reclientTagFile) &&
fs.readFileSync(reclientTagFile, 'utf8') === CREDENTIAL_HELPER_TAG
)
return;

const tmpDownload = path.resolve(reclientDir, '..', 'reclient.tar.gz');
// Clean Up
rimraf.sync(reclientDir);
rimraf.sync(tmpDownload);

const downloadURL = `https://dev-cdn.electronjs.org/reclient/credential-helper/${CREDENTIAL_HELPER_TAG}/electron-rbe-credential-helper-${targetPlatform}.tar.gz`;
console.log(`Downloading ${color.cmd(downloadURL)} into ${color.path(tmpDownload)}`);
const { status } = childProcess.spawnSync(
process.execPath,
[path.resolve(__dirname, '..', 'download.js'), downloadURL, tmpDownload],
{
stdio: 'inherit',
},
);
if (status !== 0) {
rimraf.sync(tmpDownload);
fatal(`Failure while downloading reclient`);
}

const targetDir = path.resolve(tmpDownload, '..');

fs.mkdirSync(reclientDir);
const result = childProcess.spawnSync('tar', ['zxvf', 'reclient.tar.gz', '-C', reclientDir], {
cwd: targetDir,
});
if (result.status !== 0) {
fatal('Failed to extract reclient');
}
rimraf.sync(tmpDownload);
fs.writeFileSync(reclientTagFile, CREDENTIAL_HELPER_TAG);
return;
}

function reclientEnv(config) {
if (config && config.reclient === 'none') {
return {};
}

return {
RBE_service: 'rbe.notgoma.com:443',
RBE_experimental_credentials_helper: reclientHelperPath,
};
}

function ensureHelperAuth(config) {
const result = childProcess.spawnSync(reclientHelperPath, ['status'], {
stdio: 'pipe',
});
if (result.status !== 0) {
console.error(result.stdout.toString());
console.error(
`${color.err} You do not have valid auth for Reclient, please run ${color.cmd(
'e d rbe login',
)}`,
);
process.exit(result.status || 1);
}
}

module.exports = {
env: reclientEnv,
downloadAndPrepare: downloadAndPrepareReclient,
helperPath: reclientHelperPath,
auth: ensureHelperAuth,
};
1 change: 1 addition & 0 deletions tests/evm-config-spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ const validConfig = {
},
},
goma: 'none',
reclient: 'none',
gen: {
args: [],
out: 'Testing',
Expand Down

0 comments on commit 8a3ca6f

Please sign in to comment.