Skip to content

Commit

Permalink
build: speed times with large projects using multiprocessing
Browse files Browse the repository at this point in the history
  • Loading branch information
@YoRyan
YoRyan committed Oct 25, 2023
1 parent ad9e880 commit bd6cb95
Show file tree
Hide file tree
Showing 5 changed files with 214 additions and 4,675 deletions.
88 changes: 88 additions & 0 deletions build-worker.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import * as fsp from "fs/promises";
import { glob } from "glob";
import * as path from "path";
import { Path, PathScurry } from "path-scurry";
import ts from "typescript";
import * as tstl from "typescript-to-lua";

export type Job = {
entryPathFromSrc: string;
};

export async function transpile(job: Job) {
// Create a virtual project that includes the entry point file.
const { entryPathFromSrc } = job;
const entryFile = new PathScurry("./src").cwd.resolve(entryPathFromSrc);
const bundleFiles = await globBundleFiles();
const virtualProject = Object.fromEntries(await Promise.all([entryFile, ...bundleFiles].map(readVirtualFile)));

// Call TypeScriptToLua.
const bundleFile = (entryFile.parent ?? entryFile).resolve(path.basename(entryFile.name, ".ts") + ".lua");
const result = tstl.transpileVirtualProject(virtualProject, {
...readCompilerOptions(),
types: ["lua-types/5.0", "@typescript-to-lua/language-extensions"], // Drop the jest types.
luaTarget: tstl.LuaTarget.Lua50,
sourceMapTraceback: false,
luaBundle: bundleFile.relative(),
luaBundleEntry: entryFile.relative(),
});
printDiagnostics(result.diagnostics);

// Write the result.
for (const tf of result.transpiledFiles) {
if (!tf.lua) continue;

const luaPath = path.join("./dist", path.relative("./mod", tf.outPath));
const dirPath = path.dirname(luaPath);
const outPath = path.join(dirPath, path.basename(luaPath, ".lua") + ".out");
await fsp.mkdir(dirPath, { recursive: true });
await fsp.writeFile(outPath, tf.lua);
}
}

export async function globBundleFiles() {
return [
...(await glob(
[
"node_modules/lua-types/5.0.d.ts",
"node_modules/lua-types/core/index-5.0.d.ts",
"node_modules/lua-types/core/coroutine.d.ts",
"node_modules/lua-types/core/5.0/*",
"node_modules/lua-types/special/5.0.d.ts",
],
{ withFileTypes: true }
)),
...(await glob(["@types/**/*", "lib/**/*.ts"], { cwd: "./src", withFileTypes: true })),
];
}

process.on("message", async m => {
if (process.send === undefined) return;

await transpile(m as Job);

// Signal completion to the parent.
process.send("done");
});

async function readVirtualFile(file: Path) {
const contents = await fsp.readFile(file.fullpath(), { encoding: "utf-8" });
return [file.relative(), contents] as [string, string];
}

function readCompilerOptions() {
const configJson = ts.readConfigFile("./src/tsconfig.json", ts.sys.readFile);
return ts.parseJsonConfigFileContent(configJson.config, ts.sys, ".").options;
}

function printDiagnostics(diagnostics: ts.Diagnostic[]) {
if (diagnostics.length > 0) {
console.log(
ts.formatDiagnosticsWithColorAndContext(diagnostics, {
getCurrentDirectory: () => ts.sys.getCurrentDirectory(),
getCanonicalFileName: f => f,
getNewLine: () => "\n",
})
);
}
}
236 changes: 123 additions & 113 deletions build.ts
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,42 @@
import colors from "@colors/colors";
import { ChildProcess, fork } from "child_process";
import * as fsp from "fs/promises";
import { glob } from "glob";
import minimist from "minimist";
import { parseArgs } from "node:util";
import * as path from "path";
import { Path } from "path-scurry";
import { exit } from "process";
import ts from "typescript";
import * as tstl from "typescript-to-lua";
import { parseArgs } from "util";
import { Job, globBundleFiles, transpile } from "./build-worker.js";

async function main() {
const { positionals } = parseArgs({ strict: false });
const { values, positionals } = parseArgs({
strict: false,
options: {
workers: { type: "string", short: "w", default: "0" },
},
});

const workers = parseInt(values.workers as string);
const transpiler = workers > 0 ? new MultiProcessTranspiler(workers) : new SingleProcessTranspiler();

const [mode] = positionals;
switch (mode ?? "build") {
// Watch mode transpiles files when they get changed.
case "watch":
await watch();
await watch(transpiler);
break;
// Build mode transpiles everything and then exits.
case "build":
default:
await build();
await build(transpiler);
break;
}
return 0;
}

async function watch() {
const queue = new WatchQueue();
async function watch(transpiler: Transpiler) {
const queue = new WatchQueue(transpiler);
// Transpile everything when a common library or type definition changes.
for (const bundle of await globVirtualBundle()) {
for (const bundle of await globBundleFiles()) {
(async () => {
const watcher = fsp.watch(bundle.fullpath());
for await (const _ of watcher) {
Expand All @@ -50,16 +57,20 @@ async function watch() {
}

class WatchQueue {
readonly minIntervalMs = 2 * 1000;
lastAll: number | undefined = undefined;
lastByFile: { [key: string]: number } = {};
private readonly minIntervalMs = 2 * 1000;
private lastAll: number | undefined = undefined;
private lastByFile: { [key: string]: number } = {};
private readonly transpiler: Transpiler;
constructor(transpiler: Transpiler) {
this.transpiler = transpiler;
}
async all() {
const now = nowTime();
if (this.allStale()) {
console.log("Transpiling all ...");

this.lastAll = now;
await build();
await build(this.transpiler);
}
}
async file(file: Path) {
Expand All @@ -70,130 +81,129 @@ class WatchQueue {
console.log(`Transpiling ${file.relative()} ...`);

this.lastByFile[key] = now;
await this.buildFile(file);
reportTimedResult(file, await this.transpiler.timedTranspileMs(file));
}
}
private allStale() {
return this.lastAll === undefined || nowTime() - this.lastAll > this.minIntervalMs;
}
private async buildFile(file: Path) {
const bundle = await readVirtualBundle();
await timedTranspile(readCompilerOptions(), file, bundle);
}
}

function nowTime() {
return new Date().getTime();
interface Transpiler {
timedTranspileMs(entryFile: Path): Promise<number>;
}

async function build() {
const bundle = await readVirtualBundle();
const entryPoints = await globEntryPoints();
for (const entry of entryPoints) {
await timedTranspile(readCompilerOptions(), entry, bundle);
/**
* Just calls the transpiler--no fancy workers. Includes a mutex so transpile
* times are reported correctly.
*/
class SingleProcessTranspiler implements Transpiler {
private mutex = false;
private waitingForMutex: (() => void)[] = [];
async timedTranspileMs(entryFile: Path) {
await this.getMutex();

const startMs = nowTime();
await transpile({
entryPathFromSrc: entryFile.relative(),
});
const endMs = nowTime();

this.releaseMutex();
return endMs - startMs;
}
private async getMutex() {
if (!this.mutex) {
this.mutex = true;
} else {
return await new Promise<void>(resolve => {
this.waitingForMutex.push(resolve);
});
}
}
private releaseMutex() {
const next = this.waitingForMutex.pop();
if (next !== undefined) {
next();
} else {
this.mutex = false;
}
}
}

async function readVirtualBundle() {
const bundleFiles = await globVirtualBundle();
return await Promise.all(bundleFiles.map(readVirtualFile));
/**
* Spawns other Node processes running build-worker.ts and uses them as a worker
* pool.
*/
class MultiProcessTranspiler implements Transpiler {
private workersToSpawn: number;
private waitingForWorker: ((worker: ChildProcess) => void)[] = [];
private spareWorkers: ChildProcess[] = [];
constructor(maxWorkers: number) {
this.workersToSpawn = maxWorkers;
}
async timedTranspileMs(entryFile: Path) {
const worker = await this.getWorker();
worker.send({
entryPathFromSrc: entryFile.relative(),
} as Job);

const startMs = nowTime();
await new Promise<void>(resolve => {
worker.on("message", resolve);
});
const endMs = nowTime();

this.returnWorker(worker);
return endMs - startMs;
}
private async getWorker() {
const spareWorker = this.spareWorkers.pop();
if (spareWorker !== undefined) {
return spareWorker;
} else if (this.workersToSpawn > 0) {
this.workersToSpawn--;
return fork("./build-worker.ts", { detached: true, stdio: ["ipc", "inherit", "inherit"] });
} else {
return await new Promise<ChildProcess>(resolve => {
this.waitingForWorker.push(resolve);
});
}
}
private returnWorker(worker: ChildProcess) {
worker.removeAllListeners("message");
const next = this.waitingForWorker.pop();
if (next !== undefined) {
next(worker);
} else {
this.spareWorkers.push(worker);
}
}
}

async function globVirtualBundle() {
return (
await Promise.all([
glob(
[
"node_modules/lua-types/5.0.d.ts",
"node_modules/lua-types/core/index-5.0.d.ts",
"node_modules/lua-types/core/coroutine.d.ts",
"node_modules/lua-types/core/5.0/*",
"node_modules/lua-types/special/5.0.d.ts",
],
{ withFileTypes: true }
),
glob(["@types/**/*", "lib/**/*.ts"], { cwd: "./src", withFileTypes: true }),
])
).flat();
async function build(transpiler: Transpiler) {
const entryPoints = await globEntryPoints();
await Promise.all(
entryPoints.map(async entry => {
reportTimedResult(entry, await transpiler.timedTranspileMs(entry));
})
);
}

async function globEntryPoints() {
return await glob("mod/**/*.ts", { cwd: "./src", withFileTypes: true });
}

function readCompilerOptions() {
const configJson = ts.readConfigFile("./src/tsconfig.json", ts.sys.readFile);
return ts.parseJsonConfigFileContent(configJson.config, ts.sys, ".").options;
function reportTimedResult(file: Path, ms: number) {
console.log(`${colors.gray(file.relative())} ${ms}ms`);
}

async function timedTranspile(compilerOptions: ts.CompilerOptions, entryFile: Path, virtualBundle: [string, string][]) {
const startMs = nowTime();
let err = undefined;
try {
await transpile(compilerOptions, entryFile, virtualBundle);
} catch (e) {
err = e;
}
const endMs = nowTime();

const entryPath = colors.gray(entryFile.relative());
const result = err !== undefined ? colors.red(err + "") : `${endMs - startMs}ms`;
console.log(`${entryPath} ${result}`);
}

async function transpile(compilerOptions: ts.CompilerOptions, entryFile: Path, virtualBundle: [string, string][]) {
// Create a virtual project that includes the entry point file.
const virtualProject = Object.fromEntries([await readVirtualFile(entryFile), ...virtualBundle]);

// Call TypeScriptToLua.
const bundleFile = (entryFile.parent ?? entryFile).resolve(path.basename(entryFile.name, ".ts") + ".lua");
const result = tstl.transpileVirtualProject(virtualProject, {
...compilerOptions,
// Drop the jest types here.
types: ["lua-types/5.0", "@typescript-to-lua/language-extensions"],
luaTarget: tstl.LuaTarget.Lua50,
sourceMapTraceback: false,
luaBundle: bundleFile.relative(),
luaBundleEntry: entryFile.relative(),
noResolvePaths: ["Assets/**/*"],
});
printDiagnostics(result.diagnostics);

// Write the result.
for (const tf of result.transpiledFiles) {
if (!tf.lua) continue;

const luaPath = path.join("./dist", path.relative("./mod", tf.outPath));
const dirPath = path.dirname(luaPath);
const outPath = path.join(dirPath, path.basename(luaPath, ".lua") + ".out");
await fsp.mkdir(dirPath, { recursive: true });
await fsp.writeFile(outPath, tf.lua);
}
}

async function readVirtualFile(file: Path) {
const contents = await fsp.readFile(file.fullpath(), { encoding: "utf-8" });
return [file.relative(), contents] as [string, string];
}

function printDiagnostics(diagnostics: ts.Diagnostic[]) {
if (diagnostics.length > 0) {
console.log(
ts.formatDiagnosticsWithColorAndContext(diagnostics, {
getCurrentDirectory: () => ts.sys.getCurrentDirectory(),
getCanonicalFileName: f => f,
getNewLine: () => "\n",
})
);
}
function nowTime() {
return new Date().getTime();
}

async function waitForever() {
return new Promise((_resolve, _reject) => {});
}

const argv = minimist(process.argv.slice(2), {
boolean: ["lua"],
default: { lua: false },
});
exit(await main());
Loading

0 comments on commit bd6cb95

Please sign in to comment.