diff --git a/.gitignore b/.gitignore index 1ecfb42c..25ffbb84 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ graphql/gqlgen/main tailcall-src metals.* + +analyze/* +analyze.js diff --git a/analyze.ts b/analyze.ts new file mode 100644 index 00000000..518719f9 --- /dev/null +++ b/analyze.ts @@ -0,0 +1,76 @@ +#!/usr/bin/env node +import { execSync } from 'child_process'; +import * as path from 'path'; +import * as fs from 'fs'; +import { servers, formattedServerNames } from './analyze/config'; +import { createDirectoryIfNotExists, moveFile, readFileContent } from './analyze/fileUtils'; +import { parseServerMetrics } from './analyze/parser'; +import { writeMetricsDataFiles } from './analyze/dataFileWriter'; +import { generateGnuplotScript, writeGnuplotScript } from './analyze/gnuplotGenerator'; +import { formatResults, writeResults } from './analyze/resultsFormatter'; + +const resultFiles: string[] = process.argv.slice(2); + +// Read content of result files +const resultContents: string[] = resultFiles.map(file => readFileContent(file)); + +const serverMetrics = parseServerMetrics(servers, resultContents); + +writeMetricsDataFiles(serverMetrics, servers); + +let whichBench = 1; +if (resultFiles.length > 0) { + if (resultFiles[0].startsWith("bench2")) { + whichBench = 2; + } else if (resultFiles[0].startsWith("bench3")) { + whichBench = 3; + } +} + +function getMaxValue(data: string): number { + try { + return Math.max(...data.split('\n') + .slice(1) + .map(line => { + const [, valueStr] = line.split(' '); + const value = parseFloat(valueStr); + if (isNaN(value)) { + throw new Error(`Invalid number in data: ${valueStr}`); + } + return value; + })); + } catch (error) { + console.error(`Error getting max value: ${(error as Error).message}`); + return 0; + } +} + +const reqSecMax = getMaxValue(readFileContent('/tmp/reqSec.dat')) * 1.2; +const latencyMax = getMaxValue(readFileContent('/tmp/latency.dat')) * 1.2; + +const gnuplotScript = generateGnuplotScript(whichBench, reqSecMax, latencyMax); +writeGnuplotScript(gnuplotScript); + +try { + execSync(`gnuplot /tmp/gnuplot_script.gp`, { stdio: 'inherit' }); + console.log('Gnuplot executed successfully'); +} catch (error) { + console.error('Error executing gnuplot:', (error as Error).message); +} + +const assetsDir = path.join(__dirname, "assets"); +createDirectoryIfNotExists(assetsDir); + +moveFile(`req_sec_histogram${whichBench}.png`, path.join(assetsDir, `req_sec_histogram${whichBench}.png`)); +moveFile(`latency_histogram${whichBench}.png`, path.join(assetsDir, `latency_histogram${whichBench}.png`)); + +const resultsTable = formatResults(serverMetrics, formattedServerNames, whichBench); +writeResults(resultsTable, whichBench); + +resultFiles.forEach((file) => { + try { + fs.unlinkSync(file); + } catch (error) { + console.error(`Error deleting file ${file}: ${(error as Error).message}`); + } +}); diff --git a/analyze/config.ts b/analyze/config.ts new file mode 100644 index 00000000..0b84b675 --- /dev/null +++ b/analyze/config.ts @@ -0,0 +1,14 @@ +import { FormattedServerNames } from './types'; + +export const formattedServerNames: FormattedServerNames = { + tailcall: "Tailcall", + gqlgen: "Gqlgen", + apollo: "Apollo GraphQL", + netflixdgs: "Netflix DGS", + caliban: "Caliban", + async_graphql: "async-graphql", + hasura: "Hasura", + graphql_jit: "GraphQL JIT", +}; + +export const servers: string[] = ["apollo", "caliban", "netflixdgs", "gqlgen", "tailcall", "async_graphql", "hasura", "graphql_jit"]; diff --git a/analyze/dataFileWriter.ts b/analyze/dataFileWriter.ts new file mode 100644 index 00000000..3812d5d0 --- /dev/null +++ b/analyze/dataFileWriter.ts @@ -0,0 +1,10 @@ +import { writeDataFile } from './fileUtils'; +import { ServerMetrics } from './types'; + +export function writeMetricsDataFiles(serverMetrics: Record, servers: string[]): void { + const reqSecData = "/tmp/reqSec.dat"; + const latencyData = "/tmp/latency.dat"; + + writeDataFile(reqSecData, "Server Value\n" + servers.map(server => `${server} ${serverMetrics[server].reqSec}`).join('\n')); + writeDataFile(latencyData, "Server Value\n" + servers.map(server => `${server} ${serverMetrics[server].latency}`).join('\n')); +} diff --git a/analyze/fileUtils.ts b/analyze/fileUtils.ts new file mode 100644 index 00000000..5e7ce332 --- /dev/null +++ b/analyze/fileUtils.ts @@ -0,0 +1,42 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +export function writeDataFile(filename: string, data: string): void { + try { + fs.writeFileSync(filename, data); + } catch (error) { + console.error(`Error writing data file ${filename}: ${(error as Error).message}`); + } +} + +export function readFileContent(filename: string): string { + try { + return fs.readFileSync(filename, 'utf-8'); + } catch (error) { + console.error(`Error reading file ${filename}: ${(error as Error).message}`); + return ''; + } +} + +export function moveFile(source: string, destination: string): void { + try { + if (fs.existsSync(source)) { + fs.renameSync(source, destination); + console.log(`Moved ${source} to ${destination}`); + } else { + console.log(`Source file ${source} does not exist`); + } + } catch (error) { + console.error(`Error moving file ${source}: ${(error as Error).message}`); + } +} + +export function createDirectoryIfNotExists(dir: string): void { + if (!fs.existsSync(dir)) { + try { + fs.mkdirSync(dir); + } catch (error) { + console.error(`Error creating directory: ${(error as Error).message}`); + } + } +} diff --git a/analyze/gnuplotGenerator.ts b/analyze/gnuplotGenerator.ts new file mode 100644 index 00000000..65a9f76e --- /dev/null +++ b/analyze/gnuplotGenerator.ts @@ -0,0 +1,30 @@ +import { writeDataFile } from './fileUtils'; + +export function generateGnuplotScript(whichBench: number, reqSecMax: number, latencyMax: number): string { + const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; + const latencyHistogramFile = `latency_histogram${whichBench}.png`; + + return ` +set term pngcairo size 1280,720 enhanced font 'Courier,12' +set output '${reqSecHistogramFile}' +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set title 'Requests/Sec' +set yrange [0:${reqSecMax}] +set key outside right top +plot '/tmp/reqSec.dat' using 2:xtic(1) title 'Req/Sec' + +set output '${latencyHistogramFile}' +set title 'Latency (in ms)' +set yrange [0:${latencyMax}] +plot '/tmp/latency.dat' using 2:xtic(1) title 'Latency' +`; +} + +export function writeGnuplotScript(script: string): void { + const gnuplotScriptFile = '/tmp/gnuplot_script.gp'; + writeDataFile(gnuplotScriptFile, script); +} diff --git a/analyze/parser.ts b/analyze/parser.ts new file mode 100644 index 00000000..b15e47c0 --- /dev/null +++ b/analyze/parser.ts @@ -0,0 +1,48 @@ +import { ServerMetrics } from './types'; + +export function parseMetric(input: string, metric: string): number | null { + const lines = input.split('\n'); + let metricLine: string | undefined; + + if (metric === "Latency") { + metricLine = lines.find(line => line.trim().startsWith("Latency")); + } else if (metric === "Requests/sec") { + metricLine = lines.find(line => line.trim().startsWith("Requests/sec")); + } + + if (!metricLine) return null; + + const match = metricLine.match(/([\d.]+)/); + return match ? parseFloat(match[1]) : null; +} + +export function calculateAverage(values: number[]): number { + if (values.length === 0) return 0; + const sum = values.reduce((a, b) => a + b, 0); + return sum / values.length; +} + +export function parseServerMetrics(servers: string[], inputs: string[]): Record { + const serverMetrics: Record = {}; + + servers.forEach((server, idx) => { + const startIdx = idx * 3; + const reqSecVals: number[] = []; + const latencyVals: number[] = []; + for (let j = 0; j < 3; j++) { + const inputIdx = startIdx + j; + if (inputIdx < inputs.length) { + const reqSec = parseMetric(inputs[inputIdx], "Requests/sec"); + const latency = parseMetric(inputs[inputIdx], "Latency"); + if (reqSec !== null) reqSecVals.push(reqSec); + if (latency !== null) latencyVals.push(latency); + } + } + serverMetrics[server] = { + reqSec: calculateAverage(reqSecVals), + latency: calculateAverage(latencyVals) + }; + }); + + return serverMetrics; +} diff --git a/analyze/resultsFormatter.ts b/analyze/resultsFormatter.ts new file mode 100644 index 00000000..2169c6f9 --- /dev/null +++ b/analyze/resultsFormatter.ts @@ -0,0 +1,83 @@ +import * as fs from 'fs'; +import { FormattedServerNames, ServerMetrics } from './types'; + +export function formatResults(serverMetrics: Record, formattedServerNames: FormattedServerNames, whichBench: number): string { + const sortedServers = Object.keys(serverMetrics).sort( + (a, b) => serverMetrics[b].reqSec - serverMetrics[a].reqSec + ); + const lastServer = sortedServers[sortedServers.length - 1]; + const lastServerReqSecs = serverMetrics[lastServer].reqSec; + + let resultsTable = ""; + + if (whichBench === 1) { + resultsTable += `\n| ${whichBench} | \`{ posts { id userId title user { id name email }}}\` |`; + } else if (whichBench === 2) { + resultsTable += `\n| ${whichBench} | \`{ posts { title }}\` |`; + } else if (whichBench === 3) { + resultsTable += `\n| ${whichBench} | \`{ greet }\` |`; + } + + sortedServers.forEach((server) => { + const formattedReqSecs = serverMetrics[server].reqSec.toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const formattedLatencies = serverMetrics[server].latency.toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const relativePerformance = (serverMetrics[server].reqSec / lastServerReqSecs).toFixed(2); + + resultsTable += `\n|| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; + }); + + return resultsTable; +} + +export function writeResults(resultsTable: string, whichBench: number): void { + const resultsFile = "results.md"; + + try { + if (!fs.existsSync(resultsFile) || fs.readFileSync(resultsFile, 'utf8').trim() === '') { + fs.writeFileSync(resultsFile, ` + +| Query | Server | Requests/sec | Latency (ms) | Relative | +|-------:|--------:|--------------:|--------------:|---------:|`); + } + + fs.appendFileSync(resultsFile, resultsTable); + + if (whichBench === 3) { + fs.appendFileSync(resultsFile, "\n\n"); + updateReadme(resultsFile); + } + } catch (error) { + console.error(`Error writing results: ${(error as Error).message}`); + } +} + +function updateReadme(resultsFile: string): void { + try { + const finalResults = fs + .readFileSync(resultsFile, "utf-8") + .replace(/\\/g, ''); // Remove backslashes + + const readmePath = "README.md"; + let readmeContent = fs.readFileSync(readmePath, "utf-8"); + const performanceResultsRegex = + /[\s\S]*/; + if (performanceResultsRegex.test(readmeContent)) { + readmeContent = readmeContent.replace( + performanceResultsRegex, + finalResults + ); + } else { + readmeContent += `\n${finalResults}`; + } + fs.writeFileSync(readmePath, readmeContent); + console.log("README.md updated successfully"); + } catch (error) { + console.error(`Error updating README: ${(error as Error).message}`); + } +} diff --git a/analyze/types.ts b/analyze/types.ts new file mode 100644 index 00000000..e8672961 --- /dev/null +++ b/analyze/types.ts @@ -0,0 +1,8 @@ +export interface ServerMetrics { + reqSec: number; + latency: number; +} + +export interface FormattedServerNames { + [key: string]: string; +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..85d4a4e3 --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "devDependencies": { + "@types/node": "^22.0.0" + } +} diff --git a/run_analyze_script.sh b/run_analyze_script.sh index 84f7206a..a081e1cb 100755 --- a/run_analyze_script.sh +++ b/run_analyze_script.sh @@ -2,6 +2,7 @@ # Update and install gnuplot sudo apt-get update && sudo apt-get install -y gnuplot +npm install # Remove existing results file rm -f results.md @@ -12,8 +13,9 @@ services=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "h for bench in 1 2 3; do echo "Processing files for bench${bench}:" - # Construct the command for each benchmark - cmd="bash analyze.sh" + tsc analyze.ts + # Construct the command for each benchmark + cmd="node analyze.js" # Loop through each service for service in "${services[@]}"; do @@ -33,4 +35,4 @@ for bench in 1 2 3; do # Execute the command echo "Executing: $cmd" eval $cmd -done \ No newline at end of file +done diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..9db8d78f --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "target": "ES2018", + "module": "CommonJS", + "rootDir": "./", + "strict": true, + "esModuleInterop": true + } +}