bench.js

#!/usr/bin/node

import { execSync } from "node:child_process";
import * as fs from "node:fs";
import * as path from "node:path";
import * as process from "node:process";
import * as util from "node:util";

const REPOSITORIES = {
	eslint: {
		repository: "https://github.com/eslint/eslint.git",
		lintedDirs: ["lib", "messages", "tests/lib", "tests/performance", "tools"],
	},
	prettier: {
		repository: "https://github.com/prettier/prettier.git",
		formattedDirs: ["src", "scripts"],
	},
	webpack: {
		repository: "https://github.com/webpack/webpack.git",
		formattedDirs: ["lib", "examples", "benchmark"],
		lintedDirs: ["lib"],
	},
};

const TMP_DIR = path.resolve("./target");

function benchmarkFormatter(biomeBin, options) {
	// Run Dprint once to run the installer
	execSync("npx dprint --version");

	for (const [name, config] of Object.entries(REPOSITORIES)) {
		if (
			!config.formattedDirs ||
			("repository" in options && options.repository !== name)
		) {
			continue;
		}
		console.info(`\n⌛ repository: ${name}`);
		const projectDirectory = cloneProject(
			name,
			config.repository,
			config.formattedDirs,
		);
		console.info("");

		const dirs = config.formattedDirs.join(" ");
		const dirGlobs = config.formattedDirs.map((dir) => `"${dir}/**"`).join(" ");
		const biomeCommand = `${biomeBin} format --config-path=../../ --write --max-diagnostics=0 ${dirs}`;
		const benchCommands = {
			prettier: `../../node_modules/.bin/prettier --config=../../.prettierrc.json --ignore-path=../../.prettierignore  --write --log-level=error ${dirs}`,
			// FIXME: Parallel Prettier is crashing on Node 22
			// "parallel-prettier": `../../node_modules/.bin/pprettier --write --concurrency ${os.cpus().length} ${dirGlobs}`,
			dprint: `../../node_modules/dprint/dprint fmt --config=../../dprint.json ${dirGlobs}`,
			biome: biomeCommand,
			"biome-1-thread": withEnvVariable("RAYON_NUM_THREADS", "1", biomeCommand),
		};

		let suite = "";
		for (const benchName of Object.keys(benchCommands)) {
			if ("bench" in options && !options.bench.includes(benchName)) {
				continue;
			}
			suite += ` -n "${benchName}" '${benchCommands[benchName]}'`;
		}
		if (suite.length === 0) {
			console.error(`Benchmark '${options.bench}' doesn't exist.`);
			process.exit(1);
		}

		// Run 2 warmups to make sure the files are formatted correctly
		let hyperfineCommand = `hyperfine --warmup 2 --shell=${shellOption()} ${suite}`;
		if (options.verbose) {
			hyperfineCommand += " --show-output";
			console.info(`${hyperfineCommand}\n`);
		}

		execSync(hyperfineCommand, {
			cwd: projectDirectory,
			stdio: "inherit",
		});
	}
}

function benchmarkLinter(biomeBin, options) {
	for (const [name, config] of Object.entries(REPOSITORIES)) {
		if (
			!config.lintedDirs ||
			("repository" in options && options.repository !== name)
		) {
			continue;
		}
		console.info(`\n⌛ repository: ${name}`);
		const projectDirectory = cloneProject(
			name,
			config.repository,
			config.formattedDirs,
		);
		console.info("");

		const dirs = config.lintedDirs.map((dir) => `"${dir}"`).join(" ");
		const biomeCmd = `"${biomeBin}" lint --config-path=../../ --max-diagnostics=0 ${dirs}`;
		const benchCommands = {
			eslint: `../../node_modules/.bin/eslint --quiet --config=../../eslint.config.js --no-ignore ${dirs}`,
			"ts-eslint": `../../node_modules/.bin/eslint --quiet --config=../../ts-eslint.config.js --no-ignore ${dirs}`,
			biome: biomeCmd,
			"biome-1-thread": withEnvVariable("RAYON_NUM_THREADS", "1", biomeCmd),
			"ts-biome": `"${biomeBin}" lint --config-path=../../ts-biome.json --max-diagnostics=0 ${dirs}`,
		};

		let suite = "";
		for (const benchName of Object.keys(benchCommands)) {
			if ("bench" in options && !options.bench.includes(benchName)) {
				continue;
			}
			suite += ` -n "${benchName}" '${benchCommands[benchName]}'`;
		}
		if (suite.length === 0) {
			console.error(`Benchmark '${options.bench}' doesn't exist.`);
			process.exit(1);
		}

		let hyperfineCommand = `hyperfine --ignore-failure --shell=${shellOption()} ${suite}`;
		if (options.verbose) {
			hyperfineCommand += " --show-output";
			console.info(`${hyperfineCommand}\n`);
		}

		execSync(hyperfineCommand, {
			cwd: projectDirectory,
			stdio: "inherit",
		});
	}
}

function shellOption() {
	if (process.platform === "win32") {
		// Use Powershell so that it is possible to set an environment variable for a single command (ugh!)
		return "powershell";
	}
	return "default";
}

function withEnvVariable(name, value, command) {
	switch (process.platform) {
		case "win32": {
			return `$Env:${name}=${value}; ${command}`;
		}
		default:
			return `${name}="${value}" ${command}`;
	}
}

function withDirectory(cwd) {
	return {
		run(command, options) {
			execSync(command, {
				cwd,
				...options,
			});
		},
	};
}

function cloneProject(name, repository, dirs = []) {
	const projectDirectory = path.join(TMP_DIR, name);

	const inProjectDirectory = withDirectory(projectDirectory);

	if (fs.existsSync(projectDirectory)) {
		inProjectDirectory.run("git reset --hard @{u}");
		inProjectDirectory.run("git clean -df");
		inProjectDirectory.run("git pull --quiet --depth=1 --ff-only");
	} else {
		withDirectory(TMP_DIR).run(
			`git clone --quiet --depth=1 ${dirs.length > 0 ? "--sparse" : ""} ${repository}`,
			{
				stdio: "inherit",
			},
		);
	}

	if (dirs.length > 0) {
		console.info(
			`Adding directories ${dirs.join()} to sparse checkout in ${projectDirectory}`,
		);
		inProjectDirectory.run(`git sparse-checkout add ${dirs.join(" ")}`);
	}

	return projectDirectory;
}

function buildBiome() {
	console.info("Building Biome...");
	execSync(
		withEnvVariable(
			"BIOME_VERSION",
			"0.0.0",
			"cargo build --bin biome --release",
		),
		{
			stdio: "inherit",
		},
	);
	return path.resolve("../target/release/biome");
}

function run({ positionals, values: options }) {
	fs.mkdirSync(TMP_DIR, { recursive: true });
	let biomeBinPath;
	if ("biome" in options) {
		biomeBinPath = options.biome;
	} else {
		biomeBinPath = buildBiome();
	}
	switch (positionals[0]) {
		case "formatter":
			benchmarkFormatter(biomeBinPath, options);
			break;
		case "linter":
			benchmarkLinter(biomeBinPath, options);
			break;
		default:
			console.error(
				`A positional argument among 'formatter' and 'linter' must be passed.`,
			);
			process.exit(1);
	}
}

run(
	util.parseArgs({
		allowPositionals: true,
		options: {
			bench: {
				multiple: true,
				type: "string",
			},
			biome: {
				type: "string",
			},
			repository: {
				type: "string",
			},
			verbose: {
				type: "boolean",
			},
		},
	}),
);