From ea9680b96d15101a577f8b65d2c0f81ea210314d Mon Sep 17 00:00:00 2001 From: Jono Prest Date: Mon, 9 Nov 2026 14:22:58 +0000 Subject: [PATCH 1/2] Count the build's allocations on generated projects in the benchmarks Why: The benchmarks compile runtime and Belt files one at a time, whose dependency graphs are small, and build tests/tests, whose modules use a few small modules. Costs that grow with how many modules a build has or loads barely show: a change that avoids copying the signatures of large dependencies that a module hardly reads moved them by 0.4%, and opening the map of a namespaced package, which is quadratic in its modules per build, doesn't run at all. With `--project`, `measure` now also builds two projects that the script generates, clean and after an edit, on one worker domain and counting allocated words like tests/tests: - WideDeps: 240 modules that each use one or two values of four modules with 600 types and 1,200 values each. The edit adds a value to one of them, which compiles its users again. - Namespace: a package with `"namespace": true` and 1,000 one-line modules. The edit changes a module that nothing uses. Their counts are larger than all the others together, so the report adds them up on a line of their own, `Generated projects`, and `In total` covers the other benchmarks as before. Each has its row and threshold like any other benchmark. Every module of a project build loads the runtime's interfaces, so the inputs of the generated projects and of tests/tests now include a hash of the runtime sources: a pull request that changes the runtime marks them as inputs changed rather than as regressions. Locally, the signature copy pull requests take WideDeps clean from 505M to 333M words and its edit from 232M to 146M, and building the namespace map on demand takes Namespace clean from 611M to 203M; the other benchmarks' counts are unchanged. The builds of tests/tests now share countBuild and editModule with the generated projects. Signed-off-by: Jono Prest --- .github/workflows/ci.yml | 3 +- .../generated-project-benchmarks.internal.md | 1 + scripts/compilerBenchmarks.js | 286 ++++++++++++++++-- scripts/compilerBenchmarks.test.js | 88 ++++++ 4 files changed, 347 insertions(+), 31 deletions(-) create mode 100644 changes/generated-project-benchmarks.internal.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6800b663ee..85baee8d91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -455,7 +455,8 @@ jobs: # to 0.2%. The runtime and Belt are built there too, since the # replayed builds would otherwise write interfaces that don't # match those of the files they don't replay yet. Both build the - # base's tests/tests, which the base's compiler builds already. + # base's tests/tests, which the base's compiler builds already, + # and the generated projects of this checkout's script. measure() { sudo mkdir -p /bench sudo mount --bind "$1" /bench diff --git a/changes/generated-project-benchmarks.internal.md b/changes/generated-project-benchmarks.internal.md new file mode 100644 index 0000000000..b7cee3cb79 --- /dev/null +++ b/changes/generated-project-benchmarks.internal.md @@ -0,0 +1 @@ +- Count the words the build allocates on generated projects in the CI benchmarks: many modules using little of a few large dependencies, and a namespaced package of 1,000 modules. diff --git a/scripts/compilerBenchmarks.js b/scripts/compilerBenchmarks.js index eb94d1d618..b9d24319b5 100644 --- a/scripts/compilerBenchmarks.js +++ b/scripts/compilerBenchmarks.js @@ -41,7 +41,9 @@ // of this script, so both measurements use the same ones. With `--project`, // it also builds tests/tests of the given checkout with the build system, // clean, with nothing to do and after two edits (see measureProjectBuild), -// counting allocated words only. +// and a few generated projects, clean and after an edit, for costs that grow +// with the number of modules of a build (see generatedProjects), counting +// allocated words only. // With `--playground`, it measures the size of the playground's compiler. // // `compare` reports the changes between two measurements as Markdown, @@ -329,13 +331,102 @@ export const stressInputs = { }, }; +/** + * A generated project: its files by path relative to its root, rescript.json + * among them, and the source file that its edit changes. + * + * @typedef {{ files: Record, edit: string }} GeneratedProject + */ + +/** + * The rescript.json of a generated project, with JavaScript written to lib/ + * so that removing lib/ makes the next build a clean one. + * + * @param {string} name + * @param {Record} [extra] more settings + */ +function projectConfig(name, extra = {}) { + const config = { + name, + sources: [{ dir: "src" }], + "package-specs": { module: "esmodule", "in-source": false }, + suffix: ".mjs", + ...extra, + }; + return `${JSON.stringify(config, null, 2)}\n`; +} + +/** + * Generated projects for the build benchmarks, one for each cost of a build + * that grows with the number of modules rather than with their size, which + * compiling one file at a time doesn't show. Their modules are small, so + * each builds in a few seconds at most. + * + * @type {Record GeneratedProject>} + */ +export const generatedProjects = { + // Many modules that each use one or two values of a few large + // dependencies: the cost of loading a dependency's interface, which the + // type checker copies when it reads it, against the little of it that is + // used. + WideDeps: () => { + const dependencies = 4; + const values = 600; + const users = 240; + /** @type {Record} */ + const files = { "rescript.json": projectConfig("bench-wide-deps") }; + for (let d = 0; d < dependencies; d++) { + const lines = []; + for (let v = 0; v < values; v++) { + lines.push(`type t${v} = {a${v}: int, b${v}: string}`); + lines.push( + `let make${v} = (x: int): t${v} => {a${v}: x, b${v}: "${v}"}`, + ); + lines.push(`let get${v} = (r: t${v}) => r.a${v} + ${v}`); + } + files[`src/Big${d}.res`] = `${lines.join("\n")}\n`; + } + for (let u = 0; u < users; u++) { + const first = u % dependencies; + const second = (u * 7 + 1) % dependencies; + const v = (u * 37) % values; + files[`src/User${u}.res`] = [ + `let a = Big${first}.get${v}(Big${first}.make${v}(${u}))`, + `let b = Big${second}.make${(v + 1) % values}(a).b${(v + 1) % values}`, + "", + ].join("\n"); + } + // A new value in a dependency changes its interface, so every module + // that uses it is compiled again. + return { files, edit: "src/Big0.res" }; + }, + // A namespaced package of trivial modules. Each module opens the + // namespace's map of module aliases, one for every module of the package, + // so a cost per alias of opening it is quadratic in the package's size. + Namespace: () => { + const modules = 1000; + /** @type {Record} */ + const files = { + "rescript.json": projectConfig("bench-namespace", { namespace: true }), + }; + for (let i = 0; i < modules; i++) { + // Each module but the first uses another one, through the map. + const use = i === 0 ? "0" : `Mod${Math.floor((i - 1) / 2)}.value`; + files[`src/Mod${i}.res`] = `let value = ${use} + ${i}\n`; + } + // The last module, which nothing uses. + return { files, edit: `src/Mod${modules - 1}.res` }; + }, +}; + /** * The runtime and Belt sources of a checkout, by package. * * @param {string} checkout + * @param {typeof packages} [only] the packages to list, all by default */ -function packageSources(checkout) { - return packages.flatMap(pkg => { +function packageSources(checkout, only = packages) { + return only.flatMap(pkg => { const sourceDir = path.join(checkout, pkg.dir, pkg.sources); return fs .readdirSync(sourceDir) @@ -350,6 +441,31 @@ function packageSources(checkout) { }); } +/** + * A short hash of the runtime sources of a checkout. Every module of a + * project build loads the runtime's interfaces, so a change to the runtime + * can change what building the project allocates. + * + * @param {string} checkout + */ +export function runtimeInputs(checkout) { + const runtime = packages.filter(pkg => pkg.dir.endsWith("/runtime")); + return hashFiles(packageSources(checkout, runtime).map(s => s.sourcePath)); +} + +/** + * Combines the hashes of everything a benchmark's counts depend on into + * the hash of its inputs. + * + * @param {string[]} hashes + */ +export function combineInputs(...hashes) { + return createHash("sha256") + .update(hashes.join("\0")) + .digest("hex") + .slice(0, 16); +} + /** * Builds the runtime and Belt of a checkout whose compiler is built, with * that checkout's build system. A checkout made with `git worktree add` has @@ -468,9 +584,10 @@ const sharedModule = path.join("src", "test_utils.res"); * * @param {string} checkout * @param {string} projectCheckout + * @param {string} runtime the hash of the checkout's runtime sources * @param {Results} results */ -function measureProjectBuild(checkout, projectCheckout, results) { +function measureProjectBuild(checkout, projectCheckout, runtime, results) { const rescript = path.join(checkout, relativeBinDir, "rescript.exe"); const projectDir = path.join(projectCheckout, benchmarkProject); const workDir = path.join(os.tmpdir(), "compiler-bench-project"); @@ -508,7 +625,7 @@ function measureProjectBuild(checkout, projectCheckout, results) { const sources = listFiles(workDir).filter( file => file.endsWith(".res") || file.endsWith(".resi"), ); - const inputs = hashFiles(sources); + const inputs = combineInputs(hashFiles(sources), runtime); // The largest test, which no other module uses. const leafModule = sources .filter(file => path.dirname(file) === path.join(workDir, "src")) @@ -519,17 +636,7 @@ function measureProjectBuild(checkout, projectCheckout, results) { throw new Error(`The modules to edit are missing from ${projectDir}.`); } - const build = () => ({ - ...measureRun(rescript, ["build"], workDir, false, { - env: { - REWATCH_COMPILER_DOMAINS: "1", - REWATCH_REPORT_WORKER_ALLOCATIONS: "1", - }, - }), - inputs, - }); - /** @param {string} file */ - const edit = file => fs.appendFileSync(file, "\nlet benchmarkEdit = 1\n"); + const build = () => countBuild(rescript, workDir, inputs); results["build/clean"] = build(); const outputs = outputSizes(workDir); @@ -537,15 +644,102 @@ function measureProjectBuild(checkout, projectCheckout, results) { results[`output/tests/*${ext}`] = { bytes: outputs[ext], inputs }; } results["build/no-op"] = build(); - edit(leafModule); + editModule(leafModule); results["build/leaf-edit"] = build(); - edit(path.join(workDir, sharedModule)); + editModule(path.join(workDir, sharedModule)); results["build/shared-edit"] = build(); } finally { fs.rmSync(workDir, { recursive: true, force: true }); } } +/** + * Builds a project with a build system on one worker domain, so that the + * counts don't depend on how the work is shared, and counts the words it + * and its workers allocated. + * + * @param {string} rescript + * @param {string} dir + * @param {string} inputs + * @returns {Measurement} + */ +function countBuild(rescript, dir, inputs) { + return { + ...measureRun(rescript, ["build"], dir, false, { + env: { + REWATCH_COMPILER_DOMAINS: "1", + REWATCH_REPORT_WORKER_ALLOCATIONS: "1", + }, + }), + inputs, + }; +} + +/** + * Adds a value to a module, which changes its interface when it has no + * interface file. + * + * @param {string} file + */ +function editModule(file) { + fs.appendFileSync(file, "\nlet benchmarkEdit = 1\n"); +} + +/** + * Writes a generated project to an empty directory, with the runtime of + * `checkout` as its only dependency. Returns a hash of its files. + * + * @param {GeneratedProject} project + * @param {string} dir + * @param {string} checkout + */ +function writeProject(project, dir, checkout) { + const hash = createHash("sha256"); + for (const [file, contents] of Object.entries(project.files)) { + fs.mkdirSync(path.join(dir, path.dirname(file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), contents); + hash.update(file).update("\0").update(contents).update("\0"); + } + const scope = path.join(dir, "node_modules", "@rescript"); + fs.mkdirSync(scope, { recursive: true }); + fs.symlinkSync( + path.join(checkout, "packages", "@rescript", "runtime"), + path.join(scope, "runtime"), + "junction", + ); + return hash.digest("hex").slice(0, 16); +} + +/** + * Builds each generated project with the build system of `checkout`, clean + * and after its edit, counting allocated words like measureProjectBuild. + * The projects come from the checkout of this script, and are built in a + * directory with the same name in every measurement. + * + * @param {string} checkout + * @param {string} runtime the hash of the checkout's runtime sources + * @param {Results} results + */ +function measureGeneratedProjects(checkout, runtime, results) { + const rescript = path.join(checkout, relativeBinDir, "rescript.exe"); + const workDir = path.join(os.tmpdir(), "compiler-bench-generated"); + for (const [name, generate] of Object.entries(generatedProjects)) { + fs.rmSync(workDir, { recursive: true, force: true }); + try { + const project = generate(); + const inputs = combineInputs( + writeProject(project, workDir, checkout), + runtime, + ); + results[`build/${name}/clean`] = countBuild(rescript, workDir, inputs); + editModule(path.join(workDir, project.edit)); + results[`build/${name}/edit`] = countBuild(rescript, workDir, inputs); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + } +} + /** * @param {string} checkout * @param {{ instructions: boolean, project?: string, playground?: boolean }} options @@ -665,7 +859,9 @@ export function measure(checkout, { instructions, project, playground }) { } if (project) { - measureProjectBuild(checkout, project, results); + const runtime = runtimeInputs(checkout); + measureProjectBuild(checkout, project, runtime, results); + measureGeneratedProjects(checkout, runtime, results); } return results; @@ -681,10 +877,23 @@ export function measure(checkout, { instructions, project, playground }) { * }} Change */ +/** + * Whether a benchmark builds one of the generated projects. Their counts + * are larger than all the others together, so they are added up apart. + * + * @param {string} name + */ +const isGeneratedProject = name => + Object.keys(generatedProjects).some(project => + name.startsWith(`build/${project}/`), + ); + /** * Lists the counts of one metric that differ between two measurements, * largest relative change first, and the benchmarks that only one of them - * has. Benchmarks without the metric on either side are left out. + * has. Benchmarks without the metric on either side are left out. The + * total leaves out the generated projects, which have a subtotal of their + * own, `generated`, when there are any. * * @param {Results} base * @param {Results} head @@ -715,9 +924,14 @@ export function compareResults(base, head, threshold, metric = "allocated") { } changes.sort((x, y) => Math.abs(y.percent) - Math.abs(x.percent)); - /** @param {Results} results */ - const total = results => - common.reduce((sum, name) => sum + (results[name][metric] ?? 0), 0); + /** + * @param {Results} results + * @param {string[]} names + */ + const total = (results, names) => + names.reduce((sum, name) => sum + (results[name][metric] ?? 0), 0); + const counted = common.filter(name => !isGeneratedProject(name)); + const generated = common.filter(isGeneratedProject); return { metric, @@ -729,7 +943,11 @@ export function compareResults(base, head, threshold, metric = "allocated") { added: [...headNames].filter(name => !baseNames.has(name)), removed: [...baseNames].filter(name => !headNames.has(name)), compared: common.length, - total: { base: total(base), head: total(head) }, + total: { base: total(base, counted), head: total(head, counted) }, + generated: + generated.length > 0 + ? { base: total(base, generated), head: total(head, generated) } + : undefined, }; } @@ -810,19 +1028,27 @@ function renderSection(comparison, threshold, visibleRows) { removed, compared, total, + generated, } = comparison; const words = metricWords[metric]; - const totalPercent = - total.base === 0 ? 0 : ((total.head - total.base) / total.base) * 100; - // A total that rounds to 0.00% would read as "+0.00%" or "-0.00%". - const totalChange = - Math.abs(totalPercent) < 0.005 ? "±0.00%" : formatPercent(totalPercent); + /** @param {{ base: number, head: number }} sum */ + const sumChange = sum => { + const percent = + sum.base === 0 ? 0 : ((sum.head - sum.base) / sum.base) * 100; + // A total that rounds to 0.00% would read as "+0.00%" or "-0.00%". + const change = + Math.abs(percent) < 0.005 ? "±0.00%" : formatPercent(percent); + return `${formatCount(sum.base)} → ${formatCount(sum.head)} (${change})`; + }; const lines = [ `### ${words.title}`, "", - `In total: ${formatCount(total.base)} → ${formatCount(total.head)} (${totalChange}).`, + `In total: ${sumChange(total)}.`, "", ]; + if (generated) { + lines.push(`Generated projects: ${sumChange(generated)}.`, ""); + } if (regressions.length > 0) { lines.push( `**⚠️ Regression: ${words.more(regressions.length, threshold)}.**`, diff --git a/scripts/compilerBenchmarks.test.js b/scripts/compilerBenchmarks.test.js index 1e9325455c..342310473a 100644 --- a/scripts/compilerBenchmarks.test.js +++ b/scripts/compilerBenchmarks.test.js @@ -1,11 +1,17 @@ import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; import test from "node:test"; import { + combineInputs, compareAll, compareResults, + generatedProjects, parseAllocatedWords, parseInstructions, renderSummary, + runtimeInputs, stressInputs, } from "./compilerBenchmarks.js"; @@ -348,3 +354,85 @@ test("generates the same stress inputs every time", () => { assert.equal(generate(), generate()); } }); + +test("adds up the generated projects apart from the total", () => { + const base = { + a: { allocated: 1000, inputs: "x" }, + "build/clean": { allocated: 2000, inputs: "x" }, + "build/Namespace/clean": { allocated: 100000, inputs: "x" }, + "build/WideDeps/edit": { allocated: 50000, inputs: "x" }, + }; + const head = { + a: { allocated: 1000, inputs: "x" }, + "build/clean": { allocated: 2000, inputs: "x" }, + "build/Namespace/clean": { allocated: 40000, inputs: "x" }, + "build/WideDeps/edit": { allocated: 50000, inputs: "x" }, + }; + const comparison = compareResults(base, head, 1); + assert.deepEqual(comparison.total, { base: 3000, head: 3000 }); + assert.deepEqual(comparison.generated, { base: 150000, head: 90000 }); + // The generated projects still have rows and thresholds of their own. + assert.deepEqual( + comparison.improvements.map(c => c.name), + ["build/Namespace/clean"], + ); + const summary = renderSummary([comparison], 1); + assert.match(summary, /^In total: 3,000 → 3,000 \(±0\.00%\)\.$/m); + assert.match( + summary, + /^Generated projects: 150,000 → 90,000 \(-40\.00%\)\.$/m, + ); + // Without generated projects, there is no subtotal. + assert.doesNotMatch( + renderSummary([compareResults({ a: base.a }, { a: head.a }, 1)], 1), + /Generated projects/, + ); +}); + +test("marks the project builds' inputs as changed with the runtime", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bench-runtime-")); + try { + const runtime = path.join(dir, "packages", "@rescript", "runtime"); + fs.mkdirSync(runtime, { recursive: true }); + fs.writeFileSync(path.join(runtime, "Stdlib.res"), "let x = 1\n"); + fs.writeFileSync(path.join(runtime, "README.md"), "not a source\n"); + const before = runtimeInputs(dir); + fs.writeFileSync(path.join(runtime, "README.md"), "changed\n"); + assert.equal(runtimeInputs(dir), before); + fs.writeFileSync(path.join(runtime, "Stdlib.res"), "let x = 2\n"); + const after = runtimeInputs(dir); + assert.notEqual(after, before); + assert.notEqual( + combineInputs("project", after), + combineInputs("project", before), + ); + assert.equal( + combineInputs("project", before), + combineInputs("project", before), + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("generates the same projects every time", () => { + for (const generate of Object.values(generatedProjects)) { + const project = generate(); + assert.deepEqual(generate(), project); + // The edit changes one of the project's sources. + assert.ok(project.edit.endsWith(".res")); + assert.ok(project.edit in project.files); + const config = JSON.parse(project.files["rescript.json"]); + // JavaScript goes to lib/, which a clean build removes. + assert.equal(config["package-specs"]["in-source"], false); + } +}); + +test("namespaces only the namespace benchmark's project", () => { + const { files } = generatedProjects.Namespace(); + assert.equal(JSON.parse(files["rescript.json"]).namespace, true); + // Modules use others through the namespace's map. + assert.match(files["src/Mod10.res"], /^let value = Mod4\.value \+ 10$/m); + const wide = generatedProjects.WideDeps(); + assert.equal(JSON.parse(wide.files["rescript.json"]).namespace, undefined); +}); From 66b8aa38fb1edf64a829a0e8d245d2fa33dad083 Mon Sep 17 00:00:00 2001 From: Jono Prest Date: Mon, 9 Nov 2026 14:27:03 +0000 Subject: [PATCH 2/2] Time a chain of modules with the base's and the head's build systems Why: A change to the build's scheduling, such as starting a module once the interfaces it reads are final, does the same work sooner. Counting allocated words and instructions can only show what such a change costs, never what it saves. `time ` builds a generated project of 150 modules in a chain, each using the one before it, and a module beside each, clean with the two build systems in turn on the default number of worker domains: one build each that isn't counted, then 10 rounds, the order alternating each round. `compare --timings` adds a section to the report with the base's and the head's median times and the median of the head's time over the base's in the same round, with a 95% confidence interval from a percentile bootstrap of the rounds with a fixed seed. The section never counts as a regression or an improvement; when the times' file is missing or can't be read, it says they weren't measured. In CI, the step runs after both measurements, each build may take 2 minutes and the whole command 10, and a failure only leaves the times out. Locally, with load around 5 to 9 on 8 cores, the ratio was 0.695 (0.646 to 0.825) for the pull request that starts dependents once a module's interface is final, 0.991 (0.948 to 1.055) for one that only changes namespaced packages, and 0.965 (0.921 to 1.078) with the same build system on both sides. The command took about 45 seconds. Signed-off-by: Jono Prest --- .github/workflows/ci.yml | 9 +- .../generated-project-benchmarks.internal.md | 1 + scripts/compilerBenchmarks.js | 315 +++++++++++++++++- scripts/compilerBenchmarks.test.js | 156 ++++++++- 4 files changed, 472 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85baee8d91..c3649a8d05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -467,10 +467,17 @@ jobs: } measure "$RUNNER_TEMP/base" "$RUNNER_TEMP/base.json" measure "$GITHUB_WORKSPACE" "$RUNNER_TEMP/head.json" + # Wall time, with the two build systems taking turns on this + # runner. It only indicates a change that counting can't show, so + # a failure, or a run longer than 10 minutes, leaves it out of the + # report and fails nothing. + timeout --kill-after=30s 10m \ + node scripts/compilerBenchmarks.js time "$RUNNER_TEMP/base" "$GITHUB_WORKSPACE" \ + "$RUNNER_TEMP/timings.json" || echo "::warning::The wall-time benchmark failed." status=0 report="$RUNNER_TEMP/benchmarks-report.md" node scripts/compilerBenchmarks.js compare "$RUNNER_TEMP/base.json" "$RUNNER_TEMP/head.json" \ - --threshold 1 --summary "$report" || status=$? + --threshold 1 --summary "$report" --timings "$RUNNER_TEMP/timings.json" || status=$? cat "$report" >> "$GITHUB_STEP_SUMMARY" printf '\nPull request %s compared with base %s in [this run](%s).\n' \ "$(git rev-parse --short=9 HEAD^2)" "$(git rev-parse --short=9 HEAD^1)" "$RUN_URL" \ diff --git a/changes/generated-project-benchmarks.internal.md b/changes/generated-project-benchmarks.internal.md index b7cee3cb79..f801e2d4ee 100644 --- a/changes/generated-project-benchmarks.internal.md +++ b/changes/generated-project-benchmarks.internal.md @@ -1 +1,2 @@ - Count the words the build allocates on generated projects in the CI benchmarks: many modules using little of a few large dependencies, and a namespaced package of 1,000 modules. +- Time clean builds of a generated chain of modules with the base's and the head's build systems in turn in the CI benchmarks, and report the median ratio with a 95% confidence interval. The times are only indicative and never fail the job. diff --git a/scripts/compilerBenchmarks.js b/scripts/compilerBenchmarks.js index b9d24319b5..fe76e78f21 100644 --- a/scripts/compilerBenchmarks.js +++ b/scripts/compilerBenchmarks.js @@ -26,8 +26,10 @@ // node scripts/compilerBenchmarks.js prepare // node scripts/compilerBenchmarks.js measure [--instructions] // [--project ] [--playground] +// node scripts/compilerBenchmarks.js time +// [--rounds ] // node scripts/compilerBenchmarks.js compare -// [--threshold ] [--summary ] +// [--threshold ] [--summary ] [--timings ] // // `prepare` builds the runtime and Belt of a checkout whose compiler is // built. `measure` runs the binaries of a prepared checkout: it replays the @@ -46,11 +48,17 @@ // allocated words only. // With `--playground`, it measures the size of the playground's compiler. // +// `time` builds a generated project with the build systems of two prepared +// checkouts in turn and records the wall-clock times (see timeBuilds). This +// is the one benchmark that measures time, and only to indicate the effect +// of a change that does the same work sooner. +// // `compare` reports the changes between two measurements as Markdown, // improvements and regressions alike, one section per count, and exits with // 1 when a count grew by more than the threshold. Runtime and Belt files, // whose sources the two checkouts may not share, are reported but not -// alerted on when any of them differs. +// alerted on when any of them differs. With `--timings`, the report ends +// with the times that `time` recorded, which never count as a regression. import { execFileSync, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; @@ -419,6 +427,50 @@ export const generatedProjects = { }, }; +/** + * The project the wall-time benchmark builds: a long chain of modules, each + * using the one before it, and a module beside each that uses it. A build + * spends most of its time on the chain with most of its worker domains + * idle, which is what a change to the build's scheduling would improve. + * + * @returns {GeneratedProject} + */ +export function chainProject() { + const length = 150; + /** @type {Record} */ + const files = { "rescript.json": projectConfig("bench-chain") }; + for (let i = 0; i < length; i++) { + const lines = + i === 0 + ? [ + "type t = {depth: int, items: array}", + "let base = {depth: 0, items: []}", + ] + : [ + `type t = {depth: int, items: array, prev: Chain${i - 1}.t}`, + `let base = {depth: ${i}, items: [${i}], prev: Chain${i - 1}.base}`, + ]; + // Bodies with some work for the type checker and code generation. + for (let f = 0; f < 20; f++) { + lines.push( + `let f${f} = (r: t, x: int) =>`, + " switch mod(x, 4) {", + ` | 0 => r.items->Array.map(y => y + x + ${f})->Array.length`, + ` | 1 => r.depth * ${f + 1} + x`, + ` | 2 => r.items->Array.reduce(0, (acc, y) => acc + y * ${f})`, + " | _ => -1", + " }", + ); + } + files[`src/Chain${i}.res`] = `${lines.join("\n")}\n`; + files[`src/Side${i}.res`] = [ + `let total = Chain${i}.f0(Chain${i}.base, ${i}) + Chain${i}.f1(Chain${i}.base, 1)`, + "", + ].join("\n"); + } + return { files, edit: `src/Chain${length - 1}.res` }; +} + /** * The runtime and Belt sources of a checkout, by package. * @@ -867,6 +919,217 @@ export function measure(checkout, { instructions, project, playground }) { return results; } +/** + * How many times the wall-time benchmark builds its project with each + * compiler, after one build with each that isn't counted. + */ +const timingRounds = 10; + +/** + * How long one of the wall-time benchmark's builds may take, in + * milliseconds, about 50 times as long as on a CI runner, so that a build + * that hangs, as it could with a deadlock in the scheduling, ends the + * benchmark instead of the job. + */ +const timingBuildTimeout = 2 * 60 * 1000; + +/** + * The wall-clock times of the wall-time benchmark's builds, in + * milliseconds, by benchmark: round i built with the base's compiler and + * the head's, one right after the other. + * + * @typedef {Record} Timings + */ + +/** + * Builds the wall-time benchmark's project clean with the build systems of + * two checkouts in turn, on as many worker domains as the build system + * chooses, and returns how long each build took. The two compilers take + * turns going first, so that a machine that gets slower or faster during + * the run changes both alike. + * + * Wall-clock time on a shared runner varies by several percent from one + * build to the next, and more over minutes, so this only indicates the + * effect of a change that counting doesn't show, such as starting a module + * earlier: building the same modules with idle domains doing less is + * faster, but allocates and executes as much or more. + * + * Each checkout builds a copy of the project of its own, with its own + * runtime, both under paths of the same length. + * + * @param {string} baseCheckout + * @param {string} headCheckout + * @param {number} rounds + * @returns {Timings} + */ +export function timeBuilds(baseCheckout, headCheckout, rounds) { + const workDir = path.join(os.tmpdir(), "compiler-bench-time"); + fs.rmSync(workDir, { recursive: true, force: true }); + try { + const project = chainProject(); + const sides = [ + { name: "base", checkout: baseCheckout }, + { name: "head", checkout: headCheckout }, + ].map(side => { + const dir = path.join(workDir, side.name); + writeProject(project, dir, side.checkout); + return { + ...side, + dir, + rescript: path.join(side.checkout, relativeBinDir, "rescript.exe"), + /** @type {number[]} */ + times: [], + }; + }); + /** @param {(typeof sides)[number]} side */ + const build = side => { + fs.rmSync(path.join(side.dir, "lib"), { recursive: true, force: true }); + const start = process.hrtime.bigint(); + execFileSync(side.rescript, ["build"], { + cwd: side.dir, + stdio: "pipe", + timeout: timingBuildTimeout, + killSignal: "SIGKILL", + }); + return Number(process.hrtime.bigint() - start) / 1e6; + }; + for (const side of sides) build(side); + for (let round = 0; round < rounds; round++) { + const order = round % 2 === 0 ? sides : [...sides].reverse(); + for (const side of order) side.times.push(build(side)); + } + return { + "time/Chain/clean": { base: sides[0].times, head: sides[1].times }, + }; + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } +} + +/** + * The median of numbers. + * + * @param {number[]} values + */ +export function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +} + +/** + * A pseudo-random number generator (mulberry32), so that the confidence + * interval of the same times is the same in every run. + * + * @param {number} seed + * @returns {() => number} numbers in [0, 1) + */ +function randomNumbers(seed) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * The median of the head's time divided by the base's time in the same + * round, with a 95% confidence interval from a percentile bootstrap: the + * medians of `resamples` samples of the rounds drawn with replacement, of + * which 2.5% are below `low` and 2.5% above `high`. Pairing the rounds + * leaves out a drift that slows both builds of a round alike. + * + * @param {number[]} base + * @param {number[]} head + * @param {number} [resamples] + * @returns {{ ratio: number, low: number, high: number, rounds: number }} + */ +export function timeRatio(base, head, resamples = 2000) { + if (base.length !== head.length || base.length === 0) { + throw new Error("The base and the head need the same rounds."); + } + const ratios = base.map((time, i) => head[i] / time); + const random = randomNumbers(1); + const medians = Array.from({ length: resamples }, () => + median(ratios.map(() => ratios[Math.floor(random() * ratios.length)])), + ).sort((a, b) => a - b); + /** @param {number} q */ + const quantile = q => + medians[Math.min(resamples - 1, Math.floor(q * resamples))]; + return { + ratio: median(ratios), + low: quantile(0.025), + high: quantile(0.975), + rounds: ratios.length, + }; +} + +/** + * Reads the times that `time` wrote. A file that is missing, can't be read + * or doesn't hold an object gives null, so that the report says the times + * weren't measured instead of failing. + * + * @param {string} file + * @returns {Timings | null} + */ +export function readTimings(file) { + try { + const timings = JSON.parse(fs.readFileSync(file, "utf8")); + return timings !== null && + typeof timings === "object" && + !Array.isArray(timings) + ? timings + : null; + } catch { + return null; + } +} + +/** + * Renders the wall-time benchmarks as a section of the report. They never + * count as a regression or an improvement. + * + * @param {Timings | null} timings null, or no benchmarks, when the times + * weren't measured + */ +export function renderTimings(timings) { + const heading = ["### Wall time (indicative only)", ""]; + if (!timings || Object.keys(timings).length === 0) { + return [...heading, "Not measured in this run."]; + } + const lines = [ + ...heading, + "Clean builds of a generated chain of modules with the base's and the head's build systems in turn on this runner, on several domains. Wall time on a shared runner is noisy, so this never fails the job; it shows changes such as scheduling ones that the counts above can't. The ratio is the median of each round's head time over its base time, not the ratio of the two medians.", + "", + "| Benchmark | Base, median | Head, median | Median of head / base per round | 95% CI |", + "| --- | ---: | ---: | ---: | ---: |", + ]; + /** @param {unknown} times */ + const isTimes = times => + Array.isArray(times) && + times.length > 0 && + times.every(time => Number.isFinite(time) && time > 0); + for (const [name, entry] of Object.entries(timings)) { + // The times are read from a file, and an entry that isn't two lists of + // the same number of times must not fail the report. + const { base, head } = entry ?? {}; + if (!isTimes(base) || !isTimes(head) || base.length !== head.length) { + lines.push(`| \`${name}\` | | | not comparable | |`); + continue; + } + const { ratio, low, high, rounds } = timeRatio(base, head); + lines.push( + `| \`${name}\` (${rounds} ${rounds === 1 ? "round" : "rounds"}) | ${median(base).toFixed(0)} ms | ${median(head).toFixed(0)} ms | ${ratio.toFixed(3)} | ${low.toFixed(3)} to ${high.toFixed(3)} |`, + ); + } + return lines; +} + /** * @typedef {{ * name: string, @@ -1112,8 +1375,15 @@ function renderSection(comparison, threshold, visibleRows) { * @param {number} threshold * @param {number} [visibleRows] the largest changes shown in each table; the * others are in a collapsed section under it + * @param {Timings | null} [timings] the wall-time benchmarks, shown after + * the counts; null when they were asked for but not measured */ -export function renderSummary(comparisons, threshold, visibleRows = 10) { +export function renderSummary( + comparisons, + threshold, + visibleRows = 10, + timings = undefined, +) { const shown = comparisons.filter( c => c.compared > 0 || c.added.length > 0 || c.removed.length > 0, ); @@ -1121,6 +1391,7 @@ export function renderSummary(comparisons, threshold, visibleRows = 10) { for (const comparison of shown) { lines.push("", ...renderSection(comparison, threshold, visibleRows)); } + if (timings !== undefined) lines.push("", ...renderTimings(timings)); return `${lines.join("\n")}\n`; } @@ -1137,8 +1408,12 @@ export function compareAll(base, head, threshold) { ); } -/** @param {string[]} args */ -function main(args) { +/** + * Runs a command and returns its exit status. + * + * @param {string[]} args + */ +export function main(args) { const [command, ...rest] = args; if (command === "prepare" && rest.length === 1) { prepare(path.resolve(rest[0])); @@ -1165,19 +1440,43 @@ function main(args) { console.log(`Measured ${Object.keys(results).length} benchmarks.`); return 0; } + if (command === "time" && rest.length >= 3) { + const [baseCheckout, headCheckout, out, ...options] = rest; + let rounds = timingRounds; + for (let i = 0; i < options.length; i += 2) { + if (options[i] === "--rounds") rounds = Number(options[i + 1]); + else throw new Error(`Unknown option ${options[i]}`); + } + if (!Number.isInteger(rounds) || rounds < 1) { + throw new Error("--rounds takes a positive integer."); + } + const timings = timeBuilds( + path.resolve(baseCheckout), + path.resolve(headCheckout), + rounds, + ); + fs.writeFileSync(out, `${JSON.stringify(timings, null, 2)}\n`); + for (const line of renderTimings(timings)) console.log(line); + return 0; + } if (command === "compare" && rest.length >= 2) { const [basePath, headPath, ...options] = rest; let threshold = 1; let summaryPath; + let timingsPath; for (let i = 0; i < options.length; i += 2) { if (options[i] === "--threshold") threshold = Number(options[i + 1]); else if (options[i] === "--summary") summaryPath = options[i + 1]; + else if (options[i] === "--timings") timingsPath = options[i + 1]; else throw new Error(`Unknown option ${options[i]}`); } const base = JSON.parse(fs.readFileSync(basePath, "utf8")); const head = JSON.parse(fs.readFileSync(headPath, "utf8")); + // The wall-time benchmarks are optional: when they didn't run, or + // failed, the report says so, and the exit status is the counts' alone. + const timings = timingsPath ? readTimings(timingsPath) : undefined; const comparisons = compareAll(base, head, threshold); - const summary = renderSummary(comparisons, threshold); + const summary = renderSummary(comparisons, threshold, undefined, timings); console.log(summary); if (summaryPath) fs.writeFileSync(summaryPath, summary); return comparisons.some(c => c.regressions.length > 0) ? 1 : 0; @@ -1186,8 +1485,10 @@ function main(args) { "Usage: compilerBenchmarks.js prepare \n" + " compilerBenchmarks.js measure [--instructions]\n" + " [--project ] [--playground]\n" + + " compilerBenchmarks.js time \n" + + " [--rounds ]\n" + " compilerBenchmarks.js compare [--threshold ]\n" + - " [--summary ]", + " [--summary ] [--timings ]", ); return 2; } diff --git a/scripts/compilerBenchmarks.test.js b/scripts/compilerBenchmarks.test.js index 342310473a..afcb513f2a 100644 --- a/scripts/compilerBenchmarks.test.js +++ b/scripts/compilerBenchmarks.test.js @@ -4,15 +4,20 @@ import * as os from "node:os"; import * as path from "node:path"; import test from "node:test"; import { + chainProject, combineInputs, compareAll, compareResults, generatedProjects, + main, + median, parseAllocatedWords, parseInstructions, + readTimings, renderSummary, runtimeInputs, stressInputs, + timeRatio, } from "./compilerBenchmarks.js"; const stats = `allocated_words: 628461 @@ -416,7 +421,7 @@ test("marks the project builds' inputs as changed with the runtime", () => { }); test("generates the same projects every time", () => { - for (const generate of Object.values(generatedProjects)) { + for (const generate of [...Object.values(generatedProjects), chainProject]) { const project = generate(); assert.deepEqual(generate(), project); // The edit changes one of the project's sources. @@ -436,3 +441,152 @@ test("namespaces only the namespace benchmark's project", () => { const wide = generatedProjects.WideDeps(); assert.equal(JSON.parse(wide.files["rescript.json"]).namespace, undefined); }); + +test("chains the modules of the wall-time benchmark's project", () => { + const { files } = chainProject(); + assert.match(files["src/Chain1.res"], /prev: Chain0\.t/); + assert.match(files["src/Chain149.res"], /prev: Chain148\.t/); + assert.equal(files["src/Chain150.res"], undefined); + assert.match(files["src/Side7.res"], /Chain7\.f0/); +}); + +test("takes the median of an odd and an even number of values", () => { + assert.equal(median([3, 1, 2]), 2); + assert.equal(median([4, 1, 3, 2]), 2.5); + // The values are left as they were. + const values = [2, 1]; + median(values); + assert.deepEqual(values, [2, 1]); +}); + +test("pairs the rounds of the times to compare", () => { + // The machine gets slower during the run, but the head is always 10% + // faster in the same round. + const base = [100, 120, 140, 160, 180, 200, 220, 240, 260, 280]; + const head = base.map(time => time * 0.9); + const { ratio, low, high, rounds } = timeRatio(base, head); + assert.equal(rounds, 10); + assert.ok(Math.abs(ratio - 0.9) < 1e-9); + assert.ok(Math.abs(low - 0.9) < 1e-9); + assert.ok(Math.abs(high - 0.9) < 1e-9); +}); + +test("gives a confidence interval around the median ratio", () => { + const base = [100, 100, 100, 100, 100, 100, 100, 100, 100, 100]; + const head = [85, 95, 90, 88, 92, 99, 80, 91, 89, 93]; + const result = timeRatio(base, head); + assert.equal(result.ratio, 0.905); + assert.ok(result.low <= result.ratio && result.ratio <= result.high); + assert.ok(result.low >= 0.8 && result.high <= 0.99); + // The resamples are drawn the same way every time. + assert.deepEqual(timeRatio(base, head), result); + assert.throws(() => timeRatio([1, 2], [1])); + assert.throws(() => timeRatio([], [])); +}); + +test("reports the wall time without counting it as a change", () => { + const results = { a: { allocated: 1000, inputs: "x" } }; + const timings = { + "time/Chain/clean": { base: [100, 100, 100], head: [150, 160, 155] }, + }; + const summary = renderSummary( + compareAll(results, results, 1), + 1, + undefined, + timings, + ); + // 55% slower, and still no regression. + assert.match(summary, /^## ✅ Compiler benchmarks$/m); + assert.match(summary, /^### Wall time \(indicative only\)$/m); + assert.match( + summary, + /^\| `time\/Chain\/clean` \(3 rounds\) \| 100 ms \| 155 ms \| 1\.550 \| 1\.500 to 1\.600 \|$/m, + ); + assert.doesNotMatch( + renderSummary(compareAll(results, results, 1), 1), + /Wall time/, + ); +}); + +test("leaves out wall times that can't be compared", () => { + const results = { a: { allocated: 1000, inputs: "x" } }; + const summary = renderSummary(compareAll(results, results, 1), 1, undefined, { + missing: /** @type {any} */ (null), + uneven: { base: [100, 100], head: [100] }, + empty: { base: [], head: [] }, + strings: { base: /** @type {any} */ (["100"]), head: [100] }, + "time/Chain/clean": { base: [100], head: [90] }, + }); + assert.match(summary, /^## ✅ Compiler benchmarks$/m); + for (const name of ["missing", "uneven", "empty", "strings"]) { + assert.match( + summary, + new RegExp(`^\\| \`${name}\` \\| \\| \\| not comparable \\| \\|$`, "m"), + ); + } + assert.match(summary, /\| `time\/Chain\/clean` \(1 round\) \|/); +}); + +test("says when the wall times weren't measured", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bench-timings-")); + try { + const write = (/** @type {string} */ name, /** @type {string} */ text) => { + const file = path.join(dir, name); + fs.writeFileSync(file, text); + return file; + }; + const missing = path.join(dir, "missing.json"); + const invalid = write("invalid.json", "{ not json"); + const empty = write("null.json", "null\n"); + for (const file of [missing, invalid, empty]) { + assert.equal(readTimings(file), null); + } + const valid = write( + "valid.json", + JSON.stringify({ "time/Chain/clean": { base: [100], head: [90] } }), + ); + assert.deepEqual(readTimings(valid), { + "time/Chain/clean": { base: [100], head: [90] }, + }); + + const results = { a: { allocated: 1000, inputs: "x" } }; + const notMeasured = renderSummary( + compareAll(results, results, 1), + 1, + undefined, + null, + ); + assert.match( + notMeasured, + /^### Wall time \(indicative only\)\n\nNot measured in this run\.$/m, + ); + assert.match(notMeasured, /^## ✅ Compiler benchmarks$/m); + + // The exit status is the counts' alone, whatever the times' file holds. + const base = write("base.json", JSON.stringify(results)); + const grown = write( + "head.json", + JSON.stringify({ a: { allocated: 2000, inputs: "x" } }), + ); + const summary = path.join(dir, "summary.md"); + const log = console.log; + console.log = () => {}; + try { + for (const file of [missing, invalid, empty]) { + const args = ["--summary", summary, "--timings", file]; + assert.equal(main(["compare", base, base, ...args]), 0); + assert.match( + fs.readFileSync(summary, "utf8"), + /Not measured in this run\./, + ); + assert.equal(main(["compare", base, grown, ...args]), 1); + } + assert.equal(main(["compare", base, base, "--summary", summary]), 0); + assert.doesNotMatch(fs.readFileSync(summary, "utf8"), /Wall time/); + } finally { + console.log = log; + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +});