diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6800b663ee..c3649a8d05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -455,7 +455,8 @@ jobs: # to 0.2%. The runtime and Belt are built there too, since the # replayed builds would otherwise write interfaces that don't # match those of the files they don't replay yet. Both build the - # base's tests/tests, which the base's compiler builds already. + # base's tests/tests, which the base's compiler builds already, + # and the generated projects of this checkout's script. measure() { sudo mkdir -p /bench sudo mount --bind "$1" /bench @@ -466,10 +467,17 @@ jobs: } measure "$RUNNER_TEMP/base" "$RUNNER_TEMP/base.json" measure "$GITHUB_WORKSPACE" "$RUNNER_TEMP/head.json" + # Wall time, with the two build systems taking turns on this + # runner. It only indicates a change that counting can't show, so + # a failure, or a run longer than 10 minutes, leaves it out of the + # report and fails nothing. + timeout --kill-after=30s 10m \ + node scripts/compilerBenchmarks.js time "$RUNNER_TEMP/base" "$GITHUB_WORKSPACE" \ + "$RUNNER_TEMP/timings.json" || echo "::warning::The wall-time benchmark failed." status=0 report="$RUNNER_TEMP/benchmarks-report.md" node scripts/compilerBenchmarks.js compare "$RUNNER_TEMP/base.json" "$RUNNER_TEMP/head.json" \ - --threshold 1 --summary "$report" || status=$? + --threshold 1 --summary "$report" --timings "$RUNNER_TEMP/timings.json" || status=$? cat "$report" >> "$GITHUB_STEP_SUMMARY" printf '\nPull request %s compared with base %s in [this run](%s).\n' \ "$(git rev-parse --short=9 HEAD^2)" "$(git rev-parse --short=9 HEAD^1)" "$RUN_URL" \ diff --git a/changes/generated-project-benchmarks.internal.md b/changes/generated-project-benchmarks.internal.md new file mode 100644 index 0000000000..f801e2d4ee --- /dev/null +++ b/changes/generated-project-benchmarks.internal.md @@ -0,0 +1,2 @@ +- Count the words the build allocates on generated projects in the CI benchmarks: many modules using little of a few large dependencies, and a namespaced package of 1,000 modules. +- Time clean builds of a generated chain of modules with the base's and the head's build systems in turn in the CI benchmarks, and report the median ratio with a 95% confidence interval. The times are only indicative and never fail the job. diff --git a/scripts/compilerBenchmarks.js b/scripts/compilerBenchmarks.js index eb94d1d618..fe76e78f21 100644 --- a/scripts/compilerBenchmarks.js +++ b/scripts/compilerBenchmarks.js @@ -26,8 +26,10 @@ // node scripts/compilerBenchmarks.js prepare // node scripts/compilerBenchmarks.js measure [--instructions] // [--project ] [--playground] +// node scripts/compilerBenchmarks.js time +// [--rounds ] // node scripts/compilerBenchmarks.js compare -// [--threshold ] [--summary ] +// [--threshold ] [--summary ] [--timings ] // // `prepare` builds the runtime and Belt of a checkout whose compiler is // built. `measure` runs the binaries of a prepared checkout: it replays the @@ -41,14 +43,22 @@ // of this script, so both measurements use the same ones. With `--project`, // it also builds tests/tests of the given checkout with the build system, // clean, with nothing to do and after two edits (see measureProjectBuild), -// counting allocated words only. +// and a few generated projects, clean and after an edit, for costs that grow +// with the number of modules of a build (see generatedProjects), counting +// allocated words only. // With `--playground`, it measures the size of the playground's compiler. // +// `time` builds a generated project with the build systems of two prepared +// checkouts in turn and records the wall-clock times (see timeBuilds). This +// is the one benchmark that measures time, and only to indicate the effect +// of a change that does the same work sooner. +// // `compare` reports the changes between two measurements as Markdown, // improvements and regressions alike, one section per count, and exits with // 1 when a count grew by more than the threshold. Runtime and Belt files, // whose sources the two checkouts may not share, are reported but not -// alerted on when any of them differs. +// alerted on when any of them differs. With `--timings`, the report ends +// with the times that `time` recorded, which never count as a regression. import { execFileSync, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; @@ -329,13 +339,146 @@ export const stressInputs = { }, }; +/** + * A generated project: its files by path relative to its root, rescript.json + * among them, and the source file that its edit changes. + * + * @typedef {{ files: Record, edit: string }} GeneratedProject + */ + +/** + * The rescript.json of a generated project, with JavaScript written to lib/ + * so that removing lib/ makes the next build a clean one. + * + * @param {string} name + * @param {Record} [extra] more settings + */ +function projectConfig(name, extra = {}) { + const config = { + name, + sources: [{ dir: "src" }], + "package-specs": { module: "esmodule", "in-source": false }, + suffix: ".mjs", + ...extra, + }; + return `${JSON.stringify(config, null, 2)}\n`; +} + +/** + * Generated projects for the build benchmarks, one for each cost of a build + * that grows with the number of modules rather than with their size, which + * compiling one file at a time doesn't show. Their modules are small, so + * each builds in a few seconds at most. + * + * @type {Record GeneratedProject>} + */ +export const generatedProjects = { + // Many modules that each use one or two values of a few large + // dependencies: the cost of loading a dependency's interface, which the + // type checker copies when it reads it, against the little of it that is + // used. + WideDeps: () => { + const dependencies = 4; + const values = 600; + const users = 240; + /** @type {Record} */ + const files = { "rescript.json": projectConfig("bench-wide-deps") }; + for (let d = 0; d < dependencies; d++) { + const lines = []; + for (let v = 0; v < values; v++) { + lines.push(`type t${v} = {a${v}: int, b${v}: string}`); + lines.push( + `let make${v} = (x: int): t${v} => {a${v}: x, b${v}: "${v}"}`, + ); + lines.push(`let get${v} = (r: t${v}) => r.a${v} + ${v}`); + } + files[`src/Big${d}.res`] = `${lines.join("\n")}\n`; + } + for (let u = 0; u < users; u++) { + const first = u % dependencies; + const second = (u * 7 + 1) % dependencies; + const v = (u * 37) % values; + files[`src/User${u}.res`] = [ + `let a = Big${first}.get${v}(Big${first}.make${v}(${u}))`, + `let b = Big${second}.make${(v + 1) % values}(a).b${(v + 1) % values}`, + "", + ].join("\n"); + } + // A new value in a dependency changes its interface, so every module + // that uses it is compiled again. + return { files, edit: "src/Big0.res" }; + }, + // A namespaced package of trivial modules. Each module opens the + // namespace's map of module aliases, one for every module of the package, + // so a cost per alias of opening it is quadratic in the package's size. + Namespace: () => { + const modules = 1000; + /** @type {Record} */ + const files = { + "rescript.json": projectConfig("bench-namespace", { namespace: true }), + }; + for (let i = 0; i < modules; i++) { + // Each module but the first uses another one, through the map. + const use = i === 0 ? "0" : `Mod${Math.floor((i - 1) / 2)}.value`; + files[`src/Mod${i}.res`] = `let value = ${use} + ${i}\n`; + } + // The last module, which nothing uses. + return { files, edit: `src/Mod${modules - 1}.res` }; + }, +}; + +/** + * The project the wall-time benchmark builds: a long chain of modules, each + * using the one before it, and a module beside each that uses it. A build + * spends most of its time on the chain with most of its worker domains + * idle, which is what a change to the build's scheduling would improve. + * + * @returns {GeneratedProject} + */ +export function chainProject() { + const length = 150; + /** @type {Record} */ + const files = { "rescript.json": projectConfig("bench-chain") }; + for (let i = 0; i < length; i++) { + const lines = + i === 0 + ? [ + "type t = {depth: int, items: array}", + "let base = {depth: 0, items: []}", + ] + : [ + `type t = {depth: int, items: array, prev: Chain${i - 1}.t}`, + `let base = {depth: ${i}, items: [${i}], prev: Chain${i - 1}.base}`, + ]; + // Bodies with some work for the type checker and code generation. + for (let f = 0; f < 20; f++) { + lines.push( + `let f${f} = (r: t, x: int) =>`, + " switch mod(x, 4) {", + ` | 0 => r.items->Array.map(y => y + x + ${f})->Array.length`, + ` | 1 => r.depth * ${f + 1} + x`, + ` | 2 => r.items->Array.reduce(0, (acc, y) => acc + y * ${f})`, + " | _ => -1", + " }", + ); + } + files[`src/Chain${i}.res`] = `${lines.join("\n")}\n`; + files[`src/Side${i}.res`] = [ + `let total = Chain${i}.f0(Chain${i}.base, ${i}) + Chain${i}.f1(Chain${i}.base, 1)`, + "", + ].join("\n"); + } + return { files, edit: `src/Chain${length - 1}.res` }; +} + /** * The runtime and Belt sources of a checkout, by package. * * @param {string} checkout + * @param {typeof packages} [only] the packages to list, all by default */ -function packageSources(checkout) { - return packages.flatMap(pkg => { +function packageSources(checkout, only = packages) { + return only.flatMap(pkg => { const sourceDir = path.join(checkout, pkg.dir, pkg.sources); return fs .readdirSync(sourceDir) @@ -350,6 +493,31 @@ function packageSources(checkout) { }); } +/** + * A short hash of the runtime sources of a checkout. Every module of a + * project build loads the runtime's interfaces, so a change to the runtime + * can change what building the project allocates. + * + * @param {string} checkout + */ +export function runtimeInputs(checkout) { + const runtime = packages.filter(pkg => pkg.dir.endsWith("/runtime")); + return hashFiles(packageSources(checkout, runtime).map(s => s.sourcePath)); +} + +/** + * Combines the hashes of everything a benchmark's counts depend on into + * the hash of its inputs. + * + * @param {string[]} hashes + */ +export function combineInputs(...hashes) { + return createHash("sha256") + .update(hashes.join("\0")) + .digest("hex") + .slice(0, 16); +} + /** * Builds the runtime and Belt of a checkout whose compiler is built, with * that checkout's build system. A checkout made with `git worktree add` has @@ -468,9 +636,10 @@ const sharedModule = path.join("src", "test_utils.res"); * * @param {string} checkout * @param {string} projectCheckout + * @param {string} runtime the hash of the checkout's runtime sources * @param {Results} results */ -function measureProjectBuild(checkout, projectCheckout, results) { +function measureProjectBuild(checkout, projectCheckout, runtime, results) { const rescript = path.join(checkout, relativeBinDir, "rescript.exe"); const projectDir = path.join(projectCheckout, benchmarkProject); const workDir = path.join(os.tmpdir(), "compiler-bench-project"); @@ -508,7 +677,7 @@ function measureProjectBuild(checkout, projectCheckout, results) { const sources = listFiles(workDir).filter( file => file.endsWith(".res") || file.endsWith(".resi"), ); - const inputs = hashFiles(sources); + const inputs = combineInputs(hashFiles(sources), runtime); // The largest test, which no other module uses. const leafModule = sources .filter(file => path.dirname(file) === path.join(workDir, "src")) @@ -519,17 +688,7 @@ function measureProjectBuild(checkout, projectCheckout, results) { throw new Error(`The modules to edit are missing from ${projectDir}.`); } - const build = () => ({ - ...measureRun(rescript, ["build"], workDir, false, { - env: { - REWATCH_COMPILER_DOMAINS: "1", - REWATCH_REPORT_WORKER_ALLOCATIONS: "1", - }, - }), - inputs, - }); - /** @param {string} file */ - const edit = file => fs.appendFileSync(file, "\nlet benchmarkEdit = 1\n"); + const build = () => countBuild(rescript, workDir, inputs); results["build/clean"] = build(); const outputs = outputSizes(workDir); @@ -537,15 +696,102 @@ function measureProjectBuild(checkout, projectCheckout, results) { results[`output/tests/*${ext}`] = { bytes: outputs[ext], inputs }; } results["build/no-op"] = build(); - edit(leafModule); + editModule(leafModule); results["build/leaf-edit"] = build(); - edit(path.join(workDir, sharedModule)); + editModule(path.join(workDir, sharedModule)); results["build/shared-edit"] = build(); } finally { fs.rmSync(workDir, { recursive: true, force: true }); } } +/** + * Builds a project with a build system on one worker domain, so that the + * counts don't depend on how the work is shared, and counts the words it + * and its workers allocated. + * + * @param {string} rescript + * @param {string} dir + * @param {string} inputs + * @returns {Measurement} + */ +function countBuild(rescript, dir, inputs) { + return { + ...measureRun(rescript, ["build"], dir, false, { + env: { + REWATCH_COMPILER_DOMAINS: "1", + REWATCH_REPORT_WORKER_ALLOCATIONS: "1", + }, + }), + inputs, + }; +} + +/** + * Adds a value to a module, which changes its interface when it has no + * interface file. + * + * @param {string} file + */ +function editModule(file) { + fs.appendFileSync(file, "\nlet benchmarkEdit = 1\n"); +} + +/** + * Writes a generated project to an empty directory, with the runtime of + * `checkout` as its only dependency. Returns a hash of its files. + * + * @param {GeneratedProject} project + * @param {string} dir + * @param {string} checkout + */ +function writeProject(project, dir, checkout) { + const hash = createHash("sha256"); + for (const [file, contents] of Object.entries(project.files)) { + fs.mkdirSync(path.join(dir, path.dirname(file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), contents); + hash.update(file).update("\0").update(contents).update("\0"); + } + const scope = path.join(dir, "node_modules", "@rescript"); + fs.mkdirSync(scope, { recursive: true }); + fs.symlinkSync( + path.join(checkout, "packages", "@rescript", "runtime"), + path.join(scope, "runtime"), + "junction", + ); + return hash.digest("hex").slice(0, 16); +} + +/** + * Builds each generated project with the build system of `checkout`, clean + * and after its edit, counting allocated words like measureProjectBuild. + * The projects come from the checkout of this script, and are built in a + * directory with the same name in every measurement. + * + * @param {string} checkout + * @param {string} runtime the hash of the checkout's runtime sources + * @param {Results} results + */ +function measureGeneratedProjects(checkout, runtime, results) { + const rescript = path.join(checkout, relativeBinDir, "rescript.exe"); + const workDir = path.join(os.tmpdir(), "compiler-bench-generated"); + for (const [name, generate] of Object.entries(generatedProjects)) { + fs.rmSync(workDir, { recursive: true, force: true }); + try { + const project = generate(); + const inputs = combineInputs( + writeProject(project, workDir, checkout), + runtime, + ); + results[`build/${name}/clean`] = countBuild(rescript, workDir, inputs); + editModule(path.join(workDir, project.edit)); + results[`build/${name}/edit`] = countBuild(rescript, workDir, inputs); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + } +} + /** * @param {string} checkout * @param {{ instructions: boolean, project?: string, playground?: boolean }} options @@ -665,12 +911,225 @@ export function measure(checkout, { instructions, project, playground }) { } if (project) { - measureProjectBuild(checkout, project, results); + const runtime = runtimeInputs(checkout); + measureProjectBuild(checkout, project, runtime, results); + measureGeneratedProjects(checkout, runtime, results); } return results; } +/** + * How many times the wall-time benchmark builds its project with each + * compiler, after one build with each that isn't counted. + */ +const timingRounds = 10; + +/** + * How long one of the wall-time benchmark's builds may take, in + * milliseconds, about 50 times as long as on a CI runner, so that a build + * that hangs, as it could with a deadlock in the scheduling, ends the + * benchmark instead of the job. + */ +const timingBuildTimeout = 2 * 60 * 1000; + +/** + * The wall-clock times of the wall-time benchmark's builds, in + * milliseconds, by benchmark: round i built with the base's compiler and + * the head's, one right after the other. + * + * @typedef {Record} Timings + */ + +/** + * Builds the wall-time benchmark's project clean with the build systems of + * two checkouts in turn, on as many worker domains as the build system + * chooses, and returns how long each build took. The two compilers take + * turns going first, so that a machine that gets slower or faster during + * the run changes both alike. + * + * Wall-clock time on a shared runner varies by several percent from one + * build to the next, and more over minutes, so this only indicates the + * effect of a change that counting doesn't show, such as starting a module + * earlier: building the same modules with idle domains doing less is + * faster, but allocates and executes as much or more. + * + * Each checkout builds a copy of the project of its own, with its own + * runtime, both under paths of the same length. + * + * @param {string} baseCheckout + * @param {string} headCheckout + * @param {number} rounds + * @returns {Timings} + */ +export function timeBuilds(baseCheckout, headCheckout, rounds) { + const workDir = path.join(os.tmpdir(), "compiler-bench-time"); + fs.rmSync(workDir, { recursive: true, force: true }); + try { + const project = chainProject(); + const sides = [ + { name: "base", checkout: baseCheckout }, + { name: "head", checkout: headCheckout }, + ].map(side => { + const dir = path.join(workDir, side.name); + writeProject(project, dir, side.checkout); + return { + ...side, + dir, + rescript: path.join(side.checkout, relativeBinDir, "rescript.exe"), + /** @type {number[]} */ + times: [], + }; + }); + /** @param {(typeof sides)[number]} side */ + const build = side => { + fs.rmSync(path.join(side.dir, "lib"), { recursive: true, force: true }); + const start = process.hrtime.bigint(); + execFileSync(side.rescript, ["build"], { + cwd: side.dir, + stdio: "pipe", + timeout: timingBuildTimeout, + killSignal: "SIGKILL", + }); + return Number(process.hrtime.bigint() - start) / 1e6; + }; + for (const side of sides) build(side); + for (let round = 0; round < rounds; round++) { + const order = round % 2 === 0 ? sides : [...sides].reverse(); + for (const side of order) side.times.push(build(side)); + } + return { + "time/Chain/clean": { base: sides[0].times, head: sides[1].times }, + }; + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } +} + +/** + * The median of numbers. + * + * @param {number[]} values + */ +export function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +} + +/** + * A pseudo-random number generator (mulberry32), so that the confidence + * interval of the same times is the same in every run. + * + * @param {number} seed + * @returns {() => number} numbers in [0, 1) + */ +function randomNumbers(seed) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * The median of the head's time divided by the base's time in the same + * round, with a 95% confidence interval from a percentile bootstrap: the + * medians of `resamples` samples of the rounds drawn with replacement, of + * which 2.5% are below `low` and 2.5% above `high`. Pairing the rounds + * leaves out a drift that slows both builds of a round alike. + * + * @param {number[]} base + * @param {number[]} head + * @param {number} [resamples] + * @returns {{ ratio: number, low: number, high: number, rounds: number }} + */ +export function timeRatio(base, head, resamples = 2000) { + if (base.length !== head.length || base.length === 0) { + throw new Error("The base and the head need the same rounds."); + } + const ratios = base.map((time, i) => head[i] / time); + const random = randomNumbers(1); + const medians = Array.from({ length: resamples }, () => + median(ratios.map(() => ratios[Math.floor(random() * ratios.length)])), + ).sort((a, b) => a - b); + /** @param {number} q */ + const quantile = q => + medians[Math.min(resamples - 1, Math.floor(q * resamples))]; + return { + ratio: median(ratios), + low: quantile(0.025), + high: quantile(0.975), + rounds: ratios.length, + }; +} + +/** + * Reads the times that `time` wrote. A file that is missing, can't be read + * or doesn't hold an object gives null, so that the report says the times + * weren't measured instead of failing. + * + * @param {string} file + * @returns {Timings | null} + */ +export function readTimings(file) { + try { + const timings = JSON.parse(fs.readFileSync(file, "utf8")); + return timings !== null && + typeof timings === "object" && + !Array.isArray(timings) + ? timings + : null; + } catch { + return null; + } +} + +/** + * Renders the wall-time benchmarks as a section of the report. They never + * count as a regression or an improvement. + * + * @param {Timings | null} timings null, or no benchmarks, when the times + * weren't measured + */ +export function renderTimings(timings) { + const heading = ["### Wall time (indicative only)", ""]; + if (!timings || Object.keys(timings).length === 0) { + return [...heading, "Not measured in this run."]; + } + const lines = [ + ...heading, + "Clean builds of a generated chain of modules with the base's and the head's build systems in turn on this runner, on several domains. Wall time on a shared runner is noisy, so this never fails the job; it shows changes such as scheduling ones that the counts above can't. The ratio is the median of each round's head time over its base time, not the ratio of the two medians.", + "", + "| Benchmark | Base, median | Head, median | Median of head / base per round | 95% CI |", + "| --- | ---: | ---: | ---: | ---: |", + ]; + /** @param {unknown} times */ + const isTimes = times => + Array.isArray(times) && + times.length > 0 && + times.every(time => Number.isFinite(time) && time > 0); + for (const [name, entry] of Object.entries(timings)) { + // The times are read from a file, and an entry that isn't two lists of + // the same number of times must not fail the report. + const { base, head } = entry ?? {}; + if (!isTimes(base) || !isTimes(head) || base.length !== head.length) { + lines.push(`| \`${name}\` | | | not comparable | |`); + continue; + } + const { ratio, low, high, rounds } = timeRatio(base, head); + lines.push( + `| \`${name}\` (${rounds} ${rounds === 1 ? "round" : "rounds"}) | ${median(base).toFixed(0)} ms | ${median(head).toFixed(0)} ms | ${ratio.toFixed(3)} | ${low.toFixed(3)} to ${high.toFixed(3)} |`, + ); + } + return lines; +} + /** * @typedef {{ * name: string, @@ -681,10 +1140,23 @@ export function measure(checkout, { instructions, project, playground }) { * }} Change */ +/** + * Whether a benchmark builds one of the generated projects. Their counts + * are larger than all the others together, so they are added up apart. + * + * @param {string} name + */ +const isGeneratedProject = name => + Object.keys(generatedProjects).some(project => + name.startsWith(`build/${project}/`), + ); + /** * Lists the counts of one metric that differ between two measurements, * largest relative change first, and the benchmarks that only one of them - * has. Benchmarks without the metric on either side are left out. + * has. Benchmarks without the metric on either side are left out. The + * total leaves out the generated projects, which have a subtotal of their + * own, `generated`, when there are any. * * @param {Results} base * @param {Results} head @@ -715,9 +1187,14 @@ export function compareResults(base, head, threshold, metric = "allocated") { } changes.sort((x, y) => Math.abs(y.percent) - Math.abs(x.percent)); - /** @param {Results} results */ - const total = results => - common.reduce((sum, name) => sum + (results[name][metric] ?? 0), 0); + /** + * @param {Results} results + * @param {string[]} names + */ + const total = (results, names) => + names.reduce((sum, name) => sum + (results[name][metric] ?? 0), 0); + const counted = common.filter(name => !isGeneratedProject(name)); + const generated = common.filter(isGeneratedProject); return { metric, @@ -729,7 +1206,11 @@ export function compareResults(base, head, threshold, metric = "allocated") { added: [...headNames].filter(name => !baseNames.has(name)), removed: [...baseNames].filter(name => !headNames.has(name)), compared: common.length, - total: { base: total(base), head: total(head) }, + total: { base: total(base, counted), head: total(head, counted) }, + generated: + generated.length > 0 + ? { base: total(base, generated), head: total(head, generated) } + : undefined, }; } @@ -810,19 +1291,27 @@ function renderSection(comparison, threshold, visibleRows) { removed, compared, total, + generated, } = comparison; const words = metricWords[metric]; - const totalPercent = - total.base === 0 ? 0 : ((total.head - total.base) / total.base) * 100; - // A total that rounds to 0.00% would read as "+0.00%" or "-0.00%". - const totalChange = - Math.abs(totalPercent) < 0.005 ? "±0.00%" : formatPercent(totalPercent); + /** @param {{ base: number, head: number }} sum */ + const sumChange = sum => { + const percent = + sum.base === 0 ? 0 : ((sum.head - sum.base) / sum.base) * 100; + // A total that rounds to 0.00% would read as "+0.00%" or "-0.00%". + const change = + Math.abs(percent) < 0.005 ? "±0.00%" : formatPercent(percent); + return `${formatCount(sum.base)} → ${formatCount(sum.head)} (${change})`; + }; const lines = [ `### ${words.title}`, "", - `In total: ${formatCount(total.base)} → ${formatCount(total.head)} (${totalChange}).`, + `In total: ${sumChange(total)}.`, "", ]; + if (generated) { + lines.push(`Generated projects: ${sumChange(generated)}.`, ""); + } if (regressions.length > 0) { lines.push( `**⚠️ Regression: ${words.more(regressions.length, threshold)}.**`, @@ -886,8 +1375,15 @@ function renderSection(comparison, threshold, visibleRows) { * @param {number} threshold * @param {number} [visibleRows] the largest changes shown in each table; the * others are in a collapsed section under it + * @param {Timings | null} [timings] the wall-time benchmarks, shown after + * the counts; null when they were asked for but not measured */ -export function renderSummary(comparisons, threshold, visibleRows = 10) { +export function renderSummary( + comparisons, + threshold, + visibleRows = 10, + timings = undefined, +) { const shown = comparisons.filter( c => c.compared > 0 || c.added.length > 0 || c.removed.length > 0, ); @@ -895,6 +1391,7 @@ export function renderSummary(comparisons, threshold, visibleRows = 10) { for (const comparison of shown) { lines.push("", ...renderSection(comparison, threshold, visibleRows)); } + if (timings !== undefined) lines.push("", ...renderTimings(timings)); return `${lines.join("\n")}\n`; } @@ -911,8 +1408,12 @@ export function compareAll(base, head, threshold) { ); } -/** @param {string[]} args */ -function main(args) { +/** + * Runs a command and returns its exit status. + * + * @param {string[]} args + */ +export function main(args) { const [command, ...rest] = args; if (command === "prepare" && rest.length === 1) { prepare(path.resolve(rest[0])); @@ -939,19 +1440,43 @@ function main(args) { console.log(`Measured ${Object.keys(results).length} benchmarks.`); return 0; } + if (command === "time" && rest.length >= 3) { + const [baseCheckout, headCheckout, out, ...options] = rest; + let rounds = timingRounds; + for (let i = 0; i < options.length; i += 2) { + if (options[i] === "--rounds") rounds = Number(options[i + 1]); + else throw new Error(`Unknown option ${options[i]}`); + } + if (!Number.isInteger(rounds) || rounds < 1) { + throw new Error("--rounds takes a positive integer."); + } + const timings = timeBuilds( + path.resolve(baseCheckout), + path.resolve(headCheckout), + rounds, + ); + fs.writeFileSync(out, `${JSON.stringify(timings, null, 2)}\n`); + for (const line of renderTimings(timings)) console.log(line); + return 0; + } if (command === "compare" && rest.length >= 2) { const [basePath, headPath, ...options] = rest; let threshold = 1; let summaryPath; + let timingsPath; for (let i = 0; i < options.length; i += 2) { if (options[i] === "--threshold") threshold = Number(options[i + 1]); else if (options[i] === "--summary") summaryPath = options[i + 1]; + else if (options[i] === "--timings") timingsPath = options[i + 1]; else throw new Error(`Unknown option ${options[i]}`); } const base = JSON.parse(fs.readFileSync(basePath, "utf8")); const head = JSON.parse(fs.readFileSync(headPath, "utf8")); + // The wall-time benchmarks are optional: when they didn't run, or + // failed, the report says so, and the exit status is the counts' alone. + const timings = timingsPath ? readTimings(timingsPath) : undefined; const comparisons = compareAll(base, head, threshold); - const summary = renderSummary(comparisons, threshold); + const summary = renderSummary(comparisons, threshold, undefined, timings); console.log(summary); if (summaryPath) fs.writeFileSync(summaryPath, summary); return comparisons.some(c => c.regressions.length > 0) ? 1 : 0; @@ -960,8 +1485,10 @@ function main(args) { "Usage: compilerBenchmarks.js prepare \n" + " compilerBenchmarks.js measure [--instructions]\n" + " [--project ] [--playground]\n" + + " compilerBenchmarks.js time \n" + + " [--rounds ]\n" + " compilerBenchmarks.js compare [--threshold ]\n" + - " [--summary ]", + " [--summary ] [--timings ]", ); return 2; } diff --git a/scripts/compilerBenchmarks.test.js b/scripts/compilerBenchmarks.test.js index 1e9325455c..afcb513f2a 100644 --- a/scripts/compilerBenchmarks.test.js +++ b/scripts/compilerBenchmarks.test.js @@ -1,12 +1,23 @@ import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; import test from "node:test"; import { + chainProject, + combineInputs, compareAll, compareResults, + generatedProjects, + main, + median, parseAllocatedWords, parseInstructions, + readTimings, renderSummary, + runtimeInputs, stressInputs, + timeRatio, } from "./compilerBenchmarks.js"; const stats = `allocated_words: 628461 @@ -348,3 +359,234 @@ test("generates the same stress inputs every time", () => { assert.equal(generate(), generate()); } }); + +test("adds up the generated projects apart from the total", () => { + const base = { + a: { allocated: 1000, inputs: "x" }, + "build/clean": { allocated: 2000, inputs: "x" }, + "build/Namespace/clean": { allocated: 100000, inputs: "x" }, + "build/WideDeps/edit": { allocated: 50000, inputs: "x" }, + }; + const head = { + a: { allocated: 1000, inputs: "x" }, + "build/clean": { allocated: 2000, inputs: "x" }, + "build/Namespace/clean": { allocated: 40000, inputs: "x" }, + "build/WideDeps/edit": { allocated: 50000, inputs: "x" }, + }; + const comparison = compareResults(base, head, 1); + assert.deepEqual(comparison.total, { base: 3000, head: 3000 }); + assert.deepEqual(comparison.generated, { base: 150000, head: 90000 }); + // The generated projects still have rows and thresholds of their own. + assert.deepEqual( + comparison.improvements.map(c => c.name), + ["build/Namespace/clean"], + ); + const summary = renderSummary([comparison], 1); + assert.match(summary, /^In total: 3,000 → 3,000 \(±0\.00%\)\.$/m); + assert.match( + summary, + /^Generated projects: 150,000 → 90,000 \(-40\.00%\)\.$/m, + ); + // Without generated projects, there is no subtotal. + assert.doesNotMatch( + renderSummary([compareResults({ a: base.a }, { a: head.a }, 1)], 1), + /Generated projects/, + ); +}); + +test("marks the project builds' inputs as changed with the runtime", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bench-runtime-")); + try { + const runtime = path.join(dir, "packages", "@rescript", "runtime"); + fs.mkdirSync(runtime, { recursive: true }); + fs.writeFileSync(path.join(runtime, "Stdlib.res"), "let x = 1\n"); + fs.writeFileSync(path.join(runtime, "README.md"), "not a source\n"); + const before = runtimeInputs(dir); + fs.writeFileSync(path.join(runtime, "README.md"), "changed\n"); + assert.equal(runtimeInputs(dir), before); + fs.writeFileSync(path.join(runtime, "Stdlib.res"), "let x = 2\n"); + const after = runtimeInputs(dir); + assert.notEqual(after, before); + assert.notEqual( + combineInputs("project", after), + combineInputs("project", before), + ); + assert.equal( + combineInputs("project", before), + combineInputs("project", before), + ); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("generates the same projects every time", () => { + for (const generate of [...Object.values(generatedProjects), chainProject]) { + const project = generate(); + assert.deepEqual(generate(), project); + // The edit changes one of the project's sources. + assert.ok(project.edit.endsWith(".res")); + assert.ok(project.edit in project.files); + const config = JSON.parse(project.files["rescript.json"]); + // JavaScript goes to lib/, which a clean build removes. + assert.equal(config["package-specs"]["in-source"], false); + } +}); + +test("namespaces only the namespace benchmark's project", () => { + const { files } = generatedProjects.Namespace(); + assert.equal(JSON.parse(files["rescript.json"]).namespace, true); + // Modules use others through the namespace's map. + assert.match(files["src/Mod10.res"], /^let value = Mod4\.value \+ 10$/m); + const wide = generatedProjects.WideDeps(); + assert.equal(JSON.parse(wide.files["rescript.json"]).namespace, undefined); +}); + +test("chains the modules of the wall-time benchmark's project", () => { + const { files } = chainProject(); + assert.match(files["src/Chain1.res"], /prev: Chain0\.t/); + assert.match(files["src/Chain149.res"], /prev: Chain148\.t/); + assert.equal(files["src/Chain150.res"], undefined); + assert.match(files["src/Side7.res"], /Chain7\.f0/); +}); + +test("takes the median of an odd and an even number of values", () => { + assert.equal(median([3, 1, 2]), 2); + assert.equal(median([4, 1, 3, 2]), 2.5); + // The values are left as they were. + const values = [2, 1]; + median(values); + assert.deepEqual(values, [2, 1]); +}); + +test("pairs the rounds of the times to compare", () => { + // The machine gets slower during the run, but the head is always 10% + // faster in the same round. + const base = [100, 120, 140, 160, 180, 200, 220, 240, 260, 280]; + const head = base.map(time => time * 0.9); + const { ratio, low, high, rounds } = timeRatio(base, head); + assert.equal(rounds, 10); + assert.ok(Math.abs(ratio - 0.9) < 1e-9); + assert.ok(Math.abs(low - 0.9) < 1e-9); + assert.ok(Math.abs(high - 0.9) < 1e-9); +}); + +test("gives a confidence interval around the median ratio", () => { + const base = [100, 100, 100, 100, 100, 100, 100, 100, 100, 100]; + const head = [85, 95, 90, 88, 92, 99, 80, 91, 89, 93]; + const result = timeRatio(base, head); + assert.equal(result.ratio, 0.905); + assert.ok(result.low <= result.ratio && result.ratio <= result.high); + assert.ok(result.low >= 0.8 && result.high <= 0.99); + // The resamples are drawn the same way every time. + assert.deepEqual(timeRatio(base, head), result); + assert.throws(() => timeRatio([1, 2], [1])); + assert.throws(() => timeRatio([], [])); +}); + +test("reports the wall time without counting it as a change", () => { + const results = { a: { allocated: 1000, inputs: "x" } }; + const timings = { + "time/Chain/clean": { base: [100, 100, 100], head: [150, 160, 155] }, + }; + const summary = renderSummary( + compareAll(results, results, 1), + 1, + undefined, + timings, + ); + // 55% slower, and still no regression. + assert.match(summary, /^## ✅ Compiler benchmarks$/m); + assert.match(summary, /^### Wall time \(indicative only\)$/m); + assert.match( + summary, + /^\| `time\/Chain\/clean` \(3 rounds\) \| 100 ms \| 155 ms \| 1\.550 \| 1\.500 to 1\.600 \|$/m, + ); + assert.doesNotMatch( + renderSummary(compareAll(results, results, 1), 1), + /Wall time/, + ); +}); + +test("leaves out wall times that can't be compared", () => { + const results = { a: { allocated: 1000, inputs: "x" } }; + const summary = renderSummary(compareAll(results, results, 1), 1, undefined, { + missing: /** @type {any} */ (null), + uneven: { base: [100, 100], head: [100] }, + empty: { base: [], head: [] }, + strings: { base: /** @type {any} */ (["100"]), head: [100] }, + "time/Chain/clean": { base: [100], head: [90] }, + }); + assert.match(summary, /^## ✅ Compiler benchmarks$/m); + for (const name of ["missing", "uneven", "empty", "strings"]) { + assert.match( + summary, + new RegExp(`^\\| \`${name}\` \\| \\| \\| not comparable \\| \\|$`, "m"), + ); + } + assert.match(summary, /\| `time\/Chain\/clean` \(1 round\) \|/); +}); + +test("says when the wall times weren't measured", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bench-timings-")); + try { + const write = (/** @type {string} */ name, /** @type {string} */ text) => { + const file = path.join(dir, name); + fs.writeFileSync(file, text); + return file; + }; + const missing = path.join(dir, "missing.json"); + const invalid = write("invalid.json", "{ not json"); + const empty = write("null.json", "null\n"); + for (const file of [missing, invalid, empty]) { + assert.equal(readTimings(file), null); + } + const valid = write( + "valid.json", + JSON.stringify({ "time/Chain/clean": { base: [100], head: [90] } }), + ); + assert.deepEqual(readTimings(valid), { + "time/Chain/clean": { base: [100], head: [90] }, + }); + + const results = { a: { allocated: 1000, inputs: "x" } }; + const notMeasured = renderSummary( + compareAll(results, results, 1), + 1, + undefined, + null, + ); + assert.match( + notMeasured, + /^### Wall time \(indicative only\)\n\nNot measured in this run\.$/m, + ); + assert.match(notMeasured, /^## ✅ Compiler benchmarks$/m); + + // The exit status is the counts' alone, whatever the times' file holds. + const base = write("base.json", JSON.stringify(results)); + const grown = write( + "head.json", + JSON.stringify({ a: { allocated: 2000, inputs: "x" } }), + ); + const summary = path.join(dir, "summary.md"); + const log = console.log; + console.log = () => {}; + try { + for (const file of [missing, invalid, empty]) { + const args = ["--summary", summary, "--timings", file]; + assert.equal(main(["compare", base, base, ...args]), 0); + assert.match( + fs.readFileSync(summary, "utf8"), + /Not measured in this run\./, + ); + assert.equal(main(["compare", base, grown, ...args]), 1); + } + assert.equal(main(["compare", base, base, "--summary", summary]), 0); + assert.doesNotMatch(fs.readFileSync(summary, "utf8"), /Wall time/); + } finally { + console.log = log; + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +});