diff --git a/.github/workflows/benchmark-report.yml b/.github/workflows/benchmark-report.yml new file mode 100644 index 0000000..c093a68 --- /dev/null +++ b/.github/workflows/benchmark-report.yml @@ -0,0 +1,86 @@ +name: Benchmark report + +on: + workflow_run: + workflows: [Benchmark] + types: [completed] + +permissions: + contents: read + actions: read + issues: write + pull-requests: read + +jobs: + report: + concurrency: + group: benchmark-report + cancel-in-progress: false + queue: max + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Download benchmark reports + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: benchmark-report-node* + path: ${{ runner.temp }}/benchmark-report + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Publish benchmark report + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + ARTIFACT_DIRECTORY: ${{ runner.temp }}/benchmark-report + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const path = require('node:path') + const { pathToFileURL } = require('node:url') + const { readReport, formatReports } = await import( + pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'benchmark/report.mjs')).href) + const reports = [22, 24].map((major) => { + const report = readReport(path.join(process.env.ARTIFACT_DIRECTORY, + `benchmark-report-node${major}`, 'report.json')) + if (!report.nodeVersion.startsWith(`v${major}.`)) throw new Error('Unexpected Node version') + return report + }) + const report = reports[0] + if (reports.some((entry) => entry.headSha !== context.payload.workflow_run.head_sha || + entry.headSha !== report.headSha || entry.baseSha !== report.baseSha || + entry.mergeSha !== report.mergeSha || + entry.prNumber !== report.prNumber)) throw new Error('Mismatched benchmark sources') + const pull = await github.rest.pulls.get({ + owner: context.repo.owner, repo: context.repo.repo, pull_number: report.prNumber + }) + if (pull.data.state !== 'open' || pull.data.head.sha !== report.headSha || + pull.data.merge_commit_sha !== report.mergeSha) { + core.info('The pull request is closed or its revisions have changed.') + return + } + const merge = await github.rest.repos.getCommit({ + owner: context.repo.owner, repo: context.repo.repo, ref: report.mergeSha + }) + if (merge.data.parents[0]?.sha !== report.baseSha || merge.data.parents[1]?.sha !== report.headSha) { + throw new Error('Mismatched benchmark merge parents') + } + const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.WORKFLOW_RUN_ID}` + const body = `${formatReports(reports)}\n\n[Workflow run](${runUrl})` + const marker = '' + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, repo: context.repo.repo, issue_number: report.prNumber + }) + const existing = comments.find((comment) => + comment.user?.login === 'github-actions[bot]' && comment.body?.includes(marker)) + const target = { owner: context.repo.owner, repo: context.repo.repo, body } + if (existing === undefined) { + await github.rest.issues.createComment({ ...target, issue_number: report.prNumber }) + } else { + await github.rest.issues.updateComment({ ...target, comment_id: existing.id }) + } diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..4482cdb --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,50 @@ +name: Benchmark + +on: + pull_request: + types: [opened, synchronize, reopened, edited] + +permissions: + contents: read + +jobs: + benchmark: + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + node: [22, 24] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 2 + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ matrix.node }} + - run: npm install --ignore-scripts + - name: Check out the base revision + id: base + run: | + BASE_SHA=$(git rev-parse HEAD^1) + git fetch --no-tags --depth=1 origin "$BASE_SHA" + git worktree add --detach "$RUNNER_TEMP/base" "$BASE_SHA" + npm install --ignore-scripts --prefix "$RUNNER_TEMP/base" + printf 'sha=%s\n' "$BASE_SHA" >> "$GITHUB_OUTPUT" + - name: Measure paired revisions + env: + BASE_SHA: ${{ steps.base.outputs.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + MERGE_SHA: ${{ github.sha }} + PR_NUMBER: ${{ github.event.number }} + run: | + BENCHMARK_CPU=$(awk '/^Cpus_allowed_list:/ { split($2, cpus, /[-,]/); print cpus[1] }' /proc/self/status) + taskset --cpu-list "$BENCHMARK_CPU" node benchmark/run.mjs \ + --base-directory "$RUNNER_TEMP/base" \ + --output "$RUNNER_TEMP/report.json" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-report-node${{ matrix.node }} + path: ${{ runner.temp }}/report.json + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e81ced..d5ea434 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,7 @@ jobs: node-version: 'lts/*' - run: npm install - run: npm run lint + - run: npm run test:benchmark build: runs-on: ubuntu-latest diff --git a/.npmignore b/.npmignore index 7a24988..a83a749 100644 --- a/.npmignore +++ b/.npmignore @@ -9,3 +9,4 @@ coverage release-please-config.json tsconfig.json test +benchmark diff --git a/README.md b/README.md index eae9f47..5991a0f 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,11 @@ fs.readFileSync('file.txt') node --import=./instrument.mjs ./my-app.mjs ``` +## Development benchmarks + +See [benchmark/README.md](benchmark/README.md) for the PR benchmark method and +how to check measurement noise locally. + ## Synchronous loader hooks On Node.js versions that support diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..3c10c29 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,50 @@ +# Pull request benchmarks + +Run `npm run benchmark -- --help` for the command-line options. Without a base +directory, the command compares the current revision with itself to check noise. +Both revisions use the same fixture and dependency installation. +CI compares the base with GitHub's merged PR revision. This includes current +base changes when the contributor's branch is older. Reports retain the base, +PR head, and measured merge revisions so the comparison can be reproduced. + +Each sample starts a new Node.js process. The fixture records elapsed time when +its imports and behavior checks complete. Process teardown is outside that +measurement. CPU time is retained as a diagnostic and includes loader-worker +CPU use. It does not include time spent waiting for a worker response. + +The small graph exposes fixed loader startup costs. The dependency graph uses +the pinned date-fns development dependency and a CommonJS module. It exercises +package resolution, many ESM exports, and CommonJS interoperability. These are +loader workloads, not application request-throughput measurements. + +The driver warms the filesystem cache, starts each base/head pair consecutively, +and alternates their order. It disables compilation caching. Independent trials +retain every sample, including slow samples. The reported change is the mean of +the trial estimates. Each trial estimate is the median of its paired head/base +duration ratios. This preserves the pairing when scheduling delays change over time. +CI pins both revisions and their loader workers to the same allowed CPU. This +controls CPU migration and measures startup under a single-core constraint. +Noisy scenarios receive one bounded extension of the same trials. The original +observations remain in the result. An explicit sample count disables this +extension. The spread is descriptive; the report does not claim a formal +significance test after adaptive sampling. + +The median measures typical startup and limits the effect of isolated scheduling +delays. It can miss a regression affecting only occasional starts. The report +also shows raw means, standard deviations, and p95 durations to expose those costs. +Trial CV is the standard deviation of the paired median ratios divided by their mean. +The report labels a row unstable when its trial CV exceeds the implemented +stability threshold. Base and head columns also show the raw startup standard +deviation. A stable ratio does not mean that individual startup samples have +low variation. Use repeated self-comparisons to check reproducibility on the +machine that will run the benchmark. + +The baseline is the test merge's first parent. Pull request payloads can retain +an older base SHA after the target branch advances. The publisher verifies both +merge parents against the measured baseline and PR head. + +PR execution produces artifacts with read-only permissions. A separate workflow +loads reporting code from the default branch, validates the artifact's source +and samples, and computes its own statistics before updating the bot comment. +The PR controls benchmark execution. Artifact validation does not attest to the +accuracy of its timings. diff --git a/benchmark/fixture/a.mjs b/benchmark/fixture/a.mjs new file mode 100644 index 0000000..1d772ee --- /dev/null +++ b/benchmark/fixture/a.mjs @@ -0,0 +1 @@ +export const value = 1 diff --git a/benchmark/fixture/app.mjs b/benchmark/fixture/app.mjs new file mode 100644 index 0000000..52009c8 --- /dev/null +++ b/benchmark/fixture/app.mjs @@ -0,0 +1,66 @@ +import assert from 'node:assert/strict' +import * as module from 'node:module' +import { resolve } from 'node:path' +import { performance } from 'node:perf_hooks' +import { fileURLToPath, pathToFileURL } from 'node:url' + +const root = process.env.IITM_BENCHMARK_PACKAGE_ROOT +const mode = process.env.IITM_BENCHMARK_MODE +const workload = process.env.IITM_BENCHMARK_WORKLOAD +assert.ok(root && mode && workload, 'Benchmark environment is required') +const require = module.createRequire(import.meta.url) +const active = mode.endsWith('-active') || mode.endsWith('-selective') +const target = workload === 'small' ? fileURLToPath(new URL('a.mjs', import.meta.url)) : 'date-fns' +let hooks = 0 +let calls = 0 + +if (active) { + const { Hook } = require(resolve(root, 'index.js')) + Hook([target], + /** @param {{ value: number, addDays: typeof import('date-fns').addDays }} exports */ + (exports) => { + hooks++ + if (workload === 'small') { + exports.value++ + } else { + const addDays = exports.addDays + /** + * @param {Date} date Date to advance. + * @param {number} amount Days to add. + */ + exports.addDays = function (date, amount) { + calls++ + return addDays(date, amount) + } + } + }) +} + +if (mode !== 'none') { + const options = mode.endsWith('-excluded') + ? { include: [] } + : mode.endsWith('-selective') + ? { include: [workload === 'small' ? pathToFileURL(target).href : target] } + : undefined + if (mode.startsWith('sync-')) { + const { register } = await import(pathToFileURL(resolve(root, 'register-hooks.mjs')).href) + register(options) + } else if (module.register) { + module.register(pathToFileURL(resolve(root, 'hook.mjs')), { data: options }) + } +} + +if (workload === 'small') { + const { sum } = await import('./small.mjs') + assert.equal(sum, active ? 88 : 87) +} else { + const { addDays, isValid, parseISO } = await import('date-fns') + const { default: Clock } = await import('./commonjs.cjs') + const date = parseISO('2026-09-09T00:00:00Z') + assert.equal(isValid(date), true) + assert.equal(new Clock().copy(addDays(date, 1)).getUTCDate(), 10) + assert.equal(calls, active ? 1 : 0) +} +assert.equal(hooks, active ? 1 : 0) +const cpu = process.cpuUsage() +process.stdout.write(`${JSON.stringify({ wallMs: performance.now(), cpuMs: (cpu.user + cpu.system) / 1000 })}\n`) diff --git a/benchmark/fixture/b.mjs b/benchmark/fixture/b.mjs new file mode 100644 index 0000000..3aee44a --- /dev/null +++ b/benchmark/fixture/b.mjs @@ -0,0 +1 @@ +export const value = 2 diff --git a/benchmark/fixture/c.mjs b/benchmark/fixture/c.mjs new file mode 100644 index 0000000..e2befb1 --- /dev/null +++ b/benchmark/fixture/c.mjs @@ -0,0 +1 @@ +export const value = 3 diff --git a/benchmark/fixture/commonjs.cjs b/benchmark/fixture/commonjs.cjs new file mode 100644 index 0000000..47c6c72 --- /dev/null +++ b/benchmark/fixture/commonjs.cjs @@ -0,0 +1,8 @@ +const { EventEmitter } = require('node:events') + +module.exports = class Clock extends EventEmitter { + /** @param {Date} date Date to copy. */ + copy (date) { + return new Date(date.getTime()) + } +} diff --git a/benchmark/fixture/d.mjs b/benchmark/fixture/d.mjs new file mode 100644 index 0000000..7665884 --- /dev/null +++ b/benchmark/fixture/d.mjs @@ -0,0 +1 @@ +export const value = 5 diff --git a/benchmark/fixture/e.mjs b/benchmark/fixture/e.mjs new file mode 100644 index 0000000..7d03b11 --- /dev/null +++ b/benchmark/fixture/e.mjs @@ -0,0 +1 @@ +export const value = 8 diff --git a/benchmark/fixture/f.mjs b/benchmark/fixture/f.mjs new file mode 100644 index 0000000..485ba0f --- /dev/null +++ b/benchmark/fixture/f.mjs @@ -0,0 +1 @@ +export const value = 13 diff --git a/benchmark/fixture/g.mjs b/benchmark/fixture/g.mjs new file mode 100644 index 0000000..12f6bbc --- /dev/null +++ b/benchmark/fixture/g.mjs @@ -0,0 +1 @@ +export const value = 21 diff --git a/benchmark/fixture/h.mjs b/benchmark/fixture/h.mjs new file mode 100644 index 0000000..dd52996 --- /dev/null +++ b/benchmark/fixture/h.mjs @@ -0,0 +1 @@ +export const value = 34 diff --git a/benchmark/fixture/small.mjs b/benchmark/fixture/small.mjs new file mode 100644 index 0000000..719b965 --- /dev/null +++ b/benchmark/fixture/small.mjs @@ -0,0 +1,10 @@ +import { value as a } from './a.mjs' +import { value as b } from './b.mjs' +import { value as c } from './c.mjs' +import { value as d } from './d.mjs' +import { value as e } from './e.mjs' +import { value as f } from './f.mjs' +import { value as g } from './g.mjs' +import { value as h } from './h.mjs' + +export const sum = a + b + c + d + e + f + g + h diff --git a/benchmark/report.mjs b/benchmark/report.mjs new file mode 100644 index 0000000..3668701 --- /dev/null +++ b/benchmark/report.mjs @@ -0,0 +1,89 @@ +import { readFileSync, statSync } from 'node:fs' + +import { getScenarios } from './scenarios.mjs' +import { compare, sampling } from './statistics.mjs' + +const platforms = new Set(['linux', 'darwin', 'win32']) +const architectures = new Set() +architectures.add('x64') +architectures.add('arm64') + +/** @typedef {ReturnType[number] & { trials: import('./statistics.mjs').Pair[][] }} Benchmark */ +/** + * @typedef {{ formatVersion: 2, workloadVersion: 1, prNumber: number, baseSha: string, headSha: string, mergeSha: string, + * nodeVersion: string, platform: string, arch: string, selective: boolean, synchronous: boolean, + * benchmarks: Benchmark[] }} Report + */ + +/** @param {string} filename Untrusted benchmark artifact. */ +export function readReport (filename) { + if (statSync(filename).size > 8 * 1024 * 1024) throw new Error('Benchmark artifact is too large') + const report = /** @type {Report} */ (JSON.parse(readFileSync(filename, 'utf8'))) + if (report?.formatVersion !== 2 || report.workloadVersion !== 1 || + !Number.isInteger(report.prNumber) || report.prNumber < 1 || + typeof report.baseSha !== 'string' || !/^[a-f0-9]{40}$/.test(report.baseSha) || + typeof report.headSha !== 'string' || !/^[a-f0-9]{40}$/.test(report.headSha) || + typeof report.mergeSha !== 'string' || !/^[a-f0-9]{40}$/.test(report.mergeSha) || + typeof report.nodeVersion !== 'string' || !/^v\d+\.\d+\.\d+$/.test(report.nodeVersion) || + !platforms.has(report.platform) || !architectures.has(report.arch) || + typeof report.selective !== 'boolean' || typeof report.synchronous !== 'boolean') { + throw new Error('Invalid benchmark metadata') + } + const scenarios = getScenarios(report.selective, report.synchronous) + if (!Array.isArray(report.benchmarks) || report.benchmarks.length !== scenarios.length) { + throw new Error('Invalid benchmark scenarios') + } + for (let index = 0; index < scenarios.length; index++) { + const benchmark = report.benchmarks[index] + if (benchmark?.name !== scenarios[index].name || !Array.isArray(benchmark.trials) || + benchmark.trials.length < sampling.minTrials || benchmark.trials.length > sampling.maxTrials) { + throw new Error('Invalid benchmark trials') + } + const runs = benchmark.trials[0]?.length + if (!Number.isInteger(runs) || runs < sampling.minRuns || runs > sampling.maxRuns) { + throw new Error('Invalid benchmark runs') + } + for (const trial of benchmark.trials) { + if (!Array.isArray(trial) || trial.length !== runs) throw new Error('Inconsistent benchmark runs') + for (const pair of trial) { + for (const sample of [pair?.base, pair?.head]) { + if (!Number.isFinite(sample?.wallMs) || sample.wallMs <= 0 || sample.wallMs > 30000 || + !Number.isFinite(sample?.cpuMs) || sample.cpuMs <= 0 || sample.cpuMs > 1000000) { + throw new Error('Invalid benchmark sample') + } + } + } + } + } + return report +} + +/** @param {ReturnType[]} reports Validated reports from the same workflow run. */ +export function formatReports (reports) { + const lines = [ + '', '## Benchmark report', '', + 'Fresh-process time to fixture readiness; warm filesystem cache, compilation cache disabled.', + 'Changes compare typical startup: the mean of trial median paired head/base ratios.', + 'CV is the relative standard deviation of trial ratios. Raw means, SD, and p95 expose slow starts.', + 'Raw samples remain in the workflow artifacts. Unstable rows do not support a performance conclusion.' + ] + for (const report of reports) { + lines.push('', `### Node ${report.nodeVersion} (${report.platform}/${report.arch})`, '', + '| Scenario | Base median / p95 | PR median / p95 | Paired change | Trial CV | Raw mean ± SD (base; PR) |', + '| --- | ---: | ---: | ---: | ---: | ---: |') + for (const benchmark of report.benchmarks) { + const result = compare(benchmark.trials) + const change = result.stable + ? `${result.changePercent >= 0 ? '+' : ''}${result.changePercent.toFixed(2)}%` + : 'Unstable' + lines.push(`| ${benchmark.name} | ${result.base.median.toFixed(2)} / ${result.base.p95.toFixed(2)} ms | ` + + `${result.head.median.toFixed(2)} / ${result.head.p95.toFixed(2)} ms | ${change} | ` + + `${result.ratioCvPercent.toFixed(2)}% | ` + + `${result.base.mean.toFixed(2)} ± ${result.base.stddev.toFixed(2)}; ` + + `${result.head.mean.toFixed(2)} ± ${result.head.stddev.toFixed(2)} ms |`) + } + } + lines.push('', `Base: \`${reports[0].baseSha.slice(0, 7)}\` · PR head: \`${reports[0].headSha.slice(0, 7)}\` · ` + + `Measured merge: \`${reports[0].mergeSha.slice(0, 7)}\``) + return lines.join('\n') +} diff --git a/benchmark/run.mjs b/benchmark/run.mjs new file mode 100644 index 0000000..3840a99 --- /dev/null +++ b/benchmark/run.mjs @@ -0,0 +1,161 @@ +import { spawnSync } from 'node:child_process' +import { writeFileSync } from 'node:fs' +import * as module from 'node:module' +import { resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { supportsSyncHooks } from '../supports-sync-hooks.mjs' +import { getScenarios } from './scenarios.mjs' +import { compare, sampling } from './statistics.mjs' + +/** @typedef {import('./statistics.mjs').Sample} Sample */ +/** @typedef {import('./statistics.mjs').Pair} Pair */ +/** @typedef {ReturnType[number]} Scenario */ + +if (process.argv[2] === '--help' && process.argv.length === 3) { + process.stdout.write('Usage: npm run benchmark -- [--package-directory DIR] [--base-directory DIR] ' + + '[--output FILE] [--runs PAIRS_PER_TRIAL] [--trials COUNT] [--scenario NAME]\n' + + `Default: self-comparison, ${sampling.trials} trials of ${sampling.runs} pairs; ` + + `noisy scenarios expand to ${sampling.maxRuns} pairs per trial.\n` + + 'Explicit --runs fixes the sample count. See benchmark/README.md.\n') + process.exit(0) +} +const fixture = fileURLToPath(new URL('fixture/app.mjs', import.meta.url)) +const legacyWarning = new RegExp('^\\(node:\\d+\\) ExperimentalWarning: Custom ESM Loaders ' + + 'is an experimental feature and might change at any time\\n\\(Use `[^\\n]+ --trace-warnings \\.\\.\\.` ' + + 'to show where the warning was created\\)\\n', 'gm') +const options = parseOptions(process.argv.slice(2)) +const roots = [options.baseDirectory ?? options.packageDirectory, options.packageDirectory] +const selective = typeof module.register === 'function' +const synchronous = supportsSyncHooks() +const benchmarks = [] + +const scenarios = getScenarios(selective, synchronous) +if (options.scenario !== undefined && !scenarios.some( + /** @param {Scenario} scenario */ + scenario => scenario.name === options.scenario)) { + throw new Error('Unknown or unsupported benchmark scenario') +} +for (const scenario of scenarios) { + if (options.scenario !== undefined && scenario.name !== options.scenario) continue + for (let warmup = 0; warmup < 5; warmup++) { + measure(roots[0], scenario) + measure(roots[1], scenario) + } + const trials = Array.from({ length: options.trials }, () => []) + collect(trials, scenario, options.runs ?? sampling.runs) + if (options.runs === undefined && options.trials >= sampling.trials && !compare(trials).stable) { + process.stderr.write(`${scenario.name}: increasing samples for noisy measurements\n`) + collect(trials, scenario, sampling.maxRuns - sampling.runs) + } + const summary = compare(trials) + benchmarks.push({ ...scenario, trials, summary }) + process.stderr.write(`${scenario.name}: ${summary.changePercent.toFixed(2)}%; ` + + `paired trial CV ${summary.ratioCvPercent.toFixed(2)}%${summary.stable ? '' : ' (unstable)'}\n`) +} + +const report = { + formatVersion: 2, + workloadVersion: 1, + nodeVersion: process.version, + platform: process.platform, + arch: process.arch, + selective, + synchronous, + prNumber: process.env.PR_NUMBER === undefined ? 0 : Number(process.env.PR_NUMBER), + baseSha: process.env.BASE_SHA, + headSha: process.env.HEAD_SHA, + mergeSha: process.env.MERGE_SHA, + benchmarks +} +const output = `${JSON.stringify(report)}\n` +if (options.output === undefined) process.stdout.write(output) +else writeFileSync(options.output, output) + +/** @param {string[]} argumentsList Command-line options. */ +function parseOptions (argumentsList) { + const options = { + packageDirectory: process.cwd(), + baseDirectory: undefined, + output: undefined, + scenario: undefined, + runs: undefined, + trials: sampling.trials + } + const names = new Map([ + ['--package-directory', 'packageDirectory'], ['--base-directory', 'baseDirectory'], ['--output', 'output'], + ['--runs', 'runs'], ['--trials', 'trials'], ['--scenario', 'scenario'] + ]) + for (let index = 0; index < argumentsList.length; index += 2) { + const name = names.get(argumentsList[index]) + const value = argumentsList[index + 1] + if (name === undefined || value === undefined) throw new Error('Unknown or incomplete benchmark option') + options[name] = name === 'runs' || name === 'trials' ? Number(value) : name === 'scenario' ? value : resolve(value) + } + if ((options.runs !== undefined && (!Number.isInteger(options.runs) || + options.runs < sampling.minRuns || options.runs > sampling.maxRuns)) || + !Number.isInteger(options.trials) || options.trials < sampling.minTrials || options.trials > sampling.maxTrials) { + throw new Error(`Expected ${sampling.minRuns}–${sampling.maxRuns} --runs and ` + + `${sampling.minTrials}–${sampling.maxTrials} --trials`) + } + return options +} + +/** + * @param {Pair[][]} trials Existing trials to extend without discarding observations. + * @param {{ mode: string, workload: string }} scenario Public loader invocation. + * @param {number} runs Additional pairs per trial. + */ +function collect (trials, scenario, runs) { + for (let trial = 0; trial < trials.length; trial++) { + const pairs = trials[trial] + const first = pairs.length + for (let pair = first; pair < first + runs; pair++) { + const samples = [] + for (let offset = 0; offset < 2; offset++) { + const index = (trial + pair + offset) % 2 + samples[index] = measure(roots[index], scenario) + } + pairs.push({ base: samples[0], head: samples[1] }) + } + } +} + +/** + * @param {string} packageDirectory Revision to measure. + * @param {{ mode: string, workload: string }} scenario Public loader invocation. + * @returns {Sample} + */ +function measure (packageDirectory, scenario) { + const argumentsList = !selective && scenario.mode.startsWith('async-') + ? ['--loader', resolve(packageDirectory, 'hook.mjs'), fixture] + : [fixture] + if (process.allowedNodeEnvironmentFlags.has('--disable-warning')) { + argumentsList.unshift('--disable-warning=ExperimentalWarning') + } + const environment = { + ...process.env, + NODE_OPTIONS: '', + NODE_COMPILE_CACHE: '', + NODE_DISABLE_COMPILE_CACHE: '1', + NODE_DEBUG: '', + NODE_DEBUG_NATIVE: '', + TZ: 'UTC', + IITM_BENCHMARK_PACKAGE_ROOT: packageDirectory, + IITM_BENCHMARK_MODE: scenario.mode, + IITM_BENCHMARK_WORKLOAD: scenario.workload + } + delete environment.NODE_NO_WARNINGS + delete environment.NODE_REDIRECT_WARNINGS + const result = spawnSync(process.execPath, argumentsList, { + encoding: 'utf8', + timeout: 30000, + env: environment + }) + if (result.error || result.status !== 0) { + throw new Error(`Benchmark process failed: ${result.error?.message ?? result.stderr ?? result.stdout}`) + } + const warnings = result.stderr.replace(legacyWarning, '') + if (warnings) throw new Error(`Benchmark emitted warnings: ${warnings}`) + return JSON.parse(result.stdout) +} diff --git a/benchmark/scenarios.mjs b/benchmark/scenarios.mjs new file mode 100644 index 0000000..98a03ac --- /dev/null +++ b/benchmark/scenarios.mjs @@ -0,0 +1,14 @@ +/** + * @param {boolean} selective Whether asynchronous registration is available. + * @param {boolean} synchronous Whether synchronous hooks are supported. + */ +export function getScenarios (selective, synchronous) { + const modes = ['none', 'async-inactive', 'async-active'] + if (selective) modes.push('async-selective', 'async-excluded') + if (synchronous) modes.push('sync-inactive', 'sync-active', 'sync-selective', 'sync-excluded') + const scenarios = [] + for (const workload of ['small', 'dependency']) { + for (const mode of modes) scenarios.push({ name: `${workload}/${mode}`, workload, mode }) + } + return scenarios +} diff --git a/benchmark/statistics.mjs b/benchmark/statistics.mjs new file mode 100644 index 0000000..223ef4c --- /dev/null +++ b/benchmark/statistics.mjs @@ -0,0 +1,66 @@ +/** @typedef {{ wallMs: number, cpuMs: number }} Sample */ +/** @typedef {{ base: Sample, head: Sample }} Pair */ + +export const sampling = Object.freeze({ + minTrials: 2, trials: 8, maxTrials: 20, minRuns: 2, runs: 20, maxRuns: 100, cvPercent: 2 +}) + +/** @param {number[]} values Independent observations. */ +export function summarize (values) { + const mean = values.reduce( + /** + * @param {number} sum + * @param {number} value + */ + (sum, value) => sum + value, 0) / values.length + const variance = values.reduce( + /** + * @param {number} sum + * @param {number} value + */ + (sum, value) => sum + (value - mean) ** 2, 0) / (values.length - 1) + const sorted = [...values].sort( + /** + * @param {number} left + * @param {number} right + */ + (left, right) => left - right) + const middle = Math.floor(sorted.length / 2) + const median = sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle] + return { + mean, + median, + p95: sorted[Math.ceil(sorted.length * 0.95) - 1], + stddev: Math.sqrt(variance), + cvPercent: 100 * Math.sqrt(variance) / mean + } +} + +/** @param {Pair[][]} trials Paired observations grouped by trial. */ +export function compare (trials) { + const ratios = [] + const base = [] + const head = [] + for (const trial of trials) { + const trialRatios = [] + for (const pair of trial) { + trialRatios.push(pair.head.wallMs / pair.base.wallMs) + base.push(pair.base.wallMs) + head.push(pair.head.wallMs) + } + ratios.push(summarize(trialRatios).median) + } + const ratio = summarize(ratios) + return { + base: summarize(base), + head: summarize(head), + changePercent: 100 * (ratio.mean - 1), + ratioCvPercent: ratio.cvPercent, + changeStddevPercent: 100 * ratio.stddev, + standardErrorPercent: 100 * ratio.stddev / Math.sqrt(trials.length), + stable: trials.length >= sampling.trials && trials.every( + /** @param {Pair[]} trial */ + trial => trial.length >= sampling.runs) && + ratio.cvPercent <= sampling.cvPercent + } +} diff --git a/package.json b/package.json index 87ea3ca..182cb85 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,8 @@ "test:e2e": "node test/check-exports/test.mjs && node test/integration-tests/turbopack-dev.mjs && node test/integration-tests/turbopack-build-start.mjs", "test:ts": "c8 --reporter lcov imhotap --files test/typescript/*.test.mts", "coverage": "c8 --reporter html imhotap --files test/{hook,low-level,other,get-esm-exports,register}/* && echo \"\nNow open coverage/index.html\n\"", + "benchmark": "node benchmark/run.mjs", + "test:benchmark": "node test/benchmark.mjs", "lint": "eslint .", "lint:fix": "eslint . --fix" }, @@ -44,7 +46,7 @@ "@react-email/components": "^0.0.19", "@types/node": "^18.0.6", "c8": "^7.14.0", - "date-fns": "^3.6.0", + "date-fns": "3.6.0", "eslint": "^8.57.1", "eslint-config-standard": "^17.1.0", "eslint-plugin-import": "^2.32.0", diff --git a/test/benchmark.mjs b/test/benchmark.mjs new file mode 100644 index 0000000..9829a72 --- /dev/null +++ b/test/benchmark.mjs @@ -0,0 +1,304 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { ESLint } from 'eslint' + +import { readReport, formatReports } from '../benchmark/report.mjs' +import { getScenarios } from '../benchmark/scenarios.mjs' +import { compare, summarize } from '../benchmark/statistics.mjs' + +/** @typedef {ReturnType[number]} Scenario */ +/** @typedef {import('../benchmark/report.mjs').Benchmark} Benchmark */ +/** @typedef {import('../benchmark/statistics.mjs').Pair} Pair */ +/** + * @typedef {{ owner: string, repo: string, body: string, issue_number?: number, comment_id?: number }} CommentRequest + */ +/** @typedef {{ operation: 'create' | 'update', input: CommentRequest }} CommentCall */ + +const root = fileURLToPath(new URL('..', import.meta.url)) +const require = createRequire(import.meta.url) +const directory = mkdtempSync(join(tmpdir(), 'iitm-benchmark-test-')) +const filename = join(directory, 'report.json') +const pair = { base: { wallMs: 100, cpuMs: 50 }, head: { wallMs: 110, cpuMs: 60 } } +const trials = Array.from({ length: 8 }, () => Array.from({ length: 20 }, () => structuredClone(pair))) +const original = { + formatVersion: 2, + workloadVersion: 1, + prNumber: 293, + baseSha: 'a'.repeat(40), + headSha: 'b'.repeat(40), + mergeSha: 'c'.repeat(40), + nodeVersion: 'v24.20.0', + platform: 'linux', + arch: 'x64', + selective: true, + synchronous: true, + benchmarks: getScenarios(true, true).map( + /** @param {Scenario} scenario */ + scenario => ({ ...scenario, trials: structuredClone(trials) })) +} + +/** @param {object} report Artifact to publish to the parser. */ +function parse (report) { + writeFileSync(filename, JSON.stringify(report)) + return readReport(filename) +} + +/** @param {string[]} argumentsList CLI options. */ +function run (argumentsList) { + return spawnSync(process.execPath, [join(root, 'benchmark/run.mjs'), ...argumentsList], { + cwd: root, encoding: 'utf8' + }) +} + +/** + * @param {{ head?: string, base?: string, merge?: string, state?: string, existing?: boolean, + * payloadBase?: string, parents?: string[] }} options + * @param {CommentCall[]} [calls] + */ +async function publish (options = {}, calls = []) { + const workflow = readFileSync(join(root, '.github/workflows/benchmark-report.yml'), 'utf8') + assert.match(workflow, /concurrency:\n {6}group: benchmark-report\n {6}cancel-in-progress: false/) + assert.match(workflow, /cancel-in-progress: false\n {6}queue: max/) + assert.match(workflow, /ref: \$\{\{ github.sha \}\}/) + const producer = readFileSync(join(root, '.github/workflows/benchmark.yml'), 'utf8') + assert.match(producer, /ref: \$\{\{ github.sha \}\}/) + assert.match(producer, /MERGE_SHA: \$\{\{ github.sha \}\}/) + assert.match(producer, /fetch-depth: 2/) + assert.match(producer, /BASE_SHA=\$\(git rev-parse HEAD\^1\)/) + assert.match(producer, /BASE_SHA: \$\{\{ steps.base.outputs.sha \}\}/) + const script = workflow.split(' script: |\n')[1].replace(/^ {12}/gm, '') + const wrapped = 'module.exports = async function (github, context, core) {\n' + + `${script.trimEnd().replace(/^/gm, ' ')}\n}\n` + const lint = await new ESLint().lintText(wrapped, { filePath: 'benchmark/workflow-script.cjs' }) + assert.deepEqual(lint[0].messages, []) + const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor + const execute = new AsyncFunction('require', 'github', 'context', 'core', script) + const github = { + rest: { + pulls: { + get: async () => ({ + data: { + state: options.state ?? 'open', + merge_commit_sha: options.merge ?? original.mergeSha, + head: { sha: options.head ?? original.headSha }, + base: { sha: options.payloadBase ?? original.baseSha } + } + }) + }, + repos: { + getCommit: async () => ({ + data: { + parents: (options.parents ?? [options.base ?? original.baseSha, original.headSha]).map( + /** @param {string} sha */ + sha => ({ sha })) + } + }) + }, + issues: { + listComments: async () => {}, + /** @param {CommentRequest} input */ + createComment: async input => { calls.push({ operation: 'create', input }) }, + /** @param {CommentRequest} input */ + updateComment: async input => { calls.push({ operation: 'update', input }) } + } + }, + paginate: async () => options.existing + ? [{ id: 7, user: { login: 'github-actions[bot]' }, body: '' }] + : [] + } + await execute(require, github, { + repo: { owner: 'nodejs', repo: 'import-in-the-middle' }, + payload: { workflow_run: { head_sha: original.headSha } } + }, { info: () => {} }) + return calls +} + +try { + assert.deepEqual(summarize([1, 2, 3]), { mean: 2, median: 2, p95: 3, stddev: 1, cvPercent: 50 }) + assert.equal(summarize([1, 2, 3, 100]).median, 2.5) + const paired = [ + { base: { wallMs: 1, cpuMs: 1 }, head: { wallMs: 2, cpuMs: 2 } }, + { base: { wallMs: 10, cpuMs: 10 }, head: { wallMs: 5, cpuMs: 5 } }, + { base: { wallMs: 100, cpuMs: 100 }, head: { wallMs: 200, cpuMs: 200 } } + ] + assert.equal(compare([paired, paired]).changePercent, 100) + const comparison = compare(trials) + assert.ok(Math.abs(comparison.changePercent - 10) < 1e-10) + assert.ok(comparison.stable) + assert.equal(compare(trials.slice(0, 2)).stable, false) + assert.equal(compare(trials.map( + /** @param {Pair[]} trial */ + trial => trial.slice(0, 2))).stable, false) + const noisy = structuredClone(trials) + for (let index = 0; index < noisy.length; index++) { + for (const observation of noisy[index]) observation.head.wallMs = index % 2 === 0 ? 100 : 140 + } + assert.equal(compare(noisy).stable, false) + assert.deepEqual(parse(original), original) + const text = formatReports([original]) + assert.ok(text.includes('+10.00%')) + const faster = structuredClone(original) + for (const benchmark of faster.benchmarks) { + for (const trial of benchmark.trials) { + for (const observation of trial) observation.head.wallMs = 90 + } + } + assert.ok(formatReports([parse(faster)]).includes('-10.00%')) + const slowStarts = structuredClone(trials) + for (const trial of slowStarts) trial[0].head.wallMs = 1000 + const typical = compare(slowStarts) + assert.ok(Math.abs(typical.changePercent - 10) < 1e-10) + assert.ok(typical.head.mean > 110) + assert.ok(typical.head.stddev > 100) + const forgedSummary = structuredClone(original) + forgedSummary.benchmarks[0].summary = { changePercent: -50 } + assert.ok(formatReports([parse(forgedSummary)]).includes('+10.00%')) + const unstable = structuredClone(original) + unstable.benchmarks[0].trials = noisy + assert.ok(formatReports([parse(unstable)]).includes('| Unstable |')) + + for (const field of ['formatVersion', 'workloadVersion', 'prNumber', 'baseSha', 'headSha', 'mergeSha', + 'nodeVersion', 'platform', 'arch', 'selective', 'synchronous']) { + const report = structuredClone(original) + report[field] = undefined + assert.throws(() => parse(report), /Invalid benchmark metadata/) + } + for (const value of [undefined, [], [original.benchmarks[0]]]) { + assert.throws(() => parse({ ...original, benchmarks: value }), /Invalid benchmark scenarios/) + } + for (const count of [1, 21]) { + const report = structuredClone(original) + report.benchmarks[0].trials = Array.from({ length: count }, () => [pair, pair]) + assert.throws(() => parse(report), /Invalid benchmark trials/) + } + for (const count of [2, 20]) { + const report = structuredClone(original) + report.benchmarks[0].trials = Array.from({ length: count }, () => [pair, pair]) + assert.equal(parse(report).benchmarks[0].trials.length, count) + } + for (const count of [1, 101]) { + const report = structuredClone(original) + report.benchmarks[0].trials = [Array.from({ length: count }, () => pair), [pair, pair]] + assert.throws(() => parse(report), /Invalid benchmark runs/) + } + for (const count of [2, 100]) { + const report = structuredClone(original) + report.benchmarks[0].trials = Array.from({ length: 2 }, () => Array.from({ length: count }, () => pair)) + assert.equal(parse(report).benchmarks[0].trials[0].length, count) + } + for (const field of ['wallMs', 'cpuMs']) { + const limit = field === 'wallMs' ? 30000 : 1000000 + const accepted = structuredClone(original) + accepted.benchmarks[0].trials[0][0].head[field] = limit + assert.equal(parse(accepted).benchmarks[0].trials[0][0].head[field], limit) + for (const value of [0, -1, null, '1', limit + 1]) { + const report = structuredClone(original) + report.benchmarks[0].trials[0][0].head[field] = value + assert.throws(() => parse(report), /Invalid benchmark sample/) + } + } + const mismatched = structuredClone(original) + mismatched.benchmarks[0].trials[0].pop() + assert.throws(() => parse(mismatched), /Inconsistent benchmark runs/) + writeFileSync(filename, '{}') + truncateSync(filename, 8 * 1024 * 1024 + 1) + assert.throws(() => readReport(filename), /too large/) + + process.env.GITHUB_WORKSPACE = root + process.env.ARTIFACT_DIRECTORY = directory + process.env.WORKFLOW_RUN_ID = '123' + for (const major of [22, 24]) { + const artifactDirectory = join(directory, `benchmark-report-node${major}`) + mkdirSync(artifactDirectory) + writeFileSync(join(artifactDirectory, 'report.json'), JSON.stringify({ + ...original, nodeVersion: `v${major}.23.3` + })) + } + const created = (await publish())[0] + assert.equal(created.operation, 'create') + assert.equal(created.input.issue_number, 293) + assert.match(created.input.body, /import-in-the-middle-benchmark/) + assert.match(created.input.body, /\+10\.00%/) + assert.match(created.input.body, /actions\/runs\/123/) + const updated = (await publish({ existing: true }))[0] + assert.equal(updated.operation, 'update') + assert.equal(updated.input.comment_id, 7) + assert.equal((await publish({ payloadBase: 'd'.repeat(40) }))[0].operation, 'create') + for (const options of [{ head: 'c'.repeat(40) }, { merge: 'd'.repeat(40) }, { state: 'closed' }]) { + assert.deepEqual(await publish(options), []) + } + for (const options of [{ base: 'd'.repeat(40) }, { parents: [] }, + { parents: [original.baseSha] }, { parents: [original.baseSha, 'd'.repeat(40)] }]) { + const calls = [] + await assert.rejects(publish(options, calls), + /** @param {Error} error */ + error => { + assert.deepEqual(calls, []) + return error.message === 'Mismatched benchmark merge parents' + }) + } + const artifact = join(directory, 'benchmark-report-node24/report.json') + for (const fields of [{ headSha: 'c'.repeat(40) }, { baseSha: 'c'.repeat(40) }, { prNumber: 294 }, + { mergeSha: 'd'.repeat(40) }]) { + writeFileSync(artifact, JSON.stringify({ ...original, ...fields })) + await assert.rejects(publish(), /Mismatched benchmark sources/) + } + writeFileSync(artifact, JSON.stringify({ ...original, nodeVersion: 'v22.23.3' })) + await assert.rejects(publish(), /Unexpected Node version/) + + for (const argumentsList of [['--runs', '1'], ['--runs', '101'], ['--trials', '1'], ['--trials', '21'], + ['--unknown', '2'], ['--output'], ['--scenario', 'missing']]) { + const result = run(argumentsList) + assert.notEqual(result.status, 0) + assert.match(result.stderr, /Expected|Unknown/) + } + assert.equal(run(['--help']).status, 0) + const selected = run(['--runs', '2', '--trials', '2', '--scenario', 'small/none']) + assert.equal(selected.status, 0, selected.stderr) + assert.equal(JSON.parse(selected.stdout).benchmarks.length, 1) + const missingRoot = run(['--runs', '2', '--trials', '2', '--scenario', 'small/async-inactive', + '--package-directory', join(directory, 'missing')]) + assert.notEqual(missingRoot.status, 0) + assert.match(missingRoot.stderr, /Benchmark process failed/) + const warningRoot = join(directory, 'warning') + mkdirSync(warningRoot) + writeFileSync(join(warningRoot, 'hook.mjs'), + "process.emitWarning('failed to wrap benchmark module')\nexport {}\n") + const warning = run(['--runs', '2', '--trials', '2', '--scenario', 'small/async-inactive', + '--package-directory', warningRoot]) + assert.notEqual(warning.status, 0) + assert.match(warning.stderr, /Benchmark emitted warnings.*failed to wrap benchmark module/) + const blockedRoot = join(directory, 'blocked') + mkdirSync(blockedRoot) + writeFileSync(join(blockedRoot, 'hook.mjs'), + 'Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0)\nexport {}\n') + const blocked = run(['--runs', '2', '--trials', '2', '--scenario', 'small/async-inactive', + '--package-directory', blockedRoot]) + assert.notEqual(blocked.status, 0) + assert.match(blocked.stderr, /Benchmark process failed: .*ETIMEDOUT/) + const result = run(['--runs', '2', '--trials', '2', '--output', filename]) + assert.equal(result.status, 0, result.stderr) + const raw = JSON.parse(readFileSync(filename, 'utf8')) + assert.equal(raw.formatVersion, 2) + assert.deepEqual(raw.benchmarks.map( + /** @param {Benchmark} benchmark */ + benchmark => benchmark.name), + getScenarios(raw.selective, raw.synchronous).map( + /** @param {Scenario} scenario */ + scenario => scenario.name)) + for (const benchmark of raw.benchmarks) { + assert.equal(benchmark.trials.length, 2) + assert.equal(benchmark.trials[0].length, 2) + assert.equal(benchmark.summary.stable, false) + assert.ok(benchmark.trials[0][0].base.cpuMs > 0) + } + console.log('Benchmark protocol, statistics, CLI, and all available loader modes passed.') +} finally { + rmSync(directory, { recursive: true, force: true }) +}