From f158f02c75049e742f7a9001b8a2a99d96c3b651 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 9 Sep 2026 15:55:04 +0200 Subject: [PATCH 1/2] ci: add pull request benchmark reports Pull request code runs without write permission. The artifact report runs after completion so only validated data reaches a bot comment. --- .github/workflows/benchmark-report.yml | 109 +++++++++++++++++++++ .github/workflows/benchmark.yml | 52 ++++++++++ benchmark/compare.mjs | 98 +++++++++++++++++++ benchmark/fixture/a.mjs | 1 + benchmark/fixture/active-hook.cjs | 14 +++ benchmark/fixture/app.mjs | 15 +++ benchmark/fixture/b.mjs | 1 + benchmark/fixture/c.mjs | 1 + benchmark/fixture/d.mjs | 1 + benchmark/fixture/e.mjs | 1 + benchmark/fixture/f.mjs | 1 + benchmark/fixture/g.mjs | 1 + benchmark/fixture/h.mjs | 1 + benchmark/run.mjs | 125 +++++++++++++++++++++++++ package.json | 1 + 15 files changed, 422 insertions(+) create mode 100644 .github/workflows/benchmark-report.yml create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/compare.mjs create mode 100644 benchmark/fixture/a.mjs create mode 100644 benchmark/fixture/active-hook.cjs create mode 100644 benchmark/fixture/app.mjs create mode 100644 benchmark/fixture/b.mjs create mode 100644 benchmark/fixture/c.mjs create mode 100644 benchmark/fixture/d.mjs create mode 100644 benchmark/fixture/e.mjs create mode 100644 benchmark/fixture/f.mjs create mode 100644 benchmark/fixture/g.mjs create mode 100644 benchmark/fixture/h.mjs create mode 100644 benchmark/run.mjs diff --git a/.github/workflows/benchmark-report.yml b/.github/workflows/benchmark-report.yml new file mode 100644 index 0000000..012a5f7 --- /dev/null +++ b/.github/workflows/benchmark-report.yml @@ -0,0 +1,109 @@ +name: Benchmark report + +on: + workflow_run: + workflows: [Benchmark] + types: [completed] + +permissions: + actions: read + issues: write + pull-requests: read + +jobs: + report: + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Download benchmark report + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: benchmark-report + path: ${{ runner.temp }}/benchmark-report + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Publish benchmark report + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + ARTIFACT_DIRECTORY: ${{ runner.temp }}/benchmark-report + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('node:fs') + const path = require('node:path') + const marker = '' + const names = ['No loader', 'Inactive loader', 'Active hook'] + const source = fs.readFileSync(path.join(process.env.ARTIFACT_DIRECTORY, 'report.json'), 'utf8') + const report = JSON.parse(source) + const isDuration = (value) => Number.isFinite(value) && value > 0 + + if (report.formatVersion !== 1 || !Number.isInteger(report.prNumber) || report.prNumber < 1 || + !/^[a-f0-9]{40}$/.test(report.baseSha) || !/^[a-f0-9]{40}$/.test(report.headSha) || + report.headSha !== context.payload.workflow_run.head_sha || + !/^v\d+\.\d+\.\d+$/.test(report.nodeVersion) || !Array.isArray(report.results) || + report.results.length !== names.length) { + throw new Error('Invalid benchmark report') + } + + for (let index = 0; index < names.length; index++) { + const result = report.results[index] + if (result?.name !== names[index] || !isDuration(result.baseMs) || !isDuration(result.headMs) || + !Number.isFinite(result.changePercent)) { + throw new Error('Invalid benchmark result') + } + } + + const pull = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: report.prNumber + }) + + if (pull.data.head.sha !== report.headSha) { + core.info('The pull request has a newer commit.') + return + } + + const rows = report.results.map((result) => { + const change = `${result.changePercent >= 0 ? '+' : ''}${result.changePercent.toFixed(1)}%` + return `| ${result.name} | ${result.baseMs.toFixed(2)} ms | ${result.headMs.toFixed(2)} ms | ${change} |` + }) + const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.WORKFLOW_RUN_ID}` + const body = [ + marker, + '## Benchmark report', + '', + `Node ${report.nodeVersion}. [Workflow run](${runUrl})`, + '', + '| Scenario | Base | Pull request | Change |', + '| --- | ---: | ---: | ---: |', + ...rows, + '', + `Base: \`${report.baseSha.slice(0, 7)}\` · Head: \`${report.headSha.slice(0, 7)}\`` + ].join('\n') + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: report.prNumber + }) + const existing = comments.find((comment) => + comment.user?.login === 'github-actions[bot]' && comment.body?.includes(marker)) + + if (existing === undefined) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: report.prNumber, + body + }) + } else { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body + }) + } diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..e32e30b --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,52 @@ +name: Benchmark + +on: + pull_request: + +permissions: + contents: read + +jobs: + benchmark: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: lts/* + - run: npm install --ignore-scripts + - name: Check out the base revision + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + git fetch --no-tags --depth=1 origin "$BASE_SHA" + git worktree add --detach "$RUNNER_TEMP/base" "$BASE_SHA" + npm install --ignore-scripts --prefix "$RUNNER_TEMP/base" + - name: Benchmark the base revision + run: > + node benchmark/run.mjs + --package-directory "$RUNNER_TEMP/base" + --output "$RUNNER_TEMP/base.json" + - name: Benchmark the pull request + run: > + node benchmark/run.mjs + --package-directory "$GITHUB_WORKSPACE" + --output "$RUNNER_TEMP/head.json" + - name: Build the benchmark report + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.number }} + run: > + node benchmark/compare.mjs + --base "$RUNNER_TEMP/base.json" + --head "$RUNNER_TEMP/head.json" + --output "$RUNNER_TEMP/report.json" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-report + path: ${{ runner.temp }}/report.json + if-no-files-found: error diff --git a/benchmark/compare.mjs b/benchmark/compare.mjs new file mode 100644 index 0000000..327686f --- /dev/null +++ b/benchmark/compare.mjs @@ -0,0 +1,98 @@ +import { readFileSync, writeFileSync } from 'node:fs' +import { resolve } from 'node:path' + +const options = parseOptions(process.argv.slice(2)) +const base = parseBenchmark(readFileSync(options.base, 'utf8')) +const head = parseBenchmark(readFileSync(options.head, 'utf8')) +const baseByName = new Map(base.benchmarks.map((benchmark) => [benchmark.name, benchmark.medianMs])) +const results = [] + +for (const benchmark of head.benchmarks) { + const baseMs = baseByName.get(benchmark.name) + + if (baseMs === undefined) { + throw new Error(`Base benchmark is missing ${benchmark.name}`) + } + + results.push({ + name: benchmark.name, + baseMs, + headMs: benchmark.medianMs, + changePercent: ((benchmark.medianMs - baseMs) / baseMs) * 100 + }) +} + +const report = { + formatVersion: 1, + prNumber: readPositiveInteger('PR_NUMBER'), + baseSha: readSha('BASE_SHA'), + headSha: readSha('HEAD_SHA'), + nodeVersion: head.nodeVersion, + results +} + +writeFileSync(options.output, `${JSON.stringify(report, undefined, 2)}\n`) + +/** + * @param {string[]} argumentsList Command-line arguments. + * @returns {{ base: string, head: string, output: string }} + */ +function parseOptions (argumentsList) { + const values = new Map() + + for (let index = 0; index < argumentsList.length; index += 2) { + values.set(argumentsList[index], argumentsList[index + 1]) + } + + const base = values.get('--base') + const head = values.get('--head') + const output = values.get('--output') + + if (base === undefined || head === undefined || output === undefined || values.size !== 3) { + throw new Error('Expected --base, --head, and --output') + } + + return { base: resolve(base), head: resolve(head), output: resolve(output) } +} + +/** + * @param {string} source JSON benchmark report. + * @returns {{ nodeVersion: string, benchmarks: { name: string, medianMs: number }[] }} + */ +function parseBenchmark (source) { + const report = JSON.parse(source) + + if (report.formatVersion !== 1 || typeof report.nodeVersion !== 'string' || !Array.isArray(report.benchmarks)) { + throw new Error('Invalid benchmark report') + } + + return report +} + +/** + * @param {string} name Environment variable name. + * @returns {number} + */ +function readPositiveInteger (name) { + const value = Number(process.env[name]) + + if (!Number.isInteger(value) || value < 1) { + throw new Error(`${name} must be a positive integer`) + } + + return value +} + +/** + * @param {string} name Environment variable name. + * @returns {string} + */ +function readSha (name) { + const value = process.env[name] + + if (value === undefined || !/^[a-f0-9]{40}$/.test(value)) { + throw new Error(`${name} must be a full commit SHA`) + } + + return value +} diff --git a/benchmark/fixture/a.mjs b/benchmark/fixture/a.mjs new file mode 100644 index 0000000..1d772ee --- /dev/null +++ b/benchmark/fixture/a.mjs @@ -0,0 +1 @@ +export const value = 1 diff --git a/benchmark/fixture/active-hook.cjs b/benchmark/fixture/active-hook.cjs new file mode 100644 index 0000000..32369c4 --- /dev/null +++ b/benchmark/fixture/active-hook.cjs @@ -0,0 +1,14 @@ +const { resolve } = require('node:path') + +const packageRoot = process.env.IITM_BENCHMARK_PACKAGE_ROOT +const fixtureRoot = process.env.IITM_BENCHMARK_FIXTURE_ROOT + +if (packageRoot === undefined || fixtureRoot === undefined) { + throw new Error('IITM_BENCHMARK_PACKAGE_ROOT and IITM_BENCHMARK_FIXTURE_ROOT are required') +} + +const Hook = require(resolve(packageRoot, 'index.js')) + +Hook([resolve(fixtureRoot, 'a.mjs')], (exports) => { + exports.value += 1 +}) diff --git a/benchmark/fixture/app.mjs b/benchmark/fixture/app.mjs new file mode 100644 index 0000000..cd747dc --- /dev/null +++ b/benchmark/fixture/app.mjs @@ -0,0 +1,15 @@ +import { value as a } from './a.mjs' +import { value as b } from './b.mjs' +import { value as c } from './c.mjs' +import { value as d } from './d.mjs' +import { value as e } from './e.mjs' +import { value as f } from './f.mjs' +import { value as g } from './g.mjs' +import { value as h } from './h.mjs' + +const expected = process.env.IITM_BENCHMARK_ACTIVE === '1' ? 88 : 87 +const sum = a + b + c + d + e + f + g + h + +if (sum !== expected) { + throw new Error(`Unexpected fixture result: ${sum}`) +} diff --git a/benchmark/fixture/b.mjs b/benchmark/fixture/b.mjs new file mode 100644 index 0000000..3aee44a --- /dev/null +++ b/benchmark/fixture/b.mjs @@ -0,0 +1 @@ +export const value = 2 diff --git a/benchmark/fixture/c.mjs b/benchmark/fixture/c.mjs new file mode 100644 index 0000000..e2befb1 --- /dev/null +++ b/benchmark/fixture/c.mjs @@ -0,0 +1 @@ +export const value = 3 diff --git a/benchmark/fixture/d.mjs b/benchmark/fixture/d.mjs new file mode 100644 index 0000000..7665884 --- /dev/null +++ b/benchmark/fixture/d.mjs @@ -0,0 +1 @@ +export const value = 5 diff --git a/benchmark/fixture/e.mjs b/benchmark/fixture/e.mjs new file mode 100644 index 0000000..7d03b11 --- /dev/null +++ b/benchmark/fixture/e.mjs @@ -0,0 +1 @@ +export const value = 8 diff --git a/benchmark/fixture/f.mjs b/benchmark/fixture/f.mjs new file mode 100644 index 0000000..485ba0f --- /dev/null +++ b/benchmark/fixture/f.mjs @@ -0,0 +1 @@ +export const value = 13 diff --git a/benchmark/fixture/g.mjs b/benchmark/fixture/g.mjs new file mode 100644 index 0000000..12f6bbc --- /dev/null +++ b/benchmark/fixture/g.mjs @@ -0,0 +1 @@ +export const value = 21 diff --git a/benchmark/fixture/h.mjs b/benchmark/fixture/h.mjs new file mode 100644 index 0000000..dd52996 --- /dev/null +++ b/benchmark/fixture/h.mjs @@ -0,0 +1 @@ +export const value = 34 diff --git a/benchmark/run.mjs b/benchmark/run.mjs new file mode 100644 index 0000000..9380a69 --- /dev/null +++ b/benchmark/run.mjs @@ -0,0 +1,125 @@ +import { spawnSync } from 'node:child_process' +import { writeFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const benchmarkRoot = fileURLToPath(new URL('.', import.meta.url)) +const fixtureRoot = resolve(benchmarkRoot, 'fixture') +const app = resolve(fixtureRoot, 'app.mjs') +const activeHook = resolve(fixtureRoot, 'active-hook.cjs') + +const options = parseOptions(process.argv.slice(2)) +const packageRoot = resolve(options.packageDirectory) +const results = runBenchmarks(packageRoot, options.runs) +const report = { + formatVersion: 1, + nodeVersion: process.version, + benchmarks: results +} + +if (options.output === undefined) { + process.stdout.write(`${JSON.stringify(report, undefined, 2)}\n`) +} else { + writeFileSync(options.output, `${JSON.stringify(report, undefined, 2)}\n`) +} + +/** + * @param {string[]} argumentsList Command-line arguments. + * @returns {{ output: string | undefined, packageDirectory: string, runs: number }} + */ +function parseOptions (argumentsList) { + let output + let packageDirectory = process.cwd() + let runs = 15 + + for (let index = 0; index < argumentsList.length; index++) { + const argument = argumentsList[index] + const value = argumentsList[index + 1] + + if (argument === '--output' && value !== undefined) { + output = resolve(value) + index++ + } else if (argument === '--package-directory' && value !== undefined) { + packageDirectory = value + index++ + } else if (argument === '--runs' && value !== undefined) { + runs = Number(value) + index++ + } else { + throw new Error(`Unknown or incomplete option: ${argument}`) + } + } + + if (!Number.isInteger(runs) || runs < 1) { + throw new Error('--runs must be a positive integer') + } + + return { output, packageDirectory, runs } +} + +/** + * @param {string} packageDirectory Directory that contains import-in-the-middle. + * @param {number} runs Number of process starts measured for each scenario. + * @returns {{ name: string, medianMs: number }[]} + */ +function runBenchmarks (packageDirectory, runs) { + const loader = resolve(packageDirectory, 'hook.mjs') + const scenarios = [ + { name: 'No loader', argumentsList: [app] }, + { name: 'Inactive loader', argumentsList: ['--loader', loader, app] }, + { + name: 'Active hook', + argumentsList: ['--require', activeHook, '--loader', loader, app], + environment: { IITM_BENCHMARK_ACTIVE: '1' } + } + ] + + return scenarios.map((scenario) => ({ + name: scenario.name, + medianMs: measureScenario(packageDirectory, scenario, runs) + })) +} + +/** + * @param {string} packageDirectory Directory that contains import-in-the-middle. + * @param {{ argumentsList: string[], environment?: Record }} scenario Benchmark invocation. + * @param {number} runs Number of process starts measured for the scenario. + * @returns {number} + */ +function measureScenario (packageDirectory, scenario, runs) { + const samples = [] + + for (let index = 0; index < runs; index++) { + const startedAt = process.hrtime.bigint() + const result = spawnSync(process.execPath, scenario.argumentsList, { + cwd: benchmarkRoot, + encoding: 'utf8', + env: { + ...process.env, + IITM_BENCHMARK_PACKAGE_ROOT: packageDirectory, + IITM_BENCHMARK_FIXTURE_ROOT: fixtureRoot, + ...scenario.environment, + NODE_NO_WARNINGS: '1' + } + }) + const duration = Number(process.hrtime.bigint() - startedAt) / 1e6 + + if (result.status !== 0) { + throw new Error(`Benchmark process failed: ${result.stderr || result.stdout}`) + } + + samples.push(duration) + } + + samples.sort(compareNumbers) + return samples[Math.floor(samples.length / 2)] +} + +/** + * @param {number} first First number. + * @param {number} second Second number. + * @returns {number} + */ +function compareNumbers (first, second) { + return first - second +} diff --git a/package.json b/package.json index c3362f5..ad3b2b1 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "test:e2e": "node test/check-exports/test.mjs && node test/integration-tests/turbopack-dev.mjs && node test/integration-tests/turbopack-build-start.mjs", "test:ts": "c8 --reporter lcov imhotap --files test/typescript/*.test.mts", "coverage": "c8 --reporter html imhotap --files test/{hook,low-level,other,get-esm-exports,register}/* && echo \"\nNow open coverage/index.html\n\"", + "benchmark": "node benchmark/run.mjs", "lint": "eslint .", "lint:fix": "eslint . --fix" }, From bb0740a82f2e5eacabe7adb0403a76f9ae50c33b Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Mon, 5 Oct 2026 20:47:46 +0900 Subject: [PATCH 2/2] ci: stabilize startup benchmarks and report publication Separate base and head runs can attribute scheduling drift to code changes. Paired observations retain the evidence needed to assess measurement noise. An older PR branch can lack the benchmark harness, and the PR payload can retain an outdated base SHA. Measuring the test merge against its first parent uses the landed harness and the target revision actually merged. Concurrent report jobs can both find no existing bot comment and create duplicates. Serial publication preserves one report comment for each PR. --- .github/workflows/benchmark-report.yml | 103 ++++----- .github/workflows/benchmark.yml | 44 ++-- .github/workflows/ci.yml | 1 + .npmignore | 1 + README.md | 5 + benchmark/README.md | 50 ++++ benchmark/compare.mjs | 98 -------- benchmark/fixture/active-hook.cjs | 14 -- benchmark/fixture/app.mjs | 79 +++++-- benchmark/fixture/commonjs.cjs | 8 + benchmark/fixture/small.mjs | 10 + benchmark/report.mjs | 89 ++++++++ benchmark/run.mjs | 234 +++++++++++-------- benchmark/scenarios.mjs | 14 ++ benchmark/statistics.mjs | 66 ++++++ package.json | 3 +- test/benchmark.mjs | 304 +++++++++++++++++++++++++ 17 files changed, 811 insertions(+), 312 deletions(-) create mode 100644 benchmark/README.md delete mode 100644 benchmark/compare.mjs delete mode 100644 benchmark/fixture/active-hook.cjs create mode 100644 benchmark/fixture/commonjs.cjs create mode 100644 benchmark/fixture/small.mjs create mode 100644 benchmark/report.mjs create mode 100644 benchmark/scenarios.mjs create mode 100644 benchmark/statistics.mjs create mode 100644 test/benchmark.mjs diff --git a/.github/workflows/benchmark-report.yml b/.github/workflows/benchmark-report.yml index 012a5f7..c093a68 100644 --- a/.github/workflows/benchmark-report.yml +++ b/.github/workflows/benchmark-report.yml @@ -6,21 +6,30 @@ on: types: [completed] permissions: + contents: read actions: read issues: write pull-requests: read jobs: report: + concurrency: + group: benchmark-report + cancel-in-progress: false + queue: max if: >- github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' runs-on: ubuntu-latest steps: - - name: Download benchmark report + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Download benchmark reports uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: benchmark-report + pattern: benchmark-report-node* path: ${{ runner.temp }}/benchmark-report run-id: ${{ github.event.workflow_run.id }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -32,78 +41,46 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const fs = require('node:fs') const path = require('node:path') - const marker = '' - const names = ['No loader', 'Inactive loader', 'Active hook'] - const source = fs.readFileSync(path.join(process.env.ARTIFACT_DIRECTORY, 'report.json'), 'utf8') - const report = JSON.parse(source) - const isDuration = (value) => Number.isFinite(value) && value > 0 - - if (report.formatVersion !== 1 || !Number.isInteger(report.prNumber) || report.prNumber < 1 || - !/^[a-f0-9]{40}$/.test(report.baseSha) || !/^[a-f0-9]{40}$/.test(report.headSha) || - report.headSha !== context.payload.workflow_run.head_sha || - !/^v\d+\.\d+\.\d+$/.test(report.nodeVersion) || !Array.isArray(report.results) || - report.results.length !== names.length) { - throw new Error('Invalid benchmark report') - } - - for (let index = 0; index < names.length; index++) { - const result = report.results[index] - if (result?.name !== names[index] || !isDuration(result.baseMs) || !isDuration(result.headMs) || - !Number.isFinite(result.changePercent)) { - throw new Error('Invalid benchmark result') - } - } - + const { pathToFileURL } = require('node:url') + const { readReport, formatReports } = await import( + pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'benchmark/report.mjs')).href) + const reports = [22, 24].map((major) => { + const report = readReport(path.join(process.env.ARTIFACT_DIRECTORY, + `benchmark-report-node${major}`, 'report.json')) + if (!report.nodeVersion.startsWith(`v${major}.`)) throw new Error('Unexpected Node version') + return report + }) + const report = reports[0] + if (reports.some((entry) => entry.headSha !== context.payload.workflow_run.head_sha || + entry.headSha !== report.headSha || entry.baseSha !== report.baseSha || + entry.mergeSha !== report.mergeSha || + entry.prNumber !== report.prNumber)) throw new Error('Mismatched benchmark sources') const pull = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: report.prNumber + owner: context.repo.owner, repo: context.repo.repo, pull_number: report.prNumber }) - - if (pull.data.head.sha !== report.headSha) { - core.info('The pull request has a newer commit.') + if (pull.data.state !== 'open' || pull.data.head.sha !== report.headSha || + pull.data.merge_commit_sha !== report.mergeSha) { + core.info('The pull request is closed or its revisions have changed.') return } - - const rows = report.results.map((result) => { - const change = `${result.changePercent >= 0 ? '+' : ''}${result.changePercent.toFixed(1)}%` - return `| ${result.name} | ${result.baseMs.toFixed(2)} ms | ${result.headMs.toFixed(2)} ms | ${change} |` + const merge = await github.rest.repos.getCommit({ + owner: context.repo.owner, repo: context.repo.repo, ref: report.mergeSha }) + if (merge.data.parents[0]?.sha !== report.baseSha || merge.data.parents[1]?.sha !== report.headSha) { + throw new Error('Mismatched benchmark merge parents') + } const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.WORKFLOW_RUN_ID}` - const body = [ - marker, - '## Benchmark report', - '', - `Node ${report.nodeVersion}. [Workflow run](${runUrl})`, - '', - '| Scenario | Base | Pull request | Change |', - '| --- | ---: | ---: | ---: |', - ...rows, - '', - `Base: \`${report.baseSha.slice(0, 7)}\` · Head: \`${report.headSha.slice(0, 7)}\`` - ].join('\n') + const body = `${formatReports(reports)}\n\n[Workflow run](${runUrl})` + const marker = '' const comments = await github.paginate(github.rest.issues.listComments, { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: report.prNumber + owner: context.repo.owner, repo: context.repo.repo, issue_number: report.prNumber }) const existing = comments.find((comment) => comment.user?.login === 'github-actions[bot]' && comment.body?.includes(marker)) - + const target = { owner: context.repo.owner, repo: context.repo.repo, body } if (existing === undefined) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: report.prNumber, - body - }) + await github.rest.issues.createComment({ ...target, issue_number: report.prNumber }) } else { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body - }) + await github.rest.issues.updateComment({ ...target, comment_id: existing.id }) } diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e32e30b..4482cdb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -2,6 +2,7 @@ name: Benchmark on: pull_request: + types: [opened, synchronize, reopened, edited] permissions: contents: read @@ -9,44 +10,41 @@ permissions: jobs: benchmark: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 60 + strategy: + matrix: + node: [22, 24] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.sha }} + fetch-depth: 2 + persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - node-version: lts/* + node-version: ${{ matrix.node }} - run: npm install --ignore-scripts - name: Check out the base revision - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} + id: base run: | + BASE_SHA=$(git rev-parse HEAD^1) git fetch --no-tags --depth=1 origin "$BASE_SHA" git worktree add --detach "$RUNNER_TEMP/base" "$BASE_SHA" npm install --ignore-scripts --prefix "$RUNNER_TEMP/base" - - name: Benchmark the base revision - run: > - node benchmark/run.mjs - --package-directory "$RUNNER_TEMP/base" - --output "$RUNNER_TEMP/base.json" - - name: Benchmark the pull request - run: > - node benchmark/run.mjs - --package-directory "$GITHUB_WORKSPACE" - --output "$RUNNER_TEMP/head.json" - - name: Build the benchmark report + printf 'sha=%s\n' "$BASE_SHA" >> "$GITHUB_OUTPUT" + - name: Measure paired revisions env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_SHA: ${{ steps.base.outputs.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} + MERGE_SHA: ${{ github.sha }} PR_NUMBER: ${{ github.event.number }} - run: > - node benchmark/compare.mjs - --base "$RUNNER_TEMP/base.json" - --head "$RUNNER_TEMP/head.json" - --output "$RUNNER_TEMP/report.json" + run: | + BENCHMARK_CPU=$(awk '/^Cpus_allowed_list:/ { split($2, cpus, /[-,]/); print cpus[1] }' /proc/self/status) + taskset --cpu-list "$BENCHMARK_CPU" node benchmark/run.mjs \ + --base-directory "$RUNNER_TEMP/base" \ + --output "$RUNNER_TEMP/report.json" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: benchmark-report + name: benchmark-report-node${{ matrix.node }} path: ${{ runner.temp }}/report.json if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e81ced..d5ea434 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,7 @@ jobs: node-version: 'lts/*' - run: npm install - run: npm run lint + - run: npm run test:benchmark build: runs-on: ubuntu-latest diff --git a/.npmignore b/.npmignore index 7a24988..a83a749 100644 --- a/.npmignore +++ b/.npmignore @@ -9,3 +9,4 @@ coverage release-please-config.json tsconfig.json test +benchmark diff --git a/README.md b/README.md index eae9f47..5991a0f 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,11 @@ fs.readFileSync('file.txt') node --import=./instrument.mjs ./my-app.mjs ``` +## Development benchmarks + +See [benchmark/README.md](benchmark/README.md) for the PR benchmark method and +how to check measurement noise locally. + ## Synchronous loader hooks On Node.js versions that support diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..3c10c29 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,50 @@ +# Pull request benchmarks + +Run `npm run benchmark -- --help` for the command-line options. Without a base +directory, the command compares the current revision with itself to check noise. +Both revisions use the same fixture and dependency installation. +CI compares the base with GitHub's merged PR revision. This includes current +base changes when the contributor's branch is older. Reports retain the base, +PR head, and measured merge revisions so the comparison can be reproduced. + +Each sample starts a new Node.js process. The fixture records elapsed time when +its imports and behavior checks complete. Process teardown is outside that +measurement. CPU time is retained as a diagnostic and includes loader-worker +CPU use. It does not include time spent waiting for a worker response. + +The small graph exposes fixed loader startup costs. The dependency graph uses +the pinned date-fns development dependency and a CommonJS module. It exercises +package resolution, many ESM exports, and CommonJS interoperability. These are +loader workloads, not application request-throughput measurements. + +The driver warms the filesystem cache, starts each base/head pair consecutively, +and alternates their order. It disables compilation caching. Independent trials +retain every sample, including slow samples. The reported change is the mean of +the trial estimates. Each trial estimate is the median of its paired head/base +duration ratios. This preserves the pairing when scheduling delays change over time. +CI pins both revisions and their loader workers to the same allowed CPU. This +controls CPU migration and measures startup under a single-core constraint. +Noisy scenarios receive one bounded extension of the same trials. The original +observations remain in the result. An explicit sample count disables this +extension. The spread is descriptive; the report does not claim a formal +significance test after adaptive sampling. + +The median measures typical startup and limits the effect of isolated scheduling +delays. It can miss a regression affecting only occasional starts. The report +also shows raw means, standard deviations, and p95 durations to expose those costs. +Trial CV is the standard deviation of the paired median ratios divided by their mean. +The report labels a row unstable when its trial CV exceeds the implemented +stability threshold. Base and head columns also show the raw startup standard +deviation. A stable ratio does not mean that individual startup samples have +low variation. Use repeated self-comparisons to check reproducibility on the +machine that will run the benchmark. + +The baseline is the test merge's first parent. Pull request payloads can retain +an older base SHA after the target branch advances. The publisher verifies both +merge parents against the measured baseline and PR head. + +PR execution produces artifacts with read-only permissions. A separate workflow +loads reporting code from the default branch, validates the artifact's source +and samples, and computes its own statistics before updating the bot comment. +The PR controls benchmark execution. Artifact validation does not attest to the +accuracy of its timings. diff --git a/benchmark/compare.mjs b/benchmark/compare.mjs deleted file mode 100644 index 327686f..0000000 --- a/benchmark/compare.mjs +++ /dev/null @@ -1,98 +0,0 @@ -import { readFileSync, writeFileSync } from 'node:fs' -import { resolve } from 'node:path' - -const options = parseOptions(process.argv.slice(2)) -const base = parseBenchmark(readFileSync(options.base, 'utf8')) -const head = parseBenchmark(readFileSync(options.head, 'utf8')) -const baseByName = new Map(base.benchmarks.map((benchmark) => [benchmark.name, benchmark.medianMs])) -const results = [] - -for (const benchmark of head.benchmarks) { - const baseMs = baseByName.get(benchmark.name) - - if (baseMs === undefined) { - throw new Error(`Base benchmark is missing ${benchmark.name}`) - } - - results.push({ - name: benchmark.name, - baseMs, - headMs: benchmark.medianMs, - changePercent: ((benchmark.medianMs - baseMs) / baseMs) * 100 - }) -} - -const report = { - formatVersion: 1, - prNumber: readPositiveInteger('PR_NUMBER'), - baseSha: readSha('BASE_SHA'), - headSha: readSha('HEAD_SHA'), - nodeVersion: head.nodeVersion, - results -} - -writeFileSync(options.output, `${JSON.stringify(report, undefined, 2)}\n`) - -/** - * @param {string[]} argumentsList Command-line arguments. - * @returns {{ base: string, head: string, output: string }} - */ -function parseOptions (argumentsList) { - const values = new Map() - - for (let index = 0; index < argumentsList.length; index += 2) { - values.set(argumentsList[index], argumentsList[index + 1]) - } - - const base = values.get('--base') - const head = values.get('--head') - const output = values.get('--output') - - if (base === undefined || head === undefined || output === undefined || values.size !== 3) { - throw new Error('Expected --base, --head, and --output') - } - - return { base: resolve(base), head: resolve(head), output: resolve(output) } -} - -/** - * @param {string} source JSON benchmark report. - * @returns {{ nodeVersion: string, benchmarks: { name: string, medianMs: number }[] }} - */ -function parseBenchmark (source) { - const report = JSON.parse(source) - - if (report.formatVersion !== 1 || typeof report.nodeVersion !== 'string' || !Array.isArray(report.benchmarks)) { - throw new Error('Invalid benchmark report') - } - - return report -} - -/** - * @param {string} name Environment variable name. - * @returns {number} - */ -function readPositiveInteger (name) { - const value = Number(process.env[name]) - - if (!Number.isInteger(value) || value < 1) { - throw new Error(`${name} must be a positive integer`) - } - - return value -} - -/** - * @param {string} name Environment variable name. - * @returns {string} - */ -function readSha (name) { - const value = process.env[name] - - if (value === undefined || !/^[a-f0-9]{40}$/.test(value)) { - throw new Error(`${name} must be a full commit SHA`) - } - - return value -} diff --git a/benchmark/fixture/active-hook.cjs b/benchmark/fixture/active-hook.cjs deleted file mode 100644 index 32369c4..0000000 --- a/benchmark/fixture/active-hook.cjs +++ /dev/null @@ -1,14 +0,0 @@ -const { resolve } = require('node:path') - -const packageRoot = process.env.IITM_BENCHMARK_PACKAGE_ROOT -const fixtureRoot = process.env.IITM_BENCHMARK_FIXTURE_ROOT - -if (packageRoot === undefined || fixtureRoot === undefined) { - throw new Error('IITM_BENCHMARK_PACKAGE_ROOT and IITM_BENCHMARK_FIXTURE_ROOT are required') -} - -const Hook = require(resolve(packageRoot, 'index.js')) - -Hook([resolve(fixtureRoot, 'a.mjs')], (exports) => { - exports.value += 1 -}) diff --git a/benchmark/fixture/app.mjs b/benchmark/fixture/app.mjs index cd747dc..52009c8 100644 --- a/benchmark/fixture/app.mjs +++ b/benchmark/fixture/app.mjs @@ -1,15 +1,66 @@ -import { value as a } from './a.mjs' -import { value as b } from './b.mjs' -import { value as c } from './c.mjs' -import { value as d } from './d.mjs' -import { value as e } from './e.mjs' -import { value as f } from './f.mjs' -import { value as g } from './g.mjs' -import { value as h } from './h.mjs' - -const expected = process.env.IITM_BENCHMARK_ACTIVE === '1' ? 88 : 87 -const sum = a + b + c + d + e + f + g + h - -if (sum !== expected) { - throw new Error(`Unexpected fixture result: ${sum}`) +import assert from 'node:assert/strict' +import * as module from 'node:module' +import { resolve } from 'node:path' +import { performance } from 'node:perf_hooks' +import { fileURLToPath, pathToFileURL } from 'node:url' + +const root = process.env.IITM_BENCHMARK_PACKAGE_ROOT +const mode = process.env.IITM_BENCHMARK_MODE +const workload = process.env.IITM_BENCHMARK_WORKLOAD +assert.ok(root && mode && workload, 'Benchmark environment is required') +const require = module.createRequire(import.meta.url) +const active = mode.endsWith('-active') || mode.endsWith('-selective') +const target = workload === 'small' ? fileURLToPath(new URL('a.mjs', import.meta.url)) : 'date-fns' +let hooks = 0 +let calls = 0 + +if (active) { + const { Hook } = require(resolve(root, 'index.js')) + Hook([target], + /** @param {{ value: number, addDays: typeof import('date-fns').addDays }} exports */ + (exports) => { + hooks++ + if (workload === 'small') { + exports.value++ + } else { + const addDays = exports.addDays + /** + * @param {Date} date Date to advance. + * @param {number} amount Days to add. + */ + exports.addDays = function (date, amount) { + calls++ + return addDays(date, amount) + } + } + }) +} + +if (mode !== 'none') { + const options = mode.endsWith('-excluded') + ? { include: [] } + : mode.endsWith('-selective') + ? { include: [workload === 'small' ? pathToFileURL(target).href : target] } + : undefined + if (mode.startsWith('sync-')) { + const { register } = await import(pathToFileURL(resolve(root, 'register-hooks.mjs')).href) + register(options) + } else if (module.register) { + module.register(pathToFileURL(resolve(root, 'hook.mjs')), { data: options }) + } +} + +if (workload === 'small') { + const { sum } = await import('./small.mjs') + assert.equal(sum, active ? 88 : 87) +} else { + const { addDays, isValid, parseISO } = await import('date-fns') + const { default: Clock } = await import('./commonjs.cjs') + const date = parseISO('2026-09-09T00:00:00Z') + assert.equal(isValid(date), true) + assert.equal(new Clock().copy(addDays(date, 1)).getUTCDate(), 10) + assert.equal(calls, active ? 1 : 0) } +assert.equal(hooks, active ? 1 : 0) +const cpu = process.cpuUsage() +process.stdout.write(`${JSON.stringify({ wallMs: performance.now(), cpuMs: (cpu.user + cpu.system) / 1000 })}\n`) diff --git a/benchmark/fixture/commonjs.cjs b/benchmark/fixture/commonjs.cjs new file mode 100644 index 0000000..47c6c72 --- /dev/null +++ b/benchmark/fixture/commonjs.cjs @@ -0,0 +1,8 @@ +const { EventEmitter } = require('node:events') + +module.exports = class Clock extends EventEmitter { + /** @param {Date} date Date to copy. */ + copy (date) { + return new Date(date.getTime()) + } +} diff --git a/benchmark/fixture/small.mjs b/benchmark/fixture/small.mjs new file mode 100644 index 0000000..719b965 --- /dev/null +++ b/benchmark/fixture/small.mjs @@ -0,0 +1,10 @@ +import { value as a } from './a.mjs' +import { value as b } from './b.mjs' +import { value as c } from './c.mjs' +import { value as d } from './d.mjs' +import { value as e } from './e.mjs' +import { value as f } from './f.mjs' +import { value as g } from './g.mjs' +import { value as h } from './h.mjs' + +export const sum = a + b + c + d + e + f + g + h diff --git a/benchmark/report.mjs b/benchmark/report.mjs new file mode 100644 index 0000000..3668701 --- /dev/null +++ b/benchmark/report.mjs @@ -0,0 +1,89 @@ +import { readFileSync, statSync } from 'node:fs' + +import { getScenarios } from './scenarios.mjs' +import { compare, sampling } from './statistics.mjs' + +const platforms = new Set(['linux', 'darwin', 'win32']) +const architectures = new Set() +architectures.add('x64') +architectures.add('arm64') + +/** @typedef {ReturnType[number] & { trials: import('./statistics.mjs').Pair[][] }} Benchmark */ +/** + * @typedef {{ formatVersion: 2, workloadVersion: 1, prNumber: number, baseSha: string, headSha: string, mergeSha: string, + * nodeVersion: string, platform: string, arch: string, selective: boolean, synchronous: boolean, + * benchmarks: Benchmark[] }} Report + */ + +/** @param {string} filename Untrusted benchmark artifact. */ +export function readReport (filename) { + if (statSync(filename).size > 8 * 1024 * 1024) throw new Error('Benchmark artifact is too large') + const report = /** @type {Report} */ (JSON.parse(readFileSync(filename, 'utf8'))) + if (report?.formatVersion !== 2 || report.workloadVersion !== 1 || + !Number.isInteger(report.prNumber) || report.prNumber < 1 || + typeof report.baseSha !== 'string' || !/^[a-f0-9]{40}$/.test(report.baseSha) || + typeof report.headSha !== 'string' || !/^[a-f0-9]{40}$/.test(report.headSha) || + typeof report.mergeSha !== 'string' || !/^[a-f0-9]{40}$/.test(report.mergeSha) || + typeof report.nodeVersion !== 'string' || !/^v\d+\.\d+\.\d+$/.test(report.nodeVersion) || + !platforms.has(report.platform) || !architectures.has(report.arch) || + typeof report.selective !== 'boolean' || typeof report.synchronous !== 'boolean') { + throw new Error('Invalid benchmark metadata') + } + const scenarios = getScenarios(report.selective, report.synchronous) + if (!Array.isArray(report.benchmarks) || report.benchmarks.length !== scenarios.length) { + throw new Error('Invalid benchmark scenarios') + } + for (let index = 0; index < scenarios.length; index++) { + const benchmark = report.benchmarks[index] + if (benchmark?.name !== scenarios[index].name || !Array.isArray(benchmark.trials) || + benchmark.trials.length < sampling.minTrials || benchmark.trials.length > sampling.maxTrials) { + throw new Error('Invalid benchmark trials') + } + const runs = benchmark.trials[0]?.length + if (!Number.isInteger(runs) || runs < sampling.minRuns || runs > sampling.maxRuns) { + throw new Error('Invalid benchmark runs') + } + for (const trial of benchmark.trials) { + if (!Array.isArray(trial) || trial.length !== runs) throw new Error('Inconsistent benchmark runs') + for (const pair of trial) { + for (const sample of [pair?.base, pair?.head]) { + if (!Number.isFinite(sample?.wallMs) || sample.wallMs <= 0 || sample.wallMs > 30000 || + !Number.isFinite(sample?.cpuMs) || sample.cpuMs <= 0 || sample.cpuMs > 1000000) { + throw new Error('Invalid benchmark sample') + } + } + } + } + } + return report +} + +/** @param {ReturnType[]} reports Validated reports from the same workflow run. */ +export function formatReports (reports) { + const lines = [ + '', '## Benchmark report', '', + 'Fresh-process time to fixture readiness; warm filesystem cache, compilation cache disabled.', + 'Changes compare typical startup: the mean of trial median paired head/base ratios.', + 'CV is the relative standard deviation of trial ratios. Raw means, SD, and p95 expose slow starts.', + 'Raw samples remain in the workflow artifacts. Unstable rows do not support a performance conclusion.' + ] + for (const report of reports) { + lines.push('', `### Node ${report.nodeVersion} (${report.platform}/${report.arch})`, '', + '| Scenario | Base median / p95 | PR median / p95 | Paired change | Trial CV | Raw mean ± SD (base; PR) |', + '| --- | ---: | ---: | ---: | ---: | ---: |') + for (const benchmark of report.benchmarks) { + const result = compare(benchmark.trials) + const change = result.stable + ? `${result.changePercent >= 0 ? '+' : ''}${result.changePercent.toFixed(2)}%` + : 'Unstable' + lines.push(`| ${benchmark.name} | ${result.base.median.toFixed(2)} / ${result.base.p95.toFixed(2)} ms | ` + + `${result.head.median.toFixed(2)} / ${result.head.p95.toFixed(2)} ms | ${change} | ` + + `${result.ratioCvPercent.toFixed(2)}% | ` + + `${result.base.mean.toFixed(2)} ± ${result.base.stddev.toFixed(2)}; ` + + `${result.head.mean.toFixed(2)} ± ${result.head.stddev.toFixed(2)} ms |`) + } + } + lines.push('', `Base: \`${reports[0].baseSha.slice(0, 7)}\` · PR head: \`${reports[0].headSha.slice(0, 7)}\` · ` + + `Measured merge: \`${reports[0].mergeSha.slice(0, 7)}\``) + return lines.join('\n') +} diff --git a/benchmark/run.mjs b/benchmark/run.mjs index 9380a69..3840a99 100644 --- a/benchmark/run.mjs +++ b/benchmark/run.mjs @@ -1,125 +1,161 @@ import { spawnSync } from 'node:child_process' import { writeFileSync } from 'node:fs' +import * as module from 'node:module' import { resolve } from 'node:path' import { fileURLToPath } from 'node:url' -const benchmarkRoot = fileURLToPath(new URL('.', import.meta.url)) -const fixtureRoot = resolve(benchmarkRoot, 'fixture') -const app = resolve(fixtureRoot, 'app.mjs') -const activeHook = resolve(fixtureRoot, 'active-hook.cjs') - +import { supportsSyncHooks } from '../supports-sync-hooks.mjs' +import { getScenarios } from './scenarios.mjs' +import { compare, sampling } from './statistics.mjs' + +/** @typedef {import('./statistics.mjs').Sample} Sample */ +/** @typedef {import('./statistics.mjs').Pair} Pair */ +/** @typedef {ReturnType[number]} Scenario */ + +if (process.argv[2] === '--help' && process.argv.length === 3) { + process.stdout.write('Usage: npm run benchmark -- [--package-directory DIR] [--base-directory DIR] ' + + '[--output FILE] [--runs PAIRS_PER_TRIAL] [--trials COUNT] [--scenario NAME]\n' + + `Default: self-comparison, ${sampling.trials} trials of ${sampling.runs} pairs; ` + + `noisy scenarios expand to ${sampling.maxRuns} pairs per trial.\n` + + 'Explicit --runs fixes the sample count. See benchmark/README.md.\n') + process.exit(0) +} +const fixture = fileURLToPath(new URL('fixture/app.mjs', import.meta.url)) +const legacyWarning = new RegExp('^\\(node:\\d+\\) ExperimentalWarning: Custom ESM Loaders ' + + 'is an experimental feature and might change at any time\\n\\(Use `[^\\n]+ --trace-warnings \\.\\.\\.` ' + + 'to show where the warning was created\\)\\n', 'gm') const options = parseOptions(process.argv.slice(2)) -const packageRoot = resolve(options.packageDirectory) -const results = runBenchmarks(packageRoot, options.runs) -const report = { - formatVersion: 1, - nodeVersion: process.version, - benchmarks: results +const roots = [options.baseDirectory ?? options.packageDirectory, options.packageDirectory] +const selective = typeof module.register === 'function' +const synchronous = supportsSyncHooks() +const benchmarks = [] + +const scenarios = getScenarios(selective, synchronous) +if (options.scenario !== undefined && !scenarios.some( + /** @param {Scenario} scenario */ + scenario => scenario.name === options.scenario)) { + throw new Error('Unknown or unsupported benchmark scenario') +} +for (const scenario of scenarios) { + if (options.scenario !== undefined && scenario.name !== options.scenario) continue + for (let warmup = 0; warmup < 5; warmup++) { + measure(roots[0], scenario) + measure(roots[1], scenario) + } + const trials = Array.from({ length: options.trials }, () => []) + collect(trials, scenario, options.runs ?? sampling.runs) + if (options.runs === undefined && options.trials >= sampling.trials && !compare(trials).stable) { + process.stderr.write(`${scenario.name}: increasing samples for noisy measurements\n`) + collect(trials, scenario, sampling.maxRuns - sampling.runs) + } + const summary = compare(trials) + benchmarks.push({ ...scenario, trials, summary }) + process.stderr.write(`${scenario.name}: ${summary.changePercent.toFixed(2)}%; ` + + `paired trial CV ${summary.ratioCvPercent.toFixed(2)}%${summary.stable ? '' : ' (unstable)'}\n`) } -if (options.output === undefined) { - process.stdout.write(`${JSON.stringify(report, undefined, 2)}\n`) -} else { - writeFileSync(options.output, `${JSON.stringify(report, undefined, 2)}\n`) +const report = { + formatVersion: 2, + workloadVersion: 1, + nodeVersion: process.version, + platform: process.platform, + arch: process.arch, + selective, + synchronous, + prNumber: process.env.PR_NUMBER === undefined ? 0 : Number(process.env.PR_NUMBER), + baseSha: process.env.BASE_SHA, + headSha: process.env.HEAD_SHA, + mergeSha: process.env.MERGE_SHA, + benchmarks } +const output = `${JSON.stringify(report)}\n` +if (options.output === undefined) process.stdout.write(output) +else writeFileSync(options.output, output) -/** - * @param {string[]} argumentsList Command-line arguments. - * @returns {{ output: string | undefined, packageDirectory: string, runs: number }} - */ +/** @param {string[]} argumentsList Command-line options. */ function parseOptions (argumentsList) { - let output - let packageDirectory = process.cwd() - let runs = 15 - - for (let index = 0; index < argumentsList.length; index++) { - const argument = argumentsList[index] + const options = { + packageDirectory: process.cwd(), + baseDirectory: undefined, + output: undefined, + scenario: undefined, + runs: undefined, + trials: sampling.trials + } + const names = new Map([ + ['--package-directory', 'packageDirectory'], ['--base-directory', 'baseDirectory'], ['--output', 'output'], + ['--runs', 'runs'], ['--trials', 'trials'], ['--scenario', 'scenario'] + ]) + for (let index = 0; index < argumentsList.length; index += 2) { + const name = names.get(argumentsList[index]) const value = argumentsList[index + 1] - - if (argument === '--output' && value !== undefined) { - output = resolve(value) - index++ - } else if (argument === '--package-directory' && value !== undefined) { - packageDirectory = value - index++ - } else if (argument === '--runs' && value !== undefined) { - runs = Number(value) - index++ - } else { - throw new Error(`Unknown or incomplete option: ${argument}`) - } + if (name === undefined || value === undefined) throw new Error('Unknown or incomplete benchmark option') + options[name] = name === 'runs' || name === 'trials' ? Number(value) : name === 'scenario' ? value : resolve(value) } - - if (!Number.isInteger(runs) || runs < 1) { - throw new Error('--runs must be a positive integer') + if ((options.runs !== undefined && (!Number.isInteger(options.runs) || + options.runs < sampling.minRuns || options.runs > sampling.maxRuns)) || + !Number.isInteger(options.trials) || options.trials < sampling.minTrials || options.trials > sampling.maxTrials) { + throw new Error(`Expected ${sampling.minRuns}–${sampling.maxRuns} --runs and ` + + `${sampling.minTrials}–${sampling.maxTrials} --trials`) } - - return { output, packageDirectory, runs } -} - -/** - * @param {string} packageDirectory Directory that contains import-in-the-middle. - * @param {number} runs Number of process starts measured for each scenario. - * @returns {{ name: string, medianMs: number }[]} - */ -function runBenchmarks (packageDirectory, runs) { - const loader = resolve(packageDirectory, 'hook.mjs') - const scenarios = [ - { name: 'No loader', argumentsList: [app] }, - { name: 'Inactive loader', argumentsList: ['--loader', loader, app] }, - { - name: 'Active hook', - argumentsList: ['--require', activeHook, '--loader', loader, app], - environment: { IITM_BENCHMARK_ACTIVE: '1' } - } - ] - - return scenarios.map((scenario) => ({ - name: scenario.name, - medianMs: measureScenario(packageDirectory, scenario, runs) - })) + return options } /** - * @param {string} packageDirectory Directory that contains import-in-the-middle. - * @param {{ argumentsList: string[], environment?: Record }} scenario Benchmark invocation. - * @param {number} runs Number of process starts measured for the scenario. - * @returns {number} + * @param {Pair[][]} trials Existing trials to extend without discarding observations. + * @param {{ mode: string, workload: string }} scenario Public loader invocation. + * @param {number} runs Additional pairs per trial. */ -function measureScenario (packageDirectory, scenario, runs) { - const samples = [] - - for (let index = 0; index < runs; index++) { - const startedAt = process.hrtime.bigint() - const result = spawnSync(process.execPath, scenario.argumentsList, { - cwd: benchmarkRoot, - encoding: 'utf8', - env: { - ...process.env, - IITM_BENCHMARK_PACKAGE_ROOT: packageDirectory, - IITM_BENCHMARK_FIXTURE_ROOT: fixtureRoot, - ...scenario.environment, - NODE_NO_WARNINGS: '1' +function collect (trials, scenario, runs) { + for (let trial = 0; trial < trials.length; trial++) { + const pairs = trials[trial] + const first = pairs.length + for (let pair = first; pair < first + runs; pair++) { + const samples = [] + for (let offset = 0; offset < 2; offset++) { + const index = (trial + pair + offset) % 2 + samples[index] = measure(roots[index], scenario) } - }) - const duration = Number(process.hrtime.bigint() - startedAt) / 1e6 - - if (result.status !== 0) { - throw new Error(`Benchmark process failed: ${result.stderr || result.stdout}`) + pairs.push({ base: samples[0], head: samples[1] }) } - - samples.push(duration) } - - samples.sort(compareNumbers) - return samples[Math.floor(samples.length / 2)] } /** - * @param {number} first First number. - * @param {number} second Second number. - * @returns {number} + * @param {string} packageDirectory Revision to measure. + * @param {{ mode: string, workload: string }} scenario Public loader invocation. + * @returns {Sample} */ -function compareNumbers (first, second) { - return first - second +function measure (packageDirectory, scenario) { + const argumentsList = !selective && scenario.mode.startsWith('async-') + ? ['--loader', resolve(packageDirectory, 'hook.mjs'), fixture] + : [fixture] + if (process.allowedNodeEnvironmentFlags.has('--disable-warning')) { + argumentsList.unshift('--disable-warning=ExperimentalWarning') + } + const environment = { + ...process.env, + NODE_OPTIONS: '', + NODE_COMPILE_CACHE: '', + NODE_DISABLE_COMPILE_CACHE: '1', + NODE_DEBUG: '', + NODE_DEBUG_NATIVE: '', + TZ: 'UTC', + IITM_BENCHMARK_PACKAGE_ROOT: packageDirectory, + IITM_BENCHMARK_MODE: scenario.mode, + IITM_BENCHMARK_WORKLOAD: scenario.workload + } + delete environment.NODE_NO_WARNINGS + delete environment.NODE_REDIRECT_WARNINGS + const result = spawnSync(process.execPath, argumentsList, { + encoding: 'utf8', + timeout: 30000, + env: environment + }) + if (result.error || result.status !== 0) { + throw new Error(`Benchmark process failed: ${result.error?.message ?? result.stderr ?? result.stdout}`) + } + const warnings = result.stderr.replace(legacyWarning, '') + if (warnings) throw new Error(`Benchmark emitted warnings: ${warnings}`) + return JSON.parse(result.stdout) } diff --git a/benchmark/scenarios.mjs b/benchmark/scenarios.mjs new file mode 100644 index 0000000..98a03ac --- /dev/null +++ b/benchmark/scenarios.mjs @@ -0,0 +1,14 @@ +/** + * @param {boolean} selective Whether asynchronous registration is available. + * @param {boolean} synchronous Whether synchronous hooks are supported. + */ +export function getScenarios (selective, synchronous) { + const modes = ['none', 'async-inactive', 'async-active'] + if (selective) modes.push('async-selective', 'async-excluded') + if (synchronous) modes.push('sync-inactive', 'sync-active', 'sync-selective', 'sync-excluded') + const scenarios = [] + for (const workload of ['small', 'dependency']) { + for (const mode of modes) scenarios.push({ name: `${workload}/${mode}`, workload, mode }) + } + return scenarios +} diff --git a/benchmark/statistics.mjs b/benchmark/statistics.mjs new file mode 100644 index 0000000..223ef4c --- /dev/null +++ b/benchmark/statistics.mjs @@ -0,0 +1,66 @@ +/** @typedef {{ wallMs: number, cpuMs: number }} Sample */ +/** @typedef {{ base: Sample, head: Sample }} Pair */ + +export const sampling = Object.freeze({ + minTrials: 2, trials: 8, maxTrials: 20, minRuns: 2, runs: 20, maxRuns: 100, cvPercent: 2 +}) + +/** @param {number[]} values Independent observations. */ +export function summarize (values) { + const mean = values.reduce( + /** + * @param {number} sum + * @param {number} value + */ + (sum, value) => sum + value, 0) / values.length + const variance = values.reduce( + /** + * @param {number} sum + * @param {number} value + */ + (sum, value) => sum + (value - mean) ** 2, 0) / (values.length - 1) + const sorted = [...values].sort( + /** + * @param {number} left + * @param {number} right + */ + (left, right) => left - right) + const middle = Math.floor(sorted.length / 2) + const median = sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle] + return { + mean, + median, + p95: sorted[Math.ceil(sorted.length * 0.95) - 1], + stddev: Math.sqrt(variance), + cvPercent: 100 * Math.sqrt(variance) / mean + } +} + +/** @param {Pair[][]} trials Paired observations grouped by trial. */ +export function compare (trials) { + const ratios = [] + const base = [] + const head = [] + for (const trial of trials) { + const trialRatios = [] + for (const pair of trial) { + trialRatios.push(pair.head.wallMs / pair.base.wallMs) + base.push(pair.base.wallMs) + head.push(pair.head.wallMs) + } + ratios.push(summarize(trialRatios).median) + } + const ratio = summarize(ratios) + return { + base: summarize(base), + head: summarize(head), + changePercent: 100 * (ratio.mean - 1), + ratioCvPercent: ratio.cvPercent, + changeStddevPercent: 100 * ratio.stddev, + standardErrorPercent: 100 * ratio.stddev / Math.sqrt(trials.length), + stable: trials.length >= sampling.trials && trials.every( + /** @param {Pair[]} trial */ + trial => trial.length >= sampling.runs) && + ratio.cvPercent <= sampling.cvPercent + } +} diff --git a/package.json b/package.json index ad3b2b1..9f01323 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "test:ts": "c8 --reporter lcov imhotap --files test/typescript/*.test.mts", "coverage": "c8 --reporter html imhotap --files test/{hook,low-level,other,get-esm-exports,register}/* && echo \"\nNow open coverage/index.html\n\"", "benchmark": "node benchmark/run.mjs", + "test:benchmark": "node test/benchmark.mjs", "lint": "eslint .", "lint:fix": "eslint . --fix" }, @@ -45,7 +46,7 @@ "@react-email/components": "^0.0.19", "@types/node": "^18.0.6", "c8": "^7.14.0", - "date-fns": "^3.6.0", + "date-fns": "3.6.0", "eslint": "^8.57.1", "eslint-config-standard": "^17.1.0", "eslint-plugin-import": "^2.32.0", diff --git a/test/benchmark.mjs b/test/benchmark.mjs new file mode 100644 index 0000000..9829a72 --- /dev/null +++ b/test/benchmark.mjs @@ -0,0 +1,304 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, truncateSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { ESLint } from 'eslint' + +import { readReport, formatReports } from '../benchmark/report.mjs' +import { getScenarios } from '../benchmark/scenarios.mjs' +import { compare, summarize } from '../benchmark/statistics.mjs' + +/** @typedef {ReturnType[number]} Scenario */ +/** @typedef {import('../benchmark/report.mjs').Benchmark} Benchmark */ +/** @typedef {import('../benchmark/statistics.mjs').Pair} Pair */ +/** + * @typedef {{ owner: string, repo: string, body: string, issue_number?: number, comment_id?: number }} CommentRequest + */ +/** @typedef {{ operation: 'create' | 'update', input: CommentRequest }} CommentCall */ + +const root = fileURLToPath(new URL('..', import.meta.url)) +const require = createRequire(import.meta.url) +const directory = mkdtempSync(join(tmpdir(), 'iitm-benchmark-test-')) +const filename = join(directory, 'report.json') +const pair = { base: { wallMs: 100, cpuMs: 50 }, head: { wallMs: 110, cpuMs: 60 } } +const trials = Array.from({ length: 8 }, () => Array.from({ length: 20 }, () => structuredClone(pair))) +const original = { + formatVersion: 2, + workloadVersion: 1, + prNumber: 293, + baseSha: 'a'.repeat(40), + headSha: 'b'.repeat(40), + mergeSha: 'c'.repeat(40), + nodeVersion: 'v24.20.0', + platform: 'linux', + arch: 'x64', + selective: true, + synchronous: true, + benchmarks: getScenarios(true, true).map( + /** @param {Scenario} scenario */ + scenario => ({ ...scenario, trials: structuredClone(trials) })) +} + +/** @param {object} report Artifact to publish to the parser. */ +function parse (report) { + writeFileSync(filename, JSON.stringify(report)) + return readReport(filename) +} + +/** @param {string[]} argumentsList CLI options. */ +function run (argumentsList) { + return spawnSync(process.execPath, [join(root, 'benchmark/run.mjs'), ...argumentsList], { + cwd: root, encoding: 'utf8' + }) +} + +/** + * @param {{ head?: string, base?: string, merge?: string, state?: string, existing?: boolean, + * payloadBase?: string, parents?: string[] }} options + * @param {CommentCall[]} [calls] + */ +async function publish (options = {}, calls = []) { + const workflow = readFileSync(join(root, '.github/workflows/benchmark-report.yml'), 'utf8') + assert.match(workflow, /concurrency:\n {6}group: benchmark-report\n {6}cancel-in-progress: false/) + assert.match(workflow, /cancel-in-progress: false\n {6}queue: max/) + assert.match(workflow, /ref: \$\{\{ github.sha \}\}/) + const producer = readFileSync(join(root, '.github/workflows/benchmark.yml'), 'utf8') + assert.match(producer, /ref: \$\{\{ github.sha \}\}/) + assert.match(producer, /MERGE_SHA: \$\{\{ github.sha \}\}/) + assert.match(producer, /fetch-depth: 2/) + assert.match(producer, /BASE_SHA=\$\(git rev-parse HEAD\^1\)/) + assert.match(producer, /BASE_SHA: \$\{\{ steps.base.outputs.sha \}\}/) + const script = workflow.split(' script: |\n')[1].replace(/^ {12}/gm, '') + const wrapped = 'module.exports = async function (github, context, core) {\n' + + `${script.trimEnd().replace(/^/gm, ' ')}\n}\n` + const lint = await new ESLint().lintText(wrapped, { filePath: 'benchmark/workflow-script.cjs' }) + assert.deepEqual(lint[0].messages, []) + const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor + const execute = new AsyncFunction('require', 'github', 'context', 'core', script) + const github = { + rest: { + pulls: { + get: async () => ({ + data: { + state: options.state ?? 'open', + merge_commit_sha: options.merge ?? original.mergeSha, + head: { sha: options.head ?? original.headSha }, + base: { sha: options.payloadBase ?? original.baseSha } + } + }) + }, + repos: { + getCommit: async () => ({ + data: { + parents: (options.parents ?? [options.base ?? original.baseSha, original.headSha]).map( + /** @param {string} sha */ + sha => ({ sha })) + } + }) + }, + issues: { + listComments: async () => {}, + /** @param {CommentRequest} input */ + createComment: async input => { calls.push({ operation: 'create', input }) }, + /** @param {CommentRequest} input */ + updateComment: async input => { calls.push({ operation: 'update', input }) } + } + }, + paginate: async () => options.existing + ? [{ id: 7, user: { login: 'github-actions[bot]' }, body: '' }] + : [] + } + await execute(require, github, { + repo: { owner: 'nodejs', repo: 'import-in-the-middle' }, + payload: { workflow_run: { head_sha: original.headSha } } + }, { info: () => {} }) + return calls +} + +try { + assert.deepEqual(summarize([1, 2, 3]), { mean: 2, median: 2, p95: 3, stddev: 1, cvPercent: 50 }) + assert.equal(summarize([1, 2, 3, 100]).median, 2.5) + const paired = [ + { base: { wallMs: 1, cpuMs: 1 }, head: { wallMs: 2, cpuMs: 2 } }, + { base: { wallMs: 10, cpuMs: 10 }, head: { wallMs: 5, cpuMs: 5 } }, + { base: { wallMs: 100, cpuMs: 100 }, head: { wallMs: 200, cpuMs: 200 } } + ] + assert.equal(compare([paired, paired]).changePercent, 100) + const comparison = compare(trials) + assert.ok(Math.abs(comparison.changePercent - 10) < 1e-10) + assert.ok(comparison.stable) + assert.equal(compare(trials.slice(0, 2)).stable, false) + assert.equal(compare(trials.map( + /** @param {Pair[]} trial */ + trial => trial.slice(0, 2))).stable, false) + const noisy = structuredClone(trials) + for (let index = 0; index < noisy.length; index++) { + for (const observation of noisy[index]) observation.head.wallMs = index % 2 === 0 ? 100 : 140 + } + assert.equal(compare(noisy).stable, false) + assert.deepEqual(parse(original), original) + const text = formatReports([original]) + assert.ok(text.includes('+10.00%')) + const faster = structuredClone(original) + for (const benchmark of faster.benchmarks) { + for (const trial of benchmark.trials) { + for (const observation of trial) observation.head.wallMs = 90 + } + } + assert.ok(formatReports([parse(faster)]).includes('-10.00%')) + const slowStarts = structuredClone(trials) + for (const trial of slowStarts) trial[0].head.wallMs = 1000 + const typical = compare(slowStarts) + assert.ok(Math.abs(typical.changePercent - 10) < 1e-10) + assert.ok(typical.head.mean > 110) + assert.ok(typical.head.stddev > 100) + const forgedSummary = structuredClone(original) + forgedSummary.benchmarks[0].summary = { changePercent: -50 } + assert.ok(formatReports([parse(forgedSummary)]).includes('+10.00%')) + const unstable = structuredClone(original) + unstable.benchmarks[0].trials = noisy + assert.ok(formatReports([parse(unstable)]).includes('| Unstable |')) + + for (const field of ['formatVersion', 'workloadVersion', 'prNumber', 'baseSha', 'headSha', 'mergeSha', + 'nodeVersion', 'platform', 'arch', 'selective', 'synchronous']) { + const report = structuredClone(original) + report[field] = undefined + assert.throws(() => parse(report), /Invalid benchmark metadata/) + } + for (const value of [undefined, [], [original.benchmarks[0]]]) { + assert.throws(() => parse({ ...original, benchmarks: value }), /Invalid benchmark scenarios/) + } + for (const count of [1, 21]) { + const report = structuredClone(original) + report.benchmarks[0].trials = Array.from({ length: count }, () => [pair, pair]) + assert.throws(() => parse(report), /Invalid benchmark trials/) + } + for (const count of [2, 20]) { + const report = structuredClone(original) + report.benchmarks[0].trials = Array.from({ length: count }, () => [pair, pair]) + assert.equal(parse(report).benchmarks[0].trials.length, count) + } + for (const count of [1, 101]) { + const report = structuredClone(original) + report.benchmarks[0].trials = [Array.from({ length: count }, () => pair), [pair, pair]] + assert.throws(() => parse(report), /Invalid benchmark runs/) + } + for (const count of [2, 100]) { + const report = structuredClone(original) + report.benchmarks[0].trials = Array.from({ length: 2 }, () => Array.from({ length: count }, () => pair)) + assert.equal(parse(report).benchmarks[0].trials[0].length, count) + } + for (const field of ['wallMs', 'cpuMs']) { + const limit = field === 'wallMs' ? 30000 : 1000000 + const accepted = structuredClone(original) + accepted.benchmarks[0].trials[0][0].head[field] = limit + assert.equal(parse(accepted).benchmarks[0].trials[0][0].head[field], limit) + for (const value of [0, -1, null, '1', limit + 1]) { + const report = structuredClone(original) + report.benchmarks[0].trials[0][0].head[field] = value + assert.throws(() => parse(report), /Invalid benchmark sample/) + } + } + const mismatched = structuredClone(original) + mismatched.benchmarks[0].trials[0].pop() + assert.throws(() => parse(mismatched), /Inconsistent benchmark runs/) + writeFileSync(filename, '{}') + truncateSync(filename, 8 * 1024 * 1024 + 1) + assert.throws(() => readReport(filename), /too large/) + + process.env.GITHUB_WORKSPACE = root + process.env.ARTIFACT_DIRECTORY = directory + process.env.WORKFLOW_RUN_ID = '123' + for (const major of [22, 24]) { + const artifactDirectory = join(directory, `benchmark-report-node${major}`) + mkdirSync(artifactDirectory) + writeFileSync(join(artifactDirectory, 'report.json'), JSON.stringify({ + ...original, nodeVersion: `v${major}.23.3` + })) + } + const created = (await publish())[0] + assert.equal(created.operation, 'create') + assert.equal(created.input.issue_number, 293) + assert.match(created.input.body, /import-in-the-middle-benchmark/) + assert.match(created.input.body, /\+10\.00%/) + assert.match(created.input.body, /actions\/runs\/123/) + const updated = (await publish({ existing: true }))[0] + assert.equal(updated.operation, 'update') + assert.equal(updated.input.comment_id, 7) + assert.equal((await publish({ payloadBase: 'd'.repeat(40) }))[0].operation, 'create') + for (const options of [{ head: 'c'.repeat(40) }, { merge: 'd'.repeat(40) }, { state: 'closed' }]) { + assert.deepEqual(await publish(options), []) + } + for (const options of [{ base: 'd'.repeat(40) }, { parents: [] }, + { parents: [original.baseSha] }, { parents: [original.baseSha, 'd'.repeat(40)] }]) { + const calls = [] + await assert.rejects(publish(options, calls), + /** @param {Error} error */ + error => { + assert.deepEqual(calls, []) + return error.message === 'Mismatched benchmark merge parents' + }) + } + const artifact = join(directory, 'benchmark-report-node24/report.json') + for (const fields of [{ headSha: 'c'.repeat(40) }, { baseSha: 'c'.repeat(40) }, { prNumber: 294 }, + { mergeSha: 'd'.repeat(40) }]) { + writeFileSync(artifact, JSON.stringify({ ...original, ...fields })) + await assert.rejects(publish(), /Mismatched benchmark sources/) + } + writeFileSync(artifact, JSON.stringify({ ...original, nodeVersion: 'v22.23.3' })) + await assert.rejects(publish(), /Unexpected Node version/) + + for (const argumentsList of [['--runs', '1'], ['--runs', '101'], ['--trials', '1'], ['--trials', '21'], + ['--unknown', '2'], ['--output'], ['--scenario', 'missing']]) { + const result = run(argumentsList) + assert.notEqual(result.status, 0) + assert.match(result.stderr, /Expected|Unknown/) + } + assert.equal(run(['--help']).status, 0) + const selected = run(['--runs', '2', '--trials', '2', '--scenario', 'small/none']) + assert.equal(selected.status, 0, selected.stderr) + assert.equal(JSON.parse(selected.stdout).benchmarks.length, 1) + const missingRoot = run(['--runs', '2', '--trials', '2', '--scenario', 'small/async-inactive', + '--package-directory', join(directory, 'missing')]) + assert.notEqual(missingRoot.status, 0) + assert.match(missingRoot.stderr, /Benchmark process failed/) + const warningRoot = join(directory, 'warning') + mkdirSync(warningRoot) + writeFileSync(join(warningRoot, 'hook.mjs'), + "process.emitWarning('failed to wrap benchmark module')\nexport {}\n") + const warning = run(['--runs', '2', '--trials', '2', '--scenario', 'small/async-inactive', + '--package-directory', warningRoot]) + assert.notEqual(warning.status, 0) + assert.match(warning.stderr, /Benchmark emitted warnings.*failed to wrap benchmark module/) + const blockedRoot = join(directory, 'blocked') + mkdirSync(blockedRoot) + writeFileSync(join(blockedRoot, 'hook.mjs'), + 'Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0)\nexport {}\n') + const blocked = run(['--runs', '2', '--trials', '2', '--scenario', 'small/async-inactive', + '--package-directory', blockedRoot]) + assert.notEqual(blocked.status, 0) + assert.match(blocked.stderr, /Benchmark process failed: .*ETIMEDOUT/) + const result = run(['--runs', '2', '--trials', '2', '--output', filename]) + assert.equal(result.status, 0, result.stderr) + const raw = JSON.parse(readFileSync(filename, 'utf8')) + assert.equal(raw.formatVersion, 2) + assert.deepEqual(raw.benchmarks.map( + /** @param {Benchmark} benchmark */ + benchmark => benchmark.name), + getScenarios(raw.selective, raw.synchronous).map( + /** @param {Scenario} scenario */ + scenario => scenario.name)) + for (const benchmark of raw.benchmarks) { + assert.equal(benchmark.trials.length, 2) + assert.equal(benchmark.trials[0].length, 2) + assert.equal(benchmark.summary.stable, false) + assert.ok(benchmark.trials[0][0].base.cpuMs > 0) + } + console.log('Benchmark protocol, statistics, CLI, and all available loader modes passed.') +} finally { + rmSync(directory, { recursive: true, force: true }) +}