ci: add pull request benchmark reports

.github/workflows/benchmark-report.yml

@@ -0,0 +1,86 @@
+name: Benchmark report
+
+on:
+  workflow_run:
+    workflows: [Benchmark]
+    types: [completed]
+
+permissions:
+  contents: read
+  actions: read
+  issues: write
+  pull-requests: read
+
+jobs:
+  report:
+    concurrency:
+      group: benchmark-report
+      cancel-in-progress: false
+      queue: max
+    if: >-
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - name: Download benchmark reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: benchmark-report-node*
+          path: ${{ runner.temp }}/benchmark-report
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Publish benchmark report
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        env:
+          ARTIFACT_DIRECTORY: ${{ runner.temp }}/benchmark-report
+          WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const path = require('node:path')
+            const { pathToFileURL } = require('node:url')
+            const { readReport, formatReports } = await import(
+              pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'benchmark/report.mjs')).href)
+            const reports = [22, 24].map((major) => {
+              const report = readReport(path.join(process.env.ARTIFACT_DIRECTORY,
+                `benchmark-report-node${major}`, 'report.json'))
+              if (!report.nodeVersion.startsWith(`v${major}.`)) throw new Error('Unexpected Node version')
+              return report
+            })
+            const report = reports[0]
+            if (reports.some((entry) => entry.headSha !== context.payload.workflow_run.head_sha ||
+              entry.headSha !== report.headSha || entry.baseSha !== report.baseSha ||
+              entry.mergeSha !== report.mergeSha ||
+              entry.prNumber !== report.prNumber)) throw new Error('Mismatched benchmark sources')
+            const pull = await github.rest.pulls.get({
+              owner: context.repo.owner, repo: context.repo.repo, pull_number: report.prNumber
+            })
+            if (pull.data.state !== 'open' || pull.data.head.sha !== report.headSha ||
+              pull.data.merge_commit_sha !== report.mergeSha) {
+              core.info('The pull request is closed or its revisions have changed.')
+              return
+            }
+            const merge = await github.rest.repos.getCommit({
+              owner: context.repo.owner, repo: context.repo.repo, ref: report.mergeSha
+            })
+            if (merge.data.parents[0]?.sha !== report.baseSha || merge.data.parents[1]?.sha !== report.headSha) {
+              throw new Error('Mismatched benchmark merge parents')
+            }
+            const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.WORKFLOW_RUN_ID}`
+            const body = `${formatReports(reports)}\n\n[Workflow run](${runUrl})`
+            const marker = '<!-- import-in-the-middle-benchmark -->'
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner, repo: context.repo.repo, issue_number: report.prNumber
+            })
+            const existing = comments.find((comment) =>
+              comment.user?.login === 'github-actions[bot]' && comment.body?.includes(marker))
+            const target = { owner: context.repo.owner, repo: context.repo.repo, body }
+            if (existing === undefined) {
+              await github.rest.issues.createComment({ ...target, issue_number: report.prNumber })
+            } else {
+              await github.rest.issues.updateComment({ ...target, comment_id: existing.id })
+            }

.github/workflows/benchmark.yml

@@ -0,0 +1,50 @@
+name: Benchmark
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, edited]
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        node: [22, 24]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          fetch-depth: 2
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: ${{ matrix.node }}
+      - run: npm install --ignore-scripts
+      - name: Check out the base revision
+        id: base
+        run: |
+          BASE_SHA=$(git rev-parse HEAD^1)
+          git fetch --no-tags --depth=1 origin "$BASE_SHA"
+          git worktree add --detach "$RUNNER_TEMP/base" "$BASE_SHA"
+          npm install --ignore-scripts --prefix "$RUNNER_TEMP/base"
+          printf 'sha=%s\n' "$BASE_SHA" >> "$GITHUB_OUTPUT"
+      - name: Measure paired revisions
+        env:
+          BASE_SHA: ${{ steps.base.outputs.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          MERGE_SHA: ${{ github.sha }}
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          BENCHMARK_CPU=$(awk '/^Cpus_allowed_list:/ { split($2, cpus, /[-,]/); print cpus[1] }' /proc/self/status)
+          taskset --cpu-list "$BENCHMARK_CPU" node benchmark/run.mjs \
+            --base-directory "$RUNNER_TEMP/base" \
+            --output "$RUNNER_TEMP/report.json"
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: benchmark-report-node${{ matrix.node }}
+          path: ${{ runner.temp }}/report.json
+          if-no-files-found: error

.github/workflows/ci.yml

@@ -18,6 +18,7 @@ jobs:
           node-version: 'lts/*'
       - run: npm install
       - run: npm run lint
+      - run: npm run test:benchmark
 
   build:
     runs-on: ubuntu-latest

.npmignore

@@ -9,3 +9,4 @@ coverage
 release-please-config.json
 tsconfig.json
 test
+benchmark

README.md

@@ -115,6 +115,11 @@ fs.readFileSync('file.txt')
 node --import=./instrument.mjs ./my-app.mjs
 ```
 
+## Development benchmarks
+
+See [benchmark/README.md](benchmark/README.md) for the PR benchmark method and
+how to check measurement noise locally.
+
 ## Synchronous loader hooks
 
 On Node.js versions that support

benchmark/README.md

@@ -0,0 +1,50 @@
+# Pull request benchmarks
+
+Run `npm run benchmark -- --help` for the command-line options. Without a base
+directory, the command compares the current revision with itself to check noise.
+Both revisions use the same fixture and dependency installation.
+CI compares the base with GitHub's merged PR revision. This includes current
+base changes when the contributor's branch is older. Reports retain the base,
+PR head, and measured merge revisions so the comparison can be reproduced.
+
+Each sample starts a new Node.js process. The fixture records elapsed time when
+its imports and behavior checks complete. Process teardown is outside that
+measurement. CPU time is retained as a diagnostic and includes loader-worker
+CPU use. It does not include time spent waiting for a worker response.
+
+The small graph exposes fixed loader startup costs. The dependency graph uses
+the pinned date-fns development dependency and a CommonJS module. It exercises
+package resolution, many ESM exports, and CommonJS interoperability. These are
+loader workloads, not application request-throughput measurements.
+
+The driver warms the filesystem cache, starts each base/head pair consecutively,
+and alternates their order. It disables compilation caching. Independent trials
+retain every sample, including slow samples. The reported change is the mean of
+the trial estimates. Each trial estimate is the median of its paired head/base
+duration ratios. This preserves the pairing when scheduling delays change over time.
+CI pins both revisions and their loader workers to the same allowed CPU. This
+controls CPU migration and measures startup under a single-core constraint.
+Noisy scenarios receive one bounded extension of the same trials. The original
+observations remain in the result. An explicit sample count disables this
+extension. The spread is descriptive; the report does not claim a formal
+significance test after adaptive sampling.
+
+The median measures typical startup and limits the effect of isolated scheduling
+delays. It can miss a regression affecting only occasional starts. The report
+also shows raw means, standard deviations, and p95 durations to expose those costs.
+Trial CV is the standard deviation of the paired median ratios divided by their mean.
+The report labels a row unstable when its trial CV exceeds the implemented
+stability threshold. Base and head columns also show the raw startup standard
+deviation. A stable ratio does not mean that individual startup samples have
+low variation. Use repeated self-comparisons to check reproducibility on the
+machine that will run the benchmark.
+
+The baseline is the test merge's first parent. Pull request payloads can retain
+an older base SHA after the target branch advances. The publisher verifies both
+merge parents against the measured baseline and PR head.
+
+PR execution produces artifacts with read-only permissions. A separate workflow
+loads reporting code from the default branch, validates the artifact's source
+and samples, and computes its own statistics before updating the bot comment.
+The PR controls benchmark execution. Artifact validation does not attest to the
+accuracy of its timings.

benchmark/fixture/a.mjs

@@ -0,0 +1 @@
+export const value = 1

benchmark/fixture/app.mjs

@@ -0,0 +1,66 @@
+import assert from 'node:assert/strict'
+import * as module from 'node:module'
+import { resolve } from 'node:path'
+import { performance } from 'node:perf_hooks'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+
+const root = process.env.IITM_BENCHMARK_PACKAGE_ROOT
+const mode = process.env.IITM_BENCHMARK_MODE
+const workload = process.env.IITM_BENCHMARK_WORKLOAD
+assert.ok(root && mode && workload, 'Benchmark environment is required')
+const require = module.createRequire(import.meta.url)
+const active = mode.endsWith('-active') || mode.endsWith('-selective')
+const target = workload === 'small' ? fileURLToPath(new URL('a.mjs', import.meta.url)) : 'date-fns'
+let hooks = 0
+let calls = 0
+
+if (active) {
+  const { Hook } = require(resolve(root, 'index.js'))
+  Hook([target],
+    /** @param {{ value: number, addDays: typeof import('date-fns').addDays }} exports */
+    (exports) => {
+      hooks++
+      if (workload === 'small') {
+        exports.value++
+      } else {
+        const addDays = exports.addDays
+        /**
+       * @param {Date} date Date to advance.
+       * @param {number} amount Days to add.
+       */
+        exports.addDays = function (date, amount) {
+          calls++
+          return addDays(date, amount)
+        }
+      }
+    })
+}
+
+if (mode !== 'none') {
+  const options = mode.endsWith('-excluded')
+    ? { include: [] }
+    : mode.endsWith('-selective')
+      ? { include: [workload === 'small' ? pathToFileURL(target).href : target] }
+      : undefined
+  if (mode.startsWith('sync-')) {
+    const { register } = await import(pathToFileURL(resolve(root, 'register-hooks.mjs')).href)
+    register(options)
+  } else if (module.register) {
+    module.register(pathToFileURL(resolve(root, 'hook.mjs')), { data: options })
+  }
+}
+
+if (workload === 'small') {
+  const { sum } = await import('./small.mjs')
+  assert.equal(sum, active ? 88 : 87)
+} else {
+  const { addDays, isValid, parseISO } = await import('date-fns')
+  const { default: Clock } = await import('./commonjs.cjs')
+  const date = parseISO('2026-09-09T00:00:00Z')
+  assert.equal(isValid(date), true)
+  assert.equal(new Clock().copy(addDays(date, 1)).getUTCDate(), 10)
+  assert.equal(calls, active ? 1 : 0)
+}
+assert.equal(hooks, active ? 1 : 0)
+const cpu = process.cpuUsage()
+process.stdout.write(`${JSON.stringify({ wallMs: performance.now(), cpuMs: (cpu.user + cpu.system) / 1000 })}\n`)

benchmark/fixture/b.mjs

@@ -0,0 +1 @@
+export const value = 2

benchmark/fixture/c.mjs

@@ -0,0 +1 @@
+export const value = 3

benchmark/fixture/commonjs.cjs

@@ -0,0 +1,8 @@
+const { EventEmitter } = require('node:events')
+
+module.exports = class Clock extends EventEmitter {
+  /** @param {Date} date Date to copy. */
+  copy (date) {
+    return new Date(date.getTime())
+  }
+}

benchmark/fixture/d.mjs

@@ -0,0 +1 @@
+export const value = 5

benchmark/fixture/e.mjs

@@ -0,0 +1 @@
+export const value = 8

benchmark/fixture/f.mjs

@@ -0,0 +1 @@
+export const value = 13

benchmark/fixture/g.mjs

@@ -0,0 +1 @@
+export const value = 21

benchmark/fixture/h.mjs

@@ -0,0 +1 @@
+export const value = 34

benchmark/fixture/small.mjs

@@ -0,0 +1,10 @@
+import { value as a } from './a.mjs'
+import { value as b } from './b.mjs'
+import { value as c } from './c.mjs'
+import { value as d } from './d.mjs'
+import { value as e } from './e.mjs'
+import { value as f } from './f.mjs'
+import { value as g } from './g.mjs'
+import { value as h } from './h.mjs'
+
+export const sum = a + b + c + d + e + f + g + h