lib: set exit code on failed node:bench run()

Signed-off-by: RafaelGSS <rafael.nunu@hotmail.com>

Author: RafaelGSS

doc/api/bench.md

@@ -62,8 +62,10 @@ node --experimental-bench --bench benchmark.mjs
 Benchmarks are executed serially in declaration order. Declared benchmarks are
 scheduled automatically. Call `run()` during the same turn as the declarations
 to consume the event stream or configure filtering.
-If an automatically scheduled run fails and `run()` was not called, the process
-exit code is set to `1`.
+If a benchmark run started automatically or through the module-level `run()`
+fails, the process exit code is set to `1` unless `process.exitCode` has
+already been set. Runners created with [`createRunner()`][] leave the exit code
+to their caller.
 
 ## Measurement model
 
@@ -304,7 +306,8 @@ runner and obtain its {BenchmarksStream}.
 
 Each runner can be started once. Its `run()` function accepts the same options
 as the module-level [`run()`][]. `run({ yieldBetweenSamples })` overrides the
-value passed to `createRunner()`.
+value passed to `createRunner()`. Unlike the module-level runner, a failed
+benchmark in an explicit runner does not change the process exit code.
 
 ## `bench([name][, options], fn)`
 
@@ -498,6 +501,9 @@ execution begins. Calling `run()` is optional when the returned stream is not
 needed. An explicit runner created by `createRunner()` does not run
 automatically, so its `run()` function may be called later.
 
+If any benchmark fails, the process exit code is set to `1` once the run
+finishes, unless `process.exitCode` has already been set.
+
 ```mjs
 import { bench, run } from 'node:bench';
 
@@ -843,6 +849,7 @@ A completed benchmark result contains:
   * `skewness` {number} The skewness of the scaled rate histogram.
 
 [`context.record()`]: #contextrecordsample
+[`createRunner()`]: #createrunneroptions
 [`run()`]: #runoptions
 [benchmark result]: #benchmark-result
 [command-line options documentation]: cli.md#--bench

lib/internal/bench_runner/harness.js

@@ -104,6 +104,7 @@ class Harness {
   #abortReason = null;
   #autoRun;
   #buildPromises = [];
+  #cliRun = false;
   #duplicateErrors = new SafeMap();
   #explicitRun = false;
   #fileScopeStorage = new AsyncLocalStorage();
@@ -318,6 +319,7 @@ class Harness {
     try {
       this.configure(options);
       this.#explicitRun = true;
+      this.#cliRun = force;
     } finally {
       this.#starting = false;
     }
@@ -1035,8 +1037,10 @@ class Harness {
       this.root.completion.resolve();
       this.root.emitDestroy();
       this.#storage.disable();
-      if (!this.#explicitRun && !this.success) {
-        process.exitCode = kGenericUserError;
+      // Runners created with createRunner() leave the exit code to their
+      // caller, and the CLI runner derives it from the aggregated summary.
+      if (this.#autoRun && !this.#cliRun && !this.success) {
+        process.exitCode ??= kGenericUserError;
       }
     }
   }

test/parallel/test-bench-clock-precision.js

@@ -19,4 +19,7 @@ const completion = bench('zero duration', { samples: 1 }, (b) => {
   const result = await completion;
   assert.strictEqual(result.error.code, 'ERR_INVALID_STATE');
   assert.match(result.error.message, /insufficient clock precision/);
+  // The failed run sets the exit code; reset it so the test itself passes.
+  assert.strictEqual(process.exitCode, 1);
+  process.exitCode = 0;
 })().then(common.mustCall());

test/parallel/test-bench-errors.js

@@ -81,6 +81,9 @@ stream.on('end', common.mustCall(() => {
     total: 17,
   });
   assert.strictEqual(summary.success, false);
+  // The failed run sets the exit code; reset it so the test itself passes.
+  assert.strictEqual(process.exitCode, 1);
+  process.exitCode = 0;
 
   const byName = new Map();
   for (const result of completions) {

test/parallel/test-bench-hook-errors.js

@@ -57,6 +57,9 @@ stream.on('end', common.mustCall(() => {
     total: 4,
   });
   assert.strictEqual(summary.success, false);
+  // The failed run sets the exit code; reset it so the test itself passes.
+  assert.strictEqual(process.exitCode, 1);
+  process.exitCode = 0;
 
   const byName = new Map(completions.map((result) => [result.name, result]));
   assert.strictEqual(byName.get('blocked by before').error.message,

test/parallel/test-bench-reporters.js

@@ -55,6 +55,9 @@ bench('json failed', { samples: 1 }, () => {
     skipped: 0,
     total: 2,
   });
+  // The failed run sets the exit code; reset it so the test itself passes.
+  assert.strictEqual(process.exitCode, 1);
+  process.exitCode = 0;
 
   const synthetic = [
     {

test/parallel/test-bench-run-exit-code.js

@@ -0,0 +1,68 @@
+// Flags: --experimental-bench --no-warnings
+'use strict';
+
+require('../common');
+const assert = require('assert');
+const { spawnSync } = require('child_process');
+
+function runScript(script) {
+  return spawnSync(process.execPath, [
+    '--no-warnings',
+    '--experimental-bench',
+    '-e',
+    script,
+  ]);
+}
+
+// A failing benchmark consumed through the module-level run() sets the exit
+// code.
+{
+  const child = runScript(`
+    const { bench, run } = require('node:bench');
+    bench('failure', () => { throw new Error('boom'); });
+    (async () => { for await (const record of run()); })();
+  `);
+  assert.strictEqual(child.status, 1, child.stderr.toString());
+}
+
+// A successful run through the module-level run() keeps the default exit code.
+{
+  const child = runScript(`
+    const { bench, run } = require('node:bench');
+    bench('success', { samples: 1 }, (b) => {
+      b.record({ duration_ns: 1n, operations: 1 });
+    });
+    (async () => { for await (const record of run()); })();
+  `);
+  assert.strictEqual(child.status, 0, child.stderr.toString());
+}
+
+// An exit code set by the user is preserved when the explicit run fails.
+{
+  const child = runScript(`
+    const { bench, run } = require('node:bench');
+    process.exitCode = 3;
+    bench('failure', () => { throw new Error('boom'); });
+    (async () => { for await (const record of run()); })();
+  `);
+  assert.strictEqual(child.status, 3, child.stderr.toString());
+}
+
+// An exit code set by the user is preserved when an automatic run fails.
+{
+  const child = runScript(`
+    process.exitCode = 3;
+    require('node:bench').bench('failure', () => { throw new Error(); });
+  `);
+  assert.strictEqual(child.status, 3, child.stderr.toString());
+}
+
+// Runners created with createRunner() leave the exit code to the caller.
+{
+  const child = runScript(`
+    const runner = require('node:bench').createRunner();
+    runner.bench('failure', () => { throw new Error('boom'); });
+    (async () => { for await (const record of runner.run()); })();
+  `);
+  assert.strictEqual(child.status, 0, child.stderr.toString());
+}