diff --git a/.claims.json b/.claims.json index ab1909b8..bc167f3c 100644 --- a/.claims.json +++ b/.claims.json @@ -5049,8 +5049,8 @@ ], "version": 1 }, - "generatedAt": "2026-09-28T21:23:33.437Z", - "generatedFromCommit": "1b8c7fea", + "generatedAt": "2026-09-28T22:03:05.332Z", + "generatedFromCommit": "9852c053", "generatorVersion": "1.0.0", "llmJudgeTemplates": { "count": 7, @@ -9266,7 +9266,7 @@ "passed": null, "total": 40 }, - "totalCombined": 4174, + "totalCombined": 4182, "vitestDashboard": { "failed": 0, "passed": 400, @@ -9274,8 +9274,8 @@ }, "vitestRoot": { "failed": 0, - "passed": 3734, - "total": 3734 + "passed": 3742, + "total": 3742 } }, "version": { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56f63886..507a0df3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,6 +167,72 @@ jobs: - run: npm ci - run: npx vitest run tests/unit/storage/migration-015.test.ts tests/unit/storage/trace-search.test.ts tests/unit/storage/search-query.test.ts tests/unit/tools/get-traces-search.test.ts + # The test jobs install better-sqlite3's prebuilt binary. When that download + # fails, `prebuild-install || node-gyp rebuild` compiles it here instead, + # against this Node's headers, and a binary compiled against Node 24.19+ + # headers aborts the process the first time V8 frees one of its statements + # ("Assertion failed: (env) != nullptr", nodejs/node#65446; every 24.x + # release so far). A user whose download failed gets that binary too. + # This job compiles it on purpose on Linux, macOS and Windows and requires: the binary + # aborts under collection exactly when Iris predicts it; Iris never loads + # it there (it holds the store with node:sqlite, and says why); and the + # real server, started and stopped over stdio again and again, ends every + # session cleanly. Node 22 is the control: its headers never changed, so + # its binary is safe and Iris keeps the native driver. When a Node 24 + # release carries the fix, the collect test here fails with the version + # to add to runtimeKeepsAddonHooks. + native-from-source: + name: native addon built from source (${{ matrix.os }}, Node ${{ matrix.node-version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + node-version: [22, 24] + exclude: + # The node-gyp that ships with Node 22's npm 10 cannot identify the + # runner's Visual Studio 18 ("unknown version"), so nothing compiles + # there and npm drops the optional module (the install-without- + # better-sqlite3 job covers that path). The Node 22 control runs on + # Linux and macOS. + - os: windows-latest + node-version: 22 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ matrix.node-version }} + cache: npm + cache-dependency-path: package-lock.json + - name: Install, compiling better-sqlite3 here instead of downloading it + env: + npm_config_build_from_source: 'true' + # --foreground-scripts: the compiler's output is in the log, including when the optional build fails and npm drops the module. + run: npm ci --foreground-scripts + - name: The binary was compiled on this runner, against this Node's headers + run: | + # A prebuilt download holds only build/Release; node-gyp leaves its config beside it. + test -d node_modules/better-sqlite3 || { echo "::error::better-sqlite3 did not compile here and npm dropped it (it is optional); the install log above says why"; exit 1; } + test -f node_modules/better-sqlite3/build/config.gypi || { echo "::error::better-sqlite3 was not compiled here, so this job proves nothing"; exit 1; } + node -e " + const fs = require('fs'); + const marked = fs.readFileSync('node_modules/better-sqlite3/build/Release/better_sqlite3.node').includes('RemoveEnvironmentCleanupHook'); + console.log('Node ' + process.versions.node + ': binary carries the ObjectWrap cleanup-hook call: ' + marked); + if (process.versions.node.startsWith('24.') && !marked) { console.error('::error::Node 24 headers should have compiled the call in; this job proves nothing'); process.exit(1); } + " + - run: npx vitest run tests/integration/native-addon-collect.test.ts tests/integration/native-teardown-stdio.test.ts tests/unit/storage/driver.test.ts + - name: The self-test names the driver and why + run: | + IRIS_HOME="$RUNNER_TEMP/iris-home" IRIS_DB_PATH="$RUNNER_TEMP/iris-home/iris.db" node --import tsx src/index.ts --self-test | tee self-test.txt + if [ "${{ matrix.node-version }}" = 24 ]; then + grep -q "driver node: better-sqlite3 here was compiled against Node headers that abort" self-test.txt + else + grep -q "driver better-sqlite3" self-test.txt + fi + integration: runs-on: ubuntu-latest needs: test diff --git a/CHANGELOG.md b/CHANGELOG.md index e902ae05..dd01307e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A failing test in the truthbase capture is reported as a failing test, by name, not as drift.** On `main` at 969719e the CI truthbase job said only "claims.json drifted from generator output". The tree was byte-identical to a pull-request head whose same job had passed; one test had failed during the capture (3,729 of 3,730), was recorded as `failed: 1`, and no line of the log named it. The capture now refuses a run with a failing test and lists each one with its file and the first line of its failure, and `claims:check` names every field that differs from the generator's output (for that run it would have printed `tests.vitestRoot.passed: committed 3730, generated 3729`). - **The Python client's publish workflow parses again, and every workflow is linted on every pull request.** Since the hash-pinned build landed, `publish-python.yml` held a plain YAML value with `:all: -r` in it, which YAML reads as a mapping, so GitHub refused the file on every push. Nothing noticed, because the workflow ran only on a `py-v*` tag. The step is now a block scalar. A new CI job runs actionlint 1.7.12, from an image pinned by digest, over every workflow in `.github/workflows`, including each `run:` script through shellcheck. It reported the parse error on the old file, and 6 shellcheck findings elsewhere, now fixed (four unguarded `ls *.tgz` globs in `ci.yml`, an unused loop variable and an unquoted `kill $(cat …)` in `lighthouse.yml`). `publish-python.yml` now also runs its build job on every pull request (the hashed install, the build and the fresh-environment import), and publishes only for a tag. - **`o1-mini` is priced at $1.10 in and $4.40 out per million tokens, not $3 and $12.** The judge's table carried the model's launch price; OpenAI's model page (developers.openai.com/api/docs/models/o1-mini) lists $1.10 / $4.40, and its pricing page no longer lists the model. The judge's cost cap and cost report for `o1-mini` were 2.7 times too high, so the cap refused calls that fit. Every other row was checked against claude.com/pricing and developers.openai.com/api/docs/pricing on 2026-09-28 and matches, and the table's read date is now 2026-09-28. +- **On Node 24, Iris no longer loads a better-sqlite3 that would abort the server when it frees a statement (#719).** Node 24.19.0 changed the `node::ObjectWrap` header without the runtime change it needs ([nodejs/node#65446](https://github.com/nodejs/node/issues/65446)). A better-sqlite3 compiled against that header, as npm does when the prebuilt download fails, aborts the process with `Assertion failed: (env) != nullptr` the first time V8 collects one of its statements, whether the database is open or closed. On Node 24.21.0 with 12.11.1, measured: a binary built from source aborted in 10 runs of 10, the prebuilt binary in 0 of 10. It aborted CI once, in 1 of the 62 failed jobs of the last 400 runs, the one whose install had compiled the addon. Iris now reads the binary before loading it. When the binary carries the new call and the runtime lacks the fix (every 24.x release so far, and 26.x before 26.4.0), Iris holds the store with Node's built-in SQLite and says why on stderr and in `--self-test`; `GET /health` reports `driver: "node"`. `IRIS_SQLITE_DRIVER=native` refuses instead. `npm rebuild better-sqlite3` restores the prebuilt binary, which is safe. A new CI job compiles the addon from source on Node 24 on Linux, macOS and Windows, and on Node 22 on Linux and macOS. It requires the binary to abort under collection exactly when Iris predicts it, and a real server started and stopped over stdio six times to end every session cleanly. With the check turned off, that stdio test failed 5 runs of 5 on the compiled binary. +- **A stdio server shuts down in order when the client closes its stdin.** MCP clients end a stdio session that way. The server used to drain and exit without closing the store, and a search-index build kept running for the client that had gone: 1.5 s on 80,000 traces, against 32 ms now. With the dashboard running, the end of stdin still does not stop the process. - **The published test counts are no longer recorded from a run in which a test file failed to load.** `npm run claims:capture-tests` read only vitest's totals, and a file that does not parse or throws at import counts none of its tests and no failure, so a run eleven tests short once recorded every test passing. The capture now refuses such a run, naming each file and the first line of its error, and writes nothing; it also refuses a run vitest marks failed with no failing test, and a run with a skipped test (which it used to fall back from to the committed counts). `--report root=` and `--report dashboard=` read an existing vitest JSON report instead of running the suite, which is how `tests/unit/scripts/capture-report.test.ts` proves the refusal on a real report of a file that fails to parse and one that throws at import. - **The Python recorder batches as it says it does.** `IrisRecorder` waits `flush_interval` (0.25 s by default) to gather traces into one request, but every new trace woke that wait early, so an application recording steadily sent one request per trace and a full queue was never reached. The interval is now a deadline that only `flush()` or `close()` ends early. `test_the_queue_is_bounded_and_drops_the_oldest` yields between records, and fails on the old recorder every run. - **The Failures and Moments pages no longer re-run the regression-alarm watcher once per trace (#680).** To find the alarms each trace in its window carries, the route re-ran every rule's CUSUM watcher over the agent's log up to that trace, so the cost grew with the square of the window and with the number of rules. The watchers now run once per request, and once more for each size of the rule family the log grows through; a log whose rules and runs are there from the start takes one pass. Every trace gets exactly the alarms it got before: a test checks the one pass against the per-trace answer for every trace of seeded logs where rules and runs start part-way through, where traces share a timestamp, and where alarms fire. On one agent's 500 traces the Failures and ranked Moments responses are byte-identical before and after. Measured on that window with no other load: with 25 rules, a warm request takes 286–344 ms, down from 2.5–2.6 s, and the first request after a start 4.5 s, down from 6.8 s; with 3 rules, 33–45 ms, down from 94–101 ms. The first-request cost that remains is the simulation that draws each stream's alarm line, which is memoised once drawn. diff --git a/README.md b/README.md index 2acc8dea..0bcd31bd 100644 --- a/README.md +++ b/README.md @@ -467,7 +467,7 @@ Every variable `--help` documents. CLI flags take precedence over environment va | `IRIS_PORT` | HTTP transport port (1-65535, default `3000`) | | `IRIS_HOME` | Directory for all per-user files: `config.json`, `iris.db`, `custom-rules.json`, `audit.log`, `preferences.json` (default `~/.iris`) | | `IRIS_DB_PATH` | SQLite database path (overrides `IRIS_HOME` for the DB only) | -| `IRIS_SQLITE_DRIVER` | Which SQLite driver holds the database: `native` (better-sqlite3, the default) or `node` (Node's built-in `node:sqlite`, Node 22.13+). Unset: native, and when the native module cannot load Iris warns once and falls back to the built-in | +| `IRIS_SQLITE_DRIVER` | Which SQLite driver holds the database: `native` (better-sqlite3, the default) or `node` (Node's built-in `node:sqlite`, Node 22.13+). Unset: native, and when the native module cannot load (or is a build that would abort on this Node) Iris warns once and falls back to the built-in | | `IRIS_SEARCH_BUDGET_MS` | How long one trace search (`q`) may read before it answers with the matches it found so far and `search.complete: false`, in milliseconds (50 to 60000, default `1000`). A search holds other requests while it reads, so this is also the longest it can make them wait. Also `storage.searchBudgetMs` in `config.json` | | `IRIS_LOG_LEVEL` | Log level: `debug`, `info`, `warn`, `error` | | `IRIS_DASHBOARD` | `true`/`1`/`yes`/`on` enables the web dashboard; `false`/`0`/`no`/`off` disables it (also overrides `dashboard.enabled` in `config.json`) | @@ -602,7 +602,7 @@ npm update -g @iris-eval/mcp-server **On a platform with no prebuilt `better-sqlite3`, the install still succeeds.** `better-sqlite3` is an optional dependency: when npm can neither download a prebuilt binary for your Node and platform nor compile one (compiling needs Python and a C++ toolchain — Visual Studio's C++ build tools on Windows), npm prints the build error, skips the module, and finishes the install. Iris then runs on Node's built-in SQLite, and says so: startup prints one line on stderr naming why, and `--self-test` shows `driver node: better-sqlite3 is not installed …`. To get the native driver back, install it where a prebuild or a toolchain exists (`npm install better-sqlite3` in the project; for a global install, install Iris again with `npm install -g @iris-eval/mcp-server` once a toolchain is available). CI installs the packed server with the native build forced to fail on every change, and requires the install to finish and the self-test to store and read a trace on the built-in. -Iris keeps everything in one SQLite file, opened by `better-sqlite3` — a native addon that is downloaded or compiled for your Node and platform. **When that module cannot load, Iris falls back to Node's built-in SQLite** (`node:sqlite`, Node 22.13 or later) with one warning on stderr, so a missing prebuild is a slower start rather than a dead one; `IRIS_SQLITE_DRIVER=node` chooses the built-in on purpose, `native` forbids the fallback. The built-in is opened with extension loading off and `trusted_schema` off; Node prints its own `ExperimentalWarning: SQLite is an experimental feature` line on stderr when it loads, and Iris does not silence it. `--self-test` and `GET /health` name the driver in use; every number on the proof page was measured on the native driver, and the test suite runs on both in CI. +Iris keeps everything in one SQLite file, opened by `better-sqlite3` — a native addon that is downloaded or compiled for your Node and platform. **When that module cannot load, Iris falls back to Node's built-in SQLite** (`node:sqlite`, Node 22.13 or later) with one warning on stderr, so a missing prebuild is a slower start rather than a dead one. It does the same, before loading it, for a `better-sqlite3` compiled on your machine against Node 24.19 or later headers: on every 24.x release so far such a binary aborts the whole process the first time it frees a statement (`Assertion failed: (env) != nullptr`, [nodejs/node#65446](https://github.com/nodejs/node/issues/65446)), and `npm rebuild better-sqlite3` replaces it with the prebuilt binary, which is safe. `IRIS_SQLITE_DRIVER=node` chooses the built-in on purpose, `native` forbids the fallback. The built-in is opened with extension loading off and `trusted_schema` off; Node prints its own `ExperimentalWarning: SQLite is an experimental feature` line on stderr when it loads, and Iris does not silence it. `--self-test` and `GET /health` name the driver in use; every number on the proof page was measured on the native driver, and the test suite runs on both in CI. ### Node.js version diff --git a/docs/api-reference.md b/docs/api-reference.md index 8d0afafa..e72418fe 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -1444,7 +1444,7 @@ The one health contract (0.15.0). Unauthenticated by design — no key, no sessi } ``` -- `driver` — the SQLite driver behind the store: `better-sqlite3` (the native addon, the default) or `node` (Node's built-in `node:sqlite`, chosen with `IRIS_SQLITE_DRIVER=node` or fallen back to when the native module cannot load); `null` on a transport started without storage. +- `driver` — the SQLite driver behind the store: `better-sqlite3` (the native addon, the default) or `node` (Node's built-in `node:sqlite`, chosen with `IRIS_SQLITE_DRIVER=node` or fallen back to when the native module cannot load or is a build that would abort on this Node); `null` on a transport started without storage. - `search_worker` — where searches run: `ready` (on their own thread, so a slow search never holds other requests), `not_started` (the thread starts with the first search), `unavailable` (it could not start on this machine, so searches run on the main thread; `detail` gives the reason, and the server also logs it once), or `not_used` (a store in memory, which searches on the main thread by design); `null` without storage. It is informational: search works in every case, so it never makes `status` `degraded`. `--self-test` prints the same line. - `checks.storage` — the database answered a count. The count itself is not reported: this endpoint answers without a key, so it says whether the store works, not how much it holds (the number is `total` on the authenticated `GET /api/v1/traces`); `checks.rules_store` — the deployed custom-rules file reads and parses; `checks.migrations` — every migration this build knows is applied, with the numbers so a schema that is behind is visible before a query fails. Each is `ok`, `fail`, or `absent` when there was nothing to check. - `status` is `ok` only when no check failed; otherwise `degraded`, with HTTP **503**, so a probe that reads only the status code is right. diff --git a/docs/architecture.md b/docs/architecture.md index d74c7287..7b860b2b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -216,7 +216,7 @@ dashboard/ React SPA (separate Vite build) 1. `index.ts` parses CLI args with `node:util.parseArgs`. 2. `loadConfig()` reads `~/.iris/config.json` and validates it against a strict schema (`src/config/schema.ts`) — a key Iris does not read, or a value of the wrong type, refuses startup naming it — then layers the file, the `IRIS_*` env vars and the CLI args over the defaults, in that order. -3. `createStorage()` instantiates `SqliteAdapter`, which opens the file through the driver seam (`src/storage/driver.ts`: `better-sqlite3` by default, Node's built-in `node:sqlite` when chosen with `IRIS_SQLITE_DRIVER=node` or when the native module cannot load) and calls `initialize()` to set the busy timeout, enable WAL mode, turn on foreign keys and run pending migrations — every migration is typed on the seam, not on a driver. +3. `createStorage()` instantiates `SqliteAdapter`, which opens the file through the driver seam (`src/storage/driver.ts`: `better-sqlite3` by default, Node's built-in `node:sqlite` when chosen with `IRIS_SQLITE_DRIVER=node` or when the native module cannot load or is a build that would abort on this Node) and calls `initialize()` to set the busy timeout, enable WAL mode, turn on foreign keys and run pending migrations — every migration is typed on the seam, not on a driver. 4. `createIrisServer()` creates the MCP `McpServer` instance, instantiates `EvalEngine` with the configured threshold, and registers all tools and resources. 5. Based on `config.transport.type`: - **stdio**: Creates `StdioServerTransport` and connects. diff --git a/src/index.ts b/src/index.ts index 6d0ad9aa..7da2df84 100644 --- a/src/index.ts +++ b/src/index.ts @@ -547,7 +547,11 @@ async function main(): Promise { logger.warn('HTTP transport running without API key authentication — set IRIS_API_KEY (or IRIS_API_KEY_FILE) for production'); } + let shuttingDown = false; const shutdown = async () => { + // A signal after the client closed stdin (or a second Ctrl+C) must not close the store twice. + if (shuttingDown) return; + shuttingDown = true; logger.info('Shutting down gracefully...'); const closePromises = httpServers.map( @@ -567,6 +571,19 @@ async function main(): Promise { process.on('SIGINT', shutdown); process.on('SIGTERM', shutdown); + /* + * An MCP client ends a stdio session by closing the server's stdin, and + * only signals it if it has not exited a moment later. With nothing else + * serving, the process used to drain and exit without closing the store, + * and a search-index build kept running for a client that had gone (1.5 s + * on 80,000 traces, measured). Now the end of stdin is a shutdown: the + * build stops at its next step and the store is closed in order. + * With the dashboard up, stdin ending (a server started with void shutdown()); + } } function printDemoBanner(summary: SeedDemoDataSummary, url: string): void { diff --git a/src/storage/driver.ts b/src/storage/driver.ts index 1651ebbe..cd2db446 100644 --- a/src/storage/driver.ts +++ b/src/storage/driver.ts @@ -24,7 +24,9 @@ * Selection: `IRIS_SQLITE_DRIVER=native|node` decides; unset means native * with a fallback — when the native module cannot load and the built-in * is available, Iris warns once and uses the built-in, so a bad prebuild - * is a slower start, not a dead one. A name that is neither is refused. + * is a slower start, not a dead one. The same holds for a binary that + * would load and then abort the process on this Node (nativeAbortsOnCollect, + * below). A name that is neither is refused. * * better-sqlite3 is an OPTIONAL dependency (0.20.0, #661): when npm cannot * install it — a release with no prebuilt binary for the platform, and no @@ -37,8 +39,9 @@ * parameters; `transaction(fn)` returns a callable with `.immediate()` * (nested calls become savepoints, as the native driver does). */ -import { existsSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; import { createRequire } from 'node:module'; +import { dirname } from 'node:path'; export type DriverName = 'better-sqlite3' | 'node'; @@ -87,6 +90,12 @@ export interface OpenOptions { loadNative?: () => NativeModule; /** Injectable for tests: how the built-in module is loaded. */ loadNode?: () => NodeSqliteModule; + /** + * Injectable for tests: the native binary to inspect before loading it + * (nativeAbortsOnCollect). Unset, it is the one better-sqlite3 would + * load, unless `loadNative` is injected, which loads something else. + */ + nativeBinary?: () => string | undefined; } export const DRIVER_VAR = 'IRIS_SQLITE_DRIVER'; @@ -137,6 +146,80 @@ function nativeDriver(Database: NativeModule, path: string, options: OpenOptions }; } +/* ---- A native binary that aborts when a statement is collected ---- */ + +/* + * Node 24.19.0 changed the header-only `node::ObjectWrap`: its constructor + * now registers an environment cleanup hook and its destructor removes it + * with `node::RemoveEnvironmentCleanupHook(isolate, ...)`. On a runtime + * without Node's global list of addon cleanup hooks, that call asserts + * there is a current Environment, and during a garbage collection there + * is none: the process aborts with "Assertion failed: (env) != nullptr" + * (nodejs/node#65446). better-sqlite3 12.x wraps every Database, Statement + * and iterator in an ObjectWrap, so a binary compiled against those + * headers aborts the first time V8 frees a statement, whether the + * database is open or closed. Measured on Node 24.21.0 with 12.11.1: a + * binary built from source aborted in 10 runs of 10, the prebuilt binary + * (compiled against older headers) in 0 of 10, and no exit path aborted. + * + * The prebuilt binaries are safe. The case is a binary npm compiled here: + * no prebuild for the platform, or the prebuild download failed and + * `prebuild-install || node-gyp rebuild` fell through to the compiler. + * Such a binary imports RemoveEnvironmentCleanupHook, which no + * better-sqlite3 source calls, so the name in the file is the mark. + * Iris reads the file before loading it and, where the runtime cannot + * survive it, uses Node's built-in SQLite instead of aborting later. + */ +export const OBJECTWRAP_HOOK_SYMBOL = 'RemoveEnvironmentCleanupHook'; + +/** + * Whether this Node keeps the global list of addon cleanup hooks + * (nodejs/node 1723773d), so an ObjectWrap freed during a collection is + * safe. 26.x has it from 26.4.0, the release that also changed the header. + * On 24.x it is on the v24.x-staging branch (nodejs/node#65943) and in no + * release up to 24.21.0; the CI job `native addon built from source` + * fails when a 24.x runtime stops aborting, which is when this line moves. + * Older lines never changed the header, so their binaries never carry the mark. + */ +export function runtimeKeepsAddonHooks(version: string = process.versions.node): boolean { + const [major, minor] = version.split('.').map(Number); + if (major > 26) return true; + if (major === 26) return minor >= 4; + return false; +} + +/** The better_sqlite3.node that `require('better-sqlite3')` would load, found the way it finds it; undefined when there is none. */ +export function nativeBinaryPath(): string | undefined { + try { + const pkg = require.resolve('better-sqlite3/package.json'); + const bindings = createRequire(pkg)('bindings') as (opts: { bindings: string; path: true; module_root: string }) => string; + return bindings({ bindings: 'better_sqlite3.node', path: true, module_root: dirname(pkg) }); + } catch { + return undefined; + } +} + +/** + * The binary carries the ObjectWrap cleanup hook and this runtime would + * abort on it. A file that cannot be found or read is not evidence either + * way: the load that follows reports what is wrong with it. + */ +export function nativeAbortsOnCollect(binary: string | undefined, version: string = process.versions.node): boolean { + if (binary === undefined || runtimeKeepsAddonHooks(version)) return false; + let marked = carriesHookMark.get(binary); + if (marked === undefined) { + try { + // About 2 MB, read once per process (4 ms measured); every store opened after that asks the map. + marked = readFileSync(binary).includes(OBJECTWRAP_HOOK_SYMBOL); + } catch { + return false; + } + carriesHookMark.set(binary, marked); + } + return marked; +} +const carriesHookMark = new Map(); + /* ---- The built-in driver: node:sqlite ---- */ type NodeStatement = { run(...p: unknown[]): { changes: number | bigint; lastInsertRowid: number | bigint }; get(...p: unknown[]): unknown; all(...p: unknown[]): unknown[] }; @@ -266,22 +349,8 @@ export function openDriver(path: string, options: OpenOptions = {}): Driver { return nodeDriver(loadNode(), path, options, `${DRIVER_VAR}=node chose Node's built-in SQLite`); } - let Database: NativeModule; - try { - Database = loadNative(); - // better-sqlite3 loads its binding lazily, in the constructor: an in-memory - // open is the probe that surfaces a missing, disabled or mismatched addon - // with no file involved. A file error later is a real error, never a - // reason to switch drivers. - new Database(':memory:').close(); - } catch (err) { - const absent = notInstalled(err); - const reason = err instanceof Error ? err.message.split('\n')[0] : String(err); - // Not installed and failed to load are different facts with different fixes: say which. - const what = absent - ? 'The native SQLite module (better-sqlite3) is not installed — it is optional, and npm skips it when it cannot build it for this platform' - : `The native SQLite module (better-sqlite3) could not load (${reason})`; - const fix = absent ? 'install it with npm install better-sqlite3 where a prebuilt binary or a C++ toolchain is available' : 'reinstall it with npm rebuild better-sqlite3'; + /** The native driver cannot hold the file: fall back to the built-in with one warning, or refuse, naming why and the fix. */ + const fallBack = (what: string, fix: string, reason: string): Driver => { const canFallBack = options.allowFallback !== false && choice === undefined && nodeSqliteAvailable(loadNode); if (!canFallBack) { throw new Error( @@ -295,7 +364,38 @@ export function openDriver(path: string, options: OpenOptions = {}): Driver { `[iris.storage] ${what}; using Node's built-in SQLite (node:sqlite). ` + `The store works the same; the native driver is faster and is what the proof was measured on — ${fix}, or set ${DRIVER_VAR}=node to choose the built-in on purpose.`, ); - return nodeDriver(loadNode(), path, options, absent ? 'better-sqlite3 is not installed (optional; npm skips it when it cannot build it here), so Iris uses Node\'s built-in SQLite' : `better-sqlite3 could not load (${reason}), so Iris uses Node's built-in SQLite`); + return nodeDriver(loadNode(), path, options, reason); + }; + + // Read before loading: once a statement of this binary exists, the next collection can abort the process. + const binary = (options.nativeBinary ?? (options.loadNative ? () => undefined : nativeBinaryPath))(); + if (nativeAbortsOnCollect(binary)) { + return fallBack( + `The native SQLite module (better-sqlite3) at ${binary} was compiled against Node headers that make it abort on Node ${process.versions.node} when it frees a statement (nodejs/node#65446)`, + 'reinstall the prebuilt binary with npm rebuild better-sqlite3 (it is compiled against headers without the change)', + `better-sqlite3 here was compiled against Node headers that abort on Node ${process.versions.node} when a statement is freed (nodejs/node#65446), so Iris uses Node's built-in SQLite`, + ); + } + + let Database: NativeModule; + try { + Database = loadNative(); + // better-sqlite3 loads its binding lazily, in the constructor: an in-memory + // open is the probe that surfaces a missing, disabled or mismatched addon + // with no file involved. A file error later is a real error, never a + // reason to switch drivers. + new Database(':memory:').close(); + } catch (err) { + const absent = notInstalled(err); + const reason = err instanceof Error ? err.message.split('\n')[0] : String(err); + // Not installed and failed to load are different facts with different fixes: say which. + return fallBack( + absent + ? 'The native SQLite module (better-sqlite3) is not installed — it is optional, and npm skips it when it cannot build it for this platform' + : `The native SQLite module (better-sqlite3) could not load (${reason})`, + absent ? 'install it with npm install better-sqlite3 where a prebuilt binary or a C++ toolchain is available' : 'reinstall it with npm rebuild better-sqlite3', + absent ? 'better-sqlite3 is not installed (optional; npm skips it when it cannot build it here), so Iris uses Node\'s built-in SQLite' : `better-sqlite3 could not load (${reason}), so Iris uses Node's built-in SQLite`, + ); } return nativeDriver(Database, path, options, choice === 'native' ? `${DRIVER_VAR}=native chose better-sqlite3` : 'better-sqlite3 loaded (the default)'); } diff --git a/tests/fixtures/native-teardown/collect-statements.cjs b/tests/fixtures/native-teardown/collect-statements.cjs new file mode 100644 index 00000000..c9b4b895 --- /dev/null +++ b/tests/fixtures/native-teardown/collect-statements.cjs @@ -0,0 +1,33 @@ +/* + * Prepared statements that become garbage while V8 collects on its own, + * with the database open and then after it is closed. On a better-sqlite3 + * binary compiled against Node 24.19+ headers, running on a Node without + * the global cleanup-hook list, the first collection of a statement aborts + * the process: "Assertion failed: (env) != nullptr" (nodejs/node#65446). + * An explicit global.gc() does not reach that path; the allocation below does. + * + * Usage: node collect-statements.cjs [statements] (exit 0 and "survived" when nothing aborted) + */ +'use strict'; +const Database = require('better-sqlite3'); + +const N = Number(process.argv[2] || 100000); +const churn = (db, n) => { + let junk = []; + for (let i = 0; i < n; i++) { + if (db) db.prepare('SELECT a FROM t WHERE a = ?').get(i); + junk.push({ a: i, s: 'x'.repeat(16) }); + if (junk.length > 1000) junk = []; + } +}; + +const db = new Database(':memory:'); +db.exec('CREATE TABLE t (a INTEGER)'); +churn(db, N); +// Closed: every statement is finalized by close(), and their objects are still freed later. +let kept = []; +for (let i = 0; i < 2000; i++) kept.push(db.prepare('SELECT a FROM t WHERE a = ?')); +db.close(); +kept = []; +churn(null, N); +process.stdout.write('survived\n'); diff --git a/tests/integration/native-addon-collect.test.ts b/tests/integration/native-addon-collect.test.ts new file mode 100644 index 00000000..c7d0e413 --- /dev/null +++ b/tests/integration/native-addon-collect.test.ts @@ -0,0 +1,42 @@ +/* + * Does this machine's better-sqlite3 binary abort the process when V8 + * frees one of its statements, and does Iris predict it? + * + * A binary compiled against Node 24.19+ headers aborts on the first + * collected statement on every 24.x release so far ("Assertion failed: + * (env) != nullptr", nodejs/node#65446); the prebuilt binaries do not. + * Iris reads the binary before loading it (nativeAbortsOnCollect) and uses + * Node's built-in SQLite where it would abort. This runs the real binary + * in a child process under the same allocation that aborts it, and holds + * the prediction to the outcome both ways: on the ordinary CI jobs the + * prebuilt binary survives and Iris says it would, and on the job that + * compiles better-sqlite3 from source on Node 24 it aborts and Iris says + * it would. The day a Node release carries the fix, this fails there, and + * runtimeKeepsAddonHooks learns the version. + */ +import { describe, expect, it } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { resolve } from 'node:path'; +import { nativeAbortsOnCollect, nativeBinaryPath, runtimeKeepsAddonHooks } from '../../src/storage/driver.js'; + +const FIXTURE = resolve(import.meta.dirname, '../fixtures/native-teardown/collect-statements.cjs'); + +describe('a better-sqlite3 binary that aborts on a collected statement', () => { + it('aborts exactly when Iris predicts it, on this binary and this Node', () => { + const binary = nativeBinaryPath(); + expect(binary, 'better-sqlite3 is installed in the test jobs, and the locator finds the file it loads').toBeDefined(); + const predicted = nativeAbortsOnCollect(binary); + const run = spawnSync(process.execPath, [FIXTURE, '100000'], { encoding: 'utf8', timeout: 120_000 }); + const aborted = /Assertion failed: \(env\) != nullptr/.test(run.stderr); + process.stdout.write( + `[native-collect] Node ${process.versions.node}, ${binary}: ${aborted ? 'aborted' : 'survived'} (exit ${run.status ?? run.signal}); Iris predicted ${predicted ? 'abort' : 'survive'}; runtime keeps addon hooks: ${runtimeKeepsAddonHooks()}\n`, + ); + expect( + aborted, + predicted + ? 'Iris says this binary aborts on this Node, and it did not: if this Node carries the cleanup-hook fix, add its version to runtimeKeepsAddonHooks' + : `Iris says this binary is safe on this Node, and it aborted:\n${run.stderr.slice(0, 2000)}`, + ).toBe(predicted); + expect(run.stdout.includes('survived')).toBe(!predicted); + }, 150_000); +}); diff --git a/tests/integration/native-teardown-stdio.test.ts b/tests/integration/native-teardown-stdio.test.ts new file mode 100644 index 00000000..75a53e1f --- /dev/null +++ b/tests/integration/native-teardown-stdio.test.ts @@ -0,0 +1,94 @@ +/* + * The server started and stopped over stdio, again and again, ends every + * session cleanly. + * + * Each cycle is a real server process: an MCP client connects, logs and + * evaluates traces and searches them (statements prepared and freed while + * V8 collects), then ends the session the way MCP clients do, by closing + * the server's stdin. Every process must shut down in order ("Shutdown + * complete", the store closed), exit 0 with no signal, and never print a + * native assertion. The young generation is kept small so V8 collects + * many times inside each short session: a statement freed in a session is + * collected in that session, which is where a better-sqlite3 binary + * compiled against Node 24.19+ headers aborts (nodejs/node#65446). On + * that binary Iris must hold the store with Node's built-in SQLite; the + * CI job that compiles better-sqlite3 from source runs this file there. + */ +import { describe, expect, it } from 'vitest'; +import { spawn } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { nativeAbortsOnCollect, nativeBinaryPath } from '../../src/storage/driver.js'; + +const ENTRY = resolve(import.meta.dirname, '../../src/index.ts'); +const CYCLES = 6; +const CALLS = 25; + +interface Session { + code: number | null; + signal: NodeJS.Signals | null; + stderr: string; +} + +async function session(home: string): Promise { + const child = spawn(process.execPath, ['--max-semi-space-size=1', '--import', 'tsx', ENTRY], { + env: { ...process.env, IRIS_HOME: home, IRIS_DB_PATH: join(home, 'iris.db'), IRIS_NO_AUTO_LAUNCH: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr.on('data', (d: Buffer) => (stderr += d.toString())); + const exited = new Promise((done) => child.on('exit', (code, signal) => done({ code, signal, stderr }))); + + const pending = new Map void>(); + let buffered = ''; + child.stdout.on('data', (d: Buffer) => { + buffered += d.toString(); + for (let nl = buffered.indexOf('\n'); nl >= 0; nl = buffered.indexOf('\n')) { + const line = buffered.slice(0, nl); + buffered = buffered.slice(nl + 1); + const msg = JSON.parse(line) as { id?: number; result?: unknown; error?: unknown }; + if (msg.id !== undefined) pending.get(msg.id)?.(msg); + } + }); + let nextId = 0; + const request = (method: string, params: unknown) => { + const id = ++nextId; + const answered = new Promise<{ result?: unknown; error?: unknown }>((r) => pending.set(id, r)); + child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id, method, params })}\n`); + // A process that dies mid-session answers nothing: the exit settles the wait. + return Promise.race([answered, exited.then(() => ({ error: 'the server exited' }))]); + }; + const call = async (name: string, args: Record) => { + const msg = await request('tools/call', { name, arguments: args }); + expect(msg.error, `${name} answered; the server's stderr ends:\n${stderr.slice(-2000)}`).toBeUndefined(); + }; + + await request('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'teardown', version: '1.0.0' } }); + child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' })}\n`); + for (let i = 0; i < CALLS; i++) { + await call('log_trace', { agent_name: 'support-bot', input: `question ${i} about an order`, output: `Your order ${i} ships on Tuesday; the refund for order ${i - 1} was approved.` }); + await call('evaluate_output', { output: `Order ${i} ships Tuesday. Contact support@example.com for help.`, eval_type: 'completeness' }); + await call('get_traces', { q: 'refund approved', limit: 5 }); + } + child.stdin.end(); + return exited; +} + +describe('repeated start and stop over stdio', () => { + it(`${CYCLES} sessions each end with the store closed, exit 0, and no native abort`, async () => { + const chosen = process.env.IRIS_SQLITE_DRIVER === 'node' ? 'node' : nativeAbortsOnCollect(nativeBinaryPath()) ? 'node' : 'better-sqlite3'; + for (let cycle = 0; cycle < CYCLES; cycle++) { + const home = mkdtempSync(join(tmpdir(), 'iris-teardown-')); + try { + const s = await session(home); + expect(s.stderr, `cycle ${cycle}: no native assertion`).not.toMatch(/Assertion failed|Native stack trace/); + expect({ cycle, code: s.code, signal: s.signal }).toEqual({ cycle, code: 0, signal: null }); + expect(s.stderr, `cycle ${cycle}: the end of stdin shut the server down in order`).toContain('Shutdown complete'); + expect(s.stderr, `cycle ${cycle}: the store was held by the driver Iris chose for this binary`).toContain(`driver ${chosen}:`); + } finally { + rmSync(home, { recursive: true, force: true }); + } + } + }, 180_000); +}); diff --git a/tests/unit/storage/driver.test.ts b/tests/unit/storage/driver.test.ts index 59ac1eec..3e506243 100644 --- a/tests/unit/storage/driver.test.ts +++ b/tests/unit/storage/driver.test.ts @@ -10,11 +10,22 @@ * transactions on the built-in roll back and nest as the native ones do. */ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { createRequire } from 'node:module'; import { SqliteAdapter, SQLITE_DRIVER } from '../../../src/storage/sqlite-adapter.js'; -import { openDriver, requestedDriver, nodeSqliteAvailable, DRIVER_VAR, type Driver } from '../../../src/storage/driver.js'; +import { + openDriver, + requestedDriver, + nodeSqliteAvailable, + nativeAbortsOnCollect, + nativeBinaryPath, + runtimeKeepsAddonHooks, + OBJECTWRAP_HOOK_SYMBOL, + DRIVER_VAR, + type Driver, +} from '../../../src/storage/driver.js'; import { KNOWN_MIGRATION_IDS } from '../../../src/storage/migrations/index.js'; import { buildHealth } from '../../../src/health.js'; import { LOCAL_TENANT } from '../../../src/types/tenant.js'; @@ -52,7 +63,8 @@ describe('the seam', () => { const storage = new SqliteAdapter(':memory:'); await storage.initialize(); try { - expect(storage.driver).toBe('better-sqlite3'); + // Except where this binary would abort the process on this Node: the CI job that compiles it from source on Node 24. + expect(storage.driver).toBe(nativeAbortsOnCollect(nativeBinaryPath()) ? 'node' : 'better-sqlite3'); expect(SQLITE_DRIVER).toBe('better-sqlite3'); } finally { await storage.close(); @@ -260,3 +272,118 @@ describe('the seam', () => { expect(() => openDriver(join(tempDb(), '..', 'nope.db'), { fileMustExist: true, driver: 'native' })).toThrow(); }); }); + +/* + * A better-sqlite3 binary compiled against Node 24.19+ headers aborts the + * process the first time V8 frees one of its statements, on every Node that + * lacks the global cleanup-hook list (nodejs/node#65446). The integration + * tests run the real binary (native-addon-collect) and the real server + * (native-teardown-stdio); these hold the decision itself. + */ +describe('a native binary that would abort on a collected statement', () => { + const binaryFile = (marked: boolean): string => { + const file = join(tempDb(), '..', marked ? 'marked.node' : 'plain.node'); + // The mark sits among the binary's imported names, as the linker writes it. + writeFileSync(file, Buffer.concat([Buffer.alloc(4096, 0x7f), Buffer.from(marked ? `_ZN4node28${OBJECTWRAP_HOOK_SYMBOL}EPN2v87IsolateEPFvPvES3_` : '_ZN4node25AddEnvironmentCleanupHookEPN2v87IsolateEPFvPvES3_'), Buffer.alloc(4096, 0)])); + return file; + }; + + it('the runtimes that keep the cleanup-hook list: 26.4.0 and later, and no 24.x release yet', () => { + const table = ['22.13.0', '22.23.3', '24.18.1', '24.19.0', '24.21.0', '26.3.1', '26.4.0', '26.10.0', '27.0.0'].map((v) => [v, runtimeKeepsAddonHooks(v)]); + expect(Object.fromEntries(table)).toEqual({ + '22.13.0': false, + '22.23.3': false, + '24.18.1': false, + '24.19.0': false, + '24.21.0': false, + '26.3.1': false, + '26.4.0': true, + '26.10.0': true, + '27.0.0': true, + }); + }); + + it('a binary is refused only when it carries the mark and the runtime lacks the list; an unreadable or missing one is not evidence', () => { + const marked = binaryFile(true); + const plain = binaryFile(false); + expect(nativeAbortsOnCollect(marked, '24.21.0')).toBe(true); + expect(nativeAbortsOnCollect(marked, '26.3.1')).toBe(true); + expect(nativeAbortsOnCollect(marked, '26.4.0')).toBe(false); + expect(nativeAbortsOnCollect(plain, '24.21.0')).toBe(false); + expect(nativeAbortsOnCollect(join(marked, '..', 'missing.node'), '24.21.0')).toBe(false); + expect(nativeAbortsOnCollect(undefined, '24.21.0')).toBe(false); + }); + + it('the file inspected is the one better-sqlite3 loads', () => { + const binary = nativeBinaryPath(); + expect(binary).toBeDefined(); + expect(existsSync(binary!)).toBe(true); + // Loading it here would plant a statement this process could abort on: the binary itself is the evidence then. + if (nativeAbortsOnCollect(binary)) return; + const req = createRequire(import.meta.url); + const Database = req('better-sqlite3') as new (p: string) => { close(): void }; + new Database(':memory:').close(); + expect(Object.keys(req.cache).map((k) => k.toLowerCase())).toContain(binary!.toLowerCase()); + }); + + it('by default, Iris does not load a marked binary on a runtime without the list: it warns once and holds the file with the built-in', () => { + if (!builtIn) return withoutBuiltIn(tempDb()); + const warnings: string[] = []; + let loaded = false; + const d = openDriver(tempDb(), { + nativeBinary: () => binaryFile(true), + loadNative: () => { + loaded = true; + throw new Error('the marked binary was loaded'); + }, + warn: (line) => warnings.push(line), + }); + drivers.push(d); + if (runtimeKeepsAddonHooks()) { + // This Node survives the mark: the binary is loaded as any other (the stub fails to load, and that is what the fallback names). + expect(loaded).toBe(true); + expect(d.reason).toMatch(/^better-sqlite3 could not load \(the marked binary was loaded\)/); + return; + } + expect(loaded).toBe(false); + expect(d.name).toBe('node'); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toMatch(/was compiled against Node headers that make it abort on Node \d+\.\d+\.\d+ when it frees a statement \(nodejs\/node#65446\); using Node's built-in SQLite/); + expect(warnings[0]).toMatch(/npm rebuild better-sqlite3/); + expect(warnings[0]).toMatch(/IRIS_SQLITE_DRIVER=node/); + expect(d.reason).toMatch(/^better-sqlite3 here was compiled against Node headers that abort on Node .* \(nodejs\/node#65446\), so Iris uses Node's built-in SQLite$/); + d.exec('CREATE TABLE t (a TEXT)'); + expect(d.prepare('INSERT INTO t VALUES (?)').run('x').changes).toBe(1); + }); + + it('with IRIS_SQLITE_DRIVER=native, a marked binary is refused before it loads, naming the Node issue and the fix', () => { + if (runtimeKeepsAddonHooks()) return expect(nativeAbortsOnCollect(binaryFile(true))).toBe(false); + expect(() => + openDriver(tempDb(), { + driver: 'native', + nativeBinary: () => binaryFile(true), + loadNative: () => { + throw new Error('the marked binary was loaded'); + }, + warn: () => { + throw new Error('must not warn'); + }, + }), + ).toThrow(/abort on Node .* when it frees a statement \(nodejs\/node#65446\)\. IRIS_SQLITE_DRIVER=native forbids the fallback; unset it .* or reinstall the prebuilt binary with npm rebuild better-sqlite3/); + }); + + it('a plain binary is loaded as before', () => { + let loaded = false; + expect(() => + openDriver(tempDb(), { + driver: 'native', + nativeBinary: () => binaryFile(false), + loadNative: () => { + loaded = true; + throw new Error('invalid ELF header'); + }, + }), + ).toThrow(/could not load \(invalid ELF header\)/); + expect(loaded).toBe(true); + }); +});