diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6372c9..535b425 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,8 @@ on: push: branches: [main, "feat/**"] pull_request: + # Lets the live tier (below) be run by hand; it never runs on push/PR. + workflow_dispatch: permissions: contents: read @@ -61,3 +63,82 @@ jobs: run: cmake --build build --config Release - name: Test run: ctest --test-dir build --build-config Release --output-on-failure + + # The Node addon links the C shared library, so it needs the C++ build first. + # Its tests are hermetic (a mock provider server, no API keys). + node-binding: + name: Node binding (ubuntu-latest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libboost-all-dev libssl-dev cmake ninja-build + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Build the C library + run: | + cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DAI_SDK_BUILD_TESTS=OFF \ + -DAI_SDK_BUILD_EXAMPLES=OFF \ + -DAI_SDK_BUILD_CLI=OFF + cmake --build build --config Release + + - name: Build the addon + working-directory: bindings/node + run: | + npm install + npm run build + + - name: Test + working-directory: bindings/node + run: npm run test:only + + # The live tier talks to a real provider, so it is manual-only and needs a + # secret. It skips itself (rather than failing) when the secret is absent, + # which is the case on forks. + live-binding: + name: Live provider smoke (DeepSeek) + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libboost-all-dev libssl-dev cmake ninja-build + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Build the C library + run: | + cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DAI_SDK_BUILD_TESTS=OFF \ + -DAI_SDK_BUILD_EXAMPLES=OFF \ + -DAI_SDK_BUILD_CLI=OFF + cmake --build build --config Release + + - name: Build the addon + working-directory: bindings/node + run: | + npm install + npm run build + + - name: Run the live tier + working-directory: bindings/node + env: + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + run: | + if [ -z "$DEEPSEEK_API_KEY" ]; then + echo "::warning::DEEPSEEK_API_KEY is not set — skipping the live tier" + exit 0 + fi + npm run test:live diff --git a/.github/workflows/prebuilds.yml b/.github/workflows/prebuilds.yml new file mode 100644 index 0000000..7de44d0 --- /dev/null +++ b/.github/workflows/prebuilds.yml @@ -0,0 +1,205 @@ +name: Prebuilds + +# Builds the native binaries that `npm install ai-sdk-cpp` resolves at runtime. +# +# This workflow is the difference between a package that installs and one that +# does not: the tarball ships prebuilds/, and node-gyp-build picks the one +# matching the user's platform. With nothing in prebuilds/ every install fails +# at require() with "No native build was found", on every machine. +# +# Runs on a tag so a release is built from the tag itself, and by hand so the +# matrix can be exercised without cutting one. + +on: + workflow_dispatch: + push: + tags: ["v*"] + +permissions: + contents: read + +jobs: + prebuild: + name: Prebuild (${{ matrix.target }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + # The runner image sets how old a system the artifact will run on: + # glibc versions its symbols, so one built on a newer distro refuses + # to start on an older one. 22.04 puts the floor at glibc 2.35, which + # covers Ubuntu 22.04, Debian 12 and anything later. Moving either + # Linux job to a newer image silently narrows that -- keep them + # matched, or knowingly accept the difference. + # + # The image is on a clock. GitHub put ubuntu-22.04 into deprecation on + # 2026-09-17, with brownout windows in March and April 2027 and full + # retirement on 2027-04-17. It still runs today, so 1.0.0 is not + # affected -- but the deadline is real and the obvious migration is + # the wrong one: ubuntu-24.04 raises the floor to glibc 2.39 and drops + # every Ubuntu 22.04 and Debian 12 user, who then have no prebuild and + # no source fallback, so `npm install` fails outright for them. + # + # The fix that keeps both is to run the build inside an + # `ubuntu:22.04` container on a newer runner, which decouples the + # runner label from the glibc the artifact records. That is the shape + # `scripts/` verification for this release used, so it is known to + # work: the container builds, links and loads the prebuild, and + # reports GLIBC_2.35. Do that before April 2027 rather than reacting + # to a brownout. + - runner: ubuntu-22.04 + target: linux-x64 + - runner: ubuntu-22.04-arm + target: linux-arm64 + # macos-15-intel is the last Intel image GitHub will publish; 14 and + # later are Apple Silicon. Two runners, because one prebuild cannot + # serve both architectures. + # + # This was macos-13, which GitHub retired on 2025-12-04 -- a job on a + # retired label does not build and fail, it dies with "The + # configuration 'macos-13-us-default' is not supported", so the leg + # never runs. With `assemble` behind `needs: prebuild` and no `if:`, + # one dead label means no tarball at all. + # + # Intel runners have an endpoint: GitHub stops publishing them when + # the macOS 15 image retires, currently slated for late 2027. The + # darwin-x64 prebuild disappears then unless it is cross-compiled + # from Apple Silicon, which needs an x86_64 OpenSSL to link against. + # Worth knowing a year out rather than discovering it on a tag. + - runner: macos-15-intel + target: darwin-x64 + - runner: macos-14 + target: darwin-arm64 + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + + # GCC 13, not the image's g++-11. The build refuses anything older by + # name, because GCC 11 and 12 miscompile this project's coroutines + # into a library that segfaults -- so this is a requirement, not a + # preference for a newer compiler. + sudo apt-get install -y software-properties-common + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update + sudo apt-get install -y g++-13 libssl-dev cmake ninja-build curl + echo "CC=gcc-13" >> "$GITHUB_ENV" + echo "CXX=g++-13" >> "$GITHUB_ENV" + + # Boost 1.82 or newer, which 22.04 does not carry: its repositories + # stop at 1.74 and the build needs 1.82. The headers are the entire + # requirement -- the project links Boost::headers and nothing else, so + # Boost is never compiled, and unpacking the release tarball takes a + # minute where building it from source would take ten. + # + # Not solved by moving to a 24.04 image, which has Boost 1.83: the + # image also sets the glibc floor, and 24.04 would raise it to 2.39 + # and cut off the distributions the 22.04 matrix entries exist to + # serve. See the note on the matrix above. + curl -fsSL -o /tmp/boost.tar.gz \ + https://archives.boost.io/release/1.83.0/source/boost_1_83_0.tar.gz + sudo mkdir -p /opt/boost + sudo tar xzf /tmp/boost.tar.gz -C /opt/boost --strip-components=1 + echo "BOOST_ROOT=/opt/boost" >> "$GITHUB_ENV" + + - name: Install dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install boost openssl cmake ninja + + # OPENSSL_ROOT_DIR, not OpenSSL_DIR. The project calls + # find_package(OpenSSL REQUIRED), which resolves through CMake's + # FindOpenSSL.cmake -- module mode -- and that reads OPENSSL_ROOT_DIR. + # OpenSSL_DIR is the hint for *config* mode (the directory holding + # OpenSSLConfig.cmake) and FindOpenSSL does not consult it, so setting + # it looks right and does nothing. + # + # It also cannot be caught locally, which is why this is worth the + # comment: on a developer's Mac, Homebrew is already on the default + # search path, so configure succeeds whatever this step says. CI is + # the only place Homebrew's OpenSSL is keg-only and genuinely hidden + # -- and there the failure is "Could NOT find OpenSSL" on both macOS + # legs, which skips `assemble` in turn, so the release produces no + # tarball at all rather than a broken one. + echo "OPENSSL_ROOT_DIR=$(brew --prefix openssl)" >> "$GITHUB_ENV" + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + # --ignore-scripts: the install step is for package consumers deciding + # whether a prebuild exists, not for a checkout that is about to build + # one. Skipping it also keeps this job from needing node-gyp to configure. + - name: Install dependencies (Node) + working-directory: bindings/node + run: npm install --ignore-scripts + + # The script builds the C++ library with a static OpenSSL, links the addon + # against it, and refuses to emit anything that depends on a path outside + # the package -- which is how a Homebrew OpenSSL nearly shipped. + - name: Build the prebuild + working-directory: bindings/node + run: npm run build:prebuild + + - uses: actions/upload-artifact@v4 + with: + name: prebuild-${{ matrix.target }} + path: bindings/node/prebuilds/${{ matrix.target }} + if-no-files-found: error + + # Collects every platform into one package. Publishing is deliberately left + # to a human: `npm publish` is close to irreversible once someone installs + # it, and a tag push should not be able to do that on its own. + assemble: + name: Assemble the package + needs: prebuild + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + # Each artifact lands as prebuilds/prebuild-/ because that is the + # artifact name. node-gyp-build looks for prebuilds/-, so + # the prefix has to come off. + - uses: actions/download-artifact@v4 + with: + path: bindings/node/prebuilds + pattern: prebuild-* + + - name: Strip the artifact-name prefix + working-directory: bindings/node/prebuilds + run: | + for dir in prebuild-*; do + mv "$dir" "${dir#prebuild-}" + done + echo "assembled:" + ls -1 + + - name: Install dependencies (Node) + working-directory: bindings/node + run: npm install --ignore-scripts + + # prepublishOnly runs the prebuild check and compiles the TypeScript. The + # check is the point: it fails if any platform is missing its binary, + # which is the one packaging mistake that is invisible until a user hits + # it. + - name: Build the package + working-directory: bindings/node + run: npm run prepublishOnly + + - name: Pack + working-directory: bindings/node + run: npm pack --pack-destination . + + - uses: actions/upload-artifact@v4 + with: + name: npm-tarball + path: bindings/node/ai-sdk-cpp-*.tgz + if-no-files-found: error diff --git a/.gitignore b/.gitignore index 6098a94..a7ce6a9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ build/ _deps/ .cache/ +# scripts/coverage.sh's instrumented tree (it is not covered by `build/` above). +build-coverage/ # CMake generated files CMakeCache.txt @@ -20,6 +22,10 @@ compile_commands.json *.dylib *.dll +# Coverage profiles (LLVM_PROFILE_FILE / llvm-profdata output) +*.profraw +*.profdata + # macOS .DS_Store build_py/ @@ -33,3 +39,14 @@ __pycache__/ # Node node_modules/ bindings/node/dist/ +# Per-platform binaries produced by bindings/node/scripts/make-prebuild.sh. +# Built by .github/workflows/prebuilds.yml and assembled at publish time; they +# are several megabytes per platform and do not belong in the repository. +prebuilds/ +# make-prebuild.sh's C++ build tree (not covered by `build/` above). +build-prebuild/ + +# Secrets — API keys for live tests are read from the environment, never a file +.env +.env.* +!.env.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d26f72..c0bd702 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,20 +3,224 @@ All notable changes to ai-sdk-cpp. Format loosely based on [Keep a Changelog](https://keepachangelog.com/). -## [Unreleased] +## [1.0.0] - 2026-09-18 + +The bindings stop being a preview. The Node binding is now covered by a +hermetic test suite (mock provider server, no API keys) that runs in CI, and +the defects that suite found are fixed — including six that blocked release. + +### Changed — BREAKING +- **Node: `Session.send()` and `Agent.call()` are now `async`.** They return a + promise and no longer block the JS event loop for the whole turn. A turn that + calls a tool has to hand control back to the event loop the tool's own JS + runs on; blocking there deadlocked the process. `await` both. `generateText` + and `streamText` were already async and are unchanged. + +### Added — Node binding +- **Test suite** (`bindings/node/test/`): 64 cases over the public surface — + every provider factory, tool-calling through each entry point, streaming + events, sessions, memory, batch, MCP, standard toolkit, permissions, and + tool-set merging. Hermetic: a mock provider server stands in for the vendor + APIs, so it needs no keys and no network. Cases that drive a JS tool callback + run out-of-process under a wall-clock budget, because a deadlock in one of + those blocks the event loop so completely that an in-process timeout could + never fire to fail the test. +- **CI job** (`node-binding`) building the C library and running the suite. +- `streamText` accepts `messages`, `tools`, and `maxSteps`. +- `Session` with persistent memory (auto-inject recall + auto-compaction), + the checkpoint writer, and token usage on the stream finish event. +- `mergeToolSets()`, `mcpToolsetFromServer()`, `standardToolkit()`, + `withPermissions()`, `MemoryStore`, `Batch`, and `Agent` extra tool sets. +- **Tool-set introspection**: `describeToolSet()` in Node, backed by + `ai_tool_set_describe_json()` in the C API, reports the tools in a set as + `{name, description, inputSchema}` ordered by name. Ordered because the + underlying container is an unordered map and a description that comes back in + a different order every run cannot be compared against anything — and it is + the sets you cannot otherwise see into (an MCP server's tools, a + permission-wrapped set) that this is for. +- Provider options can be passed through to agents from C and Node. ### Fixed +- **Every tool call leaked its output, in both language bindings.** The C API + documented no ownership for `ai_tool_result_t.output_json`, and the two + bindings each guessed wrong the same way: Python and Node both allocated a + fresh buffer per call with `new char[]`, and nothing on either side of the + call freed it. Measured at 155 MiB of RSS after 150 tool calls of 1 MiB — a + leak proportional to what the tools return, in a loop the SDK exists to run + for days, and invisible to every test that checks values rather than memory. + The buffer is now a `thread_local` string per binding, reused by the next + call on that thread and released when the thread exits, and the ownership + rule — borrowed, valid until the callback returns — is written on the type + in `bindings/c/ai_sdk.h`. A regression test drives 150 tool calls in a + separate process and fails on RSS growth; run against the old code it reports + the full 155 MiB, so it is measuring the leak and not the machine. +- **The shipped Linux library was going to need a newer libstdc++ than Ubuntu + 22.04 has.** The option that links the C++ runtime into `libai_sdk.so` + statically is on by default for GNU, but its guard tested `CXX_COMPILER_ID` + instead of `CMAKE_CXX_COMPILER_ID` — not a CMake variable at all, so the + comparison was false on every compiler and the flags were never applied. The + build reported the option as enabled while producing a library that depended + on the runtime of the machine that built it. Since the prebuild is built by + GCC 13 and Ubuntu 22.04 ships an older libstdc++, this failed at `require()` + on exactly the systems the 22.04 matrix entries exist to serve. The guard is + fixed, and the warning it produced ("the compiler is GNU, not GNU") was the + only sign anything was wrong. +- **A prebuild recorded an empty rpath entry.** `binding.gyp` supplied the + loader-relative rpath twice, once per-OS and once through a shared variable. + The shared one reached the Makefile unquoted, so the shell read `$O` as an + unset variable and expanded it to nothing; the linker recorded the result as + an empty element. An empty rpath element is not ignored — it means the + current working directory, so the loader would also have taken `libai_sdk.so` + from wherever the process happened to be running. Both entries looked + identical in the Makefile and only one survived; the duplicate is gone, and + the prebuild now records exactly one rpath, asserted by value after the link. +- **A failed `Session.sendStream()` turn reported success.** When the request + never got a response — refused connection, DNS failure, timeout — the + session entry point returned its error status without emitting a terminal + event, and the binding, whose fallback exists to stop a consumer waiting + forever, filled the gap with a *finish*. The caller got a turn that had + succeeded and produced nothing: zero tokens, empty `finishReason`, no error. + The status-code failures (401/429/500) were never affected — those arrive as + a response and were already surfaced from inside the stream — and neither was + an agent carrying tools, which routes the same failure through the stream. + It took a tool-less agent plus a server that never answered to reach it. + `ai_stream_text` has always emitted `AI_STREAM_ERROR` from its catch; the + session entry point now does too, and the binding's fallback fails closed + rather than reporting a finish it never received. +- **`streamText` silently dropped tool calls.** It called the model's + `do_stream()` directly, so a tool call was streamed to the caller and never + executed. It now goes through `ai::stream_text`, which runs the tool loop — + fixed in the C binding, so the Python, Rust, and Go bindings inherit it. +- **A tool call hung the blocking entry points at 100% CPU forever.** The C + binding busy-waits on the event loop while the coroutine awaits a JS tool + callback, but the callback could only run on the event loop the caller was + blocking. Tool callbacks now run on a worker thread and resolve a promise. +- **`Batch.run()` hung at 100% CPU**, having polled exactly once. `run_batch()` + waits between polls on an event-loop timer; an in-flight HTTP call left the + loop with no queued work, which *stops* it, so the timer could never fire. + The driver now restarts the loop before each `run_one()`. +- **Batch results carried the wrong `customId`** — every item but the last got + a dangling pointer, because request strings were stored via `c_str()` while + the container they lived in was still reallocating. +- **`withPermissions()` aborted the process** (`Fatal error in + v8::HandleScope::CreateHandle`): the policy callback called into JS from the + tool's worker thread, which has no handle scope. It now goes through a + `ThreadSafeFunction`. +- **Empty `text_delta` events rendered as the literal string `"undefined"`.** + Providers legitimately emit empty deltas between real tokens; events whose + payload is text now always carry a string. +- **`createGoogle` ignored `baseUrl`** and had no `GOOGLE_BASE_URL` env + fallback, so the provider could not be pointed anywhere but the default host. - **`Task` start-then-await**: `co_await` on a `Task` that was already `start()`ed re-entered the coroutine mid-await and read an empty result (silent nulls / UB). `await_suspend` now registers the continuation and suspends for in-flight tasks; only fresh tasks are launched. Move construction carries the started flag. +- **Checkpoint writer never fired for streamed turns**: `send_stream` did not + increment the session turn counter. +- **Use-after-free on GC** in the Node tool callback path (`SIGTRAP`). +- **The npm tarball carried no licence text and no README.** `license: "MIT"` + was set in `package.json`, but the MIT text lives at the repository root and + npm only auto-includes it from the package root — so the published artifact + shipped the label without the notice it refers to, which is the part MIT + actually requires. The package also rendered on the registry with no readme + at all, because there was none in `bindings/node/`. Both files are now in the + package, along with `repository`, `homepage`, `bugs` and `keywords`, which + were absent for the same reason: nobody had looked at the manifest as a + *published* artifact rather than a working directory. The README's examples + are not illustrative — every call in it was run against the binding, and the + event-type list it documents was checked against what the stream emits. +- **The prebuild matrix would not have produced a tarball.** Two independent + faults, neither visible from a developer machine, both fatal to the release. + The `darwin-x64` leg was pinned to `macos-13`, which GitHub retired on + 2025-12-04 — a job on a retired label does not fail to build, it dies before + starting. And the macOS dependency step exported `OpenSSL_DIR`, which is the + hint for CMake's *config* mode; the project resolves OpenSSL through + `FindOpenSSL.cmake`, which reads `OPENSSL_ROOT_DIR` and ignores the other. On + a Mac with Homebrew already on the default search path neither was + observable, because configure succeeds whatever the variable says. Since + `assemble` sits behind `needs: prebuild` with no `if:`, one dead leg skips it + and `npm pack` never runs — so the failure mode was not a broken package but + no package. Both are fixed and the runner labels verified against GitHub's + current lists. Separately noted: `ubuntu-22.04`, which the Linux legs use for + its glibc 2.35 floor, entered deprecation on 2026-09-17 and retires + 2027-04-17; the migration must keep that floor, which means building inside a + 22.04 container on a newer runner rather than moving to `ubuntu-24.04` and + raising it to 2.39. ### Changed - **Parallel tool execution**: `execute_tools` (generate + stream paths) launches all tool calls in a step before awaiting results, so independent calls interleave on the event loop instead of serializing behind the slowest tool. Results stay in tool-call order; per-call error isolation unchanged. +- **Provider-scoped reasoning effort**, and an opt-in checkpoint summarizer + with a configurable proactive-compaction threshold. +- **The version string has one source of truth.** `ai_sdk_version()` and the + MCP `clientInfo` handshake are compiled from the CMake project version + (`include/ai/version.hpp`) instead of a retyped literal, so they cannot drift + from the release number the package manifests carry. A test asserts the + binding's `version()` and `package.json` agree. + +### Tests +- 162 offline unit tests (`ctest`), plus the 61-case Node suite. Both run in CI + on every push; neither needs an API key or network access. +- `scripts/coverage.sh` reports what those tests actually exercise (Clang + source-based coverage). It covers the SDK but not the socket layer or the C + binding — the unit tests substitute their own `IHttpClient`, so those are + covered by the Node suite instead. The script says so in its output. + +### Notes & known limitations +- **Building requires GCC 13 or newer, or Clang.** GCC 11 and 12 miscompile the + library's coroutines: 30 of the 158 tests the suite held when this was measured + crashed on them, at `-O0` and `-O3` alike, while GCC 13 passed all 158 on the + same OS, standard library and Boost. A build that succeeded on those compilers + would therefore still be a library that segfaults, so the build now refuses + them by name instead of letting that through — + `-DAI_SDK_ALLOW_UNSUPPORTED_COMPILER=ON` overrides it. This lands hardest on + Ubuntu 22.04, RHEL 9 and Debian 12, whose default repositories ship GCC 11 or + 12. On Ubuntu 22.04 a newer GCC is one package away + (`ppa:ubuntu-toolchain-r/test` carries `g++-13`), and that is the better route + there than Clang: this project builds Clang with `-stdlib=libc++`, and libc++ + is not installed by default on any of those distributions. + `sudo apt-get install libc++-dev libc++abi-dev` is the other way, with the + caveat that anything you link against it then needs libc++ at runtime too. +- **An ambient `ANTHROPIC_AUTH_TOKEN` outranks an explicitly passed `apiKey`.** + The Anthropic provider advertises two credentials and prefers the Bearer + token (`docs/providers.md`), resolving each independently, so if the + environment carries `ANTHROPIC_AUTH_TOKEN` — which it does for anyone running + Claude Code against an Anthropic-compatible gateway — `createAnthropic({ + apiKey })` authenticates with the ambient token instead. Requests fail, or + are billed elsewhere, with nothing in the error to say why. Unset the + variable, or set `authToken` deliberately, if you meant the key you passed. + Changing the precedence is a candidate for a future release, not this one. +- **Streaming reports failures as a terminal `error` event, not a throw.** + `streamText`/`sendStream` yield `{ type: 'error' }` and end; they do not + reject. Callers must check for it. +- **Node: passing the wrong kind of object to an entry point can abort the + process instead of throwing.** Eighteen call sites across five wrapper types + (context, provider, model, tool set, agent) unwrap their arguments without + checking that the unwrap succeeded, so `withPermissions({}, policy)` or the + like reaches a null handle and dies rather than raising a catchable error. + `describeToolSet()` is guarded; the rest are unchanged as of this release, + and guarding them is a candidate for 1.0.1 — mechanical, but wide enough that + it did not belong in a release that is otherwise done. +- **Nothing retries, and `maxRetries` does not do anything.** The transport does + classify: a 408, 429 or 5xx response produces an `ApiCallError` with + `is_retryable()` set and, where the server sent one, a `Retry-After` that + reaches `retry_after()`. But nothing consumes either. `include/ai/util/retry.hpp` + holds a complete `retry_async` with exponential backoff, jitter, `should_retry` + and a `get_retry_delay` that already prefers the server's `Retry-After` over + its own backoff — and no file in `src/`, `providers/` or `bindings/` includes + it. `max_retries` is likewise copied between option structs, including in + `tool_loop_agent`, and never read by anything that could retry. So setting + `maxRetries` on any entry point is silently inert: the option is accepted, the + call is never retried, and no diagnostic says so. Stated here because the + failure mode of a retry option that does nothing is a caller who believes + they have protection they do not have — and because the signal a caller would + need to build their own is at least available on the error. +- **Google** compiles and is wired through `createGoogle` (including + `baseUrl`), but has not been exercised against the live API — treat it as + experimental. ## [0.1.0] - 2026-06-22 diff --git a/CMakeLists.txt b/CMakeLists.txt index 94162be..d159b42 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,40 @@ cmake_minimum_required(VERSION 3.20) -project(ai-sdk-cpp VERSION 0.1.0 LANGUAGES CXX) +project(ai-sdk-cpp VERSION 1.0.0 LANGUAGES CXX) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) + +# GCC 11 and 12 cannot build a working library from this source, and the way +# they fail is worth stating precisely, because it is not the way one expects. +# +# They first refuse outright: the assembler rejects the operator() of a +# coroutine lambda declared inside a function with C language linkage as a +# duplicate symbol. That failure is loud and is what bindings/c/ai_sdk.cpp was +# rearranged to avoid. Past it, they miscompile the coroutines badly enough +# that 30 of the 158 unit tests crash -- streaming, sessions, tool execution, +# the C binding's permission gates. Measured on ubuntu:22.04 and ubuntu:24.04, +# at both -O0 and -O3, against Boost 1.83: GCC 11.4 and GCC 12.3 each fail the +# same 30, and only the compiler differs between a passing and a failing run. +# GCC 13 and Clang pass all 158. +# +# So the honest outcome on GCC 11/12 is a library that builds cleanly and then +# segfaults, which is a much worse thing to hand someone than a build error. +# Refuse by name instead. These versions are still the default on Ubuntu 22.04, +# RHEL 9 and Debian 12, so this is not a rare configuration to arrive in -- +# Clang is available on all three and builds this fine. +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 13 + AND NOT AI_SDK_ALLOW_UNSUPPORTED_COMPILER) + message(FATAL_ERROR + "GCC ${CMAKE_CXX_COMPILER_VERSION} cannot build this project. Its coroutine " + "support is broken for the pattern the library relies on: the tests fail by " + "crashing, and a build that succeeds would still be a library that segfaults " + "at runtime.\n" + "Use GCC 13 or newer, or Clang.\n" + "To build anyway and accept the breakage -- for example to reproduce it -- " + "configure with -DAI_SDK_ALLOW_UNSUPPORTED_COMPILER=ON.") +endif() + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # Build all targets position-independent so the static core/providers can be # linked into the shared C binding (libai_sdk.so). Required on Linux; harmless @@ -26,6 +57,51 @@ option(AI_SDK_PROVIDER_MOONSHOTAI "Build MoonshotAI provider" ON) option(AI_SDK_PROVIDER_ZAI "Build z.ai (GLM) provider" ON) option(AI_SDK_PROVIDER_BEDROCK "Build Amazon Bedrock provider" ON) option(AI_SDK_PROVIDER_OPENAI_COMPATIBLE "Build OpenAI-Compatible provider" ON) +option(AI_SDK_COVERAGE "Instrument for Clang source-based coverage (see scripts/coverage.sh)" OFF) +# Escape hatch for the compiler gate near the top of this file. Off by default +# because the thing it permits is a library that crashes; see the comment there. +option(AI_SDK_ALLOW_UNSUPPORTED_COMPILER "Build on a compiler measured to produce a broken library (GCC < 13)" OFF) + +# Whether the shared C binding carries its own copy of the C++ runtime. +# +# This is the artifact that ships, and on Linux it has to start on distributions +# older than the one that built it. libstdc++ versions its symbols, so the usual +# shape -- a build image years ahead of the target -- produces a library that +# refuses to load, on the user's machine, which is the one place the check +# cannot run. Linking the runtime in removes the dependency: the result needs +# only libc, libm and the ELF loader. +# +# Default on for GNU, and off otherwise for a reason rather than by omission. +# This project configures Clang on Linux with -stdlib=libc++, and there is no +# -static-libc++ to match -static-libstdc++; defaulting it on there would give +# an option that reports itself enabled while doing nothing at all, which is the +# exact class of defect it exists to prevent. Turn it on with Clang and the +# build says so instead. +if(UNIX AND NOT APPLE AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + option(AI_SDK_STATIC_CXX_RUNTIME "Link the C++ runtime into the shared C binding" ON) +else() + option(AI_SDK_STATIC_CXX_RUNTIME "Link the C++ runtime into the shared C binding" OFF) +endif() + +# The version the runtime reports (ai_sdk_version(), the MCP clientInfo block). +# Sourced from the project version so it cannot drift from the release number +# the manifests carry. See include/ai/version.hpp. +add_compile_definitions(AI_SDK_VERSION="${PROJECT_VERSION}") + +# Coverage instrumentation. These flags only make the build *emit* .profraw +# files; turning those into a report needs llvm-profdata + llvm-cov, which is +# what scripts/coverage.sh drives. Clang-specific: the profile format is read +# back by llvm-cov, which will not read GCC's gcov output. +if(AI_SDK_COVERAGE) + if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + message(FATAL_ERROR + "AI_SDK_COVERAGE requires Clang (found ${CMAKE_CXX_COMPILER_ID}). " + "llvm-cov reads the profile Clang emits; GCC would need gcov instead.") + endif() + add_compile_options(-fprofile-instr-generate -fcoverage-mapping) + # The profiling runtime has to be linked in too, or nothing writes a profile. + add_link_options(-fprofile-instr-generate) +endif() # Expose provider enablement to the preprocessor so the CLI / C binding can # conditionally compile provider code (they include all providers by default). diff --git a/README.md b/README.md index 19c6ea6..73f695a 100644 --- a/README.md +++ b/README.md @@ -219,10 +219,19 @@ cd build && ./tests/unit/ai-sdk-tests "[message]" # Run a single test by name ctest --test-dir build -R "SseParser parses basic event" + +# What did those tests actually exercise? (Clang coverage; --html for a browsable report) +./scripts/coverage.sh ``` Unit tests mock HTTP responses and run without network access. Integration tests require API keys. +The unit tests substitute their own `ai::http::IHttpClient`, so they cover the +providers, the agent loop and the session layer but not the socket layer or the C +binding. Those are covered by the Node suite, which drives them against a real +server on `127.0.0.1`. The two are complementary; `scripts/coverage.sh` measures +only the former and says so in its output. + ## Language Bindings ### C API diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt index dda8604..b13ae21 100644 --- a/bindings/c/CMakeLists.txt +++ b/bindings/c/CMakeLists.txt @@ -21,6 +21,26 @@ set_target_properties(ai-sdk-c PROPERTIES PUBLIC_HEADER ai_sdk.h ) +# See AI_SDK_STATIC_CXX_RUNTIME in the root CMakeLists for why. Kept here rather +# than as a global linker flag so it applies to this one target: the tests and +# the CLI link the same code and have no reason to carry a second copy of the +# runtime. +if(AI_SDK_STATIC_CXX_RUNTIME) + # CMAKE_CXX_COMPILER_ID, spelled in full. The bare name is not a CMake + # variable at all, so it is the empty string, the comparison is false on + # every compiler, and this silently takes the branch below -- leaving the + # runtime dynamically linked while the build reports the option as ON. + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + target_link_options(ai-sdk-c PRIVATE -static-libstdc++ -static-libgcc) + else() + message(WARNING + "AI_SDK_STATIC_CXX_RUNTIME is ON but the compiler is " + "${CMAKE_CXX_COMPILER_ID}, not GNU, so -static-libstdc++ has nothing " + "to act on and will be skipped. The shared library will depend on " + "the runtime installed on the machine that loads it.") + endif() +endif() + install(TARGETS ai-sdk-c EXPORT AiSdkTargets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/bindings/c/ai_sdk.cpp b/bindings/c/ai_sdk.cpp index 8148542..f1e5b06 100644 --- a/bindings/c/ai_sdk.cpp +++ b/bindings/c/ai_sdk.cpp @@ -4,7 +4,9 @@ #include #include #include +#include #include + #if defined(AI_SDK_PROVIDER_ANTHROPIC) #include #endif @@ -22,9 +24,11 @@ #endif #include #include +#include #include #include #include +#include #include #include #include @@ -33,6 +37,53 @@ namespace json = boost::json; +namespace { + +// Drive a task to completion, pumping `ioc` while doing so. Every blocking C +// entry point waits this way, so it lives in one place. +// +// run_one() only blocks while the io_context has work armed, and most of what +// the C API drives never arms any. The HTTP client bridges its blocking I/O +// through a std::future and resumes the coroutine from a detached thread +// (FutureAwaitable in src/http/client.cpp) rather than posting the resume to +// the io_context. So `while (!task.done()) ioc.run_one();` returns from +// run_one() immediately on every iteration for the whole call, burning a full +// CPU core for as long as the request is in flight — on a server, a core per +// concurrent request. Parking when run_one() reports no work caps that at +// approximately zero; the cost is up to one sleep interval of added latency on +// a resume, and the resume is happening on another thread regardless. +// +// restart() comes first because a run_one() that finds nothing stops the +// context, and a stopped context returns 0 without executing handlers that get +// queued afterwards. The batch path arms a timer on the io_context between +// polls and depends on exactly that. +template +void drive_to_completion(TaskT& task, boost::asio::io_context& ioc) { + constexpr auto kMinIdle = std::chrono::microseconds(50); + constexpr auto kMaxIdle = std::chrono::microseconds(2000); + + auto idle = kMinIdle; + task.start(); + while (!task.done()) { + ioc.restart(); + if (ioc.run_one() == 0) { + std::this_thread::sleep_for(idle); + // Back off while the io_context stays idle. Polling only gates + // noticing that the task finished — the work itself, and every + // streamed part, is delivered from another thread — so a tight poll + // buys responsiveness only in the last moments of a call and costs a + // proportional share of a core for the whole of a long one. Start + // short so a quick call stays quick, and stretch out so a slow one + // costs almost nothing. + idle = idle * 2 < kMaxIdle ? idle * 2 : kMaxIdle; + } else { + idle = kMinIdle; + } + } +} + +} // namespace + struct ai_context { boost::asio::io_context ioc; std::string last_error; @@ -161,10 +212,7 @@ static ai_status_t consume_stream_to_callback( }(std::move(stream), cb, ud, skip_final_finish); try { - consume.start(); - while (!consume.done()) { - ioc.run_one(); - } + drive_to_completion(consume, ioc); consume.get(); return AI_OK; } catch (const std::exception& e) { @@ -178,6 +226,142 @@ static ai_status_t consume_stream_to_callback( /* -------------------------------------------------------------------------- */ +namespace { + +/// The buffer handed to a permission callback, zeroed so that a callback which +/// writes nothing leaves an empty string -- which the core reads as "no +/// explanation given". +/// +/// Declared outside the extern "C" block below: it returns std::string, which +/// is not a type C linkage can describe. +class ReasonBuffer { +public: + ReasonBuffer() { buf_[0] = '\0'; } + + char* data() { return buf_; } + + /// What the callback left behind. A callback that did not NUL-terminate + /// (a plain strcpy of AI_REASON_MAX bytes, say) would run past the end, so + /// the last byte is forced to NUL to make that impossible. + std::string str() const { + char safe[AI_REASON_MAX]; + std::memcpy(safe, buf_, sizeof(safe)); + safe[AI_REASON_MAX - 1] = '\0'; + return std::string(safe); + } + +private: + char buf_[AI_REASON_MAX]; +}; + +// The three coroutine bodies below live here, outside `extern "C"`, rather than +// as lambdas at their points of use -- and the coroutine is the whole reason. +// +// GCC 11 and 12 emit the `operator()` of a coroutine lambda declared inside a +// function with C language linkage twice, and the assembler rejects the +// duplicate symbol ("symbol ... is already defined"). GCC 13 emits it once, and +// Clang never had the problem, which is why this went unnoticed: CI builds with +// GCC 13, while GCC 11 is the default on Ubuntu 22.04 and RHEL 9 and GCC 12 on +// Debian 12 -- so on all of those the file did not build at all. The error +// lands at the assembly step, after everything else has compiled, so it reads +// like a linker or build-system fault rather than a source one. +// +// Hoisting is the entire fix. Language linkage is a property of the enclosing +// function, not of the C++ types its body happens to mention, and none of this +// is part of the C ABI -- so moving the coroutines out changes nothing except +// which compiler can parse them. The fourth coroutine in this file, the +// streaming consumer inside `drive_stream`, is a lambda too but sits above the +// block already; it has always built. + +/// The body of a tool implemented by a C callback. +struct ToolCallbackBody { + ai_tool_set::ToolBinding* binding; + std::string tool_name; + + ai::Task operator()(json::value input, ai::ToolExecutionContext) const { + std::string input_str = json::serialize(input); + auto result = binding->callback(tool_name.c_str(), input_str.c_str(), binding->user_data); + + json::value output; + if (result.output_json) { + boost::system::error_code ec; + output = json::parse(result.output_json, ec); + if (ec) output = json::value(result.output_json); + } + + if (result.is_error) { + throw std::runtime_error(result.output_json ? result.output_json : "Tool error"); + } + + co_return output; + } +}; + +/// The summarizer behind the opt-in auto-checkpoint. +struct CheckpointSummarizer { + ai::LanguageModelPtr model; + + ai::Task operator()(const ai::Prompt& history) const { + ai::GenerateTextOptions o; + o.model = model; + o.system = "Summarize the conversation so far into a concise checkpoint: " + "key decisions, current state, and next steps."; + std::string transcript; + for (auto& m : history) { + std::visit([&](auto&& msg) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + for (auto& p : msg.content) + if (auto* t = std::get_if(&p)) transcript += "User: " + t->text + "\n"; + } else if constexpr (std::is_same_v) { + for (auto& p : msg.content) + if (auto* t = std::get_if(&p)) transcript += "Assistant: " + t->text + "\n"; + } + }, m); + } + o.prompt = transcript; + auto r = co_await ai::generate_text(std::move(o)); + co_return r.text; + } +}; + +/// The bridge from the C approver callback to `ai::Approver`. +struct ApproverBody { + ai_approver_fn approver; + void* user_data; + + ai::Task operator()(const std::string& tool, + const boost::json::value& input, + const std::string& rationale) const { + std::string input_str = boost::json::serialize(input); + ReasonBuffer reason; + int d = approver(tool.c_str(), input_str.c_str(), rationale.c_str(), + reason.data(), user_data); + // Fail closed: a verdict that is not one of the three is a bug in + // the caller, and running the tool anyway is the one reading that + // turns it into a security hole. + ai::Approval approval{}; + // Left empty when the approver said nothing, which lets the + // policy's reason survive -- see the header. + approval.reason = reason.str(); + switch (d) { + case AI_PERMISSION_ALLOW: + approval.decision = ai::PermissionDecision::Allow; + break; + case AI_PERMISSION_ALLOW_ALWAYS: + approval.decision = ai::PermissionDecision::Allow; + approval.always_allow = true; + break; + default: // AI_PERMISSION_DENY, AI_PERMISSION_ASK, out of contract + approval.decision = ai::PermissionDecision::Deny; + break; + } + co_return approval; + } +}; + +} // namespace + extern "C" { const char* ai_status_message(ai_status_t status) { @@ -241,6 +425,8 @@ ai_provider_t ai_provider_create(ai_context_t ctx, const char* provider_name, ai if (!provider && name == "google") { ai::providers::google::GoogleOptions o{.io_context = ctx->ioc}; if (api_key) o.api_key = *api_key; + if (opts.base_url) o.base_url = opts.base_url; + else if (const char* env = std::getenv("GOOGLE_BASE_URL"); env && *env) o.base_url = env; provider = ai::providers::google::create_google(std::move(o)); } #endif @@ -335,28 +521,64 @@ ai_status_t ai_tool_set_add( tool_name, schema, description ? std::string(description) : "", - [binding, tool_name](json::value input, ai::ToolExecutionContext) -> ai::Task { - std::string input_str = json::serialize(input); - auto result = binding->callback(tool_name.c_str(), input_str.c_str(), binding->user_data); - - json::value output; - if (result.output_json) { - boost::system::error_code ec; - output = json::parse(result.output_json, ec); - if (ec) output = json::value(result.output_json); - } + ToolCallbackBody{binding, tool_name} + )); - if (result.is_error) { - throw std::runtime_error(result.output_json ? result.output_json : "Tool error"); - } + return AI_OK; +} - co_return output; +ai_status_t ai_tool_set_describe_json(ai_tool_set_t tools, ai_tool_set_description_t* result) { + if (!tools || !result) return AI_ERROR_INVALID_ARGUMENT; + + result->json = nullptr; + result->count = 0; + result->_storage = nullptr; + + // A ToolSet is an unordered_map underneath, so iterating it directly would + // describe the same tools in a different order from one process to the + // next. A description a caller cannot compare against another is not much + // of a description, and it would make any test of this flaky, so sort by + // name first. + auto ordered = tools->tools.all(); + std::sort(ordered.begin(), ordered.end(), + [](const ai::ToolDefinition* a, const ai::ToolDefinition* b) { + return a->name < b->name; + }); + + auto* storage = new std::string(); + try { + json::array arr; + for (const auto* definition : ordered) { + json::object entry; + entry["name"] = definition->name; + // An absent description is reported as JSON null rather than + // omitted, so every entry has the same three keys. + entry["description"] = definition->description + ? json::value(*definition->description) : json::value(); + entry["input_schema"] = definition->input_schema.raw(); + arr.push_back(std::move(entry)); } - )); + *storage = json::serialize(arr); + result->count = static_cast(arr.size()); + } catch (const std::exception&) { + delete storage; + return AI_ERROR_INTERNAL; + } + result->json = storage->c_str(); + result->_storage = storage; return AI_OK; } +void ai_tool_set_description_free(ai_tool_set_description_t* result) { + if (result && result->_storage) { + delete static_cast(result->_storage); + result->_storage = nullptr; + result->json = nullptr; + result->count = 0; + } +} + ai_status_t ai_generate_text(ai_generate_options_t opts, ai_generate_result_t* result) { if (!opts.model || !result) return AI_ERROR_INVALID_ARGUMENT; auto* ctx = opts.model->ctx; @@ -394,10 +616,7 @@ ai_status_t ai_generate_text(ai_generate_options_t opts, ai_generate_result_t* r } auto task = ai::generate_text(std::move(gen_opts)); - task.start(); - while (!task.done()) { - ctx->ioc.run_one(); - } + drive_to_completion(task, ctx->ioc); auto gen_result = task.get(); g_result_text = gen_result.text; @@ -432,38 +651,33 @@ ai_status_t ai_stream_text(ai_generate_options_t opts, ai_stream_callback_fn cal auto* ctx = opts.model->ctx; try { - ai::Prompt prompt; - if (opts.system) { - prompt.push_back(ai::SystemMessage{.content = std::string(opts.system)}); - } + // Goes through ai::stream_text rather than calling the model's + // do_stream() directly, so a tool call is executed and fed back to the + // model instead of being streamed to the caller and dropped. + ai::StreamTextOptions stream_opts; + stream_opts.model = opts.model->ptr; + stream_opts.max_steps = opts.max_steps > 0 ? opts.max_steps : 1; + if (opts.tools) stream_opts.tools = opts.tools->tools; + if (opts.system) stream_opts.system = std::string(opts.system); if (opts.messages_json) { auto messages = json::parse(opts.messages_json); - auto deserialized = ai::workflow::deserialize_prompt(messages.as_array()); - for (auto& m : deserialized) prompt.push_back(std::move(m)); + stream_opts.messages = ai::workflow::deserialize_prompt(messages.as_array()); } else if (opts.prompt) { - ai::UserContent content; - content.push_back(ai::TextPart{.text = std::string(opts.prompt)}); - prompt.push_back(ai::UserMessage{.content = std::move(content)}); + stream_opts.prompt = std::string(opts.prompt); } - - ai::CallOptions call_opts; - call_opts.prompt = std::move(prompt); - if (opts.max_output_tokens > 0) call_opts.max_output_tokens = opts.max_output_tokens; - if (opts.temperature >= 0) call_opts.temperature = opts.temperature; + if (opts.max_output_tokens > 0) stream_opts.max_output_tokens = opts.max_output_tokens; + if (opts.temperature >= 0) stream_opts.temperature = opts.temperature; if (opts.provider_options_json) { auto provider_options = json::parse(opts.provider_options_json); if (!provider_options.is_object()) { ctx->last_error = "provider_options_json must be a JSON object"; return AI_ERROR_INVALID_ARGUMENT; } - call_opts.provider_options = provider_options.as_object(); + stream_opts.provider_options = provider_options.as_object(); } - auto task = opts.model->ptr->do_stream(std::move(call_opts)); - task.start(); - while (!task.done()) { - ctx->ioc.run_one(); - } + auto task = ai::stream_text(std::move(stream_opts)); + drive_to_completion(task, ctx->ioc); auto stream_result = task.get(); return consume_stream_to_callback( std::move(stream_result.stream), ctx->ioc, callback, user_data, ctx); @@ -530,10 +744,7 @@ ai_status_t ai_agent_call(ai_agent_t agent, const char* prompt, ai_generate_resu try { auto task = agent->agent.call(std::string(prompt)); - task.start(); - while (!task.done()) { - agent->ctx->ioc.run_one(); - } + drive_to_completion(task, agent->ctx->ioc); auto agent_result = task.get(); g_result_text = agent_result.text; @@ -559,10 +770,7 @@ ai_status_t ai_agent_call_stream(ai_agent_t agent, const char* prompt, ai_stream // Run the agent's streaming tool loop, then dispatch each stream part // (text/tool/finish) to the C callback as it is produced. auto outer = agent->agent.stream(std::string(prompt)); - outer.start(); - while (!outer.done()) { - ctx->ioc.run_one(); - } + drive_to_completion(outer, ctx->ioc); auto result = outer.get(); return consume_stream_to_callback( @@ -635,10 +843,11 @@ ai_status_t ai_batch_run( }; auto task = ai::batch::run_batch(std::move(opts)); - task.start(); - while (!task.done()) { - ctx->ioc.run_one(); - } + // run_batch waits between polls on an io_context timer, so it needs + // run_one() to actually execute handlers. drive_to_completion restarts + // the context between polls, which is what lets that timer fire after + // an earlier empty poll stopped it. + drive_to_completion(task, ctx->ioc); auto run_result = task.get(); auto* storage = new BatchResultStorage(); @@ -732,30 +941,9 @@ ai_session_t ai_session_create_with_memory(ai_agent_t agent, const char* memory_ // Opt-in via `enable_checkpoint` — callers that don't want the extra API // call (e.g. DeepSeek Code) can pass 0. if (enable_checkpoint && agent->agent.model()) { - auto summarizer = [model = agent->agent.model()](const ai::Prompt& history) -> ai::Task { - ai::GenerateTextOptions o; - o.model = model; - o.system = "Summarize the conversation so far into a concise checkpoint: " - "key decisions, current state, and next steps."; - std::string transcript; - for (auto& m : history) { - std::visit([&](auto&& msg) { - using T = std::decay_t; - if constexpr (std::is_same_v) { - for (auto& p : msg.content) - if (auto* t = std::get_if(&p)) transcript += "User: " + t->text + "\n"; - } else if constexpr (std::is_same_v) { - for (auto& p : msg.content) - if (auto* t = std::get_if(&p)) transcript += "Assistant: " + t->text + "\n"; - } - }, m); - } - o.prompt = transcript; - auto r = co_await ai::generate_text(std::move(o)); - co_return r.text; - }; session.set_on_turn_finish( - ai::memory::make_checkpoint_writer(store, summarizer, 5)); + ai::memory::make_checkpoint_writer( + store, CheckpointSummarizer{agent->agent.model()}, 5)); } return new ai_session{ .session = std::move(session), .ctx = agent->ctx }; @@ -770,10 +958,7 @@ ai_status_t ai_session_send(ai_session_t session, const char* prompt, ai_generat auto* ctx = session->ctx; try { auto task = session->session.send(std::string(prompt)); - task.start(); - while (!task.done()) { - ctx->ioc.run_one(); - } + drive_to_completion(task, ctx->ioc); auto r = task.get(); g_result_text = r.text; switch (r.finish_reason) { @@ -877,10 +1062,7 @@ ai_status_t ai_session_send_stream( try { auto outer = session->session.send_stream(std::string(prompt)); - outer.start(); - while (!outer.done()) { - ctx->ioc.run_one(); - } + drive_to_completion(outer, ctx->ioc); auto result = outer.get(); auto status = consume_stream_to_callback( @@ -888,10 +1070,7 @@ ai_status_t ai_session_send_stream( if (status != AI_OK) return status; auto full_result_task = std::move(result.full_result); - full_result_task.start(); - while (!full_result_task.done()) { - ctx->ioc.run_one(); - } + drive_to_completion(full_result_task, ctx->ioc); auto gen_result = full_result_task.get(); // Dispatch executed tool results to callback @@ -918,8 +1097,7 @@ ai_status_t ai_session_send_stream( // Fire the post-turn hook (e.g. checkpoint writer) — best-effort. { auto hook = session->session.fire_turn_finish(); - hook.start(); - while (!hook.done()) ctx->ioc.run_one(); + drive_to_completion(hook, ctx->ioc); try { hook.get(); } catch (...) {} } @@ -938,6 +1116,14 @@ ai_status_t ai_session_send_stream( return AI_OK; } catch (const std::exception& e) { + // The turn failed before it could finish: a refused connection, a DNS + // failure, a request that timed out. The caller has to be told, because + // the alternative is a stream that simply ends -- which reads as a turn + // that produced nothing rather than one that never happened. + ai_stream_event_t err_event{}; + err_event.type = AI_STREAM_ERROR; + err_event.text = e.what(); + callback(err_event, user_data); return map_exception(ctx, e); } } @@ -974,17 +1160,50 @@ ai_tool_set_t ai_with_permissions(ai_tool_set_t tools, ai_permission_policy_fn p ai::PermissionPolicy p = [policy, user_data](const std::string& tool, const boost::json::value& input) { std::string input_str = boost::json::serialize(input); - int d = policy(tool.c_str(), input_str.c_str(), user_data); + ReasonBuffer reason; + int d = policy(tool.c_str(), input_str.c_str(), reason.data(), user_data); switch (d) { - case AI_PERMISSION_ALLOW: return ai::PermissionDecision::Allow; - case AI_PERMISSION_DENY: return ai::PermissionDecision::Deny; - default: return ai::PermissionDecision::Ask; + case AI_PERMISSION_ALLOW: return ai::PermissionVerdict{ai::PermissionDecision::Allow}; + case AI_PERMISSION_DENY: + return ai::PermissionVerdict{ai::PermissionDecision::Deny, reason.str()}; + default: + return ai::PermissionVerdict{ai::PermissionDecision::Ask, reason.str()}; } }; out->tools = ai::with_permissions(std::move(tools->tools), p); return out; } +ai_tool_set_t ai_with_permissions_approver(ai_tool_set_t tools, ai_permission_policy_fn policy, + void* policy_user_data, ai_approver_fn approver, + void* approver_user_data) { + if (!tools || !policy) return nullptr; + auto* out = new ai_tool_set{}; + ai::PermissionPolicy p = [policy, policy_user_data](const std::string& tool, + const boost::json::value& input) { + std::string input_str = boost::json::serialize(input); + ReasonBuffer reason; + int d = policy(tool.c_str(), input_str.c_str(), reason.data(), policy_user_data); + switch (d) { + case AI_PERMISSION_ALLOW: return ai::PermissionVerdict{ai::PermissionDecision::Allow}; + case AI_PERMISSION_DENY: + return ai::PermissionVerdict{ai::PermissionDecision::Deny, reason.str()}; + default: + return ai::PermissionVerdict{ai::PermissionDecision::Ask, reason.str()}; + } + }; + + // A NULL approver leaves `a` empty, which is exactly the fail-closed + // behaviour of ai_with_permissions. + ai::Approver a; + if (approver) { + a = ApproverBody{approver, approver_user_data}; + } + + out->tools = ai::with_permissions(std::move(tools->tools), p, std::move(a)); + return out; +} + // --- Memory store --- struct ai_memory_store { @@ -1037,6 +1256,22 @@ ai_tool_set_t ai_mcp_toolset_from_server(ai_context_t ctx, const char* config_js ai::mcp::McpServerConfig cfg; cfg.transport = (o.contains("transport") && o["transport"].is_string()) ? std::string(o["transport"].as_string()) : "stdio"; + if (o.contains("name") && o["name"].is_string()) + cfg.name = std::string(o["name"].as_string()); + // Required: the name is what qualifies every tool this server exposes + // (mcp____), which is the string a caller keys permission + // rules and approval prompts on. Without it the tools are addressable + // only by bare name, so one server can shadow another's tools -- or a + // built-in "Bash" -- and no rule can name the one it means. Refusing + // here is the fail-closed reading: an ambiguous toolset is worse than + // no toolset. + if (cfg.name.empty()) { + ctx->last_error = + "MCP config must include a non-empty \"name\": it qualifies every tool the " + "server exposes as mcp____, which is what permission rules and " + "approvers are keyed on."; + return nullptr; + } if (o.contains("command") && o["command"].is_string()) cfg.command = std::string(o["command"].as_string()); if (o.contains("args") && o["args"].is_array()) @@ -1053,14 +1288,12 @@ ai_tool_set_t ai_mcp_toolset_from_server(ai_context_t ctx, const char* config_js auto client = std::make_shared(cfg); auto t = client->connect(); - t.start(); - while (!t.done()) ctx->ioc.run_one(); + drive_to_completion(t, ctx->ioc); t.get(); // throws on connect failure // List tools (async, drive on ioc). auto lt = client->list_tools(); - lt.start(); - while (!lt.done()) ctx->ioc.run_one(); + drive_to_completion(lt, ctx->ioc); auto mcp_tools = lt.get(); auto tools = ai::mcp::mcp_tools_to_toolset(client, mcp_tools); @@ -1072,7 +1305,7 @@ ai_tool_set_t ai_mcp_toolset_from_server(ai_context_t ctx, const char* config_js } const char* ai_sdk_version(void) { - return "0.1.0"; + return ai::version(); } } // extern "C" diff --git a/bindings/c/ai_sdk.h b/bindings/c/ai_sdk.h index 3dbb29b..4de4a9a 100644 --- a/bindings/c/ai_sdk.h +++ b/bindings/c/ai_sdk.h @@ -71,7 +71,19 @@ void ai_model_destroy(ai_model_t model); * -------------------------------------------------------------------------- */ typedef struct { - const char* output_json; /* JSON string of the tool output */ + /* JSON string of the tool output. + * + * Borrowed, not transferred. The library parses it into its own storage + * before ai_tool_fn returns and keeps no reference afterwards, so the + * callback owns this buffer and is free to reuse it -- a static or + * thread-local buffer is the intended shape, and a string literal is fine. + * + * It must stay valid until the callback has returned, which rules out a + * pointer into a local that has gone out of scope by then. Allocating a + * fresh buffer per call is also wrong, in a way that is invisible: nothing + * on either side of this contract frees it, so each call leaks the output + * for the life of the process. Both language bindings shipped that way. */ + const char* output_json; int is_error; /* 0 = success, 1 = error */ } ai_tool_result_t; @@ -89,6 +101,21 @@ ai_status_t ai_tool_set_add( void* user_data ); +/* What a tool set contains, as JSON: an array of objects with the three fields + * a model is told about -- "name", "description" and "input_schema". Same + * ownership rule as ai_batch_result_t: the string belongs to `_storage` and is + * released by ai_tool_set_description_free. Tools are ordered by name, so the + * same set always describes itself the same way and callers can compare two + * descriptions. */ +typedef struct { + const char* json; /* array of {name, description, input_schema} */ + int count; /* number of tools described */ + void* _storage; /* internal; freed by ai_tool_set_description_free */ +} ai_tool_set_description_t; + +ai_status_t ai_tool_set_describe_json(ai_tool_set_t tools, ai_tool_set_description_t* result); +void ai_tool_set_description_free(ai_tool_set_description_t* result); + /* -------------------------------------------------------------------------- * generate_text — synchronous (blocks until complete) * -------------------------------------------------------------------------- */ @@ -253,16 +280,63 @@ enum { AI_PERMISSION_ALLOW = 0, AI_PERMISSION_DENY = 1, AI_PERMISSION_ASK = 2, + /// Approver-only: allow this call and skip the approver for this tool for + /// the rest of the session. + AI_PERMISSION_ALLOW_ALWAYS = 3, }; +/// Bytes available in the `reason` buffer handed to a policy or approver, +/// including the terminating NUL. Longer text is truncated. +#define AI_REASON_MAX 512 + /// Synchronous permission policy: inspect (tool, input_json) and return a /// decision. `user_data` is passed through from ai_with_permissions. -typedef int (*ai_permission_policy_fn)(const char* tool, const char* input_json, void* user_data); +/// +/// To explain a refusal, write a sentence into `reason` with snprintf and +/// return AI_PERMISSION_DENY. It reaches the model as part of the tool result, +/// so it should say what would have to change — a model told only that a call +/// was refused will try variations of it. +/// +/// `reason` is a writable AI_REASON_MAX-byte buffer, empty on entry; writing +/// nothing means "no explanation". It belongs to the SDK, so there is nothing +/// to allocate and nothing to free. +/// +/// C++ callers: annotate the callback `-> int`. The AI_PERMISSION_* constants +/// belong to an unnamed enum, and a lambda returning one deduces that enum as +/// its return type; conversion to a function pointer requires an identical +/// return type, so the lambda will not convert. (Naming the enum would not +/// help — it would still be a distinct type from int.) +typedef int (*ai_permission_policy_fn)(const char* tool, const char* input_json, + char* reason, void* user_data); /// Return a copy of `tools` where each execute is gated by `policy`. /// (Ask with no interactive approver fails closed -> Deny.) ai_tool_set_t ai_with_permissions(ai_tool_set_t tools, ai_permission_policy_fn policy, void* user_data); +/// Interactive approver, called only when the policy returns AI_PERMISSION_ASK. +/// Returns AI_PERMISSION_ALLOW, AI_PERMISSION_ALLOW_ALWAYS or AI_PERMISSION_DENY; +/// anything else is treated as Deny. `rationale` is human-readable text from the +/// engine. Called on the engine's worker thread, so an implementation may block +/// while a UI collects the answer — but it must not call back into the agent. +/// +/// `reason` works as it does on ai_permission_policy_fn, with one rule for a +/// DENY that writes nothing: the policy's reason, if it gave one, is used +/// instead. That is usually what a UI wants — the prompt it showed the user +/// explains the refusal better than the bare fact that they clicked no. Write +/// to it to say something else ("the user declined", "rate limited, try later"). +/// +/// C++ callers: annotate the callback `-> int`, for the reason given on +/// ai_permission_policy_fn. +typedef int (*ai_approver_fn)(const char* tool, const char* input_json, + const char* rationale, char* reason, void* user_data); + +/// As ai_with_permissions, but `Ask` escalates to `approver`, which decides, +/// instead of failing closed. A NULL `approver` behaves exactly like +/// ai_with_permissions. +ai_tool_set_t ai_with_permissions_approver(ai_tool_set_t tools, + ai_permission_policy_fn policy, void* policy_user_data, + ai_approver_fn approver, void* approver_user_data); + /* -------------------------------------------------------------------------- * Memory — persistent cross-session project knowledge. * -------------------------------------------------------------------------- */ @@ -282,8 +356,15 @@ ai_status_t ai_memory_save(ai_memory_store_t store, const char* scope, void ai_tool_set_merge(ai_tool_set_t dest, ai_tool_set_t src); /// Connect to an MCP server described by `config_json` and return its tools as -/// a ToolSet. config_json fields: transport ("stdio"|"http"), command, args[], -/// env{}, url, headers{}. Returns NULL on failure (see ai_last_error). +/// a ToolSet. config_json fields: name, transport ("stdio"|"http"), command, +/// args[], env{}, url, headers{}. Returns NULL on failure (see ai_last_error). +/// +/// `name` is required and is not sent to the server: it qualifies every tool +/// the server exposes, as `mcp____`. That qualified string is what +/// the model is offered and what permission policies and approvers receive, so +/// it is what a caller keys rules on. Bare tool names are not exposed, because +/// two servers may both define `add_numbers` (or `Bash`) and a rule could not +/// then say which one it meant. ai_tool_set_t ai_mcp_toolset_from_server(ai_context_t ctx, const char* config_json); /* -------------------------------------------------------------------------- diff --git a/bindings/node/LICENSE b/bindings/node/LICENSE new file mode 100644 index 0000000..a0a561d --- /dev/null +++ b/bindings/node/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Mihal Dimo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bindings/node/README.md b/bindings/node/README.md new file mode 100644 index 0000000..7b2c9d3 --- /dev/null +++ b/bindings/node/README.md @@ -0,0 +1,139 @@ +# ai-sdk-cpp + +Node.js bindings for [ai-sdk-cpp](https://github.com/mhdimo/ai-sdk-cpp), a C++20 +AI agent framework. A N-API addon over the framework's C ABI — the orchestration +layer between your application and LLM providers, as native code rather than a +JS reimplementation. + +```bash +npm install ai-sdk-cpp +``` + +Requires Node 18 or newer. Prebuilt binaries ship for `darwin-arm64`, +`darwin-x64`, `linux-arm64` and `linux-x64`, so installing does not need a +compiler, a C++ toolchain, or a network fetch of one. On any other platform the +install fails loudly and names the platforms that are supported, rather than +leaving you with a package that breaks at `require()`. + +## Quick start + +```js +const { createAnthropic, generateText } = require('ai-sdk-cpp'); + +const anthropic = createAnthropic(); // reads ANTHROPIC_API_KEY +const model = anthropic('claude-sonnet-4-20250514'); + +const result = await generateText({ + model, + prompt: 'Summarise what a coroutine is in two sentences.', +}); + +console.log(result.text); +console.log(result.usage); // { inputTokens, outputTokens } +``` + +Pass credentials explicitly when you would rather not depend on the ambient +environment: + +```js +const openai = createOpenAI({ apiKey: 'sk-…', baseUrl: 'https://api.openai.com/v1' }); +``` + +Available provider factories: `createAnthropic`, `createOpenAI`, `createGoogle`, +`createDeepSeek`, `createZai`, `createDeepSeekAnthropic`, `createZaiOpenAI`. +Each reads its own environment variable (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, +`GOOGLE_GENERATIVE_AI_API_KEY`, `DEEPSEEK_API_KEY`, `ZAI_API_KEY`) and honours a +`_BASE_URL` override where the provider supports one. + +## Streaming + +```js +const { streamText } = require('ai-sdk-cpp'); + +for await (const event of streamText({ model, prompt: 'Write a haiku.' })) { + if (event.type === 'text_delta') process.stdout.write(event.text); + if (event.type === 'error') console.error('stream failed:', event.text); +} +``` + +Event types: `text_delta`, `tool_call_start`, `tool_call_delta`, +`tool_call_end`, `reasoning_start`, `reasoning_delta`, `reasoning_end`, +`tool_result`, `step_finish`, `finish`, `error`. Token usage arrives on the +`finish` event. + +**Streaming reports failures as a terminal `error` event; it does not throw.** +Always check for it — a stream that ends without one is a stream that succeeded. + +## Tools and agents + +```js +const { tool, Agent } = require('ai-sdk-cpp'); + +const add = tool( + 'add', + { type: 'object', properties: { a: { type: 'number' }, b: { type: 'number' } }, + required: ['a', 'b'], additionalProperties: false }, + 'Add two numbers.', + ({ a, b }) => a + b +); + +const agent = new Agent({ model, tools: [add], instructions: 'Be concise.' }); +const result = await agent.call('What is 17 + 25?'); +``` + +The agent runs the tool loop itself: it calls the model, executes the tools, +feeds the results back, and repeats until the model finishes or `maxSteps` is +reached. Independent tool calls within a single step run concurrently. + +## Sessions + +A `Session` holds conversation history across turns and applies a context +strategy, so long conversations keep working without you managing the window. + +```js +const { Session } = require('ai-sdk-cpp'); + +const session = new Session(agent, { memoryDir: './.memory', maxContextTokens: 100_000 }); +await session.send('My name is Liang.'); +const answer = await session.send('What did I just tell you?'); + +// Streaming turns deliver events on the event loop, so a UI stays responsive. +for await (const event of session.sendStream('And now a long answer…')) { + if (event.type === 'text_delta') process.stdout.write(event.text); +} +``` + +With `memoryDir` set, relevant persisted memory is injected before each turn and +history auto-compacts near the token budget. `enableCheckpoint` (default `true`) +writes a summary into memory every five turns; that costs an extra model call, +so set it to `false` if you do not want it. + +## Other entry points + +- `standardToolkit()` — `read_file`, `write_file`, `edit_file`, `glob`, `grep`, `bash`. +- `withPermissions(tools, policy, approver?)` — wrap a tool set in an approval + policy, with a per-session allow-always cache and a fail-closed default. +- `mcpToolsetFromServer(configJson)` — tools from an MCP server. +- `mergeToolSets(dest, src)` — combine tool sets. +- `describeToolSet(tools)` — introspect a set as `{ name, description, inputSchema }`. +- `MemoryStore`, `Batch`, `version()`. + +## Behaviour worth knowing before you file a bug + +- **An ambient `ANTHROPIC_AUTH_TOKEN` outranks an explicitly passed `apiKey`.** + The Anthropic provider offers two credentials and prefers the Bearer token, so + if your environment carries `ANTHROPIC_AUTH_TOKEN` — as it does for anyone + running Claude Code against an Anthropic-compatible gateway — + `createAnthropic({ apiKey })` authenticates with the ambient token instead. + Unset the variable, or pass `authToken` deliberately. +- **Passing the wrong kind of object to an entry point can abort the process + rather than throw.** Several wrapper types unwrap their arguments without + checking the unwrap succeeded. `describeToolSet()` is guarded; the rest are + not, as of 1.0.0. +- **`Session.send()` and `Agent.call()` are async.** They return a promise and + do not block the event loop for the turn. A turn that calls a tool has to hand + control back to the event loop that the tool's own JS runs on. + +## License + +MIT — see [LICENSE](LICENSE). diff --git a/bindings/node/binding.gyp b/bindings/node/binding.gyp index 8f94a63..ea4eafc 100644 --- a/bindings/node/binding.gyp +++ b/bindings/node/binding.gyp @@ -1,23 +1,56 @@ { + # gyp parses this file as a Python literal: comments are #, not //. + "variables": { + # Where libai_sdk is linked from. A source checkout uses the in-repo build. + # The prebuild script sets AI_SDK_LIBDIR to a static-OpenSSL build instead, + # so the shipped dylib has no Homebrew or distro path baked into it. + "ai_sdk_libdir": " and the C + # header `ai_sdk.h` -- it never mentions ai::Task or any other C++20-only + # core type, because it links libai_sdk and crosses the C ABI. So 17 costs + # nothing, and it is what lets `node-gyp rebuild` succeed on a user's + # machine with a toolchain whose C++20 support is partial (VS2019, older + # Xcode). Raising this to 20 would narrow the set of machines that can + # build the addon from source without buying anything. "cflags_cc": ["-std=c++17"], "sources": ["src/addon.cpp"], "include_dirs": [ "-/node.napi.node, which is what a published tarball +// ships. The same require therefore works in-repo and after npm install. +// +// The argument is the package root, not __dirname: this file is emitted to +// dist/, and both locations sit beside that directory rather than inside it. +// The addon's own rpath is @loader_path, so whatever directory it is found in +// must also hold libai_sdk. +const native = require('node-gyp-build')( + require('path').join(__dirname, '..') +) as NativeBinding; interface NativeBinding { Context: new () => NativeContext; @@ -29,9 +38,12 @@ interface NativeBinding { MemoryStore: new (dir: string) => NativeMemoryStore; Batch: new (provider: NativeProvider, modelId: string) => NativeBatch; standardToolkit(): NativeToolSet; - withPermissions(tools: NativeToolSet, policy: PermissionPolicy): NativeToolSet; + withPermissions(tools: NativeToolSet, policy: PermissionPolicy, approver?: Approver): NativeToolSet; + /** Present only on addons that accept the third argument above. */ + supportsApprover?: boolean; version(): string; mergeToolSets(dest: NativeToolSet, src: NativeToolSet): void; + describeToolSet(tools: NativeToolSet): string; mcpToolsetFromServer(ctx: NativeContext, configJson: string): NativeToolSet; } @@ -46,7 +58,7 @@ interface NativeAgentOptions { } interface NativeSession { - send(prompt: string): NativeResult; + send(prompt: string): Promise; sendStream(prompt: string, callback: StreamCallback): void; addUser(text: string): void; addAssistant(text: string): void; @@ -69,7 +81,7 @@ interface NativeToolSet { add(name: string, description: string, schemaJson: string, callback: ToolCallback): void; } interface NativeAgent { - call(prompt: string): NativeResult; + call(prompt: string): Promise; } type ToolCallback = (toolName: string, inputJson: string) => Promise<{ output: string; isError: boolean }> | { output: string; isError: boolean }; type StreamCallback = (type: string, text: string | null, toolName: string | null, toolCallId: string | null, usage: { inputTokens: number; outputTokens: number } | null) => void; @@ -211,7 +223,7 @@ export async function generateText(opts: GenerateTextOptions): Promise = new Set([ + 'text_delta', + 'tool_call_delta', + 'reasoning_delta', + 'tool_result', + 'error', +]); + export async function* streamText(opts: GenerateTextOptions): AsyncGenerator { const queue: StreamEvent[] = []; let resolveWait: (() => void) | null = null; @@ -269,7 +292,7 @@ export async function* streamText(opts: GenerateTextOptions): AsyncGenerator { const event: StreamEvent = { type: type as StreamEvent['type'] }; - if (text) event.text = text; + if (text || TEXT_EVENTS.has(event.type)) event.text = text ?? ''; if (toolName) event.toolName = toolName; if (toolCallId) event.toolCallId = toolCallId; if (usage) event.usage = usage; @@ -334,7 +357,7 @@ export class Agent { } async call(prompt: string): Promise { - const result = this._native.call(prompt); + const result = await this._native.call(prompt); return { text: result.text, finishReason: result.finishReason, @@ -358,6 +381,37 @@ export function mergeToolSets(dest: StandardToolSet, src: StandardToolSet): void native.mergeToolSets(dest, src); } +/** One tool, as a model is told about it when the set is passed to a call. */ +export interface ToolDescription { + name: string; + /** null when the tool was registered without a description. */ + description: string | null; + inputSchema: unknown; +} + +/** + * The tools in a set, ordered by name. + * + * The order is stable, so two descriptions can be compared — which is the + * point of having this at all: it is how you find out what an MCP server or a + * merged set actually exposes before handing it to an agent. + */ +export function describeToolSet(tools: StandardToolSet): ToolDescription[] { + // The C API speaks snake_case; everything this package hands back is + // camelCase (see the usage object on a stream event), so map it at the + // boundary rather than leaking the C spelling into the public types. + const raw = JSON.parse(native.describeToolSet(tools)) as Array<{ + name: string; + description: string | null; + input_schema: unknown; + }>; + return raw.map((t) => ({ + name: t.name, + description: t.description, + inputSchema: t.input_schema, + })); +} + /** Connect to an MCP server and return its tools as a ToolSet. */ export function mcpToolsetFromServer(configJson: string): StandardToolSet { return native.mcpToolsetFromServer(getCtx(), configJson); @@ -365,15 +419,77 @@ export function mcpToolsetFromServer(configJson: string): StandardToolSet { // --- Standard toolkit + permissions + session --- -export type PermissionPolicy = (tool: string, inputJson: string) => number; +/** Mirrors the C enum: what a policy or approver answers with. */ +export const PermissionDecision = { + Allow: 0, + Deny: 1, + /** Policy only: escalate to the approver (without one this fails closed). */ + Ask: 2, + /** Approver only: allow now and stop asking for this tool this session. */ + AllowAlways: 3, +} as const; +export type PermissionDecision = (typeof PermissionDecision)[keyof typeof PermissionDecision]; + +/** A decision plus the sentence to show the model when refusing a call. + * Returning a bare number means "no explanation". */ +export interface PermissionVerdict { + decision: number; + /** Shown to the model as part of the tool result. Say what would have to + * change — a model told only that a call was refused retries variants of + * it, and every retry is another prompt the user already answered. */ + reason?: string; +} + +/** Synchronous rule: answer Allow/Deny/Ask with no I/O. */ +export type PermissionPolicy = ( + tool: string, + inputJson: string, +) => number | PermissionVerdict; + +/** Interactive approver, called only when the policy answers `Ask`. It is + * expected to be async — the usual implementation awaits a UI prompt — and + * its verdict is what allows or refuses the call. `rationale` is a + * human-readable line from the engine, suitable for showing to the user. + * + * Anything that is not one of the `PermissionDecision` values (or a rejected + * promise) refuses the call: an approver is a permission gate, so a missing + * `return` must not read as approval. + * + * Caveat: a prompt can only be awaited from the streaming/async entry points + * (`Session.sendStream`, `streamText`, and the promise-returning `call`s), + * which run the loop off the JS thread. A blocking main-thread call — the C + * API's `ai_agent_call`, reached from `generateText`/`Agent.call` on the + * synchronous path — cannot park for a prompt, so a promise-returning + * approver there refuses the call and its late verdict is discarded. */ +export type Approver = ( + tool: string, + inputJson: string, + rationale: string, +) => number | PermissionVerdict | Promise; + export type StandardToolSet = NativeToolSet; export function standardToolkit(): StandardToolSet { return native.standardToolkit(); } -export function withPermissions(tools: StandardToolSet, policy: PermissionPolicy): StandardToolSet { - return native.withPermissions(tools, policy); +/** True when this addon understands the third `withPermissions` argument. + * An older addon would ignore it and silently fail closed on every `Ask`, + * so callers that need a prompt should check rather than assume. */ +export const supportsApprover: boolean = native.supportsApprover === true; + +export function withPermissions( + tools: StandardToolSet, + policy: PermissionPolicy, + approver?: Approver, +): StandardToolSet { + if (approver && !supportsApprover) { + throw new Error( + 'ai-sdk-cpp: this native addon predates interactive approvals (rebuild it), ' + + 'so an `Ask` could not reach your approver', + ); + } + return native.withPermissions(tools, policy, approver); } export interface SessionOptions { @@ -405,14 +521,13 @@ export class Session { } } - send(prompt: string): GenerateResult { - const result = this._native.send(prompt); - return { + send(prompt: string): Promise { + return this._native.send(prompt).then((result) => ({ text: result.text, finishReason: result.finishReason, usage: { inputTokens: result.inputTokens, outputTokens: result.outputTokens }, steps: result.steps, - }; + })); } addUser(text: string): void { @@ -443,7 +558,7 @@ export class Session { this._native.sendStream(prompt, (type, text, toolName, toolCallId, usage) => { const ev: StreamEvent = { type: type as StreamEvent['type'] }; - if (text) ev.text = text; + if (text || TEXT_EVENTS.has(ev.type)) ev.text = text ?? ''; if (toolName) ev.toolName = toolName; if (toolCallId) ev.toolCallId = toolCallId; if (usage) ev.usage = usage; diff --git a/bindings/node/package-lock.json b/bindings/node/package-lock.json index 9bccdb7..b226e64 100644 --- a/bindings/node/package-lock.json +++ b/bindings/node/package-lock.json @@ -1,13 +1,14 @@ { "name": "ai-sdk-cpp", - "version": "0.1.0", + "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ai-sdk-cpp", - "version": "0.1.0", - "license": "Apache-2.0", + "version": "1.0.0", + "hasInstallScript": true, + "license": "MIT", "dependencies": { "node-addon-api": "^7.0.0", "node-gyp-build": "^4.8.0" diff --git a/bindings/node/package.json b/bindings/node/package.json index d3b1bc1..cf52a01 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,17 +1,22 @@ { "name": "ai-sdk-cpp", - "version": "0.1.0", + "version": "1.0.0", "description": "Native C++ AI agent framework with Node.js bindings", "main": "dist/index.js", "types": "dist/index.d.ts", "scripts": { "build": "node-gyp rebuild && tsc", - "prepublishOnly": "npm run build" + "build:prebuild": "scripts/make-prebuild.sh", + "install": "node scripts/install.cjs", + "test": "npm run build && node --test test/*.test.cjs", + "test:only": "node --test test/*.test.cjs", + "test:live": "node --test test/live/*.test.cjs", + "prepublishOnly": "node scripts/check-prebuilds.cjs && tsc" }, "files": [ "dist/", - "binding.gyp", - "src/" + "prebuilds/", + "scripts/install.cjs" ], "dependencies": { "node-addon-api": "^7.0.0", @@ -25,5 +30,23 @@ "engines": { "node": ">=18.0.0" }, + "repository": { + "type": "git", + "url": "git+https://github.com/mhdimo/ai-sdk-cpp.git", + "directory": "bindings/node" + }, + "homepage": "https://github.com/mhdimo/ai-sdk-cpp#readme", + "bugs": { + "url": "https://github.com/mhdimo/ai-sdk-cpp/issues" + }, + "keywords": [ + "ai", + "llm", + "agent", + "anthropic", + "openai", + "tool-calling", + "napi" + ], "license": "MIT" } diff --git a/bindings/node/scripts/check-prebuilds.cjs b/bindings/node/scripts/check-prebuilds.cjs new file mode 100644 index 0000000..fe5bea6 --- /dev/null +++ b/bindings/node/scripts/check-prebuilds.cjs @@ -0,0 +1,101 @@ +// Refuses to publish a package nobody could install. +// +// The failure this guards against is silent and total. `npm publish` is happy +// with no prebuilds/ directory, and every user then gets "No native build was +// found" from node-gyp-build. Nothing upstream catches it either, because the +// publisher's own machine has build/Release from `npm run build` and loads +// fine -- the package only breaks on machines that are not this one, which is +// all of them. +// +// Run by prepublishOnly. CI assembles prebuilds/ from the per-platform +// artifacts before publishing; this checks that assembly actually happened. + +const fs = require('node:fs'); +const path = require('node:path'); + +const DIR = path.join(__dirname, '..', 'prebuilds'); + +// The platforms this release means to support, checked as a set. +// +// Checking only the directories that happen to be present answers a different +// question -- "is what arrived complete?" -- and answers it yes for a tarball +// that arrived with one platform in it. Such a tarball installs perfectly for +// whoever built it, and for everyone else `npm install` exits 1: install.cjs +// finds no matching prebuild and has no source fallback to fall back to. That +// is the failure this script's own header describes, and it does not catch it. +// +// Adding a platform here is a commitment: the matrix has to build it, and this +// will refuse to publish until it does. Windows has no recipe yet. +const SUPPORTED = ['darwin-arm64', 'darwin-x64', 'linux-arm64', 'linux-x64']; + +const SHARED_LIB = { + darwin: 'libai_sdk.dylib', + linux: 'libai_sdk.so', + win32: 'ai_sdk.dll', +}; + +/** The addon and its shared library, both of which an install needs. */ +function expectedFiles(tuple) { + const platform = tuple.split('-')[0]; + const lib = SHARED_LIB[platform]; + if (!lib) return null; // a platform we have no recipe for + return ['node.napi.node', lib]; +} + +if (!fs.existsSync(DIR)) { + console.error( + 'refusing to publish: no prebuilds/ directory.\n' + + 'Run `npm run build:prebuild` for this platform, or publish from CI,\n' + + 'which builds every platform and assembles them here.' + ); + process.exit(1); +} + +const tuples = fs + .readdirSync(DIR, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => e.name); + +if (tuples.length === 0) { + console.error('refusing to publish: prebuilds/ is empty.'); + process.exit(1); +} + +let failed = false; +for (const tuple of tuples) { + const wanted = expectedFiles(tuple); + if (!wanted) { + console.error(` ${tuple}: unrecognised platform, cannot check`); + failed = true; + continue; + } + const missing = wanted.filter((f) => !fs.existsSync(path.join(DIR, tuple, f))); + if (missing.length > 0) { + console.error(` ${tuple}: missing ${missing.join(', ')}`); + failed = true; + } else { + console.log(` ${tuple}: ok`); + } +} + +// Completeness of what arrived, then whether what arrived is the whole set. +// Separate checks because they fail for different reasons -- an upload that was +// cut short, versus a matrix leg that never ran -- and the second is the one +// that decides whether the package is installable. +const absent = SUPPORTED.filter((t) => !tuples.includes(t)); +if (absent.length > 0) { + console.error( + `refusing to publish: no prebuild for ${absent.join(', ')}.\n` + + 'Those platforms are claimed by this release, and a package without them\n' + + 'fails at install time there -- see the matrix in prebuilds.yml. If a\n' + + 'platform is being dropped, remove it from SUPPORTED deliberately.' + ); + failed = true; +} + +if (failed) { + console.error('refusing to publish: a prebuild is missing or incomplete (see above).'); + process.exit(1); +} + +console.log(`prebuilds ok: ${tuples.join(', ')}`); diff --git a/bindings/node/scripts/install.cjs b/bindings/node/scripts/install.cjs new file mode 100644 index 0000000..d2393dd --- /dev/null +++ b/bindings/node/scripts/install.cjs @@ -0,0 +1,101 @@ +// Install step for the Node binding. +// +// npm runs `node-gyp rebuild` by default for any package containing a +// binding.gyp, and that default is wrong for this one. The addon links +// libai_sdk, which exists only in a source checkout that has already built the +// C++ library -- so a user installing from npm would watch a compiler fail on +// a file they do not have. Shipping prebuilds is the whole point of the +// package, and declaring an install script is the only way to stop npm from +// compiling instead. +// +// Three cases: +// +// 1. A source checkout. Build the addon if the C++ library is already there, +// which is what npm used to do by default; if it is not, say so and +// succeed anyway, because failing `npm install` on a fresh clone would +// block a contributor before they have read the README. +// +// This is checked before the prebuild, deliberately. A checkout can have +// both -- make-prebuild.sh leaves prebuilds/ behind -- and node-gyp-build +// prefers build/Release at require time. Deciding on the prebuild here +// would leave a developer who is editing addon.cpp running a stale +// binary from their last release build, with nothing to say so. +// +// 2. A prebuild matches this platform. Nothing to do -- every normal +// install. +// +// 3. Neither. Fail loudly and name the platforms that do have prebuilds, +// rather than letting the require() fail later with node-gyp-build's +// platform list and no hint about what to do. + +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +const ROOT = path.join(__dirname, '..'); +const REPO = path.join(ROOT, '..', '..'); + +const tuple = `${process.platform}-${process.arch}`; +const prebuild = path.join(ROOT, 'prebuilds', tuple, 'node.napi.node'); + +/** The C++ library the addon links, in either its shared or static form. */ +function findBuiltLibrary() { + const dir = path.join(REPO, 'build', 'bindings', 'c'); + if (!fs.existsSync(dir)) return null; + const wanted = ['libai_sdk.dylib', 'libai_sdk.so', 'ai_sdk.dll', 'libai_sdk.a']; + for (const name of wanted) { + const full = path.join(dir, name); + if (fs.existsSync(full)) return full; + } + return null; +} + +/** Platform tuples this package actually carries a prebuild for. */ +function availablePlatforms() { + const dir = path.join(ROOT, 'prebuilds'); + if (!fs.existsSync(dir)) return []; + return fs + .readdirSync(dir, { withFileTypes: true }) + .filter((e) => e.isDirectory()) + .map((e) => e.name); +} + +const inSourceCheckout = + fs.existsSync(path.join(ROOT, 'binding.gyp')) && + fs.existsSync(path.join(REPO, 'CMakeLists.txt')); + +if (inSourceCheckout) { + // Match what npm would have done on its own, but only when it can succeed. + const lib = findBuiltLibrary(); + if (!lib) { + console.log( + 'ai-sdk-cpp: the C++ library is not built yet, so the native addon was skipped.\n' + + ' Build it, then the addon:\n' + + ' cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j\n' + + ' (cd bindings/node && npm run build)' + ); + process.exit(0); + } + + const gyp = spawnSync( + process.execPath, + [require.resolve('node-gyp/bin/node-gyp.js'), 'rebuild'], + { cwd: ROOT, stdio: 'inherit' } + ); + process.exit(gyp.status ?? 1); +} + +if (fs.existsSync(prebuild)) { + process.exit(0); +} + +const have = availablePlatforms(); +console.error( + `\nai-sdk-cpp has no prebuilt binary for ${tuple}.\n\n` + + (have.length + ? `Prebuilds are available for: ${have.join(', ')}\n\n` + : 'This package was published without any prebuilds, which is a packaging bug.\n\n') + + 'To build it for this platform, install from source:\n' + + ' git clone https://github.com/mhdimo/ai-sdk-cpp && cd ai-sdk-cpp/bindings/node\n' +); +process.exit(1); diff --git a/bindings/node/scripts/make-prebuild.sh b/bindings/node/scripts/make-prebuild.sh new file mode 100755 index 0000000..90cb79f --- /dev/null +++ b/bindings/node/scripts/make-prebuild.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +# +# Build a relocatable Node prebuild for the current platform into +# bindings/node/prebuilds/-/. +# +# Run from anywhere; the repo is found relative to this script. On CI this is +# the whole per-platform story: the workflow runs it once per runner and +# uploads the directory. +# +# Two properties make the result relocatable, and both are load-bearing: +# +# 1. libai_sdk is linked against a *static* OpenSSL. A dynamic one would +# either record the build machine's Homebrew or distro path -- unloadable +# on anyone else's machine -- or, if bundled, put a second libssl into a +# Node process that already has its own. Node links OpenSSL statically, +# so a bundled dynamic copy is a well known source of symbol conflicts. +# +# 2. The addon is linked with -Wl,-rpath,@loader_path, so the loader looks +# for the shared library in the same directory as the .node file, wherever +# the package manager happened to put it. +# +# The output is one self-contained directory: node.napi.node with the shared +# library beside it. node-gyp-build finds the first by platform tuple; the +# dynamic loader finds the second by rpath. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +NODE_DIR="$ROOT/bindings/node" +BUILD_DIR="${AI_SDK_PREBUILD_BUILD_DIR:-$ROOT/build-prebuild}" + +# process.stdout.write everywhere rather than `node -p`: -p formats through +# util.inspect, which quotes strings and may colourise when stdout is a tty. +# Either would corrupt the tuple and the -j argument. +PLATFORM="$(node -e 'process.stdout.write(process.platform)')" +ARCH="$(node -e 'process.stdout.write(process.arch)')" +TUPLE="$PLATFORM-$ARCH" +JOBS="$(node -e 'process.stdout.write(String(require("os").cpus().length))')" + +# Must match MACOSX_DEPLOYMENT_TARGET in binding.gyp. If they differ, the dylib +# records the build machine's macOS as its minimum and the prebuild advertises +# a floor it cannot honour -- it links fine here and fails on the user's older +# machine, which is the one place the failure is expensive. +MIN_MACOS="12.0" + +case "$PLATFORM" in + darwin) LIBNAME="libai_sdk.dylib" ;; + linux) LIBNAME="libai_sdk.so" ;; + *) + echo "make-prebuild: no prebuild recipe for platform '$PLATFORM'" >&2 + exit 1 + ;; +esac + +# Setting AI_SDK_LIBDIR is what tells binding.gyp this is a prebuild build, and +# that is the only thing it decides: with it set the addon records just the +# loader-relative rpath, and without it the in-repo build tree is added ahead of +# it for a source checkout. Nothing else needs to be passed down. + +echo "==> $TUPLE: building libai_sdk with a static OpenSSL" +cmake -S "$ROOT" -B "$BUILD_DIR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DOPENSSL_USE_STATIC_LIBS=TRUE \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$MIN_MACOS" \ + -DAI_SDK_BUILD_TESTS=OFF \ + -DAI_SDK_BUILD_EXAMPLES=OFF \ + -DAI_SDK_BUILD_CLI=OFF \ + -DAI_SDK_BUILD_PYTHON=OFF \ + >/dev/null +cmake --build "$BUILD_DIR" --target ai-sdk-c -j"$JOBS" >/dev/null + +LIBDIR="$BUILD_DIR/bindings/c" +[ -f "$LIBDIR/$LIBNAME" ] || { echo "make-prebuild: $LIBDIR/$LIBNAME missing" >&2; exit 1; } + +echo "==> $TUPLE: building the addon against it" +(cd "$NODE_DIR" && AI_SDK_LIBDIR="$LIBDIR" npx node-gyp rebuild >/dev/null) + +echo "==> $TUPLE: compiling the JavaScript" +(cd "$NODE_DIR" && npx tsc) + +OUT="$NODE_DIR/prebuilds/$TUPLE" +rm -rf "$OUT" +mkdir -p "$OUT" +cp "$NODE_DIR/build/Release/ai_sdk_native.node" "$OUT/node.napi.node" +cp "$LIBDIR/$LIBNAME" "$OUT/$LIBNAME" + +# A prebuild that depends on anything outside the platform's system libraries +# is broken for everyone but the machine that built it -- which is exactly how +# the Homebrew OpenSSL path went unnoticed until the package was packed. +echo "==> $TUPLE: checking the result is self-contained" +if [ "$PLATFORM" = "darwin" ]; then + deps="$(otool -L "$OUT/node.napi.node" "$OUT/$LIBNAME" | awk '/^\t/ {print $1}')" +else + # An unresolvable dependency reads `libai_sdk.so => not found`, which has no + # path in it at all. Matched as though it were one, "not" lands in the + # whitelist check below and comes back as `NOT RELOCATABLE: not` -- a message + # that names neither the library nor the problem. + ldd_out="$(ldd "$OUT/node.napi.node" "$OUT/$LIBNAME" 2>&1)" + missing="$(printf '%s\n' "$ldd_out" | awk '/=>[[:space:]]+not found/ {print $1}')" + if [ -n "$missing" ]; then + echo "make-prebuild: these dependencies cannot be resolved, so the prebuild" >&2 + echo " would not load on any machine but the one that built it:" >&2 + printf ' %s\n' $missing >&2 + exit 1 + fi + deps="$(printf '%s\n' "$ldd_out" | awk '/=>/ {print $3}')" +fi +bad=0 +while IFS= read -r dep; do + [ -n "$dep" ] || continue + case "$dep" in + # A dependency that resolved to a file inside the package. That is $ORIGIN + # doing its job, and it is what self-contained means rather than a + # violation of it: ldd reports the path it *resolved*, so a correctly + # rpath'd libai_sdk.so arrives here as an absolute path in $OUT. Without + # this arm the check rejects a working Linux prebuild while passing the + # macOS one, because otool -L prints the install name (@rpath/...) rather + # than a resolved path -- so the same defect hides on the platform where + # the check would have been read. + "$OUT"/*) ;; + # The system library directories as each platform's toolchain actually + # spells them. Linux is the other trap: Ubuntu 22.04 prints system + # libraries as /lib//... , not /usr/lib/... . A whitelist written + # from memory rejects every library on the machine, so this check fails on + # a prebuild that is perfectly fine -- and, worse, it fails the same way on + # one that is not, which is how it goes unread. + /usr/lib/*|/lib/*|/lib64/*|/System/*|@rpath/*|@loader_path/*) ;; + *) echo " NOT RELOCATABLE: $dep" >&2; bad=1 ;; + esac +done <<< "$deps" +if [ "$bad" -ne 0 ]; then + echo "make-prebuild: the prebuild depends on a path outside the package." >&2 + echo " Link that dependency statically instead." >&2 + exit 1 +fi + +# The rpath is asserted by its recorded value rather than inferred from the load +# test further down, because the load test cannot see this class of defect. A +# malformed entry does not fail to load -- it fails to *match*, and the loader +# moves on to the next entry. So an addon whose only relocatable rpath is inert +# still loads here, where the library is where the build put it, and breaks on a +# user's machine, which is the one place the check cannot run. +if [ "$PLATFORM" = "linux" ]; then + runpath="$(objdump -p "$OUT/node.napi.node" | awk '/RUNPATH|RPATH/ {print $2}')" + case "$runpath" in + *'$ORIGIN'*) echo " rpath: $runpath" ;; + *) + echo "make-prebuild: the addon's rpath is '${runpath:-}', with no" >&2 + echo " \$ORIGIN entry, so it will not find $LIBNAME beside it" >&2 + echo " on a user's machine. See the OS=='linux' condition in" >&2 + echo " binding.gyp." >&2 + exit 1 + ;; + esac +fi + +# Both binaries must record the same minimum macOS, or the prebuild advertises +# a floor it cannot honour: it links and loads here, then fails on the older +# machine. Nothing else in this script can catch that, because the build host +# is always new enough to run it. +if [ "$PLATFORM" = "darwin" ]; then + for f in "$OUT/node.napi.node" "$OUT/$LIBNAME"; do + minos="$(otool -l "$f" | awk '/LC_BUILD_VERSION/{found=1} found && /minos/{print $2; exit}')" + [ -n "$minos" ] || { echo "make-prebuild: no LC_BUILD_VERSION in $f" >&2; exit 1; } + if awk -v a="$minos" -v b="$MIN_MACOS" 'BEGIN{ + split(a, x, "."); split(b, y, ".") + exit !(x[1]+0 > y[1]+0 || (x[1]+0 == y[1]+0 && x[2]+0 > y[2]+0)) + }'; then + echo " $(basename "$f") requires macOS $minos, above the $MIN_MACOS target" >&2 + exit 1 + fi + echo " $(basename "$f") minos $minos" + done +fi + +# Two runtimes version their symbols here, and the one that actually decides +# whether the artifact starts is not the one named after the C library. +# `GLIBC_` and `GLIBCXX_` are unrelated namespaces -- a pattern for the first +# cannot match the second, whatever it is anchored to -- so reporting only the +# glibc floor answers a question nobody asked while the real bar goes unsaid. +# libstdc++ also moves faster than glibc does, by years, so an artifact can +# clear a comfortable glibc floor and still refuse to start. +# +# Stating both is all this does: the floor is set by the build image, not by +# this script, which is why the workflow matrix is named in the log. +if [ "$PLATFORM" = "linux" ]; then + # The `|| true` is load-bearing on each of these. An absent namespace is a + # *result*, not a failure -- and the one that goes absent is the one that says + # the artifact is portable. A statically linked C++ runtime leaves no GLIBCXX_ + # references at all, so grep finds nothing, exits 1, pipefail carries that + # through the pipeline, and `set -e` stops the script at the exact moment it + # was about to report the good news. The run then ends in failure with no + # message, which reads as a broken build rather than a portable one. + syms="$( { objdump -T "$OUT/$LIBNAME"; objdump -T "$OUT/node.napi.node"; } 2>/dev/null || true )" + glibc_floor="$(printf '%s\n' "$syms" | grep -o 'GLIBC_[0-9.]*' | sort -u -V | tail -1 || true)" + cxx_floor="$(printf '%s\n' "$syms" | grep -o 'GLIBCXX_[0-9.]*' | sort -u -V | tail -1 || true)" + echo " glibc floor: ${glibc_floor:-unknown}" + echo " libstdc++ floor: ${cxx_floor:-unknown}" + echo " (both set by the build image -- see the workflow matrix)" +fi + +# Load the prebuild the way a user's install would, with PREBUILDS_ONLY so the +# in-repo build/Release is not what answers. This is the only check that +# actually exercises the shipped layout: node-gyp-build's tuple lookup, the +# rpath, and the shared library all have to line up for it to pass. +echo "==> $TUPLE: loading it the way an install would" +(cd "$NODE_DIR" && PREBUILDS_ONLY=1 node -e " + const ai = require('.'); + console.log(' loaded, version ' + ai.version()); + // The addon holds a Context open, so exit explicitly rather than waiting + // for a loop that will never drain. + process.exit(0); +") + +echo "==> $TUPLE: wrote $OUT" +ls -la "$OUT" + +# The addon was built in prebuild mode, which is to say linked with only +# @loader_path -- correct for the shipped directory, wrong for build/Release, +# where the dylib is not beside the addon but in build/bindings/c. That build +# overwrote whatever was there, so put the source-tree addon back. Skipped when +# there is no in-repo library to link against, which is the case on a CI runner +# that only builds prebuilds. +DEV_LIBDIR="$ROOT/build/bindings/c" +if [ -d "$DEV_LIBDIR" ]; then + echo "==> $TUPLE: restoring the source-tree addon" + (cd "$NODE_DIR" && npx node-gyp rebuild >/dev/null) +else + echo "==> $TUPLE: no $DEV_LIBDIR, leaving build/Release as the prebuild" +fi diff --git a/bindings/node/src/addon.cpp b/bindings/node/src/addon.cpp index a5a7e4a..6f1cfae 100644 --- a/bindings/node/src/addon.cpp +++ b/bindings/node/src/addon.cpp @@ -1,9 +1,11 @@ #include #include "ai_sdk.h" +#include #include #include #include #include +#include #include #include #include @@ -182,12 +184,27 @@ struct ToolCallPayload { std::string input; }; +// The C layer copies output_json into its own storage before the callback +// returns and keeps no reference to it past that point (see the ownership note +// on ai_tool_result_t in bindings/c/ai_sdk.h). So the buffer has to outlive the +// return statement, and only that: a thread_local is exactly that lifetime, +// where a local would be a dangling read and a fresh allocation per call leaks +// one buffer per tool call for the life of the process. +// +// A long-running agent is the case that makes this worth stating -- the leak is +// proportional to tool output, so a TUI session that runs for a day accumulates +// every byte it ever produced, and nothing ever reports it. +// +// Reuse is safe because nothing can interleave between this returning and the C +// layer copying: the copy is straight-line code in the same call frame with no +// co_await in between, so a second tool call cannot reach this on the same +// thread inside that window even when calls run concurrently. +static thread_local std::string g_tool_result_buffer; + static ai_tool_result_t make_tool_result(const std::string& output, bool is_error) { ai_tool_result_t res = {}; - char* copy = new char[output.size() + 1]; - std::copy(output.begin(), output.end(), copy); - copy[output.size()] = '\0'; - res.output_json = copy; + g_tool_result_buffer = output; + res.output_json = g_tool_result_buffer.c_str(); res.is_error = is_error ? 1 : 0; return res; } @@ -347,6 +364,84 @@ void ToolSetWrapper::AddTool(const Napi::CallbackInfo& info) { tool_callbacks_.push_back(cb_data); } +// --- Async blocking calls: worker thread + Promise --- +// +// generateText / agent.call / session.send run their C call on a worker thread +// and return a Promise. The C call drives an io_context to completion, so it +// occupies its thread for the whole turn — and a tool callback is a JS function +// that can only run on the JS thread. Blocking the JS thread therefore +// deadlocks as soon as a model returns a tool call: the tool waits for the JS +// thread, which is waiting for the call. The streaming entry points already +// avoid this by running on a worker thread; these now match. +// +// The C result's char* fields die with it, so the worker copies them out and +// the JS thread turns them into an object. + +struct BlockingCallResult { + explicit BlockingCallResult(napi_env env) : deferred(env) {} + + Napi::Promise::Deferred deferred; + bool ok = false; + std::string error; + std::string text; + std::string finishReason; + int inputTokens = 0; + int outputTokens = 0; + int steps = 0; + + /** Copy out a finished C call and free it (every path frees exactly once). */ + void take(ai_status_t status, ai_generate_result_t& r, const char* fallback) { + ok = (status == AI_OK); + if (ok) { + text = r.text ? r.text : ""; + finishReason = r.finish_reason ? r.finish_reason : "stop"; + inputTokens = r.input_tokens; + outputTokens = r.output_tokens; + steps = r.steps; + } else { + error = r.text ? r.text : fallback; + } + ai_generate_result_free(&r); + } +}; + +// Settles the promise on the JS thread once the worker is done. +static void ResolveBlockingCall(Napi::Env env, Napi::Function, BlockingCallResult* p) { + if (!p->ok) { + p->deferred.Reject(Napi::Error::New(env, p->error).Value()); + } else { + Napi::Object obj = Napi::Object::New(env); + obj.Set("text", Napi::String::New(env, p->text)); + obj.Set("finishReason", Napi::String::New(env, p->finishReason)); + obj.Set("inputTokens", Napi::Number::New(env, p->inputTokens)); + obj.Set("outputTokens", Napi::Number::New(env, p->outputTokens)); + obj.Set("steps", Napi::Number::New(env, p->steps)); + p->deferred.Resolve(obj); + } + delete p; +} + +// Run `work` on a worker thread and settle the returned promise on the JS +// thread. `work` receives the result slot to fill and must not touch Napi +// values off the JS thread. +template +static Napi::Promise RunBlockingAsync(Napi::Env env, F work) { + auto* p = new BlockingCallResult(env); + + Napi::ThreadSafeFunction tsfn = Napi::ThreadSafeFunction::New( + env, + Napi::Function::New(env, [](const Napi::CallbackInfo& i) { return i.Env().Undefined(); }), + "ai-blocking", 0, 1); + + std::thread([p, tsfn, work = std::move(work)]() mutable { + work(*p); + tsfn.NonBlockingCall(p, ResolveBlockingCall); + tsfn.Release(); + }).detach(); + + return p->deferred.Promise(); +} + // --- AgentWrapper --- Napi::Object AgentWrapper::Init(Napi::Env env, Napi::Object exports) { @@ -415,25 +510,13 @@ Napi::Value AgentWrapper::Call(const Napi::CallbackInfo& info) { } std::string prompt = info[0].As().Utf8Value(); - ai_generate_result_t result = {}; - - ai_status_t status = ai_agent_call(agent_, prompt.c_str(), &result); - if (status != AI_OK) { - std::string msg = result.text ? result.text : "Agent call failed"; - ai_generate_result_free(&result); - Napi::Error::New(env, msg).ThrowAsJavaScriptException(); - return env.Undefined(); - } + ai_agent_t agent = agent_; - Napi::Object obj = Napi::Object::New(env); - obj.Set("text", Napi::String::New(env, result.text ? result.text : "")); - obj.Set("finishReason", Napi::String::New(env, result.finish_reason ? result.finish_reason : "stop")); - obj.Set("inputTokens", Napi::Number::New(env, result.input_tokens)); - obj.Set("outputTokens", Napi::Number::New(env, result.output_tokens)); - obj.Set("steps", Napi::Number::New(env, result.steps)); - - ai_generate_result_free(&result); - return obj; + return RunBlockingAsync(env, [agent, prompt](BlockingCallResult& out) { + ai_generate_result_t result = {}; + ai_status_t status = ai_agent_call(agent, prompt.c_str(), &result); + out.take(status, result, "Agent call failed"); + }); } // --- Async streaming: background thread + ThreadSafeFunction --- @@ -509,19 +592,27 @@ static ai_stream_callback_fn stream_callback = [](ai_stream_event_t event, void* }; // Run a blocking C stream call on a worker thread; return immediately. `cCall` -// is invoked as cCall(StreamSession* s, ai_stream_callback_fn cb) and must drive -// the stream to completion. After it returns, the TSFN is released. +// is invoked as cCall(StreamSession* s, ai_stream_callback_fn cb), must drive +// the stream to completion, and returns the call's status. After it returns, +// the TSFN is released. template static void RunStreamAsync(Napi::Env env, Napi::Function callback, F cCall) { auto* s = new StreamSession(); s->tsfn = Napi::ThreadSafeFunction::New(env, callback, "ai-stream", 0, 1); std::thread([s, cCall = std::move(cCall)]() mutable { - cCall(s, stream_callback); + ai_status_t status = cCall(s, stream_callback); if (!s->terminal) { - // The C call returned without a finish/error event (e.g. it failed - // before streaming) — synthesize one so the JS consumer terminates. + // The call ended without a finish or error event. That only happens + // when it bailed out early, so the turn failed — and emitting a + // finish here would tell the caller a turn succeeded when no answer + // was ever produced. Fail closed. ai_stream_event_t ev = {}; - ev.type = AI_STREAM_FINISH; + if (status == AI_OK) { + ev.type = AI_STREAM_FINISH; + } else { + ev.type = AI_STREAM_ERROR; + ev.text = ai_status_message(status); + } emit_stream_event(*s, ev); } s->tsfn.Release(); @@ -531,6 +622,14 @@ static void RunStreamAsync(Napi::Env env, Napi::Function callback, F cCall) { // --- Free functions --- +// Owns the strings ai_generate_options_t points into, so they stay alive for +// the whole worker-thread call. Heap-allocated and never copied, so the +// char* pointers taken here stay valid. +struct GenerateRequest { + std::string prompt, system, messages, providerOptions; + ai_generate_options_t opts = {}; +}; + Napi::Value GenerateText(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); @@ -543,65 +642,51 @@ Napi::Value GenerateText(const Napi::CallbackInfo& info) { auto* model_wrap = Napi::ObjectWrap::Unwrap(model_obj); auto opts_obj = info[1].As(); - ai_generate_options_t opts = {}; - opts.model = model_wrap->handle(); - opts.temperature = -1; - - std::string prompt_str, system_str, messages_str, provider_options_str; + auto req = std::make_unique(); + req->opts.model = model_wrap->handle(); + req->opts.temperature = -1; if (opts_obj.Has("prompt") && !opts_obj.Get("prompt").IsUndefined()) { - prompt_str = opts_obj.Get("prompt").As().Utf8Value(); - opts.prompt = prompt_str.c_str(); + req->prompt = opts_obj.Get("prompt").As().Utf8Value(); + req->opts.prompt = req->prompt.c_str(); } if (opts_obj.Has("system") && !opts_obj.Get("system").IsUndefined()) { - system_str = opts_obj.Get("system").As().Utf8Value(); - opts.system = system_str.c_str(); + req->system = opts_obj.Get("system").As().Utf8Value(); + req->opts.system = req->system.c_str(); } if (opts_obj.Has("messagesJson") && !opts_obj.Get("messagesJson").IsUndefined()) { - messages_str = opts_obj.Get("messagesJson").As().Utf8Value(); - opts.messages_json = messages_str.c_str(); + req->messages = opts_obj.Get("messagesJson").As().Utf8Value(); + req->opts.messages_json = req->messages.c_str(); } if (opts_obj.Has("providerOptions") && !opts_obj.Get("providerOptions").IsUndefined()) { auto json = env.Global().Get("JSON").As(); auto stringify = json.Get("stringify").As(); - provider_options_str = stringify.Call(json, {opts_obj.Get("providerOptions")}) + req->providerOptions = stringify.Call(json, {opts_obj.Get("providerOptions")}) .As().Utf8Value(); - opts.provider_options_json = provider_options_str.c_str(); + req->opts.provider_options_json = req->providerOptions.c_str(); } if (opts_obj.Has("maxSteps") && !opts_obj.Get("maxSteps").IsUndefined()) { - opts.max_steps = opts_obj.Get("maxSteps").As().Int32Value(); + req->opts.max_steps = opts_obj.Get("maxSteps").As().Int32Value(); } if (opts_obj.Has("maxOutputTokens") && !opts_obj.Get("maxOutputTokens").IsUndefined()) { - opts.max_output_tokens = opts_obj.Get("maxOutputTokens").As().Int32Value(); + req->opts.max_output_tokens = opts_obj.Get("maxOutputTokens").As().Int32Value(); } if (opts_obj.Has("temperature") && !opts_obj.Get("temperature").IsUndefined()) { - opts.temperature = opts_obj.Get("temperature").As().DoubleValue(); + req->opts.temperature = opts_obj.Get("temperature").As().DoubleValue(); } if (opts_obj.Has("toolSet") && !opts_obj.Get("toolSet").IsUndefined()) { auto ts_obj = opts_obj.Get("toolSet").As(); auto* ts_wrap = Napi::ObjectWrap::Unwrap(ts_obj); - opts.tools = ts_wrap->handle(); - } - - ai_generate_result_t result = {}; - ai_status_t status = ai_generate_text(opts, &result); - - if (status != AI_OK) { - std::string msg = result.text ? result.text : ai_status_message(status); - ai_generate_result_free(&result); - Napi::Error::New(env, msg).ThrowAsJavaScriptException(); - return env.Undefined(); + req->opts.tools = ts_wrap->handle(); } - Napi::Object obj = Napi::Object::New(env); - obj.Set("text", Napi::String::New(env, result.text ? result.text : "")); - obj.Set("finishReason", Napi::String::New(env, result.finish_reason ? result.finish_reason : "stop")); - obj.Set("inputTokens", Napi::Number::New(env, result.input_tokens)); - obj.Set("outputTokens", Napi::Number::New(env, result.output_tokens)); - obj.Set("steps", Napi::Number::New(env, result.steps)); - - ai_generate_result_free(&result); - return obj; + auto* raw = req.release(); + return RunBlockingAsync(env, [raw](BlockingCallResult& out) { + std::unique_ptr req(raw); + ai_generate_result_t result = {}; + ai_status_t status = ai_generate_text(req->opts, &result); + out.take(status, result, ai_status_message(status)); + }); } Napi::Value StreamText(const Napi::CallbackInfo& info) { @@ -669,7 +754,7 @@ Napi::Value StreamText(const Napi::CallbackInfo& info) { opts.system = system_str.empty() ? nullptr : system_str.c_str(); opts.messages_json = messages_str.empty() ? nullptr : messages_str.c_str(); opts.provider_options_json = provider_options_str.empty() ? nullptr : provider_options_str.c_str(); - ai_stream_text(opts, cb, s); + return ai_stream_text(opts, cb, s); }); return env.Undefined(); @@ -770,22 +855,13 @@ Napi::Value SessionWrapper::Send(const Napi::CallbackInfo& info) { return env.Undefined(); } std::string prompt = info[0].As().Utf8Value(); - ai_generate_result_t result = {}; - ai_status_t status = ai_session_send(session_, prompt.c_str(), &result); - if (status != AI_OK) { - std::string msg = result.text ? result.text : "Session send failed"; - ai_generate_result_free(&result); - Napi::Error::New(env, msg).ThrowAsJavaScriptException(); - return env.Undefined(); - } - Napi::Object obj = Napi::Object::New(env); - obj.Set("text", Napi::String::New(env, result.text ? result.text : "")); - obj.Set("finishReason", Napi::String::New(env, result.finish_reason ? result.finish_reason : "stop")); - obj.Set("inputTokens", Napi::Number::New(env, result.input_tokens)); - obj.Set("outputTokens", Napi::Number::New(env, result.output_tokens)); - obj.Set("steps", Napi::Number::New(env, result.steps)); - ai_generate_result_free(&result); - return obj; + ai_session_t session = session_; + + return RunBlockingAsync(env, [session, prompt](BlockingCallResult& out) { + ai_generate_result_t result = {}; + ai_status_t status = ai_session_send(session, prompt.c_str(), &result); + out.take(status, result, "Session send failed"); + }); } Napi::Value SessionWrapper::SendStream(const Napi::CallbackInfo& info) { @@ -800,7 +876,7 @@ Napi::Value SessionWrapper::SendStream(const Napi::CallbackInfo& info) { RunStreamAsync(env, callback, [session, prompt](StreamSession* s, ai_stream_callback_fn cb) { - ai_session_send_stream(session, prompt.c_str(), cb, s); + return ai_session_send_stream(session, prompt.c_str(), cb, s); }); return env.Undefined(); @@ -848,6 +924,215 @@ Napi::Value StandardToolkit(const Napi::CallbackInfo& info) { return obj; } +// Permission policies are JS functions, but the tool they gate executes on the +// call's worker thread. Calling into JS from there without a HandleScope is a +// hard V8 abort, so the policy goes through a ThreadSafeFunction and the worker +// blocks on the result — the same bridge the tool callbacks use. +struct PermissionCallbackData { + Napi::ThreadSafeFunction tsfn; + Napi::FunctionReference callback; +}; + +/** A verdict plus the sentence to show the model when it refuses. */ +struct Verdict { + int decision = AI_PERMISSION_DENY; + std::string reason; +}; + +/// Read a decision, and an optional explanation, out of whatever the JS side +/// returned. Accepts a bare number — the original shape, and still the common +/// one — or `{ decision, reason }`. Anything else is not a decision, and an +/// absent one must fail closed. +static Verdict verdict_from(const Napi::Value& v) { + Verdict out; + if (v.IsNumber()) { + out.decision = v.As().Int32Value(); + return out; + } + if (v.IsObject()) { + Napi::Object obj = v.As(); + Napi::Value d = obj.Get("decision"); + out.decision = d.IsNumber() ? d.As().Int32Value() : AI_PERMISSION_DENY; + Napi::Value r = obj.Get("reason"); + if (r.IsString()) { + out.reason = r.As().Utf8Value(); + } + return out; + } + return out; // not a number, not an object -> deny +} + +/// Copy a JS verdict's reason into the SDK's buffer, truncated to fit. +static void set_reason(char* buf, const std::string& text) { + if (!buf) return; + std::snprintf(buf, AI_REASON_MAX, "%s", text.c_str()); +} + + +static std::mutex g_permission_mutex; +static std::map>> g_permission_pending; + +struct PermissionPayload { + uint64_t id; + std::string tool; + std::string input; +}; + +// Runs on the JS thread. +static void PermissionReceiver(Napi::Env env, Napi::Function jsPolicyFn, PermissionPayload* p) { + Napi::Value result = jsPolicyFn.Call({ + Napi::String::New(env, p->tool), + Napi::String::New(env, p->input), + }); + Verdict verdict = verdict_from(result); + + std::shared_ptr> prom; + { + std::lock_guard lk(g_permission_mutex); + auto it = g_permission_pending.find(p->id); + if (it != g_permission_pending.end()) prom = it->second; + } + if (prom) prom->set_value(std::move(verdict)); + delete p; +} + +// --- Interactive approver: the optional third argument to withPermissions --- +// +// The policy above answers Allow/Deny/Ask synchronously. Ask is where the +// interesting case lives (an MCP tool no settings rule mentions), and the +// engine hands it to an Approver — which, unlike the policy, is expected to be +// async: it awaits a UI prompt. So it goes over the same bridge the JS tools +// use: the worker thread parks on a promise, the JS call runs on the JS +// thread, and the verdict comes back through __resolveApproval. +// +// Without this, `Ask` fails closed to Deny, which is why every MCP tool call +// used to be refused no matter what. + +struct ApprovalCallbackData { + Napi::ThreadSafeFunction tsfn; + Napi::FunctionReference callback; +}; + +static std::mutex g_approval_mutex; +static std::map>> g_approval_pending; +static Napi::FunctionReference g_resolve_approval_ref; + +struct ApprovalPayload { + uint64_t id; + std::string tool; + std::string input; + std::string rationale; +}; + + +static void settle_approval(uint64_t id, Verdict verdict) { + std::shared_ptr> prom; + { + std::lock_guard lk(g_approval_mutex); + auto it = g_approval_pending.find(id); + if (it != g_approval_pending.end()) prom = it->second; + } + if (prom) prom->set_value(std::move(verdict)); +} + +/** __resolveApproval(id, decision) — the JS side's answer to a parked ask. */ +static Napi::Value ResolveApproval(const Napi::CallbackInfo& info) { + uint64_t id = (uint64_t)info[0].As().DoubleValue(); + settle_approval(id, info.Length() > 1 ? verdict_from(info[1]) : Verdict{}); + return info.Env().Undefined(); +} + +/** Runs on the JS thread. */ +static void ApproverReceiver(Napi::Env env, Napi::Function jsApproverFn, ApprovalPayload* p) { + uint64_t id = p->id; + Napi::Value ret = jsApproverFn.Call({Napi::String::New(env, p->tool), + Napi::String::New(env, p->input), + Napi::String::New(env, p->rationale)}); + delete p; + + if (ret.IsObject() && ret.As().Has("then")) { + // The usual shape: a prompt that resolves when the user answers. + Napi::Object promiseObj = ret.As(); + Napi::Function then = promiseObj.Get("then").As(); + Napi::Function onFulfill = Napi::Function::New(env, [id](const Napi::CallbackInfo& info) { + settle_approval(id, info.Length() > 0 ? verdict_from(info[0]) : Verdict{}); + return info.Env().Undefined(); + }); + // A prompt that threw is not an approval. + Napi::Function onReject = Napi::Function::New(env, [id](const Napi::CallbackInfo& info) { + settle_approval(id, Verdict{}); + return info.Env().Undefined(); + }); + then.Call(promiseObj, { onFulfill, onReject }); + return; + } + settle_approval(id, verdict_from(ret)); +} + +/** Build the bridge state for one approver. Deliberately never freed, same + * reasoning as PermissionCallbackData: the gated tool set outlives this call + * and holds this pointer. */ +static ApprovalCallbackData* MakeApprovalCallbackData(Napi::Env env, Napi::Function approver_fn) { + auto* data = new ApprovalCallbackData(); + data->callback = Napi::Persistent(approver_fn); + data->tsfn = Napi::ThreadSafeFunction::New(env, approver_fn, "ai-approver", 0, 1); + data->tsfn.Unref(env); + return data; +} + +/** The C side of the bridge. `ud` is the ApprovalCallbackData built above — + * a captureless function pointer, so the pointer has to travel as user data + * rather than through the closure. */ +static int ApproverBridge(const char* tool, const char* input_json, const char* rationale, + char* reason_buf, void* ud) { + auto* data = static_cast(ud); + std::string tool_s = tool ? tool : ""; + std::string input_s = input_json ? input_json : ""; + std::string rationale_s = rationale ? rationale : ""; + + // A blocking C API call runs on the JS thread, so a prompt awaited + // here would deadlock the thread that has to render it. Answer from + // the value if it is already one, and otherwise fail closed: the + // streaming path — where prompts actually happen — arrives on a + // worker thread and takes the parking branch below. + if (std::this_thread::get_id() == g_main_thread_id) { + Napi::Env e = data->callback.Env(); + Napi::Value result = data->callback.Call({Napi::String::New(e, tool_s), + Napi::String::New(e, input_s), + Napi::String::New(e, rationale_s)}); + if (result.IsObject() && result.As().Has("then")) { + // Nobody is waiting for this verdict, and an unhandled + // rejection would take the host process down with it. + Napi::Object p = result.As(); + Napi::Function noop = Napi::Function::New(e, [](const Napi::CallbackInfo& i) { + return i.Env().Undefined(); + }); + p.Get("then").As().Call(p, { noop, noop }); + return AI_PERMISSION_DENY; + } + Verdict v = verdict_from(result); + set_reason(reason_buf, v.reason); + return v.decision; + } + + auto promise = std::make_shared>(); + auto fut = promise->get_future(); + uint64_t id = g_call_id.fetch_add(1); + { + std::lock_guard lk(g_approval_mutex); + g_approval_pending[id] = promise; + } + data->tsfn.NonBlockingCall(new ApprovalPayload{ id, tool_s, input_s, rationale_s }, + ApproverReceiver); + Verdict verdict = fut.get(); + { + std::lock_guard lk(g_approval_mutex); + g_approval_pending.erase(id); + } + set_reason(reason_buf, verdict.reason); + return verdict.decision; +} + Napi::Value WithPermissions(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (info.Length() < 2) { @@ -857,20 +1142,59 @@ Napi::Value WithPermissions(const Napi::CallbackInfo& info) { auto tools_obj = info[0].As(); auto* tools_wrap = Napi::ObjectWrap::Unwrap(tools_obj); Napi::Function policy_fn = info[1].As(); - auto* fn_ref = new Napi::FunctionReference(); - *fn_ref = Napi::Persistent(policy_fn); + + // Deliberately never freed: the gated tool set outlives this call and its + // execute functions hold this pointer (same reasoning as ToolCallbackData). + auto* data = new PermissionCallbackData(); + data->callback = Napi::Persistent(policy_fn); + data->tsfn = Napi::ThreadSafeFunction::New(env, policy_fn, "ai-permission", 0, 1); + data->tsfn.Unref(env); ai_permission_policy_fn c_fn = [](const char* tool, const char* input_json, - void* ud) -> int { - auto* ref = static_cast(ud); - Napi::Env e = ref->Env(); - Napi::Value result = ref->Call({Napi::String::New(e, tool), - Napi::String::New(e, input_json)}); - return result.IsNumber() ? result.As().Int32Value() - : AI_PERMISSION_DENY; + char* reason_buf, void* ud) -> int { + auto* data = static_cast(ud); + std::string tool_s = tool ? tool : ""; + std::string input_s = input_json ? input_json : ""; + + // Main-thread callers (e.g. a blocking C API call) can run the policy + // inline; a TSFN round trip there would deadlock the JS thread. + if (std::this_thread::get_id() == g_main_thread_id) { + Napi::Env e = data->callback.Env(); + Napi::Value result = data->callback.Call({Napi::String::New(e, tool_s), + Napi::String::New(e, input_s)}); + Verdict v = verdict_from(result); + set_reason(reason_buf, v.reason); + return v.decision; + } + + auto promise = std::make_shared>(); + auto fut = promise->get_future(); + uint64_t id = g_call_id.fetch_add(1); + { + std::lock_guard lk(g_permission_mutex); + g_permission_pending[id] = promise; + } + data->tsfn.NonBlockingCall(new PermissionPayload{ id, tool_s, input_s }, + PermissionReceiver); + Verdict verdict = fut.get(); + { + std::lock_guard lk(g_permission_mutex); + g_permission_pending.erase(id); + } + set_reason(reason_buf, verdict.reason); + return verdict.decision; }; - ai_tool_set_t gated = ai_with_permissions(tools_wrap->handle(), c_fn, fn_ref); + // Third argument opt-in: without it an `Ask` stays a fail-closed Deny, so + // existing two-argument callers keep the behaviour they already have. + ai_tool_set_t gated = nullptr; + if (info.Length() > 2 && info[2].IsFunction()) { + gated = ai_with_permissions_approver( + tools_wrap->handle(), c_fn, data, ApproverBridge, + MakeApprovalCallbackData(env, info[2].As())); + } else { + gated = ai_with_permissions(tools_wrap->handle(), c_fn, data); + } Napi::Object obj = g_toolset_constructor.New({}); auto* out_wrap = Napi::ObjectWrap::Unwrap(obj); out_wrap->SetTools(gated); @@ -891,6 +1215,37 @@ Napi::Value MergeToolSets(const Napi::CallbackInfo& info) { return env.Undefined(); } +// What the tool set contains, as a JSON string. Parsed on the JS side, which is +// where the shape belongs: this side only has to hand over the bytes. +Napi::Value DescribeToolSet(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + if (info.Length() < 1 || !info[0].IsObject()) { + Napi::TypeError::New(env, "Expected (toolSet)").ThrowAsJavaScriptException(); + return env.Undefined(); + } + auto* wrap = Napi::ObjectWrap::Unwrap(info[0].As()); + // Unwrap hands back null for an object that was never wrapped -- a plain + // {} -- and it has already raised a JS exception by then. Returning without + // raising another is the whole fix: a second ThrowAsJavaScriptException + // lands on top of the pending one, napi_throw refuses it, and node-addon-api + // turns that refusal into a fatal error that takes the process down. Left + // unguarded the null would instead reach ->handle() as a null deref. + if (wrap == nullptr) return env.Undefined(); + + ai_tool_set_description_t desc{}; + const ai_status_t status = ai_tool_set_describe_json(wrap->handle(), &desc); + if (status != AI_OK) { + Napi::Error::New(env, "Could not describe the tool set").ThrowAsJavaScriptException(); + return env.Undefined(); + } + + // Copied into a V8 string before the storage is released, since `desc.json` + // points into it. + Napi::String json = Napi::String::New(env, desc.json); + ai_tool_set_description_free(&desc); + return json; +} + Napi::Value McpToolsetFromServer(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (info.Length() < 2) { @@ -1013,8 +1368,13 @@ Napi::Value BatchWrapper::Run(const Napi::CallbackInfo& info) { int count = reqs.Length(); int poll_ms = (info.Length() > 1 && info[1].IsNumber()) ? info[1].As().Int32Value() : 5000; - // String storage must outlive the call. + // String storage must outlive the call, and the c_str() pointers below are + // taken before the vectors stop growing — so reserve up front, or every + // entry but the last dangles once push_back reallocates. std::vector custom_ids, prompts, systems; + custom_ids.reserve(count); + prompts.reserve(count); + systems.reserve(count); std::vector creqs(count); for (int i = 0; i < count; ++i) { auto r = reqs.Get(i).As(); @@ -1060,6 +1420,8 @@ Napi::Value BatchWrapper::Run(const Napi::CallbackInfo& info) { Napi::Object Init(Napi::Env env, Napi::Object exports) { g_main_thread_id = std::this_thread::get_id(); g_resolve_ref = Napi::Persistent(Napi::Function::New(env, ResolveToolCall, "__resolveToolCall")); + g_resolve_approval_ref = + Napi::Persistent(Napi::Function::New(env, ResolveApproval, "__resolveApproval")); ContextWrapper::Init(env, exports); ProviderWrapper::Init(env, exports); @@ -1075,8 +1437,12 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) { exports.Set("standardToolkit", Napi::Function::New(env, StandardToolkit)); exports.Set("withPermissions", Napi::Function::New(env, WithPermissions)); exports.Set("mergeToolSets", Napi::Function::New(env, MergeToolSets)); + exports.Set("describeToolSet", Napi::Function::New(env, DescribeToolSet)); exports.Set("mcpToolsetFromServer", Napi::Function::New(env, McpToolsetFromServer)); exports.Set("version", Napi::Function::New(env, GetVersion)); + // Feature flag for the third `withPermissions` argument — a JS shim that + // passed an approver to an older addon would have it silently ignored. + exports.Set("supportsApprover", Napi::Boolean::New(env, true)); return exports; } diff --git a/bindings/node/test/generate-text.test.cjs b/bindings/node/test/generate-text.test.cjs new file mode 100644 index 0000000..be5e643 --- /dev/null +++ b/bindings/node/test/generate-text.test.cjs @@ -0,0 +1,118 @@ +// generateText (non-streaming) against a mock provider. No API keys, no network. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const ai = require('../dist/index.js'); +const { startMock } = require('./helpers/harness.cjs'); + +let mock; +test.before(async () => { + mock = await startMock(); +}); +test.after(() => mock && mock.stop()); + +function model(name, provider = ai.createOpenAI) { + return provider({ apiKey: 'test-key', baseUrl: mock.baseUrl })(name); +} + +test('generateText returns text, finish reason and usage', async () => { + const r = await ai.generateText({ model: model('mock-stop'), prompt: 'say pong' }); + + assert.equal(r.text, 'pong'); + assert.equal(r.finishReason, 'stop'); + assert.equal(r.usage.inputTokens, 12); + assert.equal(r.usage.outputTokens, 2); + assert.equal(r.steps, 1); +}); + +test('generateText surfaces a length finish reason', async () => { + const r = await ai.generateText({ model: model('mock-length'), prompt: 'write a lot' }); + assert.equal(r.finishReason, 'length'); +}); + +test('generateText sends the prompt and system message on the wire', async () => { + await mock.reset(); + await ai.generateText({ + model: model('mock-stop'), + prompt: 'the user question', + system: 'be terse', + }); + + const [req] = await mock.requests(); + assert.equal(req.url, '/chat/completions'); + assert.equal(req.body.model, 'mock-stop'); + assert.deepEqual(req.body.messages.at(-1), { role: 'user', content: 'the user question' }); + assert.ok( + req.body.messages.some((m) => m.role === 'system' && m.content === 'be terse'), + 'system message should be sent' + ); +}); + +test('generateText forwards maxOutputTokens and temperature', async () => { + await mock.reset(); + await ai.generateText({ + model: model('mock-stop'), + prompt: 'hi', + maxOutputTokens: 42, + temperature: 0.25, + }); + + const [req] = await mock.requests(); + assert.equal(req.body.max_tokens, 42); + assert.equal(req.body.temperature, 0.25); +}); + +test('generateText accepts a messages array', async () => { + await mock.reset(); + await ai.generateText({ + model: model('mock-stop'), + messages: [ + { role: 'user', content: 'first' }, + { role: 'assistant', content: 'second' }, + { role: 'user', content: 'third' }, + ], + }); + + const [req] = await mock.requests(); + assert.deepEqual( + req.body.messages.map((m) => m.content), + ['first', 'second', 'third'] + ); +}); + +test('generateText maps a 401 to an authentication error', async () => { + await assert.rejects( + () => ai.generateText({ model: model('mock-401'), prompt: 'hi' }), + /auth/i + ); +}); + +test('generateText maps a 429 to an error mentioning the rate limit', async () => { + await assert.rejects( + () => ai.generateText({ model: model('mock-429'), prompt: 'hi' }), + /rate limit/i + ); +}); + +test('generateText reports a 500 without hanging', async () => { + await assert.rejects(() => ai.generateText({ model: model('mock-500'), prompt: 'hi' })); +}); + +test('generateText reports a non-JSON body instead of crashing', async () => { + await assert.rejects(() => ai.generateText({ model: model('mock-not-json'), prompt: 'hi' })); +}); + +test('generateText works through the Anthropic-shaped path', async () => { + const m = model('mock-stop', ai.createAnthropic); + const r = await ai.generateText({ model: m, prompt: 'hi' }); + + assert.equal(r.text, 'pong'); +}); + +test('generateText sends Anthropic requests to /v1/messages', async () => { + await mock.reset(); + await ai.generateText({ model: model('mock-stop', ai.createAnthropic), prompt: 'hi' }); + + const [req] = await mock.requests(); + assert.equal(req.url, '/v1/messages'); + assert.ok(req.headers['anthropic-version'], 'anthropic-version header should be set'); +}); diff --git a/bindings/node/test/helpers/harness.cjs b/bindings/node/test/helpers/harness.cjs new file mode 100644 index 0000000..4f15c10 --- /dev/null +++ b/bindings/node/test/helpers/harness.cjs @@ -0,0 +1,161 @@ +// Parent-side test harness: boots the mock provider server in a child process +// and gives tests a baseUrl to point a provider at. +// +// The mock MUST be out-of-process. The binding's synchronous entry points +// (generateText / Agent.call / Session.send) run the C++ io_context on the JS +// main thread, so an in-process server could never answer a request whose +// response the caller is blocking on. + +const { spawn } = require('node:child_process'); +const path = require('node:path'); + +const SERVER = path.join(__dirname, 'mock-server.cjs'); +const DYLIB_DIR = path.join(__dirname, '..', '..', '..', 'build', 'bindings', 'c'); + +/** + * Start the mock server. Resolves once it reports its port. + * @returns {Promise<{baseUrl: string, requests: Function, reset: Function, stop: Function}>} + */ +function startMock() { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [SERVER, '0'], { + stdio: ['ignore', 'pipe', 'pipe'], + // The addon links @rpath/libai_sdk.dylib; make sure the mock process can + // resolve it too (harmless on Linux, where rpath already covers it). + env: { ...process.env, DYLD_LIBRARY_PATH: [DYLIB_DIR, process.env.DYLD_LIBRARY_PATH].filter(Boolean).join(':') }, + }); + + let settled = false; + let stdout = ''; + let stderr = ''; + + const timer = setTimeout(() => { + if (settled) return; + settled = true; + child.kill('SIGKILL'); + reject(new Error(`mock server did not report a port in time\nstderr: ${stderr}`)); + }, 10000); + + child.stdout.on('data', (buf) => { + stdout += buf.toString(); + const m = stdout.match(/MOCK_PORT=(\d+)/); + if (m && !settled) { + settled = true; + clearTimeout(timer); + const port = Number(m[1]); + const baseUrl = `http://127.0.0.1:${port}`; + + const call = async (method, route) => { + const res = await fetch(`${baseUrl}${route}`, { method }); + if (!res.ok) throw new Error(`${route} -> ${res.status}`); + return res; + }; + + resolve({ + baseUrl, + child, + requests: async () => (await call('GET', '/__requests')).json(), + reset: async () => { + await call('POST', '/__reset'); + }, + stop: () => { + child.kill('SIGKILL'); + }, + }); + } + }); + + child.stderr.on('data', (b) => { + stderr += b.toString(); + }); + + child.on('error', (err) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(err); + }); + + child.on('exit', (code) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(new Error(`mock server exited early with code ${code}\nstderr: ${stderr}`)); + }); + }); +} + +/** + * Run a script in a separate process with a hard wall-clock budget. + * + * Used for scenarios that block the JS event loop: a timer inside the test + * process would never fire, so the only way to fail fast instead of wedging + * the whole run is to isolate the call and kill it from out here. + * + * @returns {Promise<{completed: boolean, code: number|null, signal: string|null, stdout: string, stderr: string}>} + */ +function runIsolated(scriptPath, { env = {}, timeoutMs = 20000 } = {}) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [scriptPath], { + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, ...env }, + }); + + let stdout = ''; + let stderr = ''; + let completed = false; + let killed = false; + + child.stdout.on('data', (b) => (stdout += b.toString())); + child.stderr.on('data', (b) => (stderr += b.toString())); + + const timer = setTimeout(() => { + killed = true; + child.kill('SIGKILL'); + }, timeoutMs); + + child.on('exit', (code, signal) => { + clearTimeout(timer); + completed = !killed; + resolve({ completed, killed, code, signal, stdout, stderr }); + }); + }); +} + +const RUNNER = path.join(__dirname, 'scenario-runner.cjs'); + +/** + * Run one scenario in a child process and return its `OK {...}` payload. + * + * Throws a descriptive failure if the child had to be killed: that is the + * signature of a hang, and the whole point of isolating these calls. + * + * @param {string} scenario name understood by scenario-runner.cjs + * @param {{env?: object, timeoutMs?: number}} opts + */ +async function runScenario(scenario, { env = {}, timeoutMs = 15000 } = {}) { + const r = await runIsolated(RUNNER, { env: { SCENARIO: scenario, ...env }, timeoutMs }); + + if (!r.completed) { + throw new Error( + `scenario "${scenario}" did not finish within ${timeoutMs}ms ` + + `(killed=${r.killed} signal=${r.signal}) — it hung.\n` + + `stdout: ${r.stdout.trim() || ''}\n` + + `stderr: ${r.stderr.trim() || ''}` + ); + } + + const line = r.stdout.split('\n').find((l) => l.startsWith('OK ') || l.startsWith('ERR ')); + if (!line) { + throw new Error( + `scenario "${scenario}" produced no result line\nstdout: ${r.stdout}\nstderr: ${r.stderr}` + ); + } + if (line.startsWith('ERR ')) { + throw new Error(`scenario "${scenario}" threw: ${line.slice(4)}\nstderr: ${r.stderr}`); + } + + return JSON.parse(line.slice(3)); +} + +module.exports = { startMock, runIsolated, runScenario, DYLIB_DIR, RUNNER }; diff --git a/bindings/node/test/helpers/mcp-server.cjs b/bindings/node/test/helpers/mcp-server.cjs new file mode 100644 index 0000000..380a052 --- /dev/null +++ b/bindings/node/test/helpers/mcp-server.cjs @@ -0,0 +1,85 @@ +// A minimal MCP server over stdio, so the binding's MCP path can be exercised +// end-to-end without a real server binary or a network. +// +// Framing is newline-delimited JSON-RPC (see src/mcp/transport.cpp) — not the +// Content-Length framing some MCP servers use. +// +// Exposes one tool, `add_numbers`, whose result embeds the sum so a test can +// assert that the value actually made it back to the model. + +let buffer = ''; + +function send(message) { + process.stdout.write(JSON.stringify(message) + '\n'); +} + +function handle(req) { + switch (req.method) { + case 'initialize': + return send({ + jsonrpc: '2.0', + id: req.id, + result: { + protocolVersion: '2024-11-05', + capabilities: { tools: {} }, + serverInfo: { name: 'mock-mcp', version: '1.0.0' }, + }, + }); + + // Notifications carry no id and must not be answered. + case 'notifications/initialized': + return undefined; + + case 'tools/list': + return send({ + jsonrpc: '2.0', + id: req.id, + result: { + tools: [ + { + name: 'add_numbers', + description: 'Add two numbers.', + inputSchema: { + type: 'object', + properties: { a: { type: 'number' }, b: { type: 'number' } }, + required: ['a', 'b'], + }, + }, + ], + }, + }); + + case 'tools/call': { + const args = (req.params && req.params.arguments) || {}; + const sum = (args.a || 0) + (args.b || 0); + return send({ + jsonrpc: '2.0', + id: req.id, + result: { content: [{ type: 'text', text: `sum=${sum}` }], isError: false }, + }); + } + + default: + return send({ + jsonrpc: '2.0', + id: req.id, + error: { code: -32601, message: `unknown method ${req.method}` }, + }); + } +} + +process.stdin.on('data', (chunk) => { + buffer += chunk.toString(); + let idx; + while ((idx = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, idx).trim(); + buffer = buffer.slice(idx + 1); + if (!line) continue; + try { + handle(JSON.parse(line)); + } catch (e) { + // A malformed line is the server's problem to report, not to crash on. + send({ jsonrpc: '2.0', id: null, error: { code: -32700, message: e.message } }); + } + } +}); diff --git a/bindings/node/test/helpers/mock-server.cjs b/bindings/node/test/helpers/mock-server.cjs new file mode 100644 index 0000000..4316d76 --- /dev/null +++ b/bindings/node/test/helpers/mock-server.cjs @@ -0,0 +1,547 @@ +// Mock provider server. Canned Anthropic/OpenAI-shaped responses, no network. +// +// MUST run in its own process: the binding's synchronous entry points +// (generateText / Agent.call / Session.send) block the Node event loop while +// they drive the C++ io_context. An in-process server could never answer a +// request whose response the caller is blocking on — it would deadlock. +// +// Scenarios are selected by the `model` field in the request body, so tests +// just ask for a model named after the behaviour they want. Requests are +// routed by path: /chat/completions (OpenAI-compatible) vs /v1/messages +// (Anthropic). Note the Anthropic provider ignores the SSE `event:` name and +// reads only the JSON `type` field inside `data:`. + +const http = require('node:http'); + +// --- OpenAI-compatible (/chat/completions) ------------------------------- + +const OPENAI_STOP = { + id: 'chatcmpl-mock', + object: 'chat.completion', + created: 1700000000, + model: 'mock', + choices: [ + { + index: 0, + message: { role: 'assistant', content: 'pong' }, + logprobs: null, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 12, completion_tokens: 2, total_tokens: 14 }, +}; + +function openaiToolCall(name, args, id = 'call_mock_1') { + return { + id: 'chatcmpl-mock', + object: 'chat.completion', + created: 1700000000, + model: 'mock', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'I will check that for you.', + tool_calls: [ + { + index: 0, + id, + type: 'function', + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }, + logprobs: null, + finish_reason: 'tool_calls', + }, + ], + usage: { prompt_tokens: 30, completion_tokens: 9, total_tokens: 39 }, + }; +} + +const OPENAI_LENGTH = { + ...OPENAI_STOP, + choices: [{ ...OPENAI_STOP.choices[0], finish_reason: 'length' }], +}; + +/** Build an SSE body the way DeepSeek/OpenAI actually send it. */ +function openaiSse(chunks) { + return chunks.map((c) => `data: ${JSON.stringify(c)}\n\n`).join(''); +} + +function openaiChunk(delta, finishReason = null, usage = null) { + return { + id: 'chatcmpl-mock', + object: 'chat.completion.chunk', + created: 1700000000, + model: 'mock', + choices: [{ index: 0, delta, logprobs: null, finish_reason: finishReason }], + usage, + }; +} + +const USAGE_CHUNK = openaiChunk( + { content: '' }, + 'stop', + { prompt_tokens: 7, completion_tokens: 6, total_tokens: 13 } +); + +/** Text stream including EMPTY deltas, exactly as the live API produces them. */ +const OPENAI_STREAM_EMPTY_DELTAS = openaiSse([ + openaiChunk({ role: 'assistant', content: '' }), + openaiChunk({ content: 'po' }), + openaiChunk({ content: '' }), + openaiChunk({ content: 'ng' }), + USAGE_CHUNK, +]) + 'data: [DONE]\n\n'; + +/** A stream that emits content then stops for length. */ +const OPENAI_STREAM_LENGTH = openaiSse([ + openaiChunk({ role: 'assistant', content: '' }), + openaiChunk({ content: 'trunc' }), + openaiChunk({ content: '' }, 'length', { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 }), +]) + 'data: [DONE]\n\n'; + +/** Tool-call stream: argument fragments arrive split across deltas. */ +function toolCallSse(name) { + return ( + openaiSse([ + openaiChunk({ role: 'assistant', content: '' }), + openaiChunk({ + tool_calls: [ + { index: 0, id: 'call_mock_1', type: 'function', function: { name, arguments: '' } }, + ], + }), + openaiChunk({ tool_calls: [{ index: 0, function: { arguments: '{"city":' } }] }), + openaiChunk({ tool_calls: [{ index: 0, function: { arguments: '"Paris"}' } }] }), + ]) + + openaiSse([ + openaiChunk({ content: '' }, 'tool_calls', { + prompt_tokens: 20, + completion_tokens: 7, + total_tokens: 27, + }), + ]) + + 'data: [DONE]\n\n' + ); +} + +/** Reasoning deltas (reasoning models emit these before content). */ +const OPENAI_STREAM_REASONING = openaiSse([ + openaiChunk({ role: 'assistant', content: '' }), + { ...openaiChunk({ reasoning_content: 'thinking...' }) }, + openaiChunk({ content: '4' }), + USAGE_CHUNK, +]) + 'data: [DONE]\n\n'; + +// --- Google (/models/:generateContent) ------------------------------- + +function googleText(text) { + return { + candidates: [ + { + content: { role: 'model', parts: [{ text }] }, + finishReason: 'STOP', + }, + ], + usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 1 }, + }; +} + +// --- Anthropic (/v1/messages) -------------------------------------------- + +function anthropicText(text, stopReason = 'end_turn', content = null) { + return { + id: 'msg_mock', + type: 'message', + role: 'assistant', + model: 'mock', + content: content ?? [{ type: 'text', text }], + stop_reason: stopReason, + usage: { input_tokens: 10, output_tokens: 2 }, + }; +} + +const ANTHROPIC_TOOL_USE = anthropicText(null, 'tool_use', [ + { + type: 'tool_use', + id: 'toolu_mock_1', + name: 'get_weather', + input: { city: 'Paris' }, + }, +]); + +function anthropicSse(events) { + // The provider ignores the `event:` name, but real servers send it — include + // it so the fixture stays honest about the wire format. + return events + .map(([name, data]) => `event: ${name}\ndata: ${JSON.stringify(data)}\n\n`) + .join(''); +} + +function anthropicStream(text, stopReason = 'end_turn') { + return anthropicStreamDeltas([text], stopReason); +} + +/** Anthropic also emits empty text deltas between real tokens. */ +function anthropicStreamDeltas(deltas, stopReason = 'end_turn') { + return anthropicSse([ + [ + 'message_start', + { + type: 'message_start', + message: { + id: 'msg_mock', + type: 'message', + role: 'assistant', + model: 'mock', + usage: { input_tokens: 10, output_tokens: 0 }, + }, + }, + ], + ['content_block_start', { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }], + ...deltas.map((text) => [ + 'content_block_delta', + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }, + ]), + ['content_block_stop', { type: 'content_block_stop', index: 0 }], + ['message_delta', { type: 'message_delta', delta: { stop_reason: stopReason }, usage: { output_tokens: 2 } }], + ['message_stop', { type: 'message_stop' }], + ]); +} + +function anthropicToolStream(name) { + return anthropicSse([ + [ + 'message_start', + { + type: 'message_start', + message: { + id: 'msg_mock', + type: 'message', + role: 'assistant', + model: 'mock', + usage: { input_tokens: 20, output_tokens: 0 }, + }, + }, + ], + [ + 'content_block_start', + { + type: 'content_block_start', + index: 0, + content_block: { type: 'tool_use', id: 'toolu_mock_1', name }, + }, + ], + ['content_block_delta', { type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: '{"city":' } }], + ['content_block_delta', { type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: '"Paris"}' } }], + ['content_block_stop', { type: 'content_block_stop', index: 0 }], + ['message_delta', { type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 7 } }], + ['message_stop', { type: 'message_stop' }], + ]); +} + +// --- Scenario selection --------------------------------------------------- + +function hasToolResult(messages) { + return Array.isArray(messages) && messages.some((m) => m && m.role === 'tool'); +} + +function countToolResults(messages) { + return Array.isArray(messages) ? messages.filter((m) => m && m.role === 'tool').length : 0; +} + +/** Anthropic sends tool results as a *user* message with tool_result blocks. */ +function anthropicHasToolResult(messages) { + if (!Array.isArray(messages)) return false; + return messages.some( + (m) => + m && + Array.isArray(m.content) && + m.content.some((b) => b && b.type === 'tool_result') + ); +} + +/** Text of the most recent user message, whatever shape `content` takes. */ +function lastUserText(messages) { + if (!Array.isArray(messages)) return ''; + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i]; + if (!m || m.role !== 'user') continue; + if (typeof m.content === 'string') return m.content; + if (Array.isArray(m.content)) { + return m.content.filter((b) => b && b.type === 'text').map((b) => b.text).join(''); + } + } + return ''; +} + +function pickOpenAiScenario(model, body) { + const messages = body.messages ?? []; + switch (model) { + case 'mock-tool-call': + // First turn asks for the tool; second turn (with the tool result + // present) returns prose, exercising the loop-back to the model. + return hasToolResult(messages) + ? { kind: 'json', value: OPENAI_STOP } + : { kind: 'json', value: openaiToolCall('get_weather', { city: 'Paris' }) }; + case 'mock-read-file': + // The prompt IS the path to read, so a test can point this at a temp + // file without the mock needing to know where that file lives. + return hasToolResult(messages) + ? { kind: 'json', value: OPENAI_STOP } + : { kind: 'json', value: openaiToolCall('read_file', { path: lastUserText(messages) }) }; + case 'mock-read-file-twice': + // Asks for the same tool twice in one turn. That is the shape a + // "always allow" needs: the first call is what the approver decides, + // the second is the same tool asked for again after its result came + // back, and must not reach the approver. + return countToolResults(messages) < 2 + ? { kind: 'json', value: openaiToolCall('read_file', { path: lastUserText(messages) }) } + : { kind: 'json', value: OPENAI_STOP }; + case 'mock-add-numbers': + // Asks the model to use the MCP server's `add_numbers` tool. MCP tools are + // offered under their server-qualified name, so that is the name the model + // must use — the bare `add_numbers` is deliberately not exposed. + return hasToolResult(messages) + ? { kind: 'json', value: OPENAI_STOP } + : { kind: 'json', value: openaiToolCall('mcp__numbers__add_numbers', { a: 2, b: 3 }) }; + case 'mock-length': + return { kind: 'json', value: OPENAI_LENGTH }; + case 'mock-stream': + case 'mock-stream-empty-deltas': + return { kind: 'sse', value: OPENAI_STREAM_EMPTY_DELTAS }; + case 'mock-stream-length': + return { kind: 'sse', value: OPENAI_STREAM_LENGTH }; + case 'mock-stream-reasoning': + return { kind: 'sse', value: OPENAI_STREAM_REASONING }; + case 'mock-stream-tool': + return hasToolResult(messages) + ? { kind: 'sse', value: OPENAI_STREAM_EMPTY_DELTAS } + : { kind: 'sse', value: toolCallSse('get_weather') }; + case 'mock-401': + return { + kind: 'json', + status: 401, + value: { error: { message: 'Invalid API key', type: 'authentication_error' } }, + }; + case 'mock-429': + return { + kind: 'json', + status: 429, + headers: { 'retry-after': '3' }, + value: { error: { message: 'Rate limit exceeded', type: 'rate_limit_error' } }, + }; + case 'mock-500': + return { kind: 'json', status: 500, value: { error: { message: 'Server exploded' } } }; + case 'mock-400': + return { kind: 'json', status: 400, value: { error: { message: 'Bad request' } } }; + case 'mock-malformed': { + // Top-level body is not a JSON object -> provider's as_object() throws. + return { kind: 'raw', value: '"just a string"' }; + } + case 'mock-not-json': + return { kind: 'raw', value: 'this is not json at all' }; + case 'mock-truncated-sse': + // Stream ends without a finish chunk or [DONE]. + return { + kind: 'sse', + value: `data: ${JSON.stringify(openaiChunk({ role: 'assistant', content: 'partial' }))}\n\n`, + }; + default: + return { kind: 'json', value: OPENAI_STOP }; + } +} + +function pickAnthropicScenario(model, body) { + const messages = body.messages ?? []; + switch (model) { + case 'mock-tool-call': + return anthropicHasToolResult(messages) + ? { kind: 'json', value: anthropicText('pong') } + : { kind: 'json', value: ANTHROPIC_TOOL_USE }; + case 'mock-stream': + return { kind: 'sse', value: anthropicStream('pong') }; + case 'mock-stream-empty-deltas': + return { kind: 'sse', value: anthropicStreamDeltas(['', 'po', '', 'ng', '']) }; + case 'mock-stream-tool': + return anthropicHasToolResult(messages) + ? { kind: 'sse', value: anthropicStream('done') } + : { kind: 'sse', value: anthropicToolStream('get_weather') }; + case 'mock-401': + return { + kind: 'json', + status: 401, + value: { type: 'error', error: { type: 'authentication_error', message: 'Invalid API key' } }, + }; + case 'mock-429': + return { + kind: 'json', + status: 429, + headers: { 'retry-after': '3' }, + value: { type: 'error', error: { type: 'rate_limit_error', message: 'Rate limited' } }, + }; + default: + return { kind: 'json', value: anthropicText('pong') }; + } +} + +/** + * Route by URL path, then pick the scenario by model name. + * + * Path, not body, distinguishes the providers: Google puts the model in the + * URL (`/models/:generateContent`) and never sends a `model` field. + */ +function pickScenario(path, model, body) { + if (path.includes(':generateContent')) { + return { kind: 'json', value: googleText('pong') }; + } + if (path.startsWith('/v1/messages')) { + return pickAnthropicScenario(model, body); + } + return pickOpenAiScenario(model, body); +} + +// --- Anthropic batch (/v1/messages/batches) ------------------------------ + +// A batch is submit -> poll until terminal -> fetch JSONL results. The mock +// reports `in_progress` on the first poll and `ended` on the next, so the +// caller's wait loop runs exactly twice. +let batchRequests = []; +let batchPolls = 0; + +/** First text block of the first user message, for echoing back. */ +function promptFromParams(params) { + const messages = params && params.messages; + if (!Array.isArray(messages)) return ''; + const text = lastUserText(messages); + return text || '(no prompt)'; +} + +function batchResultsJsonl() { + return ( + batchRequests + .map((r) => + JSON.stringify({ + custom_id: r.custom_id, + result: { + type: 'succeeded', + message: { + id: 'msg_mock_batch', + type: 'message', + role: 'assistant', + model: 'mock', + content: [{ type: 'text', text: `echo:${promptFromParams(r.params)}` }], + stop_reason: 'end_turn', + usage: { input_tokens: 3, output_tokens: 1 }, + }, + }, + }) + ) + .join('\n') + '\n' + ); +} + +function handleBatch(req, res, body, path) { + const send = (obj) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify(obj)); + }; + + if (req.method === 'POST' && path === '/v1/messages/batches') { + batchRequests = Array.isArray(body.requests) ? body.requests : []; + batchPolls = 0; + send({ + id: 'msgbatch_mock', + processing_status: 'in_progress', + request_counts: { processing: batchRequests.length }, + }); + return; + } + + if (req.method === 'GET' && path === '/v1/messages/batches/msgbatch_mock') { + const done = batchPolls++ > 0; + send({ + id: 'msgbatch_mock', + processing_status: done ? 'ended' : 'in_progress', + created_at: '2024-01-01T00:00:00Z', + ended_at: done ? '2024-01-01T00:01:00Z' : undefined, + request_counts: done + ? { succeeded: batchRequests.length } + : { processing: batchRequests.length }, + }); + return; + } + + if (req.method === 'GET' && path === '/v1/messages/batches/msgbatch_mock/results') { + res.writeHead(200, { 'content-type': 'application/x-jsonl' }); + res.end(batchResultsJsonl()); + return; + } + + res.writeHead(404, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: `no batch route for ${req.method} ${path}` } })); +} + +// --- Server --------------------------------------------------------------- + +let requestLog = []; + +const server = http.createServer((req, res) => { + // Control endpoint: lets tests assert what the binding actually put on the + // wire (request path, body, provider options) without touching the mock. + if (req.method === 'GET' && req.url === '/__requests') { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify(requestLog)); + return; + } + if (req.method === 'POST' && req.url === '/__reset') { + requestLog = []; + res.writeHead(200); + res.end('ok'); + return; + } + + const chunks = []; + req.on('data', (c) => chunks.push(c)); + req.on('end', () => { + const raw = Buffer.concat(chunks).toString(); + let body = {}; + try { + body = JSON.parse(raw); + } catch { + /* leave body empty; scenario selection falls back to the default */ + } + + const path = req.url.split('?')[0]; + requestLog.push({ url: req.url, headers: req.headers, body, raw }); + + // Batch routes share the /v1/messages prefix, so they come first. + if (path.startsWith('/v1/messages/batches')) { + handleBatch(req, res, body, path); + return; + } + + const model = typeof body.model === 'string' ? body.model : ''; + const scenario = pickScenario(path, model, body); + const status = scenario.status ?? 200; + const headers = { 'content-type': scenario.kind === 'sse' ? 'text/event-stream' : 'application/json', ...(scenario.headers ?? {}) }; + + if (scenario.kind === 'sse') { + res.writeHead(status, headers); + res.write(scenario.value); + res.end(); + return; + } + res.writeHead(status, headers); + res.end(scenario.kind === 'raw' ? scenario.value : JSON.stringify(scenario.value)); + }); +}); + +server.listen(Number(process.argv[2] || 0), '127.0.0.1', () => { + // The parent reads this line to learn the ephemeral port. + console.log(`MOCK_PORT=${server.address().port}`); +}); + +module.exports = { server }; diff --git a/bindings/node/test/helpers/scenario-runner.cjs b/bindings/node/test/helpers/scenario-runner.cjs new file mode 100644 index 0000000..e4ddf68 --- /dev/null +++ b/bindings/node/test/helpers/scenario-runner.cjs @@ -0,0 +1,422 @@ +// Runs ONE binding operation in its own process so the parent can apply a +// wall-clock budget. Required for the synchronous entry points: when they hang +// they block the JS event loop, so no in-process timer can ever fire. +// +// Reads SCENARIO and MOCK_BASE_URL from the environment. +// Prints a single line to stdout: `OK ` or `ERR `. +// Exits 0 on success, 1 on error. + +const ai = require('../../dist/index.js'); + +const SCENARIO = process.env.SCENARIO; +const BASE = process.env.MOCK_BASE_URL; + +const provider = ai.createOpenAI({ apiKey: 'test-key', baseUrl: BASE }); +const model = (name) => provider(name); + +const SCHEMA = { + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], +}; + +/** A tool that returns synchronously. */ +const getWeather = () => + ai.tool('get_weather', SCHEMA, 'Get the current weather for a city.', (input) => ({ + city: input.city, + weather: 'sunny', + })); + +/** + * A tool that does real async work (a timer stands in for any I/O). This only + * completes if the JS event loop is still turning while the native call is in + * flight — which is the whole point of running the call on a worker thread. + */ +const getWeatherSlowly = () => + ai.tool('get_weather', SCHEMA, 'Get the current weather for a city.', async (input) => { + await new Promise((r) => setTimeout(r, 50)); + return { city: input.city, weather: 'sunny' }; + }); + +/** A tool that rejects; the loop should record the failure rather than die. */ +const getWeatherBadly = () => + ai.tool('get_weather', SCHEMA, 'Get the current weather for a city.', () => { + throw new Error('weather service is down'); + }); + +async function run() { + switch (SCENARIO) { + case 'generateText-tool': { + const r = await ai.generateText({ + model: model('mock-tool-call'), + tools: [getWeather()], + prompt: 'What is the weather in Paris?', + maxSteps: 3, + }); + return { text: r.text, steps: r.steps }; + } + + case 'agent-call-tool': { + const agent = new ai.Agent({ + model: model('mock-tool-call'), + tools: [getWeather()], + instructions: 'Use the tool.', + maxSteps: 3, + }); + const r = await agent.call('What is the weather in Paris?'); + return { text: r.text, steps: r.steps }; + } + + case 'session-send-tool': { + const agent = new ai.Agent({ + model: model('mock-tool-call'), + tools: [getWeather()], + instructions: 'Use the tool.', + maxSteps: 3, + }); + const session = new ai.Session(agent); + const r = await session.send('What is the weather in Paris?'); + return { text: r.text, steps: r.steps }; + } + + case 'session-stream-tool': { + const agent = new ai.Agent({ + model: model('mock-stream-tool'), + tools: [getWeather()], + instructions: 'Use the tool.', + maxSteps: 3, + }); + const session = new ai.Session(agent); + const types = []; + for await (const ev of session.sendStream('What is the weather in Paris?')) { + types.push(ev.type); + } + return { events: types }; + } + + // The same turn, but the provider rejects at request time. A 401 lands + // before the first stream part, which is the case the session entry point + // handles differently from streamText. It has to arrive as a terminal + // error event; anything else means the caller is told the turn succeeded. + // + // SESSION_NO_TOOLS drops the tool set. The two shapes take different routes + // through the C entry point -- with tools the failure surfaces from inside + // the stream, without them it reaches the outer catch -- so a test that + // only ever uses tools cannot see the second one. + case 'session-stream-error': { + const useTools = process.env.SESSION_NO_TOOLS !== '1'; + const agent = new ai.Agent({ + model: model(process.env.MOCK_ERROR_MODEL || 'mock-401'), + tools: useTools ? [getWeather()] : [], + instructions: useTools ? 'Use the tool.' : 'Answer directly.', + maxSteps: 3, + }); + const session = new ai.Session(agent); + const types = []; + let errorText = null; + for await (const ev of session.sendStream('What is the weather in Paris?')) { + types.push(ev.type); + if (ev.type === 'error') errorText = ev.text; + } + return { events: types, errorText }; + } + + case 'generateText-async-tool': { + const r = await ai.generateText({ + model: model('mock-tool-call'), + tools: [getWeatherSlowly()], + prompt: 'What is the weather in Paris?', + maxSteps: 3, + }); + return { text: r.text, steps: r.steps }; + } + + // How much the process grows while running tool calls. + // + // The binding has to report each tool result through a buffer, and the + // question this measures is whose memory that buffer is and when it is + // released. Reporting one that is allocated per call and freed by nobody + // leaks the full output of every call for the life of the process: nothing + // at all in a short run, and unbounded in the long-lived one this SDK is + // for. It shows up here and nowhere else, because every other test in this + // suite looks at values rather than at what was retained to produce them. + // + // Each iteration is a *separate* generateText call rather than one long + // agent loop, and that is what makes the measurement mean anything: a loop + // keeps every tool result in the conversation, which is real retained + // memory that grows with the same shape as the leak and would drown it. + // Ending each call discards the history, so anything still growing is + // growing in the binding. + case 'tool-result-memory': { + const bytes = Number(process.env.TOOL_RESULT_BYTES || 1 << 20); + const calls = Number(process.env.TOOL_RESULT_CALLS || 150); + const payload = 'x'.repeat(bytes); + const tool = ai.tool('get_weather', SCHEMA, 'Get the current weather for a city.', () => payload); + + const ask = () => + ai.generateText({ + model: model('mock-tool-call'), + tools: [tool], + prompt: 'What is the weather in Paris?', + maxSteps: 3, + }); + + // One call before the baseline, so one-time work -- loading the addon's + // lazily-built pieces, the first allocation of the buffer -- is already + // in `before` rather than counted as growth. + const warm = await ask(); + const before = process.memoryUsage().rss; + + let steps = warm.steps; + for (let i = 0; i < calls; i++) { + steps += (await ask()).steps; + } + + const after = process.memoryUsage().rss; + return { calls, bytes, steps, before, after, growth: after - before }; + } + + case 'generateText-throwing-tool': { + const r = await ai.generateText({ + model: model('mock-tool-call'), + tools: [getWeatherBadly()], + prompt: 'What is the weather in Paris?', + maxSteps: 3, + }); + return { text: r.text, steps: r.steps }; + } + + case 'session-send-twice': { + const agent = new ai.Agent({ + model: model('mock-tool-call'), + tools: [getWeather()], + instructions: 'Use the tool.', + maxSteps: 3, + }); + const session = new ai.Session(agent); + const first = await session.send('What is the weather in Paris?'); + const second = await session.send('And again?'); + return { first: first.text, second: second.text, turns: 2 }; + } + + // --- Standard toolkit + permissions --------------------------------- + // + // The prompt is a path, and `mock-read-file` asks the model to read it + // back — so the assertion is that the file's *contents* reach the model. + + case 'standard-toolkit-read-file': { + const agent = new ai.Agent({ + model: model('mock-read-file'), + tools: [], + extraToolSets: [ai.standardToolkit()], + maxSteps: 3, + }); + const r = await agent.call(process.env.TARGET_FILE); + return { text: r.text, steps: r.steps }; + } + + case 'permissions-allow': { + const seen = []; + const gated = ai.withPermissions(ai.standardToolkit(), (tool, inputJson) => { + seen.push(tool); + return 0; // AI_PERMISSION_ALLOW + }); + const agent = new ai.Agent({ + model: model('mock-read-file'), + tools: [], + extraToolSets: [gated], + maxSteps: 3, + }); + const r = await agent.call(process.env.TARGET_FILE); + return { text: r.text, steps: r.steps, policyCalls: seen }; + } + + case 'permissions-deny-reason': { + // The object form: a refusal plus the sentence the model gets to read. + const seen = []; + const gated = ai.withPermissions(ai.standardToolkit(), (tool) => { + seen.push(tool); + return { + decision: ai.PermissionDecision.Deny, + reason: 'that path is outside the project root', + }; + }); + const agent = new ai.Agent({ + model: model('mock-read-file'), + tools: [], + extraToolSets: [gated], + maxSteps: 3, + }); + const r = await agent.call(process.env.TARGET_FILE); + return { text: r.text, steps: r.steps, policyCalls: seen }; + } + + case 'permissions-deny': { + const seen = []; + const gated = ai.withPermissions(ai.standardToolkit(), (tool) => { + seen.push(tool); + return 1; // AI_PERMISSION_DENY + }); + const agent = new ai.Agent({ + model: model('mock-read-file'), + tools: [], + extraToolSets: [gated], + maxSteps: 3, + }); + const r = await agent.call(process.env.TARGET_FILE); + return { text: r.text, steps: r.steps, policyCalls: seen }; + } + + // The interactive approver (third argument to withPermissions). Ask is + // what an unlisted tool gets, and the approver is where a UI prompt would + // go — so it is async here on purpose, and only an approver that is + // *awaited* can answer. + case 'permissions-ask-allow': + case 'permissions-ask-deny': + case 'permissions-ask-deny-reason': + case 'permissions-ask-always': + case 'permissions-ask-junk': + case 'permissions-ask-no-approver': + case 'permissions-allow-skips-approver': { + const decides = SCENARIO !== 'permissions-allow-skips-approver'; + const verdict = { + 'permissions-ask-allow': ai.PermissionDecision.Allow, + 'permissions-ask-deny': ai.PermissionDecision.Deny, + // The object form on the approver side too: a user who clicks "no" + // often has a reason, and it is worth more to the model than the + // bare fact of the refusal. + 'permissions-ask-deny-reason': { + decision: ai.PermissionDecision.Deny, + reason: 'the user declined this particular call', + }, + 'permissions-ask-always': ai.PermissionDecision.AllowAlways, + // Not a decision at all: a gate has to read this as a refusal. + 'permissions-ask-junk': 42, + 'permissions-ask-no-approver': ai.PermissionDecision.Allow, // never reached + 'permissions-allow-skips-approver': ai.PermissionDecision.Deny, // never reached + }[SCENARIO]; + + const policyCalls = []; + const approverCalls = []; + const gated = ai.withPermissions( + ai.standardToolkit(), + (tool) => { + policyCalls.push(tool); + return decides ? ai.PermissionDecision.Ask : ai.PermissionDecision.Allow; + }, + SCENARIO === 'permissions-ask-no-approver' + ? undefined + : async (tool, inputJson, rationale) => { + approverCalls.push({ tool, inputJson, rationale }); + // A prompt is not instant, and the worker thread has to still be + // parked when this resolves. + await new Promise((r) => setTimeout(r, 20)); + return verdict; + } + ); + + const agent = new ai.Agent({ + model: model(SCENARIO === 'permissions-ask-always' ? 'mock-read-file-twice' : 'mock-read-file'), + tools: [], + extraToolSets: [gated], + maxSteps: 5, + }); + const r = await agent.call(process.env.TARGET_FILE); + return { + text: r.text, + steps: r.steps, + policyCalls, + approverCalls, + rationale: approverCalls.length ? approverCalls[0].rationale : null, + }; + } + + case 'merge-toolsets': { + // Merging the standard toolkit into itself must not lose definitions: + // the read still has to work through the merged set. + const dest = ai.standardToolkit(); + ai.mergeToolSets(dest, ai.standardToolkit()); + const agent = new ai.Agent({ + model: model('mock-read-file'), + tools: [], + extraToolSets: [dest], + maxSteps: 3, + }); + const r = await agent.call(process.env.TARGET_FILE); + return { text: r.text, steps: r.steps }; + } + + case 'session-with-memory': { + const agent = new ai.Agent({ + model: model('mock-read-file'), + tools: [], + extraToolSets: [ai.standardToolkit()], + maxSteps: 3, + }); + const session = new ai.Session(agent, { + memoryDir: process.env.MEMORY_DIR, + enableCheckpoint: false, // no extra summarizer call to mock + }); + const r = await session.send(process.env.TARGET_FILE); + return { text: r.text, steps: r.steps }; + } + + case 'mcp-add-numbers': { + // `name` is required and qualifies every tool the server exposes as + // mcp____; it is what permission rules key on. The mock model + // asks for the qualified name for the same reason. + const toolset = ai.mcpToolsetFromServer( + JSON.stringify({ + name: 'numbers', + transport: 'stdio', + command: process.execPath, + args: [process.env.MCP_SERVER], + }) + ); + const agent = new ai.Agent({ + model: model('mock-add-numbers'), + tools: [], + extraToolSets: [toolset], + maxSteps: 3, + }); + const r = await agent.call('What is 2 + 3?'); + return { text: r.text, steps: r.steps }; + } + + case 'mcp-bad-command': { + // A server that cannot be spawned must fail fast and loudly — not spin + // waiting on a pipe that will never open. + let message = null; + try { + ai.mcpToolsetFromServer( + JSON.stringify({ + name: 'unspawnable', + transport: 'stdio', + command: '/nonexistent/definitely-not-a-real-binary-xyz', + }) + ); + } catch (e) { + message = e && e.message ? e.message : String(e); + } + return { threw: message !== null, message }; + } + + case 'generateText-plain': { + const r = await ai.generateText({ model: model('mock-stop'), prompt: 'hi' }); + return { text: r.text }; + } + + default: + throw new Error(`unknown scenario: ${SCENARIO}`); + } +} + +run() + .then((result) => { + process.stdout.write(`OK ${JSON.stringify(result)}\n`); + process.exit(0); + }) + .catch((err) => { + process.stdout.write(`ERR ${err && err.message ? err.message : String(err)}\n`); + process.exit(1); + }); diff --git a/bindings/node/test/live/live-runner.cjs b/bindings/node/test/live/live-runner.cjs new file mode 100644 index 0000000..929e313 --- /dev/null +++ b/bindings/node/test/live/live-runner.cjs @@ -0,0 +1,162 @@ +// Runs ONE binding operation against a real provider, in its own process, so +// the parent can apply a wall-clock budget. The hermetic suite covers behaviour +// with a mock; this covers the part a mock cannot — that a real provider's +// wire format is parsed into the shape the binding promises. +// +// Reads LIVE_SCENARIO, DEEPSEEK_API_KEY (and optionally LIVE_MODEL / +// LIVE_BASE_URL) from the environment. The key is only ever read here and +// handed to the provider — never written anywhere. +// +// Prints a single line to stdout: `OK ` or `ERR `. +// Exits 0 on success, 1 on error. +// +// Scenarios report *facts* (counts, flags, the strings involved); the parent +// decides what those facts have to satisfy. Assertions about model wording +// would be flaky, so there are none. + +const crypto = require('node:crypto'); +const ai = require('../../dist/index.js'); + +const SCENARIO = process.env.LIVE_SCENARIO; +const API_KEY = process.env.DEEPSEEK_API_KEY; + +if (!API_KEY) { + process.stdout.write('ERR DEEPSEEK_API_KEY is not set\n'); + process.exit(1); +} + +const MODEL = process.env.LIVE_MODEL || 'deepseek-chat'; +const provider = ai.createDeepSeek( + process.env.LIVE_BASE_URL ? { apiKey: API_KEY, baseUrl: process.env.LIVE_BASE_URL } : { apiKey: API_KEY } +); + +const token = () => 'TOKEN-' + crypto.randomBytes(4).toString('hex'); + +/** A tool whose result the model cannot guess: it has to actually call it. */ +function lookupTool(secret) { + return ai.tool( + 'lookup_release_token', + { + type: 'object', + properties: { key: { type: 'string', description: 'The key to look up.' } }, + required: ['key'], + additionalProperties: false, + }, + 'Look up the release token for a key.', + (input) => ({ key: input.key, token: secret }) + ); +} + +async function run() { + switch (SCENARIO) { + case 'generate': { + const r = await ai.generateText({ model: provider(MODEL), prompt: 'Say hello in exactly three words.' }); + return { + text: r.text, + finishReason: r.finishReason, + inputTokens: r.usage.inputTokens, + outputTokens: r.usage.outputTokens, + steps: r.steps, + }; + } + + case 'stream': { + const deltas = []; + const types = []; + let finishUsage = null; + let finishReason = null; + + for await (const ev of ai.streamText({ model: provider(MODEL), prompt: 'Say hello in exactly five words.' })) { + types.push(ev.type); + if (ev.type === 'text_delta') deltas.push(ev.text); + if (ev.type === 'finish') { + finishUsage = ev.usage || null; + // The reason rides along inside `usage` rather than as a top-level + // field — see the finish handling in index.ts. + finishReason = (ev.usage && ev.usage.finishReason) || null; + } + } + + return { + types, + deltaCount: deltas.length, + // Every delta must be a string — a `undefined` here is the bug that + // used to render as the literal word "undefined" downstream. + nonStringDeltas: deltas.filter((d) => typeof d !== 'string').length, + text: deltas.join(''), + finishUsage, + finishReason, + }; + } + + case 'tool-call': { + const secret = token(); + const r = await ai.generateText({ + model: provider(MODEL), + tools: [lookupTool(secret)], + prompt: + "Call the lookup_release_token tool with key 'release'. " + + 'Then reply with the token value it returned, exactly as given, and nothing else.', + maxSteps: 3, + }); + + return { text: r.text, steps: r.steps, secret, containsSecret: r.text.includes(secret) }; + } + + case 'stream-tool': { + const secret = token(); + const types = []; + const deltas = []; + + for await (const ev of ai.streamText({ + model: provider(MODEL), + tools: [lookupTool(secret)], + prompt: + "Call the lookup_release_token tool with key 'release'. " + + 'Then reply with the token value it returned, exactly as given, and nothing else.', + maxSteps: 3, + })) { + types.push(ev.type); + if (ev.type === 'text_delta') deltas.push(ev.text); + } + + const text = deltas.join(''); + return { types, text, secret, containsSecret: text.includes(secret) }; + } + + case 'session-two-turns': { + const codeword = 'CODENAME-' + crypto.randomBytes(3).toString('hex'); + const agent = new ai.Agent({ + model: provider(MODEL), + tools: [], + instructions: 'Answer in as few words as possible.', + maxSteps: 3, + }); + const session = new ai.Session(agent); + + const first = await session.send(`Remember this codeword: ${codeword}. Reply with just OK.`); + const second = await session.send('What was the codeword? Reply with just the codeword.'); + + return { + firstText: first.text, + secondText: second.text, + codeword, + // Proves the session carried history into the second request. + recalled: second.text.includes(codeword), + }; + } + + default: + throw new Error(`unknown scenario: ${SCENARIO}`); + } +} + +run() + .then((result) => { + process.stdout.write(`OK ${JSON.stringify(result)}\n`); + process.exit(0); + }) + .catch((err) => { + process.stdout.write(`ERR ${err && err.message ? err.message : String(err)}\n`); + process.exit(1); + }); diff --git a/bindings/node/test/live/live.deepseek.test.cjs b/bindings/node/test/live/live.deepseek.test.cjs new file mode 100644 index 0000000..6b18837 --- /dev/null +++ b/bindings/node/test/live/live.deepseek.test.cjs @@ -0,0 +1,125 @@ +// Live tier: the same binding paths as the hermetic suite, against a real +// provider. A mock can only prove the binding is self-consistent — it cannot +// prove that a real wire format parses into the shape the binding promises. +// +// Opt-in. Without DEEPSEEK_API_KEY every case skips, so `npm run test:live` is +// safe to run anywhere. The key is read from the environment only; it is never +// written to a file or reported in an assertion message. +// +// Assertions here are about *shape and causality*, never wording: a live model +// may phrase anything any way it likes, but a token it could not have known +// without calling the tool must appear in the answer, and a session must still +// remember what it was told a turn ago. +// +// DEEPSEEK_API_KEY=... npm run test:live +// +// Each case runs in its own process under a wall-clock budget — the same +// reasoning as the hermetic suite: a hang in a native call blocks the JS event +// loop, so only a timer out here can fail it. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { runIsolated } = require('../helpers/harness.cjs'); + +const RUNNER = path.join(__dirname, 'live-runner.cjs'); + +const KEY = process.env.DEEPSEEK_API_KEY; +const MODEL = process.env.LIVE_MODEL || 'deepseek-chat'; + +// Real models are slow, and the tool cases take two round trips. Generous. +const TIMEOUT_MS = 90000; + +/** Run one live scenario out-of-process and return its reported facts. */ +async function live(scenario) { + const r = await runIsolated(RUNNER, { + env: { LIVE_SCENARIO: scenario }, + timeoutMs: TIMEOUT_MS, + }); + + if (!r.completed) { + throw new Error( + `live scenario "${scenario}" did not finish within ${TIMEOUT_MS}ms ` + + `(killed=${r.killed} signal=${r.signal}) — it hung.\n` + + `stdout: ${r.stdout.trim() || ''}\n` + + `stderr: ${r.stderr.trim() || ''}` + ); + } + + const line = r.stdout.split('\n').find((l) => l.startsWith('OK ') || l.startsWith('ERR ')); + if (!line) { + throw new Error(`live scenario "${scenario}" produced no result\nstdout: ${r.stdout}\nstderr: ${r.stderr}`); + } + if (line.startsWith('ERR ')) { + throw new Error(`live scenario "${scenario}" failed: ${line.slice(4)}`); + } + return JSON.parse(line.slice(3)); +} + +// Everything in this file needs a key; without one, skip the lot with a reason +// that says how to opt in. +const skip = KEY ? false : `set DEEPSEEK_API_KEY to run the live tier (model: ${MODEL})`; + +test('live: generateText returns text and token usage', { skip }, async () => { + const r = await live('generate'); + + assert.ok(typeof r.text === 'string' && r.text.trim().length > 0, `expected text, got: ${JSON.stringify(r.text)}`); + assert.ok(r.inputTokens > 0, `expected a positive input token count, got ${r.inputTokens}`); + assert.ok(r.outputTokens > 0, `expected a positive output token count, got ${r.outputTokens}`); + assert.ok(r.steps >= 1, `expected at least one step, got ${r.steps}`); + assert.ok( + typeof r.finishReason === 'string' && r.finishReason.length > 0, + `expected a finish reason, got: ${JSON.stringify(r.finishReason)}` + ); +}); + +test('live: streamText delivers string deltas and a finish with usage', { skip }, async () => { + const r = await live('stream'); + + assert.ok(r.deltaCount > 0, `expected text deltas, got events: ${r.types.join(', ')}`); + assert.equal(r.nonStringDeltas, 0, 'every text delta should be a string (an undefined renders as "undefined")'); + assert.ok(r.text.trim().length > 0, `the deltas should concatenate to text, got: ${JSON.stringify(r.text)}`); + assert.ok(r.types.includes('finish'), `expected a finish event, got: ${r.types.join(', ')}`); + assert.ok( + r.finishUsage?.inputTokens > 0, + `expected usage on the finish event, got: ${JSON.stringify(r.finishUsage)}` + ); +}); + +test('live: a tool call round-trips through the model', { skip }, async () => { + const r = await live('tool-call'); + + // The token is generated locally and only ever crosses the wire inside the + // tool result. If it comes back in the answer, the whole loop ran: the call + // was parsed from the response, executed in JS, and fed back. + assert.equal( + r.containsSecret, + true, + `the model should not have been able to answer without calling the tool; it said: ${JSON.stringify(r.text)}` + ); + assert.equal(r.steps, 2, 'a tool call and the answer are two steps'); +}); + +test('live: a streamed tool call round-trips through the model', { skip }, async () => { + const r = await live('stream-tool'); + + assert.ok(r.types.includes('tool_call_start'), `expected a tool_call_start event, got: ${r.types.join(', ')}`); + assert.ok(r.types.includes('tool_call_end'), `expected a tool_call_end event, got: ${r.types.join(', ')}`); + assert.equal( + r.containsSecret, + true, + `the streamed tool result should reach the model; it said: ${JSON.stringify(r.text)}` + ); +}); + +test('live: a session remembers the previous turn', { skip }, async () => { + const r = await live('session-two-turns'); + + assert.ok(r.firstText.trim().length > 0, 'the first turn should answer'); + // A codeword invented locally cannot be recalled unless the first turn's + // messages were carried into the second request. + assert.equal( + r.recalled, + true, + `the session should have carried history; the second turn said: ${JSON.stringify(r.secondText)}` + ); +}); diff --git a/bindings/node/test/stream-text.test.cjs b/bindings/node/test/stream-text.test.cjs new file mode 100644 index 0000000..87ab101 --- /dev/null +++ b/bindings/node/test/stream-text.test.cjs @@ -0,0 +1,179 @@ +// Streaming against a mock provider. No API keys, no network. +// +// Unlike the synchronous entry points these never block the JS event loop, so +// they can be driven in-process with an ordinary `for await` and need no +// watchdog. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const ai = require('../dist/index.js'); +const { startMock } = require('./helpers/harness.cjs'); + +let mock; +test.before(async () => { + mock = await startMock(); +}); +test.after(() => mock && mock.stop()); + +function model(name, provider = ai.createOpenAI) { + return provider({ apiKey: 'test-key', baseUrl: mock.baseUrl })(name); +} + +async function collect(opts) { + const events = []; + for await (const ev of ai.streamText(opts)) events.push(ev); + return events; +} + +const textOf = (events) => + events + .filter((e) => e.type === 'text_delta') + .map((e) => e.text) + .join(''); + +test('streamText concatenates text deltas', async () => { + const events = await collect({ model: model('mock-stream-empty-deltas'), prompt: 'say pong' }); + assert.equal(textOf(events), 'pong'); +}); + +test('every text_delta carries a string, so concatenation never yields "undefined"', async () => { + // Providers emit empty deltas between real tokens. If the wrapper omits the + // field for those, `out += ev.text` appends the literal string "undefined". + const events = await collect({ model: model('mock-stream-empty-deltas'), prompt: 'say pong' }); + + const deltas = events.filter((e) => e.type === 'text_delta'); + assert.ok(deltas.length > 0, 'expected some text_delta events'); + assert.ok( + deltas.every((e) => typeof e.text === 'string'), + `every text_delta must have a string .text, got: ${JSON.stringify(deltas.map((e) => e.text))}` + ); + assert.ok( + deltas.some((e) => e.text === ''), + 'the mock sends empty deltas; they should survive as empty strings' + ); + assert.ok(!textOf(events).includes('undefined')); +}); + +test('streamText reports usage and finish reason on the final event', async () => { + const events = await collect({ model: model('mock-stream-empty-deltas'), prompt: 'say pong' }); + + const finish = events.at(-1); + assert.equal(finish.type, 'finish'); + assert.equal(finish.usage.inputTokens, 7); + assert.equal(finish.usage.outputTokens, 6); + assert.equal(finish.usage.finishReason, 'stop'); +}); + +test('streamText surfaces a length finish reason', async () => { + const events = await collect({ model: model('mock-stream-length'), prompt: 'write a lot' }); + + assert.equal(textOf(events), 'trunc'); + assert.equal(events.at(-1).usage.finishReason, 'length'); +}); + +test('streamText emits reasoning before the answer', async () => { + const events = await collect({ model: model('mock-stream-reasoning'), prompt: 'think' }); + const types = events.map((e) => e.type); + + assert.ok(types.includes('reasoning_start')); + assert.ok(types.includes('reasoning_delta')); + assert.ok(types.includes('reasoning_end')); + + const reasoning = events.find((e) => e.type === 'reasoning_delta'); + assert.equal(reasoning.text, 'thinking...'); + + // Reasoning must be complete before the answer starts. + assert.ok( + types.indexOf('reasoning_end') < types.indexOf('finish'), + 'reasoning should end before the turn finishes' + ); + assert.equal(textOf(events), '4'); +}); + +test('streamText streams a tool call as start/delta/end', async () => { + const events = await collect({ + model: model('mock-stream-tool'), + tools: [ + ai.tool('get_weather', { type: 'object', properties: { city: { type: 'string' } } }, 'w', (i) => i), + ], + prompt: 'weather?', + maxSteps: 3, + }); + + const start = events.find((e) => e.type === 'tool_call_start'); + assert.ok(start, 'expected a tool_call_start event'); + assert.equal(start.toolName, 'get_weather'); + assert.ok(start.toolCallId, 'tool_call_start should carry a call id'); + + const args = events + .filter((e) => e.type === 'tool_call_delta') + .map((e) => e.text) + .join(''); + assert.deepEqual(JSON.parse(args), { city: 'Paris' }); + + assert.ok(events.some((e) => e.type === 'tool_call_end')); +}); + +test('streamText runs the tool and loops back to the model', async () => { + await mock.reset(); + const events = await collect({ + model: model('mock-stream-tool'), + tools: [ + ai.tool('get_weather', { type: 'object', properties: { city: { type: 'string' } } }, 'w', (i) => ({ + city: i.city, + weather: 'sunny', + })), + ], + prompt: 'weather?', + maxSteps: 3, + }); + + assert.equal(textOf(events), 'pong', 'the second turn should stream the final answer'); + + const reqs = await mock.requests(); + assert.equal(reqs.length, 2, 'the tool result should be sent back to the model'); + assert.match(String(reqs[1].body.messages.find((m) => m.role === 'tool').content), /sunny/); +}); + +test('a truncated stream still yields the text it received', async () => { + const events = await collect({ model: model('mock-truncated-sse'), prompt: 'hi' }); + + assert.equal(textOf(events), 'partial'); + assert.equal(events.at(-1).type, 'finish'); +}); + +test('streaming failures surface as a terminal error event', async () => { + for (const [name, status] of [ + ['mock-401', '401'], + ['mock-429', '429'], + ['mock-500', '500'], + ]) { + const events = await collect({ model: model(name), prompt: 'hi' }); + const err = events.find((e) => e.type === 'error'); + + assert.ok(err, `${name} should produce an error event`); + assert.match(err.text, new RegExp(status), `${name} error should name the status`); + assert.equal(events.at(-1).type, 'error', 'the error event terminates the stream'); + } +}); + +test('streamText works through the Anthropic-shaped path', async () => { + const events = await collect({ model: model('mock-stream-empty-deltas', ai.createAnthropic), prompt: 'hi' }); + assert.equal(textOf(events), 'pong'); +}); + +test('streamText sends the system message and tool schemas on the wire', async () => { + await mock.reset(); + await collect({ + model: model('mock-stream-empty-deltas'), + system: 'be terse', + prompt: 'the user question', + }); + + const [req] = await mock.requests(); + assert.ok( + req.body.messages.some((m) => m.role === 'system' && m.content === 'be terse'), + 'system message should be sent' + ); + assert.deepEqual(req.body.messages.at(-1), { role: 'user', content: 'the user question' }); + assert.equal(req.body.stream, true, 'the streaming path should ask for a stream'); +}); diff --git a/bindings/node/test/surface.test.cjs b/bindings/node/test/surface.test.cjs new file mode 100644 index 0000000..30fb72e --- /dev/null +++ b/bindings/node/test/surface.test.cjs @@ -0,0 +1,474 @@ +// The rest of the public surface: version, every provider factory, the +// standard toolkit, permissions, memory, batch, and MCP. +// +// Two safety models, chosen by whether the call can deadlock: +// - nothing that calls back into JS: plain in-process tests +// - anything that runs a JS tool callback: the isolated scenario runner, +// because a deadlock there blocks the JS event loop and an in-process +// timer could never fire to fail the test +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const ai = require('../dist/index.js'); +const { startMock, runScenario } = require('./helpers/harness.cjs'); + +const pkg = require('../package.json'); +const MCP_SERVER = path.join(__dirname, 'helpers', 'mcp-server.cjs'); + +let mock; +test.before(async () => { + mock = await startMock(); +}); +test.after(() => mock && mock.stop()); + +function tmpdir(prefix) { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +/** Every file under `dir`, recursively. */ +function walk(dir) { + const out = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...walk(full)); + else out.push(full); + } + return out; +} + +/** The tool-result message from the model's follow-up request. */ +async function toolResultContent() { + const reqs = await mock.requests(); + assert.equal(reqs.length, 2, 'expected a follow-up request carrying the tool result'); + const msg = reqs[1].body.messages.find((m) => m.role === 'tool'); + assert.ok(msg, 'follow-up request should carry a tool result message'); + return String(msg.content); +} + +// --- version -------------------------------------------------------------- + +test('version() agrees with the published package version', () => { + // A release that bumps one and not the other ships an SDK that lies about + // which version it is. + assert.equal(ai.version(), pkg.version); +}); + +// --- providers ------------------------------------------------------------ + +// Each provider must actually reach its own endpoint. Path, not body, +// identifies Google: it puts the model in the URL and sends no `model` field. +const PROVIDERS = [ + ['createAnthropic', (u) => u.startsWith('/v1/messages')], + ['createDeepSeekAnthropic', (u) => u.startsWith('/v1/messages')], + ['createZai', (u) => u.startsWith('/v1/messages')], + ['createOpenAI', (u) => u === '/chat/completions'], + ['createDeepSeek', (u) => u === '/chat/completions'], + ['createZaiOpenAI', (u) => u === '/chat/completions'], + ['createGoogle', (u) => u.includes(':generateContent')], +]; + +for (const [factory, matchesUrl] of PROVIDERS) { + test(`${factory}() generates text against its own endpoint`, async (t) => { + await mock.reset(); + assert.equal(typeof ai[factory], 'function', `${factory} should be exported`); + + let result; + try { + const provider = ai[factory]({ apiKey: 'test-key', baseUrl: mock.baseUrl }); + result = await ai.generateText({ model: provider('mock-plain'), prompt: 'hi' }); + } catch (e) { + // A provider that is compiled out of the build cannot be constructed. + // That is a build-configuration choice, not a defect — but say so + // loudly rather than quietly passing. + if (/Failed to create provider/.test(e.message)) { + t.skip(`${factory} is not compiled into this build (AI_SDK_PROVIDER_* off)`); + return; + } + throw e; + } + + assert.equal(result.text, 'pong', `${factory} should return the model's text`); + + const reqs = await mock.requests(); + assert.equal(reqs.length, 1, `${factory} should make exactly one request`); + assert.ok(matchesUrl(reqs[0].url), `${factory} hit an unexpected path: ${reqs[0].url}`); + }); +} + +test('provider factories expose both call and .model() forms', () => { + const provider = ai.createOpenAI({ apiKey: 'test-key', baseUrl: mock.baseUrl }); + + const called = provider('my-model'); + const viaMethod = provider.model('my-model'); + + assert.equal(called.modelId, 'my-model'); + assert.equal(viaMethod.modelId, 'my-model'); + assert.equal(called.provider, 'openai'); +}); + +test('an API error rejects the promise instead of hanging', async () => { + const model = ai.createOpenAI({ apiKey: 'test-key', baseUrl: mock.baseUrl })('mock-401'); + + await assert.rejects( + () => ai.generateText({ model, prompt: 'hi' }), + /401|Invalid API key|Authentication failed/i, + 'a 401 should surface as a rejection the caller can catch' + ); +}); + +// --- standard toolkit ----------------------------------------------------- + +test('standardToolkit() reads a real file and hands it to the model', async () => { + const file = path.join(tmpdir('ai-sdk-toolkit-'), 'notes.txt'); + fs.writeFileSync(file, 'the secret handshake is pineapple'); + + await mock.reset(); + const r = await runScenario('standard-toolkit-read-file', { + env: { MOCK_BASE_URL: mock.baseUrl, TARGET_FILE: file }, + }); + + assert.equal(r.text, 'pong'); + assert.equal(r.steps, 2, 'the tool call and the answer are two steps'); + assert.match(await toolResultContent(), /pineapple/, 'the file contents should reach the model'); +}); + +// --- permissions ---------------------------------------------------------- + +test('withPermissions() runs a tool the policy allows', async () => { + const file = path.join(tmpdir('ai-sdk-perm-'), 'notes.txt'); + fs.writeFileSync(file, 'the secret handshake is pineapple'); + + await mock.reset(); + const r = await runScenario('permissions-allow', { + env: { MOCK_BASE_URL: mock.baseUrl, TARGET_FILE: file }, + }); + + assert.equal(r.text, 'pong'); + assert.deepEqual(r.policyCalls, ['read_file'], 'the policy should be consulted for the tool'); + assert.match(await toolResultContent(), /pineapple/); +}); + +test('withPermissions() blocks a denied tool and tells the model why', async () => { + const file = path.join(tmpdir('ai-sdk-perm-'), 'notes.txt'); + fs.writeFileSync(file, 'the secret handshake is pineapple'); + + await mock.reset(); + const r = await runScenario('permissions-deny', { + env: { MOCK_BASE_URL: mock.baseUrl, TARGET_FILE: file }, + }); + + assert.deepEqual(r.policyCalls, ['read_file']); + + const content = await toolResultContent(); + assert.match(content, /permission_denied/, 'the model should learn the call was denied'); + assert.doesNotMatch(content, /pineapple/, 'a denied tool must not disclose file contents'); +}); + +// --- the interactive approver -------------------------------------------- +// +// `Ask` is the decision a policy gives when it does not want to answer — the +// case every tool no settings rule mentions lands in. Without an approver that +// is a Deny, which is what these tests would find if the approver never made +// it across the binding. + +/** A file whose contents only ever reach the model if the read really ran. */ +function secretFile() { + const file = path.join(tmpdir('ai-sdk-ask-'), 'notes.txt'); + fs.writeFileSync(file, 'the secret handshake is pineapple'); + return file; +} + +const ask = (scenario, file) => + runScenario(scenario, { env: { MOCK_BASE_URL: mock.baseUrl, TARGET_FILE: file } }); + +test('an approver can allow what the policy left undecided', async () => { + const file = secretFile(); + await mock.reset(); + + const r = await ask('permissions-ask-allow', file); + + assert.equal(r.text, 'pong'); + assert.equal(r.steps, 2, 'the call should have run: tool call, then the answer'); + assert.deepEqual(r.policyCalls, ['read_file'], 'the policy is consulted first'); + assert.equal(r.approverCalls.length, 1, 'and the ask reached the approver exactly once'); + assert.equal(r.approverCalls[0].tool, 'read_file'); + assert.match(r.approverCalls[0].inputJson, /notes\.txt/, 'the approver sees the input'); + assert.ok(r.rationale, 'the engine supplies a rationale to show the user'); + assert.match(await toolResultContent(), /pineapple/, 'an approved tool actually ran'); +}); + +test('an approver can refuse', async () => { + const file = secretFile(); + await mock.reset(); + + const r = await ask('permissions-ask-deny', file); + + assert.equal(r.approverCalls.length, 1, 'the refusal came from the approver, not a missing one'); + const content = await toolResultContent(); + assert.match(content, /permission_denied/); + assert.doesNotMatch(content, /pineapple/, 'a refused tool must not disclose file contents'); +}); + +test('a policy can hand the model a reason for its refusal', async () => { + const file = secretFile(); + await mock.reset(); + + const r = await runScenario('permissions-deny-reason', { + env: { MOCK_BASE_URL: mock.baseUrl, TARGET_FILE: file }, + }); + + assert.deepEqual(r.policyCalls, ['read_file']); + assert.equal(r.steps, 2, 'a refusal ends the loop, it does not abort it'); + + const content = await toolResultContent(); + assert.match(content, /permission_denied/); + assert.match( + content, + /that path is outside the project root/, + 'the reason has to survive the trip through the C API into the tool result', + ); + assert.doesNotMatch(content, /pineapple/); +}); + +test("an approver's reason overrides the policy's", async () => { + const file = secretFile(); + await mock.reset(); + + const r = await ask('permissions-ask-deny-reason', file); + + assert.equal(r.approverCalls.length, 1); + const content = await toolResultContent(); + assert.match(content, /permission_denied/); + assert.match( + content, + /the user declined this particular call/, + 'the approver answered second, so its reason is the one worth showing', + ); + assert.doesNotMatch(content, /pineapple/); +}); + +test('with no approver an undecided call still fails closed', async () => { + const file = secretFile(); + await mock.reset(); + + const r = await ask('permissions-ask-no-approver', file); + + assert.deepEqual(r.approverCalls, [], 'there was no approver to call'); + const content = await toolResultContent(); + assert.match(content, /permission_denied/, 'Ask without an approver is a Deny'); + assert.doesNotMatch(content, /pineapple/); +}); + +test('a verdict that is not a decision refuses the call', async () => { + const file = secretFile(); + await mock.reset(); + + const r = await ask('permissions-ask-junk', file); + + assert.equal(r.approverCalls.length, 1, 'the approver was the thing that answered'); + const content = await toolResultContent(); + assert.match(content, /permission_denied/, 'a gate reads an unknown verdict as a refusal'); + assert.doesNotMatch(content, /pineapple/); +}); + +test('a policy that decides on its own never reaches the approver', async () => { + const file = secretFile(); + await mock.reset(); + + const r = await ask('permissions-allow-skips-approver', file); + + assert.deepEqual(r.approverCalls, [], 'only Ask escalates'); + assert.match(await toolResultContent(), /pineapple/); +}); + +test('always-allow stops asking for that tool for the rest of the session', async () => { + const file = secretFile(); + await mock.reset(); + + const r = await ask('permissions-ask-always', file); + + assert.equal(r.text, 'pong'); + assert.equal( + r.approverCalls.length, + 1, + 'the second call of the same tool must not ask again' + ); + + // …and the second call still ran: both results carry the file's contents. + const reqs = await mock.requests(); + assert.equal(reqs.length, 3, 'tool call, tool call again, then the answer'); + const results = reqs[2].body.messages.filter((m) => m.role === 'tool'); + assert.equal(results.length, 2, 'both calls produced a result'); + for (const msg of results) { + assert.match(String(msg.content), /pineapple/, 'an allowed call ran, not just the first'); + } +}); + +// --- merging tool sets ---------------------------------------------------- + +test('mergeToolSets() keeps the merged definitions usable', async () => { + const file = path.join(tmpdir('ai-sdk-merge-'), 'notes.txt'); + fs.writeFileSync(file, 'the secret handshake is pineapple'); + + await mock.reset(); + const r = await runScenario('merge-toolsets', { + env: { MOCK_BASE_URL: mock.baseUrl, TARGET_FILE: file }, + }); + + assert.equal(r.text, 'pong'); + assert.match(await toolResultContent(), /pineapple/); +}); + +// --- describing tool sets ------------------------------------------------- + +test('describeToolSet() reports what a set exposes, in name order', () => { + const described = ai.describeToolSet(ai.standardToolkit()); + + assert.ok(Array.isArray(described)); + assert.ok(described.length >= 5, 'the standard toolkit has several tools'); + + const names = described.map((t) => t.name); + assert.deepEqual( + names, + [...names].sort(), + 'the order is stable, or two descriptions cannot be compared' + ); + assert.ok(names.includes('read_file')); + + const readFile = described.find((t) => t.name === 'read_file'); + assert.equal(typeof readFile.description, 'string'); + assert.equal(typeof readFile.inputSchema, 'object'); + assert.equal(readFile.inputSchema.type, 'object'); +}); + +test('describeToolSet() throws on something that is not a tool set', () => { + // The assertion is only that it throws. Unwrapping a plain {} yields a null + // handle, so without the guard this is a segfault that takes the host process + // down rather than an exception the caller can catch -- and the message + // belongs to node-addon-api, not to us. + assert.throws(() => ai.describeToolSet({})); +}); + +test('describeToolSet() reads through a wrapped tool set', () => { + // ToolSet has no public constructor, so the interesting case is a set the + // public API derives rather than builds: if this only worked on a raw + // toolkit it would be useless for the MCP and permissions sets, which are + // exactly the ones you cannot otherwise see inside. + const gated = ai.withPermissions(ai.standardToolkit(), () => ai.PermissionDecision.Allow); + + const names = ai.describeToolSet(gated).map((t) => t.name); + assert.ok(names.includes('read_file'), 'the wrapper does not hide the tools it wraps'); + assert.deepEqual(names, [...names].sort(), 'order stays stable through a wrapper'); +}); + +// --- sessions with memory ------------------------------------------------- + +test('a session with memoryDir runs the turn and creates its memory directory', async () => { + const dir = tmpdir('ai-sdk-session-mem-'); + const file = path.join(tmpdir('ai-sdk-session-file-'), 'notes.txt'); + fs.writeFileSync(file, 'the secret handshake is pineapple'); + + await mock.reset(); + const r = await runScenario('session-with-memory', { + env: { MOCK_BASE_URL: mock.baseUrl, TARGET_FILE: file, MEMORY_DIR: dir }, + }); + + assert.equal(r.text, 'pong'); + assert.ok(fs.existsSync(dir), 'the session should have created its memory directory'); +}); + +// --- MemoryStore ---------------------------------------------------------- + +test('MemoryStore writes a record to disk as markdown', () => { + const dir = tmpdir('ai-sdk-memory-'); + const store = new ai.MemoryStore(dir); + store.save('project', 'build-command', 'cmake --build build -j8'); + + const files = walk(dir); + assert.ok(files.length > 0, `expected a file under ${dir}`); + assert.ok( + files.some((f) => f.endsWith('.md')), + `expected a markdown record, got: ${files.join(', ')}` + ); + + const contents = files.map((f) => fs.readFileSync(f, 'utf8')).join('\n'); + assert.match(contents, /cmake --build build -j8/); + assert.match(contents, /build-command/); +}); + +// --- Batch ---------------------------------------------------------------- + +test('Batch submits, polls, and returns one result per request', async () => { + await mock.reset(); + const provider = ai.createAnthropic({ apiKey: 'test-key', baseUrl: mock.baseUrl }); + const batch = new ai.Batch(provider, 'claude-sonnet-4-20250514'); + + const r = batch.run( + [ + { customId: 'first', prompt: 'one' }, + { customId: 'second', prompt: 'two' }, + ], + 1 // poll immediately; the mock ends the batch on the second poll + ); + + assert.equal(r.batchId, 'msgbatch_mock'); + assert.equal(r.status, 'completed'); + assert.deepEqual( + r.items.map((i) => i.customId), + ['first', 'second'], + 'results should line up with the requests' + ); + assert.equal(r.items[0].result, 'echo:one', 'each result should carry its own prompt'); + assert.equal(r.items[1].result, 'echo:two'); + assert.equal(r.items[0].error, null); +}); + +test('Batch rejects a provider that cannot batch', () => { + const provider = ai.createDeepSeek({ apiKey: 'test-key', baseUrl: mock.baseUrl }); + assert.throws( + () => new ai.Batch(provider, 'deepseek-chat'), + /does not support batching/, + 'an unsupported provider should fail at construction, not at run' + ); +}); + +// --- MCP ------------------------------------------------------------------ + +test('mcpToolsetFromServer() connects over stdio and runs the server tool', async () => { + await mock.reset(); + const r = await runScenario('mcp-add-numbers', { + env: { MOCK_BASE_URL: mock.baseUrl, MCP_SERVER }, + timeoutMs: 20000, // spawning a child process costs more than the default + }); + + assert.equal(r.text, 'pong'); + assert.match( + await toolResultContent(), + /sum=5/, + 'the MCP server result should reach the model' + ); +}); + +test('mcpToolsetFromServer() rejects an unusable config', () => { + assert.throws(() => ai.mcpToolsetFromServer('{ not json'), /MCP config/i); +}); + +test('mcpToolsetFromServer() fails fast when the server cannot be spawned', async () => { + const r = await runScenario('mcp-bad-command', { + env: { MOCK_BASE_URL: mock.baseUrl }, + timeoutMs: 20000, + }); + + assert.equal(r.threw, true, 'an unspawnable server should raise, not wait forever'); + // Assert *why* it threw, not just that it did. Without this the test still + // passes when the config is rejected earlier for an unrelated reason — which + // is exactly what happened when `name` became required and this config did + // not have one yet. + assert.match( + r.message, + /spawn/i, + 'the failure should come from spawning the process, not from validating the config' + ); +}); diff --git a/bindings/node/test/tool-calls.test.cjs b/bindings/node/test/tool-calls.test.cjs new file mode 100644 index 0000000..042648f --- /dev/null +++ b/bindings/node/test/tool-calls.test.cjs @@ -0,0 +1,245 @@ +// Tool calls through the synchronous entry points. +// +// Every case here runs in its own process under a wall-clock budget. That is +// not incidental: when one of these entry points hangs it does so at 100% CPU +// with the JS event loop blocked, so an in-process `setTimeout` would never +// fire and the run would wedge instead of failing. The parent kills the child +// from outside. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const net = require('node:net'); +const { startMock, runScenario } = require('./helpers/harness.cjs'); + +let mock; +test.before(async () => { + mock = await startMock(); +}); +test.after(() => mock && mock.stop()); + +/** Run one scenario in a child process, under a wall-clock budget. */ +const run = (scenario, opts = {}) => + runScenario(scenario, { env: { MOCK_BASE_URL: mock.baseUrl }, ...opts }); + +/** + * A port with nothing listening on it. Binding to 0 and immediately closing + * hands back a port the kernel just proved was free, so connecting to it is + * refused at once instead of hanging on a firewall. + */ +const deadPort = () => + new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.on('error', reject); + srv.listen(0, '127.0.0.1', () => { + const { port } = srv.address(); + srv.close(() => resolve(port)); + }); + }); + +test('generateText runs a tool call and loops back to the model', async () => { + await mock.reset(); + const r = await run('generateText-tool'); + + assert.equal(r.steps, 2, 'should take two steps: tool call, then the answer'); + assert.equal(r.text, 'pong'); + + // The tool result must actually reach the model on the follow-up request. + const reqs = await mock.requests(); + assert.equal(reqs.length, 2, 'expected the tool result to be sent back to the model'); + + const followUp = reqs[1].body.messages; + const toolMsg = followUp.find((m) => m.role === 'tool'); + assert.ok(toolMsg, 'follow-up request should carry a tool result message'); + assert.match(String(toolMsg.content), /sunny/, 'tool result should contain what the tool returned'); +}); + +test('Agent.call runs a tool call and loops back to the model', async () => { + await mock.reset(); + const r = await run('agent-call-tool'); + + assert.equal(r.steps, 2); + assert.equal(r.text, 'pong'); + + const reqs = await mock.requests(); + assert.equal(reqs.length, 2); + assert.ok( + reqs[1].body.messages.some((m) => m.role === 'tool'), + 'follow-up request should carry a tool result message' + ); +}); + +test('Session.send runs a tool call', async () => { + await mock.reset(); + const r = await run('session-send-tool'); + + assert.equal(r.text, 'pong'); +}); + +test('a tool that awaits real async work still completes', async () => { + // The tool's timer can only fire if the JS event loop is free while the + // native call is in flight. This is the case a blocked main thread loses. + await mock.reset(); + const r = await run('generateText-async-tool'); + + assert.equal(r.steps, 2); + assert.equal(r.text, 'pong'); + + const reqs = await mock.requests(); + assert.match(String(reqs[1].body.messages.find((m) => m.role === 'tool').content), /sunny/); +}); + +test('a throwing tool is reported to the model instead of killing the turn', async () => { + await mock.reset(); + const r = await run('generateText-throwing-tool'); + + assert.equal(r.text, 'pong', 'the loop should continue after a tool throws'); + + const reqs = await mock.requests(); + const toolMsg = reqs[1].body.messages.find((m) => m.role === 'tool'); + assert.match(String(toolMsg.content), /weather service is down/, 'the error should reach the model'); +}); + +test('a session survives more than one tool-calling turn', async () => { + await mock.reset(); + const r = await run('session-send-twice'); + + assert.equal(r.first, 'pong'); + assert.equal(r.second, 'pong'); +}); + +test('Session.sendStream runs a tool call', async () => { + await mock.reset(); + const r = await run('session-stream-tool'); + + assert.ok( + r.events.includes('tool_call_start'), + `expected a tool_call_start event, got: ${r.events.join(', ')}` + ); + assert.ok( + r.events.includes('tool_call_end'), + `expected a tool_call_end event, got: ${r.events.join(', ')}` + ); + + const reqs = await mock.requests(); + assert.equal(reqs.length, 2, 'expected the streamed tool result to be sent back to the model'); +}); + +// The counterpart to the tool-call case above: the provider rejects the request +// itself. Unlike streamText, which has always surfaced this as an error event, +// the session entry point returned the failure status without emitting +// anything, and the binding then synthesized a *finish* event for the missing +// terminal event. The caller saw an empty turn that had succeeded — the one +// outcome a consumer cannot detect, because checking for an error is exactly +// what finds nothing. +test('Session.sendStream surfaces a request failure as an error event', async () => { + for (const [name, status] of [ + ['mock-401', '401'], + ['mock-429', '429'], + ['mock-500', '500'], + ]) { + await mock.reset(); + // `run` spreads opts after its own default env, so the whole env has to be + // passed here -- a partial one would drop MOCK_BASE_URL. + const r = await run('session-stream-error', { + env: { MOCK_BASE_URL: mock.baseUrl, MOCK_ERROR_MODEL: name }, + }); + + assert.ok( + r.events.includes('error'), + `${name}: expected an error event, got: ${r.events.join(', ') || '(none)'}` + ); + assert.match( + String(r.errorText), + new RegExp(status), + `${name}: the error event should name the status, not be a placeholder` + ); + assert.equal( + r.events.at(-1), + 'error', + `${name}: the error has to terminate the stream, or the caller keeps waiting` + ); + assert.ok( + !r.events.includes('finish'), + `${name}: a failed turn must not also report a clean finish` + ); + } +}); + +// The transport case, which the status-code cases above structurally cannot +// reach: the request never gets a response at all. That covers a refused +// connection, DNS failure, TLS failure and timeouts — the failures a machine +// on a flaky network actually sees. Nothing in the stream ever errors, so the +// failure has to be reported by the call itself, and before this was fixed the +// caller got a `finish` carrying zero tokens and an empty finishReason: a +// successful-looking empty turn, which is the one outcome a consumer cannot +// detect, because looking for the error is exactly what finds nothing. +test('Session.sendStream reports a connection failure instead of an empty success', async () => { + const port = await deadPort(); + await mock.reset(); + + // Both agent shapes, because they take different routes through the C entry + // point: with tools the failure surfaces from inside the stream, without them + // it lands in the outer catch. A tool set is the *less* common shape for a + // plain chat session, so covering only the tool-bearing case misses the bug. + for (const noTools of ['1', '0']) { + const what = noTools === '1' ? 'without tools' : 'with tools'; + const r = await run('session-stream-error', { + env: { MOCK_BASE_URL: `http://127.0.0.1:${port}`, SESSION_NO_TOOLS: noTools }, + }); + + assert.ok( + r.events.includes('error'), + `${what}: expected an error event, got: ${r.events.join(', ') || '(none)'}` + ); + assert.equal( + r.events.at(-1), + 'error', + `${what}: the error has to terminate the stream, or the caller keeps waiting` + ); + assert.ok( + !r.events.includes('finish'), + `${what}: a turn whose request never reached the provider must not report a clean finish` + ); + assert.ok( + r.errorText && r.errorText.length > 0, + `${what}: the error event needs a message, not a placeholder: ${JSON.stringify(r.errorText)}` + ); + } +}); + +// What the tool-result path retains, rather than what it returns. +// +// The failure this is for is invisible to every other test here: a binding that +// allocates a buffer per tool call and frees none of them returns exactly the +// right values and leaks the entire output of every call for the life of the +// process. Nothing fails, nothing logs, and the process simply grows -- which +// is why it is worth a test that measures memory instead of assertions. +test('tool results do not accumulate in the process', async () => { + await mock.reset(); + // A long budget on purpose: this runs 150 round trips through the mock, and + // the failure it looks for is a slow growth rather than a hang. + const r = await run('tool-result-memory', { timeoutMs: 180000 }); + + // First, that the loop did the work. A scenario that quietly stopped calling + // the tool would leak nothing and pass the assertion below for the wrong + // reason -- and the leak would go on being unmeasured. + assert.equal( + r.steps, + (r.calls + 1) * 2, + `expected every one of the ${r.calls} calls to run the tool and loop back` + ); + + // 150 calls of 1 MiB with a buffer per call is 150 MiB of growth. The fixed + // version reuses one buffer per thread, so it stays flat. The bar sits at a + // fifth of the leak: far above allocator and GC noise, so it reports the + // defect rather than the machine, and far below the defect, so it cannot be + // passed by accident. + const limit = (r.bytes * r.calls) / 5; + const mib = (n) => `${(n / 1048576).toFixed(1)} MiB`; + assert.ok( + r.growth < limit, + `RSS grew ${mib(r.growth)} over ${r.calls} tool calls of ${mib(r.bytes)} each ` + + `(limit ${mib(limit)}). The tool-result buffer is being allocated per call ` + + `and never freed; see the ownership note on ai_tool_result_t in ` + + `bindings/c/ai_sdk.h.` + ); +}); diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 56aeb9c..243c01c 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -35,7 +35,7 @@ def build_extension(self, ext): setup( name="ai-sdk-cpp", - version="0.1.0", + version="1.0.0", author="AI SDK Contributors", description="Native C++ AI agent framework with Python bindings", long_description=open("README.md").read() if os.path.exists("README.md") else "", diff --git a/bindings/python/src/module.cpp b/bindings/python/src/module.cpp index 438ab25..bda2d0d 100644 --- a/bindings/python/src/module.cpp +++ b/bindings/python/src/module.cpp @@ -2,6 +2,7 @@ #include #include #include "ai_sdk.h" +#include #include #include #include @@ -76,6 +77,21 @@ class PyModel { ai_model_t model_; }; +/// The buffer a Python tool's output is reported through. +/// +/// See the ownership note on ai_tool_result_t in bindings/c/ai_sdk.h: the +/// library copies output_json into its own storage before the callback returns +/// and keeps no reference afterwards, so the callback owns the buffer and only +/// has to keep it alive across the return. A thread_local is that lifetime +/// exactly. This used to allocate a fresh buffer per call with `new char[]` +/// and free it nowhere, which leaks the output of every tool call for the life +/// of the process -- unbounded for a long-running agent, and proportional to +/// the data the tools return. +static std::string& thread_buf() { + static thread_local std::string buf; + return buf; +} + struct PyToolEntry { std::string name; std::string description; @@ -125,17 +141,12 @@ class PyToolSet { output = json.attr("dumps")(result).cast(); } - char* out_copy = new char[output.size() + 1]; - std::copy(output.begin(), output.end(), out_copy); - out_copy[output.size()] = '\0'; - res.output_json = out_copy; + thread_buf() = std::move(output); + res.output_json = thread_buf().c_str(); res.is_error = 0; } catch (const py::error_already_set& e_py) { - std::string err = e_py.what(); - char* err_copy = new char[err.size() + 1]; - std::copy(err.begin(), err.end(), err_copy); - err_copy[err.size()] = '\0'; - res.output_json = err_copy; + thread_buf() = e_py.what(); + res.output_json = thread_buf().c_str(); res.is_error = 1; } return res; @@ -430,16 +441,34 @@ static std::unique_ptr py_standard_toolkit() { return PyToolSet::wrap(ai_standard_toolkit_create()); } +/// Copy a Python string into the SDK's reason buffer, truncating to fit. +static void set_reason(char* buf, const std::string& text) { + if (!buf) return; + std::snprintf(buf, AI_REASON_MAX, "%s", text.c_str()); +} + static std::unique_ptr py_with_permissions(PyToolSet& tools, py::function policy_fn) { // policy_fn(tool: str, input_json: str) -> int (0=allow, 1=deny, 2=ask) + // or (decision: int, reason: str) to explain a refusal. The reason reaches + // the model as part of the tool result, so it should say what would have to + // change -- a model told only that a call was refused retries variants. // NOTE: the function is heap-allocated and kept alive for the toolset's // lifetime (one allocation per call; not GC'd). auto* fn = new py::function(std::move(policy_fn)); - ai_permission_policy_fn c_fn = [](const char* tool, const char* input_json, void* ud) -> int { + ai_permission_policy_fn c_fn = [](const char* tool, const char* input_json, char* reason, + void* ud) -> int { auto* f = static_cast(ud); py::gil_scoped_acquire acquire; try { - return f->operator()(tool, input_json).cast(); + py::object result = f->operator()(tool, input_json); + if (py::isinstance(result)) { + py::tuple t = result.cast(); + if (t.size() > 1 && !t[1].is_none()) { + set_reason(reason, t[1].cast()); + } + return t.empty() ? AI_PERMISSION_DENY : t[0].cast(); + } + return result.cast(); } catch (...) { return AI_PERMISSION_DENY; // fail closed } @@ -502,7 +531,10 @@ PYBIND11_MODULE(_native, m) { "Returns a ToolSet with read_file/write_file/edit_file/glob/grep/bash."); m.def("with_permissions", &py_with_permissions, py::arg("tools"), py::arg("policy"), - "Wrap a ToolSet with a permission policy(policy)(tool, input_json) -> int (0=allow,1=deny,2=ask)."); + "Wrap a ToolSet with a permission policy.\n\n" + "policy(tool, input_json) -> int (0=allow, 1=deny, 2=ask), or a\n" + "(decision, reason) tuple to explain a refusal. The reason is shown\n" + "to the model as part of the tool result."); m.def("generate_text", &py_generate_text, py::arg("model"), diff --git a/bindings/rust/Cargo.toml b/bindings/rust/Cargo.toml index b79bab7..2763c17 100644 --- a/bindings/rust/Cargo.toml +++ b/bindings/rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ai-sdk-cpp" -version = "0.1.0" +version = "1.0.0" edition = "2021" description = "Rust FFI bindings for ai-sdk-cpp" license = "MIT" diff --git a/include/ai/ai.hpp b/include/ai/ai.hpp index 7d2919e..1d87a7a 100644 --- a/include/ai/ai.hpp +++ b/include/ai/ai.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include diff --git a/include/ai/permission/permission.hpp b/include/ai/permission/permission.hpp index aab7e6a..32238cd 100644 --- a/include/ai/permission/permission.hpp +++ b/include/ai/permission/permission.hpp @@ -16,18 +16,40 @@ enum class PermissionDecision { Ask, ///< escalate to the interactive Approver (if any) }; +/// A policy's answer: the decision, and optionally why. +/// +/// The reason is shown to the model when the call is refused, so it should say +/// what would have to change rather than restate the refusal. A model that is +/// told "the user declined" stops; one that is told nothing retries variants, +/// and every retry is another approval prompt for a user who already answered. +/// +/// Implicitly constructible from a bare PermissionDecision, so a rule with +/// nothing to explain can keep returning Allow/Deny/Ask directly. +struct PermissionVerdict { + PermissionDecision decision; + std::string reason; + + PermissionVerdict() : decision(PermissionDecision::Allow) {} + // NOLINTNEXTLINE(google-explicit-constructor) — deliberate, see above. + PermissionVerdict(PermissionDecision d) : decision(d) {} + PermissionVerdict(PermissionDecision d, std::string why) + : decision(d), reason(std::move(why)) {} +}; + /// Result of an interactive approval. When `decision == Allow` and /// `always_allow` is true, the tool is added to a session-local allowlist so /// subsequent calls skip the approver. struct Approval { PermissionDecision decision; bool always_allow = false; + /// Shown to the model when this refuses a call. Empty for none. + std::string reason; }; /// Synchronous permission rule: inspect (tool, input) and return a decision /// with no I/O. Acts as a fast path before any interactive Approver. A missing /// (empty) policy means "Allow everything". -using PermissionPolicy = std::function; @@ -45,13 +67,17 @@ using Approver = std::function( /// wrapped in a permission gate. The gate applies `policy` first: /// - Allow -> run the tool; /// - Deny -> return `{ "error": "permission_denied", "tool": }` -/// without running; +/// without running, plus `"message": ` when a reason was +/// given; /// - Ask -> `co_await approver` (if provided). An approval of /// `{Allow, always_allow}` adds the tool to a session-local /// allowlist so later calls skip the approver. With no approver /// wired, `Ask` is treated as `Deny` (safe default). /// Tools without an `execute` (schema-only) are copied unchanged. A missing /// (empty) policy makes the gate pass-through (Allow). +/// +/// The denial reaches the model as the tool's result, so `reason` is how it +/// learns that retrying will not help. ToolSet with_permissions(ToolSet tools, PermissionPolicy policy, Approver approver = {}); } // namespace ai diff --git a/include/ai/version.hpp b/include/ai/version.hpp new file mode 100644 index 0000000..097dac3 --- /dev/null +++ b/include/ai/version.hpp @@ -0,0 +1,18 @@ +#pragma once + +// The version the runtime reports — ai_sdk_version() in the C API, the MCP +// clientInfo handshake, and anything else that has to tell a peer what it is +// talking to. +// +// The build defines AI_SDK_VERSION from the CMake project version +// (CMakeLists.txt), so the native string is derived rather than retyped. The +// fallback only applies outside that build system. +#ifndef AI_SDK_VERSION +#define AI_SDK_VERSION "0.0.0-unknown" +#endif + +namespace ai { + +constexpr const char* version() noexcept { return AI_SDK_VERSION; } + +} // namespace ai diff --git a/scripts/coverage.sh b/scripts/coverage.sh new file mode 100755 index 0000000..cb63912 --- /dev/null +++ b/scripts/coverage.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# What do the tests actually exercise? +# +# Clang source-based coverage: builds instrumented, runs ctest, merges the +# profiles, reports per file, then names the source files no test touched. +# +# Scope: the OFFLINE C++ SUITE only. The Node binding suite is not part of this +# profile, and that leaves a real hole — the C++ tests substitute their own +# ai::http::IHttpClient, so src/http/* and the C binding are only reached +# incidentally. The Node suite drives both for real. Read the two together. +# +# The instrumented build goes to its OWN tree (build-coverage/), never the +# default build/. That matters: the Node addon links build/bindings/c at +# runtime, so instrumenting that directory in place would slow every Node test +# down and litter .profraw files through it. +# +# Usage: +# ./scripts/coverage.sh # headline + full report +# ./scripts/coverage.sh --html # ... and browsable HTML +# +# Env: +# AI_SDK_COVERAGE_BUILD_DIR build tree to use (default: build-coverage) +# AI_SDK_COVERAGE_CMAKE_ARGS extra cmake args, e.g. to disable a slow provider +set -euo pipefail +cd "$(dirname "$0")/.." + +BUILD_DIR="${AI_SDK_COVERAGE_BUILD_DIR:-build-coverage}" +COV_DIR="$BUILD_DIR/coverage" +PROF_DIR="$COV_DIR/profraw" +MERGED="$COV_DIR/merged.profdata" +HTML=0 + +for arg in "$@"; do + case "$arg" in + --html) HTML=1 ;; + -h|--help) sed -n '2,22p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown option: $arg (try --help)" >&2; exit 2 ;; + esac +done + +# The profile format is version-tied to the compiler, so prefer the toolchain +# that ships with it — on macOS that is the Xcode one, not some Homebrew LLVM +# that happens to be on PATH. +find_llvm_tool() { + local name="$1" + if [[ "$(uname)" == "Darwin" ]] && xcrun --find "$name" >/dev/null 2>&1; then + xcrun --find "$name"; return 0 + fi + if command -v "$name" >/dev/null 2>&1; then + command -v "$name"; return 0 + fi + local p + for p in /opt/homebrew/opt/llvm/bin /usr/local/opt/llvm/bin /usr/lib/llvm-*/bin; do + if [[ -x "$p/$name" ]]; then echo "$p/$name"; return 0; fi + done + echo "error: $name not found. Install LLVM (macOS: xcode-select --install, or brew install llvm)." >&2 + return 1 +} + +PROFDATA="$(find_llvm_tool llvm-profdata)" +LLVM_COV="$(find_llvm_tool llvm-cov)" +JOBS="$( (sysctl -n hw.ncpu 2>/dev/null || nproc) )" + +echo "==> Configuring $BUILD_DIR with AI_SDK_COVERAGE=ON" +# Compile each source file into exactly ONE image. The default build makes both a +# static and a shared core from the same sources under the same output name, so +# every core function would appear twice and llvm-cov would double-count it. +# BUILD_SHARED_LIBS=ON pulls the providers and the C binding into shared objects +# too, which is also what makes their coverage visible at all — see the note on +# OBJECTS below. +COVERAGE_CMAKE_ARGS="-DAI_SDK_BUILD_STATIC=OFF -DBUILD_SHARED_LIBS=ON" +# shellcheck disable=SC2086 # both the defaults and the override are split on purpose +cmake -B "$BUILD_DIR" -DCMAKE_BUILD_TYPE=Release -DAI_SDK_COVERAGE=ON \ + $COVERAGE_CMAKE_ARGS ${AI_SDK_COVERAGE_CMAKE_ARGS:-} >/dev/null + +echo "==> Building" +cmake --build "$BUILD_DIR" -j"$JOBS" >/dev/null + +rm -rf "$COV_DIR" +mkdir -p "$PROF_DIR" + +# %p keeps concurrent test processes from clobbering each other; %m separates +# the profile written by each linked image (the test binary and every dylib). +echo "==> Running ctest" +LLVM_PROFILE_FILE="$PWD/$PROF_DIR/%p-%m.profraw" \ + ctest --test-dir "$BUILD_DIR" --output-on-failure + +if ! compgen -G "$PROF_DIR/*.profraw" >/dev/null; then + echo "error: no .profraw files were written — was the build instrumented?" >&2 + exit 1 +fi + +echo "==> Merging profiles" +"$PROFDATA" merge -sparse "$PROF_DIR"/*.profraw -o "$MERGED" + +# Every instrumented image has to be named, or the coverage recorded inside it is +# silently dropped. Two traps here, both of which produce a plausible-but-wrong +# number rather than an error: +# +# 1. Each image must be passed with its own -object= flag. Passing them as +# positional arguments looks like it works, but llvm-cov uses only the FIRST +# one and ignores the rest, so you silently get one library's coverage +# reported as the project's. +# 2. A static archive is not enough. ld64 does not carry the __LLVM_COV +# sections of archive members into the linked binary, so a library that is +# only ever linked statically has to be passed as a .a and read directly. +# The coverage build avoids the situation by making everything shared. +OBJECTS=() +add_object() { OBJECTS+=("-object=$1"); } +while IFS= read -r -d '' f; do add_object "$f"; done < <( + find "$BUILD_DIR" -type f \( -name '*.dylib' -o -name '*.so' \) -not -path '*/_deps/*' -print0 +) +# The test binary carries the tests plus any header-only code they instantiate. +if [[ -x "$BUILD_DIR/tests/unit/ai-sdk-tests" ]]; then + add_object "$BUILD_DIR/tests/unit/ai-sdk-tests" +fi + +if [[ ${#OBJECTS[@]} -eq 0 ]]; then + echo "error: no instrumented binaries found under $BUILD_DIR" >&2 + exit 1 +fi + +# Everything that is not this project's library code. +EXTERNAL='(^|/)(_deps|build|build-coverage)/|/usr/|/opt/homebrew/|/Applications/Xcode|/Library/Developer/' +# The shipped library: core, each provider, and the C binding. +LIBRARY="$EXTERNAL|(^|/)(tests|cli|examples)/" +# Public headers show up as one-line rows (a default argument, an inline +# accessor). They are real coverage but they drown the table. +IMPL="$LIBRARY|\.(h|hpp|ipp|inl)$" + +# llvm-cov notes "N functions have mismatched data" whenever the same header-only +# function is instantiated in more than one image, which is the normal case here. +# Collect those instead of letting them land in the middle of a table. +LLVM_NOTES="$COV_DIR/llvm-cov.log" +: > "$LLVM_NOTES" + +report() { + "$LLVM_COV" report "${OBJECTS[@]}" \ + -instr-profile="$MERGED" \ + -ignore-filename-regex="$1" 2>>"$LLVM_NOTES" +} + +echo +echo "============= library implementation, exercised by the offline tests =============" +report "$IMPL" + +echo +echo "Note: src/http/* and bindings/c/* look untouched above because the C++ tests" +echo "mock at ai::http::IHttpClient, one layer above them. The Node suite exercises" +echo "both against a real socket; it is not included in this profile." + +echo +echo "============== the same, with the public headers folded in ==============" +# Headers only: the CLI and example binaries are never run by ctest, so they have +# no profile to contribute and passing them would abort llvm-cov. +report "$LIBRARY" | tail -1 + +# The actionable half of the report: what did no test touch at all? +echo +echo "=========================== never executed by any test ===========================" +BODY="$(report "$IMPL" | sed -n '3,$p')" +TOTAL_FILES="$(echo "$BODY" | awk 'NF>1 && $2>0' | wc -l | tr -d ' ')" +UNTOUCHED="$(echo "$BODY" | awk 'NF>1 && $8>0 && $8==$9 {print $1}' | sort)" +if [[ -z "$UNTOUCHED" ]]; then + echo " (none — every source file ran at least once)" +else + echo "$UNTOUCHED" | sed 's/^/ /' + echo + echo " $(echo "$UNTOUCHED" | wc -l | tr -d ' ') of $TOTAL_FILES source files" +fi + +if [[ "$HTML" == 1 ]]; then + echo + echo "==> Writing HTML to $COV_DIR/html" + "$LLVM_COV" show "${OBJECTS[@]}" \ + -instr-profile="$MERGED" \ + -ignore-filename-regex="$EXTERNAL" \ + -format=html -output-dir="$COV_DIR/html" >/dev/null 2>>"$LLVM_NOTES" + echo " open $COV_DIR/html/index.html" +fi + +if [[ -s "$LLVM_NOTES" ]]; then + echo + echo "==> llvm-cov notes" + sed 's/\x1b\[[0-9;]*m//g' "$LLVM_NOTES" | sort -u | sed 's/^/ /' +fi diff --git a/src/core/stream_text.cpp b/src/core/stream_text.cpp index 37e88b4..1168a67 100644 --- a/src/core/stream_text.cpp +++ b/src/core/stream_text.cpp @@ -403,6 +403,10 @@ AsyncGenerator stream_text_multi_step( final_result.warnings = steps.back().result.warnings; } final_result.steps = std::move(steps); + // Every tool call the whole run executed, matching what generate_text + // reports. Without this the same field is empty or populated depending only + // on which entry point the caller used. + final_result.tool_calls = std::move(all_tool_calls); sink->fulfill(std::move(final_result)); } diff --git a/src/mcp/mcp_client.cpp b/src/mcp/mcp_client.cpp index bb1a5c9..d6bf8e6 100644 --- a/src/mcp/mcp_client.cpp +++ b/src/mcp/mcp_client.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -191,7 +192,7 @@ Task McpClient::connect() { json::object init_params; init_params["protocolVersion"] = "2024-11-05"; - init_params["clientInfo"] = json::object{{"name", "ai-sdk-cpp"}, {"version", "0.1.0"}}; + init_params["clientInfo"] = json::object{{"name", "ai-sdk-cpp"}, {"version", ai::version()}}; init_params["capabilities"] = json::object{}; impl_->send_request("initialize", json::value(init_params)); impl_->send_notification("notifications/initialized"); @@ -377,19 +378,33 @@ Task> McpClient::get_prompt( ai::ToolSet mcp_tools_to_toolset(std::shared_ptr client, const std::vector& tools) { + const std::string& server = client->server_name(); ToolSet toolset; for (auto& mcp_tool : tools) { - auto tool_name = mcp_tool.name; + // Server-qualified, as mcp____. A bare name is ambiguous + // the moment two servers expose the same tool, which makes a permission + // rule keyed on that name unexpressible — "allow add_numbers" cannot say + // which server it means, and an "always allow" the user grants for one + // server would silently cover the other. It is also a shadowing hazard: + // a server offering a tool called "Bash" or "Write" is then + // indistinguishable from the real one in a rule or an approval prompt. + // + // The prefix is ours, not the server's, so call_tool below still speaks + // the bare name. An unnamed client cannot be qualified this way and + // keeps bare names; the C entry point refuses to build one. + const std::string bare_name = mcp_tool.name; + std::string exposed = + server.empty() ? bare_name : "mcp__" + server + "__" + bare_name; auto captured_client = client; ToolDefinition def{ - .name = mcp_tool.name, + .name = std::move(exposed), .description = mcp_tool.description, .input_schema = mcp_tool.input_schema, .strict = false, - .execute = [captured_client, tool_name]( + .execute = [captured_client, bare_name]( boost::json::value input, ToolExecutionContext ) -> Task { - co_return co_await captured_client->call_tool(tool_name, std::move(input)); + co_return co_await captured_client->call_tool(bare_name, std::move(input)); }, }; toolset.add(std::move(def)); diff --git a/src/mcp/transport.cpp b/src/mcp/transport.cpp index 5a9544f..5651f86 100644 --- a/src/mcp/transport.cpp +++ b/src/mcp/transport.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -12,6 +13,7 @@ #ifdef _WIN32 #include #else +#include #include #include #include @@ -105,9 +107,23 @@ struct StdioTransport::Impl { if (pipe(stdin_pipe) != 0 || pipe(stdout_pipe) != 0) { throw std::runtime_error("MCP: failed to create pipes"); } + + // Reports why the child never became the server. Without this a wrong + // command surfaces as "EOF reading from server": execvp fails in the + // child, the parent sees only a closed pipe, and the user is told about + // the symptom instead of the cause. FD_CLOEXEC means a *successful* exec + // closes the write end, so a parent read returning 0 is proof the server + // started, and a full int in return means exec failed with that errno. + int exec_status[2]; + if (pipe(exec_status) != 0) { + throw std::runtime_error("MCP: failed to create pipes"); + } + fcntl(exec_status[1], F_SETFD, FD_CLOEXEC); + pid_t pid = fork(); if (pid < 0) throw std::runtime_error("MCP: failed to fork"); if (pid == 0) { + close(exec_status[0]); close(stdin_pipe[1]); close(stdout_pipe[0]); dup2(stdin_pipe[0], STDIN_FILENO); @@ -120,10 +136,27 @@ struct StdioTransport::Impl { for (auto& a : args) argv.push_back(a.c_str()); argv.push_back(nullptr); execvp(command.c_str(), const_cast(argv.data())); + int exec_errno = errno; + // If even this write fails there is nothing useful left to say; the + // 127 exit status remains as the fallback signal. + ssize_t reported = write(exec_status[1], &exec_errno, sizeof(exec_errno)); + (void)reported; _exit(127); } close(stdin_pipe[0]); close(stdout_pipe[1]); + close(exec_status[1]); + + int exec_errno = 0; + ssize_t exec_report = read(exec_status[0], &exec_errno, sizeof(exec_errno)); + close(exec_status[0]); + if (exec_report == static_cast(sizeof(exec_errno))) { + int status = 0; + waitpid(pid, &status, 0); + throw std::runtime_error(std::string("MCP: failed to spawn server process '") + + command + "': " + std::strerror(exec_errno)); + } + child_pid = pid; child_stdin = fdopen(stdin_pipe[1], "w"); child_stdout = fdopen(stdout_pipe[0], "r"); diff --git a/src/permission/permission.cpp b/src/permission/permission.cpp index f1fd9d4..1d33924 100644 --- a/src/permission/permission.cpp +++ b/src/permission/permission.cpp @@ -60,30 +60,44 @@ ToolSet with_permissions(ToolSet tools, PermissionPolicy policy, Approver approv co_return co_await (*original)(std::move(input), std::move(ctx)); } - PermissionDecision decision = PermissionDecision::Allow; + PermissionVerdict verdict{PermissionDecision::Allow}; if (state->policy) { - decision = state->policy(tool_name, input); + verdict = state->policy(tool_name, input); } + std::string reason = std::move(verdict.reason); - if (decision == PermissionDecision::Ask) { + if (verdict.decision == PermissionDecision::Ask) { if (state->approver) { Approval approval = co_await state->approver( tool_name, input, rationale_for(tool_name, input)); - decision = approval.decision; + verdict.decision = approval.decision; + // An approver that explains itself overrides the policy's + // reason: it saw the same input and answered second. + if (!approval.reason.empty()) { + reason = std::move(approval.reason); + } if (approval.decision == PermissionDecision::Allow && approval.always_allow) { state->always_allowed.insert(tool_name); } } else { - // No interactive approver: fail closed. - decision = PermissionDecision::Deny; + // No interactive approver: fail closed. Say why -- this is + // the one denial with nobody to explain it, and a model + // given no reason will keep trying variations of the call. + verdict.decision = PermissionDecision::Deny; + reason = "no approver is configured to authorize this tool call"; } } - if (decision == PermissionDecision::Deny) { + if (verdict.decision == PermissionDecision::Deny) { boost::json::object err; err["error"] = "permission_denied"; err["tool"] = tool_name; + // Omitted rather than sent empty, so a consumer can tell "no + // reason was given" from "the reason was a blank string". + if (!reason.empty()) { + err["message"] = reason; + } co_return boost::json::value(std::move(err)); } diff --git a/src/stream/data_stream.cpp b/src/stream/data_stream.cpp index b6d013b..39d9192 100644 --- a/src/stream/data_stream.cpp +++ b/src/stream/data_stream.cpp @@ -1,6 +1,8 @@ #include +#include #include #include +#include namespace ai::stream { @@ -69,52 +71,60 @@ std::optional DataStreamDecoder::decode_line(std::string_view line) char type = line[0]; auto payload = line.substr(2); + // These frames arrive off a socket, so a truncated or non-conforming one is + // the expected case rather than an exceptional one. Every parse and every + // field lookup below therefore has to fail as nullopt: an exception here + // unwinds out of the caller's read loop, which is the one place it cannot be + // handled. ai::json wraps boost::json's non-throwing forms for exactly this. + auto val = ai::json::safe_parse(payload); + if (!val) return std::nullopt; + switch (type) { case '0': { - auto val = json::parse(payload); - return TextDelta{.id = "0", .delta = std::string(val.as_string())}; + if (!val->is_string()) return std::nullopt; + return TextDelta{.id = "0", .delta = std::string(val->as_string())}; } case '1': { - auto val = json::parse(payload); - auto& obj = val.as_object(); - return ToolInputStart{ - .id = std::string(obj.at("toolCallId").as_string()), - .tool_name = std::string(obj.at("toolName").as_string()), - }; + auto id = ai::json::get_string(*val, "toolCallId"); + auto name = ai::json::get_string(*val, "toolName"); + if (!id || !name) return std::nullopt; + return ToolInputStart{.id = std::move(*id), .tool_name = std::move(*name)}; } case '2': { - auto val = json::parse(payload); - auto& obj = val.as_object(); - return ToolInputDelta{ - .id = std::string(obj.at("toolCallId").as_string()), - .delta = std::string(obj.at("argsTextDelta").as_string()), - }; + auto id = ai::json::get_string(*val, "toolCallId"); + auto delta = ai::json::get_string(*val, "argsTextDelta"); + if (!id || !delta) return std::nullopt; + return ToolInputDelta{.id = std::move(*id), .delta = std::move(*delta)}; } case '3': { - auto val = json::parse(payload); - auto& obj = val.as_object(); - return ToolInputEnd{.id = std::string(obj.at("toolCallId").as_string())}; + auto id = ai::json::get_string(*val, "toolCallId"); + if (!id) return std::nullopt; + return ToolInputEnd{.id = std::move(*id)}; } case 'd': { - auto val = json::parse(payload); - auto& obj = val.as_object(); + if (!val->is_object()) return std::nullopt; FinishReason reason = FinishReason::Stop; - if (obj.contains("finishReason")) { - auto fr = std::string(obj.at("finishReason").as_string()); - if (fr == "length") reason = FinishReason::Length; - else if (fr == "tool-calls") reason = FinishReason::ToolCalls; + if (auto fr = ai::json::get_string(*val, "finishReason")) { + if (*fr == "length") reason = FinishReason::Length; + else if (*fr == "tool-calls") reason = FinishReason::ToolCalls; } Usage usage; - if (obj.contains("usage")) { - auto& u = obj.at("usage").as_object(); - if (u.contains("promptTokens")) usage.input_tokens.total = (int)u.at("promptTokens").as_int64(); - if (u.contains("completionTokens")) usage.output_tokens.total = (int)u.at("completionTokens").as_int64(); + // `usage` is nested one level down, so it needs its own lookup + // before the accessors apply. + auto& obj = val->as_object(); + if (auto it = obj.find("usage"); it != obj.end() && it->value().is_object()) { + if (auto p = ai::json::get_int(it->value(), "promptTokens")) { + usage.input_tokens.total = static_cast(*p); + } + if (auto c = ai::json::get_int(it->value(), "completionTokens")) { + usage.output_tokens.total = static_cast(*c); + } } return FinishPart{.reason = reason, .usage = usage}; } case 'e': { - auto val = json::parse(payload); - return ErrorPart{.message = std::string(val.as_string())}; + if (!val->is_string()) return std::nullopt; + return ErrorPart{.message = std::string(val->as_string())}; } } return std::nullopt; diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index f417ff5..2e5c865 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -1,6 +1,7 @@ add_executable(ai-sdk-tests test_json_schema.cpp test_sse_parser.cpp + test_base64.cpp test_validator.cpp test_messages.cpp test_cancellation.cpp @@ -19,6 +20,7 @@ add_executable(ai-sdk-tests test_embeddings.cpp test_rerank.cpp test_middleware.cpp + test_data_stream.cpp ) target_link_libraries(ai-sdk-tests PRIVATE @@ -49,6 +51,17 @@ if(AI_SDK_PROVIDER_OPENAI) target_link_libraries(ai-sdk-tests PRIVATE ai-sdk-openai) endif() +# Provider construction and option forwarding. Every provider named here has to +# be linked for its headers to resolve, so the test compiles only when the set +# of enabled providers covers what it exercises. +if(AI_SDK_PROVIDER_OPENAI AND AI_SDK_PROVIDER_ANTHROPIC + AND AI_SDK_PROVIDER_DEEPSEEK AND AI_SDK_PROVIDER_ZAI + AND AI_SDK_PROVIDER_MOONSHOTAI AND AI_SDK_PROVIDER_OPENAI_COMPATIBLE) + target_sources(ai-sdk-tests PRIVATE test_providers.cpp) + target_link_libraries(ai-sdk-tests PRIVATE + ai-sdk-deepseek ai-sdk-zai ai-sdk-moonshotai ai-sdk-openai-compatible) +endif() + include(CTest) include(Catch) catch_discover_tests(ai-sdk-tests) diff --git a/tests/unit/local_http_server.hpp b/tests/unit/local_http_server.hpp new file mode 100644 index 0000000..81458ee --- /dev/null +++ b/tests/unit/local_http_server.hpp @@ -0,0 +1,250 @@ +#pragma once + +// A real HTTP/1.1 server bound to 127.0.0.1, for tests that cannot inject a +// fake ai::http::IHttpClient. +// +// The provider unit tests all substitute IHttpClient, which is why src/http/* +// sits at a few percent coverage: nothing in this suite ever opens a socket. +// Anything reached through the C API is in that position permanently — the C +// binding constructs its own HttpClient and offers no seam — so testing the +// permission gate end-to-end means serving real bytes to a real client. +// +// Scripted, not smart: the Nth request gets the Nth body. The agent loop under +// test makes exactly two calls (one that asks for a tool, one that answers), so +// a list of two bodies is a complete conversation. + +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ai::test { + +class LocalHttpServer { +public: + /// Serves `bodies` in order, one per request. `think_time` is how long to + /// wait before each response, which is how a test reaches the state where + /// the client is idle waiting on the network rather than on its own work. + explicit LocalHttpServer(std::vector bodies, + std::chrono::milliseconds think_time = std::chrono::milliseconds{0}) + : bodies_(std::move(bodies)), think_time_(think_time), acceptor_(ioc_) { + using boost::asio::ip::tcp; + tcp::endpoint endpoint(boost::asio::ip::make_address("127.0.0.1"), 0); + acceptor_.open(endpoint.protocol()); + acceptor_.set_option(boost::asio::socket_base::reuse_address(true)); + acceptor_.bind(endpoint); + acceptor_.listen(); + port_ = acceptor_.local_endpoint().port(); + thread_ = std::thread([this] { serve(); }); + } + + ~LocalHttpServer() { + // Closing the acceptor does not cancel a blocking accept(). POSIX leaves + // that unspecified, and Linux -- unlike BSD, where this harness was + // written -- does not wake the blocked thread: the syscall is holding a + // reference to the listening socket, so closing the descriptor never + // reaches it and join() below waits forever. macOS passing while both + // Linux jobs hung for the full ctest timeout was exactly this. + // + // Two blocking calls need waking, and neither is woken by a close(): + // + // accept(): a connection of our own. The kernel completes the + // handshake into the backlog whether or not anyone is in accept(), so + // this returns from accept() even if the serving thread had not got + // there yet. It then sees `stopping_` and leaves rather than serving + // the connection it just accepted. + // + // read_some(): shutdown(), which makes the socket readable-at-EOF for + // the reader. Today's tests never need this -- the model, and with it + // the client's half of the connection, is destroyed before the server + // -- but read_some() on a connection a client left open is unbounded, + // and a test that forgot to close one would hang here the same way. + stopping_.store(true); + poke(); + if (std::intptr_t fd = live_fd_.load(); fd >= 0) { + shutdown_socket(fd); + } + try { + acceptor_.close(); + } catch (const std::exception&) { + // Already closed, or never opened. + } + if (thread_.joinable()) { + thread_.join(); + } + } + + LocalHttpServer(const LocalHttpServer&) = delete; + LocalHttpServer& operator=(const LocalHttpServer&) = delete; + + /// Point a provider's base_url at this. The path the client appends is + /// ignored, so any suffix works. + std::string base_url() const { return "http://127.0.0.1:" + std::to_string(port_); } + + int requests_served() const { return served_.load(); } + + /// The bodies of the requests received so far, in order. This is how a test + /// sees what the client actually sent — the tool result handed back to the + /// model, for instance, which is otherwise invisible from outside the loop. + std::vector request_bodies() const { + std::lock_guard lk(bodies_mutex_); + return request_bodies_; + } + + /// The nth request body (0-based), or "" if that request has not arrived. + std::string request_body(size_t n) const { + std::lock_guard lk(bodies_mutex_); + return n < request_bodies_.size() ? request_bodies_[n] : std::string{}; + } + +private: + static size_t content_length_of(std::string header_block) { + for (char& c : header_block) { + c = static_cast(std::tolower(static_cast(c))); + } + const std::string key = "content-length:"; + auto pos = header_block.find(key); + if (pos == std::string::npos) { + return 0; + } + pos += key.size(); + auto end = header_block.find('\r', pos); + try { + return static_cast(std::stoul(header_block.substr(pos, end - pos))); + } catch (const std::exception&) { + return 0; + } + } + + /// Open a connection to ourselves, to give a blocked accept() something to + /// return. Best-effort: if it fails the acceptor is closed anyway, and an + /// accept() that has not already been reached will fail on its own. + void poke() { + boost::system::error_code ec; + boost::asio::ip::tcp::socket wake(ioc_); + wake.connect(boost::asio::ip::tcp::endpoint( + boost::asio::ip::address_v4::loopback(), port_), + ec); + } + + /// Wake a thread blocked in read_some() on this descriptor. intptr_t rather + /// than int because a Windows SOCKET is 64-bit. + static void shutdown_socket(std::intptr_t fd) { +#ifdef _WIN32 + ::shutdown(static_cast(fd), SD_BOTH); +#else + ::shutdown(static_cast(fd), SHUT_RDWR); +#endif + } + + void serve() { + using boost::asio::ip::tcp; + boost::system::error_code ec; + while (true) { + tcp::socket sock(ioc_); + acceptor_.accept(sock, ec); + if (ec) { + return; // acceptor closed -> shutdown + } + if (stopping_.load()) { + return; // the wakeup connection from ~LocalHttpServer + } + + live_fd_.store(static_cast(sock.native_handle())); + + std::string pending; + char chunk[4096]; + while (true) { + auto header_end = pending.find("\r\n\r\n"); + while (header_end == std::string::npos) { + size_t n = sock.read_some(boost::asio::buffer(chunk), ec); + if (ec) { + break; // peer closed -> back to accept + } + pending.append(chunk, n); + header_end = pending.find("\r\n\r\n"); + } + if (ec) { + break; + } + + size_t total = header_end + 4 + content_length_of(pending.substr(0, header_end)); + while (pending.size() < total) { + size_t n = sock.read_some(boost::asio::buffer(chunk), ec); + if (ec) { + break; + } + pending.append(chunk, n); + } + if (ec) { + break; + } + { + size_t body_len = total - (header_end + 4); + std::lock_guard lk(bodies_mutex_); + request_bodies_.push_back(pending.substr(header_end + 4, body_len)); + } + pending.erase(0, total); + + // Past the end of the script the last body repeats, so a test + // that makes one call too many fails on an assertion rather + // than on a dead socket. + size_t index = std::min(served_.load(), bodies_.size() - 1); + const std::string& body = bodies_[index]; + served_.fetch_add(1); + + if (think_time_.count() > 0) { + std::this_thread::sleep_for(think_time_); + } + + std::string response = + "HTTP/1.1 200 OK\r\n" + "Content-Type: application/json\r\n" + "Content-Length: " + std::to_string(body.size()) + "\r\n" + "Connection: keep-alive\r\n\r\n" + body; + + boost::asio::write(sock, boost::asio::buffer(response), ec); + if (ec) { + break; + } + } + + // Cleared here rather than by a scope guard so the descriptor is + // never published as shutdown-able after the socket behind it is + // gone. + live_fd_.store(-1); + } + } + + std::vector bodies_; + std::chrono::milliseconds think_time_; + boost::asio::io_context ioc_; + boost::asio::ip::tcp::acceptor acceptor_; + std::thread thread_; + /// Set by the destructor; the serving thread leaves as soon as it sees it. + std::atomic stopping_{false}; + /// The descriptor of the connection being served, or -1. A bare descriptor + /// rather than the socket, so the destructor can shutdown() it without + /// touching an object the serving thread is blocked in. + std::atomic live_fd_{-1}; + std::atomic served_{0}; + // Written by the serving thread, read by the test thread. + mutable std::mutex bodies_mutex_; + std::vector request_bodies_; + unsigned short port_ = 0; +}; + +} // namespace ai::test diff --git a/tests/unit/temp_dir.hpp b/tests/unit/temp_dir.hpp new file mode 100644 index 0000000..a56c4fb --- /dev/null +++ b/tests/unit/temp_dir.hpp @@ -0,0 +1,60 @@ +#pragma once + +// A scratch directory that is unique for the life of one *process*, not merely +// one test case. +// +// The distinction is the whole reason this is a shared header. Every test file +// used to name its directory from a counter in a function-local static, which +// is per-process -- and `catch_discover_tests()` registers each TEST_CASE as a +// separate ctest entry, so `ctest -j` runs them as separate processes. Every +// process therefore handed out the same first few names: two tests running side +// by side picked the same directory, and whichever finished first deleted it +// while the other was still writing to it. That shows up as an intermittent +// failure of whichever test lost the race, with nothing in the output to say +// why -- "MarkdownMemoryStore round-trips records with fidelity" failed under +// `ctest -j` and passed on its own, which is what led here. +// +// The pid makes the name unique across processes; the counter keeps it unique +// within one. Both are needed. + +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#define AI_SDK_TEST_GETPID _getpid +#else +#include +#define AI_SDK_TEST_GETPID getpid +#endif + +namespace ai::test { + +inline std::filesystem::path make_unique_temp_dir(const std::string& prefix) { + static std::atomic counter{0}; + auto dir = std::filesystem::temp_directory_path() / + (prefix + std::to_string(AI_SDK_TEST_GETPID()) + "-" + + std::to_string(++counter)); + std::filesystem::create_directories(dir); + return dir; +} + +/// Removes its directory when it goes out of scope. +struct TempDir { + std::filesystem::path path; + + explicit TempDir(const std::string& prefix = "ai-sdk-test-") + : path(make_unique_temp_dir(prefix)) {} + + ~TempDir() { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } + + TempDir(const TempDir&) = delete; + TempDir& operator=(const TempDir&) = delete; +}; + +} // namespace ai::test diff --git a/tests/unit/test_base64.cpp b/tests/unit/test_base64.cpp new file mode 100644 index 0000000..c183ffc --- /dev/null +++ b/tests/unit/test_base64.cpp @@ -0,0 +1,109 @@ +#include + +#include + +#include +#include +#include +#include + +// base64_encode ships to users through three providers — Anthropic, OpenAI and +// Google all call it to inline image and file bytes as `data:;base64,...` +// — and until this file existed nothing executed it. A padding bug here does not +// throw; it produces a payload the API rejects, or worse, accepts as corrupt. +// +// The vectors below are RFC 4648 §10, which is the only authority worth testing +// against: hand-written expectations would just restate the implementation. + +namespace { + +std::vector bytes_of(std::string_view s) { + return std::vector(s.begin(), s.end()); +} + +/// '=' count at the end of `s` (0 when there is none). +size_t trailing_padding(const std::string& s) { + size_t n = 0; + while (n < s.size() && s[s.size() - 1 - n] == '=') { + ++n; + } + return n; +} + +/// True if a '=' is followed by anything other than more '=' — i.e. padding +/// leaked into the middle of the output. +bool has_interior_padding(const std::string& s) { + auto first = s.find('='); + return first != std::string::npos && s.find_first_not_of('=', first) != std::string::npos; +} + +/// Encoded length of `n` bytes: four characters per three, rounded up. +size_t encoded_length(size_t n) { + return 4 * ((n + 2) / 3); +} + +} // namespace + +TEST_CASE("base64_encode matches the RFC 4648 vectors", "[base64]") { + REQUIRE(ai::util::base64_encode(bytes_of("")) == ""); + REQUIRE(ai::util::base64_encode(bytes_of("f")) == "Zg=="); + REQUIRE(ai::util::base64_encode(bytes_of("fo")) == "Zm8="); + REQUIRE(ai::util::base64_encode(bytes_of("foo")) == "Zm9v"); + REQUIRE(ai::util::base64_encode(bytes_of("foob")) == "Zm9vYg=="); + REQUIRE(ai::util::base64_encode(bytes_of("fooba")) == "Zm9vYmE="); + REQUIRE(ai::util::base64_encode(bytes_of("foobar")) == "Zm9vYmFy"); +} + +TEST_CASE("base64_encode uses the right length and padding for every remainder", "[base64]") { + // Sweeping lengths catches an off-by-one in the `i + 2 < size` loop bound, + // which would mis-place the boundary between full and partial quanta. + for (size_t n = 0; n < 10; ++n) { + std::vector data(n, 'A'); + auto encoded = ai::util::base64_encode(data); + + INFO("input length " << n); + REQUIRE(encoded.size() == encoded_length(n)); + REQUIRE(encoded.size() % 4 == 0); + REQUIRE_FALSE(has_interior_padding(encoded)); + REQUIRE(trailing_padding(encoded) == (3 - n % 3) % 3); + } +} + +TEST_CASE("base64_encode handles the byte values that sit on bit boundaries", "[base64]") { + // 0x00 and 0xFF are where a shift-by-6 goes wrong: all-zero spans the whole + // alphabet, all-ones exercises the `+` and `/` end of the table. + REQUIRE(ai::util::base64_encode(std::vector{0x00}) == "AA=="); + REQUIRE(ai::util::base64_encode(std::vector{0xFF}) == "/w=="); + REQUIRE(ai::util::base64_encode(std::vector{0xFF, 0xFF, 0xFF}) == "////"); + REQUIRE(ai::util::base64_encode(std::vector{0x00, 0x00, 0x00}) == "AAAA"); + REQUIRE(ai::util::base64_encode(std::vector{0xFB, 0xFF}) == "+/8="); +} + +TEST_CASE("base64_encode emits only alphabet characters plus padding", "[base64]") { + // Every possible byte, in every position of the quantum, so no shift can + // index outside base64_chars. + const std::string alphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; + + std::vector all_bytes(256); + for (int b = 0; b < 256; ++b) { + all_bytes[static_cast(b)] = static_cast(b); + } + + for (size_t len : {1U, 2U, 3U, 4U, 255U, 256U}) { + std::vector data(all_bytes.begin(), all_bytes.begin() + len); + auto encoded = ai::util::base64_encode(data); + + INFO("length " << len); + REQUIRE(encoded.find_first_not_of(alphabet) == std::string::npos); + REQUIRE(encoded.size() == encoded_length(len)); + } +} + +TEST_CASE("base64_encode pointer overload agrees with the vector overload", "[base64]") { + auto data = bytes_of("the quick brown fox"); + REQUIRE(ai::util::base64_encode(data.data(), data.size()) == + ai::util::base64_encode(data)); + // A zero-length pointer range must not read through the pointer. + REQUIRE(ai::util::base64_encode(data.data(), 0) == ""); +} diff --git a/tests/unit/test_c_binding.cpp b/tests/unit/test_c_binding.cpp index bc7ba58..550079d 100644 --- a/tests/unit/test_c_binding.cpp +++ b/tests/unit/test_c_binding.cpp @@ -1,8 +1,17 @@ #include #include "ai_sdk.h" +#include "local_http_server.hpp" +#include + +#include +#include +#include #include +#include + +namespace json = boost::json; namespace { @@ -12,11 +21,14 @@ struct Stack { ai_model_t model = nullptr; ai_tool_set_t tools = nullptr; - Stack() { + /// `base_url` NULL keeps the provider's default host; a test that needs to + /// watch a call fail points it at an address with nothing behind it. + explicit Stack(const char* base_url = nullptr) { ctx = ai_context_create(); if (!ctx) return; ai_provider_options_t popts{}; popts.api_key = "test-key"; + popts.base_url = base_url; provider = ai_provider_create(ctx, "anthropic", popts); if (!provider) return; model = ai_model_create(provider, "claude-sonnet-4-5"); @@ -44,6 +56,52 @@ struct Stack { } }; +/// A base URL with nothing listening on it. Binding to port 0 and closing hands +/// back a port the OS just proved was free, so the connect is refused at once +/// rather than hanging until the request times out. +std::string unreachable_base_url() { + boost::asio::io_context ioc; + const boost::asio::ip::tcp::acceptor acc( + ioc, boost::asio::ip::tcp::endpoint(boost::asio::ip::tcp::v4(), 0)); + return "http://127.0.0.1:" + std::to_string(acc.local_endpoint().port()); +} + +/// What a stream callback was handed, reduced to the questions that matter: +/// was it told the turn failed, was it told the turn finished, and what did +/// the failure say. +struct StreamTally { + std::vector types; + std::string error_text; + + static void on_event(ai_stream_event_t event, void* user_data) { + auto* self = static_cast(user_data); + self->types.push_back(event.type); + if (event.type == AI_STREAM_ERROR && event.text) { + self->error_text = event.text; + } + } + + bool saw(ai_stream_event_type_t type) const { + return std::find(types.begin(), types.end(), type) != types.end(); + } + + /// A turn that failed has to say so, and has to *end* on the failure: an + /// error followed by a finish would leave a consumer that stops at the + /// first terminal event believing the turn completed. + void check_reports_failure() const { + INFO("events emitted: " << types.size() << ", error text: " << error_text); + // REQUIRE, not CHECK: back() below needs an element, and the failure + // this is here to catch is a stream that emitted nothing at all. Left + // as a CHECK the test does not report that, it segfaults on the empty + // vector and takes the rest of the run with it. + REQUIRE_FALSE(types.empty()); + CHECK(saw(AI_STREAM_ERROR)); + CHECK_FALSE(saw(AI_STREAM_FINISH)); + CHECK(types.back() == AI_STREAM_ERROR); + CHECK_FALSE(error_text.empty()); + } +}; + } // namespace TEST_CASE("ai_agent_create accepts provider_options_json", "[c_binding]") { @@ -86,3 +144,533 @@ TEST_CASE("ai_agent_create rejects non-object provider_options_json", "[c_bindin REQUIRE(ai_last_error(s.ctx) != nullptr); CHECK(std::string(ai_last_error(s.ctx)).find("JSON object") != std::string::npos); } + +TEST_CASE("an MCP config without a usable name cannot produce a toolset", "[c_binding][mcp]") { + // The name qualifies every tool the server exposes, and that qualified + // string is what a policy or approver is keyed on. A toolset whose tools + // have only bare names is one another server can shadow, so the binding + // refuses to build it rather than handing back something unaddressable. + ai_context_t ctx = ai_context_create(); + REQUIRE(ctx != nullptr); + + for (const char* config : { + R"({"transport":"stdio","command":"/bin/cat"})", // absent + R"({"name":"","transport":"stdio","command":"/bin/cat"})", // empty + R"({"name":42,"transport":"stdio","command":"/bin/cat"})", // wrong type + }) { + INFO("config: " << config); + CHECK(ai_mcp_toolset_from_server(ctx, config) == nullptr); + REQUIRE(ai_last_error(ctx) != nullptr); + CHECK(std::string(ai_last_error(ctx)).find("name") != std::string::npos); + } + + ai_context_destroy(ctx); +} + +/* --------------------------------------------------------------------------- + * Permission gating, driven end to end through the C API. + * + * The decision constants are an enum in the header and an int in the callback + * signature, so every mapping between the two is a place where a comparison can + * be inverted without anything failing to compile. The only way to catch that + * is to run a tool and see whether it ran — which needs a model that asks for + * one, which needs a socket. Hence the loopback server. + * ------------------------------------------------------------------------ */ + +namespace { + +/// Canned OpenAI chat-completion bodies. The agent loop is generate-only +/// (ToolLoopAgent::generate -> generate_text), so these are plain JSON rather +/// than SSE frames. +const char* kToolCallBody = R"({ + "id": "chatcmpl-1", "object": "chat.completion", "created": 1, "model": "gpt-4o", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": null, + "tool_calls": [{"id": "call_1", "type": "function", + "function": {"name": "probe", "arguments": "{}"}}]}, + "finish_reason": "tool_calls" + }], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} +})"; + +const char* kFinalBody = R"({ + "id": "chatcmpl-2", "object": "chat.completion", "created": 2, "model": "gpt-4o", + "choices": [{"index": 0, + "message": {"role": "assistant", "content": "done"}, + "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 3, "total_tokens": 15} +})"; + +const char* kProbeSchema = R"({"type":"object","properties":{},"additionalProperties":false})"; + +/// What the tool itself records. `runs` is the whole point: "was the tool +/// allowed to execute" is not otherwise observable from outside the agent. +struct Probe { + std::atomic runs{0}; +}; + +ai_tool_result_t probe_tool(const char*, const char*, void* user_data) { + static_cast(user_data)->runs.fetch_add(1); + return ai_tool_result_t{R"({"ok":true})", 0}; +} + +TEST_CASE("a tool set describes its tools as JSON", "[c_binding][toolset]") { + ai_tool_set_t tools = ai_tool_set_create(); + REQUIRE(tools != nullptr); + REQUIRE(ai_tool_set_add(tools, "probe", "records that it ran", kProbeSchema, probe_tool, + nullptr) == AI_OK); + + ai_tool_set_description_t desc{}; + REQUIRE(ai_tool_set_describe_json(tools, &desc) == AI_OK); + CHECK(desc.count == 1); + + // Parsed rather than substring-matched: the claim is about the shape a + // consumer can rely on, and "does the text contain this word" would pass + // for output that is not JSON at all. + auto parsed = json::parse(desc.json); + REQUIRE(parsed.is_array()); + REQUIRE(parsed.as_array().size() == 1); + + const auto& entry = parsed.as_array()[0].as_object(); + CHECK(entry.at("name") == "probe"); + CHECK(entry.at("description") == "records that it ran"); + CHECK(entry.at("input_schema").as_object().at("type") == "object"); + + ai_tool_set_description_free(&desc); + CHECK(desc.json == nullptr); + CHECK(desc.count == 0); + ai_tool_set_destroy(tools); +} + +TEST_CASE("a tool set describes its tools in name order", "[c_binding][toolset]") { + // A ToolSet is an unordered_map underneath, so without the sort this comes + // out in a different order every run and no two descriptions can be + // compared. + ai_tool_set_t tools = ai_tool_set_create(); + REQUIRE(tools != nullptr); + for (const char* name : {"zeta", "alpha", "mid"}) { + REQUIRE(ai_tool_set_add(tools, name, "", kProbeSchema, probe_tool, nullptr) == AI_OK); + } + + ai_tool_set_description_t desc{}; + REQUIRE(ai_tool_set_describe_json(tools, &desc) == AI_OK); + REQUIRE(desc.count == 3); + + auto parsed = json::parse(desc.json).as_array(); + CHECK(parsed[0].as_object().at("name") == "alpha"); + CHECK(parsed[1].as_object().at("name") == "mid"); + CHECK(parsed[2].as_object().at("name") == "zeta"); + + ai_tool_set_description_free(&desc); + ai_tool_set_destroy(tools); +} + +TEST_CASE("an empty tool set describes itself as an empty array", "[c_binding][toolset]") { + ai_tool_set_t tools = ai_tool_set_create(); + REQUIRE(tools != nullptr); + + ai_tool_set_description_t desc{}; + REQUIRE(ai_tool_set_describe_json(tools, &desc) == AI_OK); + CHECK(desc.count == 0); + CHECK(std::string(desc.json) == "[]"); + + ai_tool_set_description_free(&desc); + ai_tool_set_destroy(tools); +} + +TEST_CASE("describing a tool set rejects null arguments and frees safely", + "[c_binding][toolset]") { + ai_tool_set_t tools = ai_tool_set_create(); + REQUIRE(tools != nullptr); + + ai_tool_set_description_t desc{}; + CHECK(ai_tool_set_describe_json(nullptr, &desc) == AI_ERROR_INVALID_ARGUMENT); + CHECK(ai_tool_set_describe_json(tools, nullptr) == AI_ERROR_INVALID_ARGUMENT); + + // Free gets called on a zeroed struct by callers who checked the status and + // bailed, and on an already-freed one by callers who did not. Neither may + // touch the heap. + ai_tool_set_description_t zeroed{}; + ai_tool_set_description_free(&zeroed); + ai_tool_set_description_free(nullptr); + + REQUIRE(ai_tool_set_describe_json(tools, &desc) == AI_OK); + ai_tool_set_description_free(&desc); + ai_tool_set_description_free(&desc); + + ai_tool_set_destroy(tools); +} + +/// An agent whose provider talks to a loopback server, wired up with a +/// permission-gated copy of a single `probe` tool. +struct GatedAgent { + ai::test::LocalHttpServer server; + ai_context_t ctx = nullptr; + ai_provider_t provider = nullptr; + ai_model_t model = nullptr; + ai_tool_set_t raw_tools = nullptr; + ai_tool_set_t gated_tools = nullptr; + ai_agent_t agent = nullptr; + Probe probe; + + GatedAgent(std::vector bodies, ai_permission_policy_fn policy, void* policy_ud, + ai_approver_fn approver, void* approver_ud, + std::chrono::milliseconds think_time = std::chrono::milliseconds{0}) + : server(std::move(bodies), think_time) { + ctx = ai_context_create(); + if (!ctx) return; + + // base_url has to outlive the create call: the C struct takes a bare + // const char*, so a temporary .c_str() here would dangle. + std::string base = server.base_url(); + ai_provider_options_t popts{}; + popts.api_key = "test-key"; + popts.base_url = base.c_str(); + provider = ai_provider_create(ctx, "openai", popts); + if (!provider) return; + + model = ai_model_create(provider, "gpt-4o"); + if (!model) return; + + raw_tools = ai_tool_set_create(); + if (!raw_tools) return; + if (ai_tool_set_add(raw_tools, "probe", "records that it ran", kProbeSchema, probe_tool, + &probe) != AI_OK) { + return; + } + + gated_tools = ai_with_permissions_approver(raw_tools, policy, policy_ud, approver, + approver_ud); + if (!gated_tools) return; + + ai_agent_options_t aopts{}; + aopts.model = model; + aopts.tools = gated_tools; + aopts.instructions = "test"; + aopts.max_steps = 4; + agent = ai_agent_create(aopts); + } + + ~GatedAgent() { + if (agent) ai_agent_destroy(agent); + if (gated_tools) ai_tool_set_destroy(gated_tools); + if (raw_tools) ai_tool_set_destroy(raw_tools); + if (model) ai_model_destroy(model); + if (provider) ai_provider_destroy(provider); + if (ctx) ai_context_destroy(ctx); + } + + GatedAgent(const GatedAgent&) = delete; + GatedAgent& operator=(const GatedAgent&) = delete; + + bool ready() const { return agent != nullptr; } + + /// One full agent run: tool-call turn, then a final answer. + ai_status_t call(std::string* out_text = nullptr) { + ai_generate_result_t r{}; + ai_status_t status = ai_agent_call(agent, "go", &r); + if (out_text && r.text) *out_text = r.text; + ai_generate_result_free(&r); + return status; + } +}; + +/// A policy that always asks, so the approver is the only thing deciding. +int always_ask(const char*, const char*, char*, void*) { return AI_PERMISSION_ASK; } + +/// Writes into the SDK's reason buffer the way the header tells callers to. +void write_reason(char* buf, const std::string& text) { + if (buf) { + std::snprintf(buf, AI_REASON_MAX, "%s", text.c_str()); + } +} + +/// Counts how many times the approver was consulted, on top of the answer. +struct ApproverState { + std::atomic calls{0}; + int answer = AI_PERMISSION_DENY; + /// Written into the reason buffer when non-empty. + std::string reason; +}; + +int counting_approver(const char*, const char*, const char*, char* reason, void* user_data) { + auto* st = static_cast(user_data); + st->calls.fetch_add(1); + if (!st->reason.empty()) { + write_reason(reason, st->reason); + } + return st->answer; +} + +} // namespace + +TEST_CASE("a policy that allows runs the tool", "[c_binding][permission]") { + int policy_calls = 0; + // The `-> int` is load-bearing: AI_PERMISSION_ALLOW belongs to an unnamed + // enum, so without it the lambda deduces that enum as its return type and + // stops converting to ai_permission_policy_fn. + auto allow = [](const char*, const char*, char*, void* ud) -> int { + ++*static_cast(ud); + return AI_PERMISSION_ALLOW; + }; + + GatedAgent a({kToolCallBody, kFinalBody}, allow, &policy_calls, nullptr, nullptr); + REQUIRE(a.ready()); + + CHECK(a.call() == AI_OK); + CHECK(a.probe.runs.load() == 1); + CHECK(policy_calls == 1); +} + +TEST_CASE("a policy that denies keeps the tool from running", "[c_binding][permission]") { + auto deny = [](const char*, const char*, char*, void*) -> int { return AI_PERMISSION_DENY; }; + + GatedAgent a({kToolCallBody, kFinalBody}, deny, nullptr, nullptr, nullptr); + REQUIRE(a.ready()); + + // The run still succeeds: a denied tool comes back to the model as an error + // result, it does not abort the agent. + CHECK(a.call() == AI_OK); + CHECK(a.probe.runs.load() == 0); +} + +TEST_CASE("a reason written to the buffer reaches the model", "[c_binding][permission]") { + // The reason is produced inside the callback and consumed by the core, so + // the only proof that the C layer transfers it correctly is to read the + // request the model was sent. A stub that echoes the buffer back would pass + // even if the binding dropped it or copied the wrong bytes. + auto deny = [](const char*, const char*, char* reason, void*) -> int { + write_reason(reason, "the path is outside the project root"); + return AI_PERMISSION_DENY; + }; + + GatedAgent a({kToolCallBody, kFinalBody}, deny, nullptr, nullptr, nullptr); + REQUIRE(a.ready()); + + CHECK(a.call() == AI_OK); + CHECK(a.probe.runs.load() == 0); + REQUIRE(a.server.requests_served() == 2); + + // Second request = the one carrying the tool result back to the model. + const std::string followup = a.server.request_body(1); + REQUIRE(followup.find("permission_denied") != std::string::npos); + CHECK(followup.find("the path is outside the project root") != std::string::npos); +} + +TEST_CASE("a callback that fills the buffer exactly does not over-read", "[c_binding][permission]") { + // The buffer arrives zeroed but a caller can legitimately write every byte + // of it with no NUL -- strncpy of a full-length string, say. Reading that + // with strlen would run off the end, so the SDK caps the copy. This asserts + // the cap holds and the run survives it. + auto deny = [](const char*, const char*, char* reason, void*) -> int { + if (reason) { + for (int i = 0; i < AI_REASON_MAX; ++i) { + reason[i] = 'x'; + } + } + return AI_PERMISSION_DENY; + }; + + GatedAgent a({kToolCallBody, kFinalBody}, deny, nullptr, nullptr, nullptr); + REQUIRE(a.ready()); + + CHECK(a.call() == AI_OK); + CHECK(a.probe.runs.load() == 0); + + // Truncated to one byte short of the buffer, and stopping at the first NUL + // the SDK guarantees. + const std::string followup = a.server.request_body(1); + REQUIRE(followup.find("permission_denied") != std::string::npos); + CHECK(followup.find(std::string(AI_REASON_MAX, 'x')) == std::string::npos); + CHECK(followup.find(std::string(AI_REASON_MAX - 1, 'x')) != std::string::npos); +} + +TEST_CASE("an approver decides when the policy asks", "[c_binding][permission]") { + SECTION("allow") { + ApproverState st; + st.answer = AI_PERMISSION_ALLOW; + + GatedAgent a({kToolCallBody, kFinalBody}, always_ask, nullptr, counting_approver, &st); + REQUIRE(a.ready()); + CHECK(a.call() == AI_OK); + CHECK(a.probe.runs.load() == 1); + CHECK(st.calls.load() == 1); + } + + SECTION("deny") { + ApproverState st; + st.answer = AI_PERMISSION_DENY; + + GatedAgent a({kToolCallBody, kFinalBody}, always_ask, nullptr, counting_approver, &st); + REQUIRE(a.ready()); + CHECK(a.call() == AI_OK); + CHECK(a.probe.runs.load() == 0); + CHECK(st.calls.load() == 1); + } + + SECTION("no approver at all fails closed") { + // Documented behaviour of ai_with_permissions: Ask with nothing to ask + // is a Deny, not an Allow. + GatedAgent a({kToolCallBody, kFinalBody}, always_ask, nullptr, nullptr, nullptr); + REQUIRE(a.ready()); + CHECK(a.call() == AI_OK); + CHECK(a.probe.runs.load() == 0); + } +} + +TEST_CASE("ALLOW_ALWAYS approves the tool for the rest of the toolset's life", + "[c_binding][permission]") { + ApproverState st; + st.answer = AI_PERMISSION_ALLOW_ALWAYS; + + // Four bodies: two complete runs, each asking for the tool once. + GatedAgent a({kToolCallBody, kFinalBody, kToolCallBody, kFinalBody}, always_ask, nullptr, + counting_approver, &st); + REQUIRE(a.ready()); + + CHECK(a.call() == AI_OK); + CHECK(a.probe.runs.load() == 1); + CHECK(st.calls.load() == 1); + + // Second run, same toolset: the allowlist short-circuits before the policy + // and the approver, so both counts stay put while the tool still executes. + CHECK(a.call() == AI_OK); + CHECK(a.probe.runs.load() == 2); + CHECK(st.calls.load() == 1); +} + +TEST_CASE("an out-of-contract approver answer is treated as deny", "[c_binding][permission]") { + // The header promises Allow / Allow-Always / Deny and says anything else is + // Deny. These are the values a caller reaches by accident — a -1 default, a + // stray status code, or the policy's own ASK constant returned from the + // wrong callback. Each must fail closed. + for (const int answer : std::array{-1, 42, 7, static_cast(AI_PERMISSION_ASK)}) { + ApproverState st; + st.answer = answer; + + GatedAgent a({kToolCallBody, kFinalBody}, always_ask, nullptr, counting_approver, &st); + REQUIRE(a.ready()); + + INFO("approver returned: " << answer); + CHECK(a.call() == AI_OK); + CHECK(a.probe.runs.load() == 0); + CHECK(st.calls.load() == 1); + } +} + +TEST_CASE("the approver is given the tool name and a rationale", "[c_binding][permission]") { + struct Seen { + std::string tool; + std::string rationale; + }; + Seen seen; + + auto approver = [](const char* tool, const char*, const char* rationale, char*, + void* ud) -> int { + auto* s = static_cast(ud); + s->tool = tool ? tool : ""; + s->rationale = rationale ? rationale : ""; + return AI_PERMISSION_ALLOW; + }; + + GatedAgent a({kToolCallBody, kFinalBody}, always_ask, nullptr, approver, &seen); + REQUIRE(a.ready()); + CHECK(a.call() == AI_OK); + + CHECK(seen.tool == "probe"); + CHECK(seen.rationale.find("probe") != std::string::npos); +} + +TEST_CASE("waiting on a slow model costs almost no CPU", "[c_binding][perf]") { + // The C API drives a coroutine to completion by polling an io_context, + // because a step that is waiting on the network is resumed from the thread + // that owns the pending future rather than from the loop. That makes the + // loop unable to simply block, and the failure mode when it is written + // naively is a tight spin: right answers, a full core for every concurrent + // request, for as long as the model takes to answer. + // + // Measured at ~0.5% of a core against a server that holds each response. + // The bound is deliberately loose — this is a smoke alarm, not a + // benchmark. What it catches is the ~100% of a core a spin costs. + auto allow = [](const char*, const char*, char*, void*) -> int { + return AI_PERMISSION_ALLOW; + }; + + constexpr auto kThink = std::chrono::milliseconds{200}; + GatedAgent a({kToolCallBody, kFinalBody}, allow, nullptr, nullptr, nullptr, kThink); + REQUIRE(a.ready()); + + const auto cpu_before = std::clock(); + const auto wall_before = std::chrono::steady_clock::now(); + CHECK(a.call() == AI_OK); + const std::chrono::duration wall = std::chrono::steady_clock::now() - wall_before; + const double cpu_s = static_cast(std::clock() - cpu_before) / CLOCKS_PER_SEC; + const double wall_s = wall.count(); + + CHECK(a.probe.runs.load() == 1); + + // Two turns, each held up by the server. Without this the ratio below is + // only testing that a fast machine is fast. + INFO("wall " << wall_s << "s, cpu " << cpu_s << "s"); + REQUIRE(wall_s > 0.3); + CHECK(cpu_s < wall_s / 2); +} + +// Every streaming entry point has to report a failure that the stream itself +// cannot see. A transport failure produces no response at all, so nothing +// downstream can turn it into an event; if the entry point returns its error +// status and emits nothing, the consumer is handed a stream that simply ended. +// +// That is what `ai_session_send_stream` did. The Node binding, whose fallback +// exists to stop a consumer waiting forever, filled the gap with a *finish* — +// so the caller got a turn that had succeeded and produced nothing: zero +// tokens, empty finishReason, no error, and nothing to tell it apart from a +// model that answered with silence. +// +// Only *this* entry point was wrong; the other two already emitted. They are +// covered anyway, because the contract is what is being pinned here, and a +// future edit to either of them should have to fail a test. +TEST_CASE("a stream that cannot reach the provider reports the failure", + "[c_binding][stream]") { + const std::string url = unreachable_base_url(); + Stack s(url.c_str()); + REQUIRE(s.ctx); + REQUIRE(s.model); + + SECTION("ai_stream_text") { + StreamTally tally; + ai_generate_options_t opts{}; + opts.model = s.model; + opts.prompt = "hello"; + opts.max_steps = 1; + + CHECK(ai_stream_text(opts, &StreamTally::on_event, &tally) != AI_OK); + tally.check_reports_failure(); + } + + SECTION("ai_agent_call_stream") { + ai_agent_t agent = s.make_agent(nullptr); + REQUIRE(agent); + + StreamTally tally; + CHECK(ai_agent_call_stream(agent, "hello", &StreamTally::on_event, &tally) != AI_OK); + tally.check_reports_failure(); + + ai_agent_destroy(agent); + } + + SECTION("ai_session_send_stream") { + ai_agent_t agent = s.make_agent(nullptr); + REQUIRE(agent); + ai_session_t session = ai_session_create(agent); + REQUIRE(session); + + StreamTally tally; + CHECK(ai_session_send_stream(session, "hello", &StreamTally::on_event, &tally) != AI_OK); + tally.check_reports_failure(); + + ai_session_destroy(session); + ai_agent_destroy(agent); + } +} diff --git a/tests/unit/test_coding_agent.cpp b/tests/unit/test_coding_agent.cpp index 5343b36..5b9f422 100644 --- a/tests/unit/test_coding_agent.cpp +++ b/tests/unit/test_coding_agent.cpp @@ -6,6 +6,8 @@ #include +#include "temp_dir.hpp" + #include #include #include @@ -26,20 +28,7 @@ T run(ai::Task task, boost::asio::io_context& ioc) { return task.get(); } -fs::path make_unique_temp_dir() { - static std::atomic counter{0}; - auto dir = fs::temp_directory_path() / ("ai-sdk-facade-test-" + std::to_string(++counter)); - fs::create_directories(dir); - return dir; -} -struct TempDir { - fs::path path; - TempDir() : path(make_unique_temp_dir()) {} - ~TempDir() { - std::error_code ec; - fs::remove_all(path, ec); - } -}; +using ai::test::TempDir; } // namespace diff --git a/tests/unit/test_data_stream.cpp b/tests/unit/test_data_stream.cpp new file mode 100644 index 0000000..31ba83a --- /dev/null +++ b/tests/unit/test_data_stream.cpp @@ -0,0 +1,215 @@ +#include + +#include +#include + +#include + +#include +#include +#include + +// The data stream protocol is a wire format, so the exact bytes matter: a +// client written against the documented codes breaks if a prefix or a key name +// drifts, and nothing else in the suite would notice. The encoder and decoder +// below are each other's only consumer in-tree, which means a symmetric mistake +// (both sides agreeing on the wrong thing) would round-trip cleanly and still be +// wrong. The literal expectations are therefore spelled out rather than derived +// from the encoder. + +namespace { + +using ai::stream::DataStreamDecoder; +using ai::stream::DataStreamEncoder; + +std::string encoded(const ai::StreamPart& part) { + DataStreamEncoder encoder; + return encoder.encode(part); +} + +/// Decode, requiring that the line was understood at all. +ai::StreamPart decoded(const std::string& line) { + DataStreamDecoder decoder; + auto part = decoder.decode_line(line); + REQUIRE(part.has_value()); + return *part; +} + +/// What the encoder emits for a part it has no wire representation for. The +/// caller cannot distinguish this from an empty payload, which is worth knowing. +bool has_no_encoding(const ai::StreamPart& part) { + return encoded(part).empty(); +} + +ai::Usage usage_of(int input, int output) { + ai::Usage usage; + usage.input_tokens.total = input; + usage.output_tokens.total = output; + return usage; +} + +} // namespace + +TEST_CASE("the encoder emits the documented code prefix for each part", "[data_stream]") { + // The prefix is the whole contract with the client. + CHECK(encoded(ai::TextDelta{.id = "t", .delta = "hi"}) == "0:\"hi\"\n"); + CHECK(encoded(ai::ToolInputEnd{.id = "call-1"}) == "3:{\"toolCallId\":\"call-1\"}\n"); + CHECK(encoded(ai::ErrorPart{.message = "boom"}) == "e:\"boom\"\n"); + + CHECK(encoded(ai::ToolInputStart{.id = "call-1", .tool_name = "search"}) + == "1:{\"toolCallId\":\"call-1\",\"toolName\":\"search\"}\n"); + CHECK(encoded(ai::ToolInputDelta{.id = "call-1", .delta = "{\"q\":"}) + == "2:{\"toolCallId\":\"call-1\",\"argsTextDelta\":\"{\\\"q\\\":\"}\n"); +} + +TEST_CASE("text deltas round-trip through the encoder and decoder", "[data_stream]") { + // Including the characters that have to be escaped for the framing to hold: + // a literal newline inside a delta must not be mistaken for a line break. + for (const std::string text : {std::string("hello"), std::string(""), + std::string("line\nbreak"), std::string("\"quoted\""), + std::string("uni\u00e9code"), std::string("0:1:2:")}) { + INFO("delta: " << text); + auto part = decoded(encoded(ai::TextDelta{.id = "t", .delta = text})); + REQUIRE(std::holds_alternative(part)); + CHECK(std::get(part).delta == text); + } +} + +TEST_CASE("tool call parts round-trip through the encoder and decoder", "[data_stream]") { + SECTION("start carries the id and the tool name") { + auto part = decoded(encoded(ai::ToolInputStart{.id = "c1", .tool_name = "read_file"})); + REQUIRE(std::holds_alternative(part)); + CHECK(std::get(part).id == "c1"); + CHECK(std::get(part).tool_name == "read_file"); + } + + SECTION("delta carries the raw argument fragment") { + auto part = decoded(encoded(ai::ToolInputDelta{.id = "c1", .delta = "{\"path\":\"a\"}"})); + REQUIRE(std::holds_alternative(part)); + CHECK(std::get(part).id == "c1"); + CHECK(std::get(part).delta == "{\"path\":\"a\"}"); + } + + SECTION("end carries only the id") { + auto part = decoded(encoded(ai::ToolInputEnd{.id = "c1"})); + REQUIRE(std::holds_alternative(part)); + CHECK(std::get(part).id == "c1"); + } +} + +TEST_CASE("finish parts round-trip for every reason the protocol can express", "[data_stream]") { + // Only three of the six FinishReason values have a wire spelling. The other + // three collapse to "other" and come back as Stop, so the round trip is + // lossy — pinned here so the collapse is a known property, not a surprise. + struct Case { + ai::FinishReason sent; + const char* wire; + ai::FinishReason received; + }; + + for (const Case& c : { + Case{ai::FinishReason::Stop, "stop", ai::FinishReason::Stop}, + Case{ai::FinishReason::Length, "length", ai::FinishReason::Length}, + Case{ai::FinishReason::ToolCalls, "tool-calls", ai::FinishReason::ToolCalls}, + Case{ai::FinishReason::ContentFilter, "other", ai::FinishReason::Stop}, + Case{ai::FinishReason::Error, "other", ai::FinishReason::Stop}, + Case{ai::FinishReason::Other, "other", ai::FinishReason::Stop}, + }) { + INFO("wire spelling: " << c.wire); + auto line = encoded(ai::FinishPart{.reason = c.sent, .usage = usage_of(1, 2)}); + CHECK(line.find("\"finishReason\":\"" + std::string(c.wire) + "\"") != std::string::npos); + + auto part = decoded(line); + REQUIRE(std::holds_alternative(part)); + CHECK(std::get(part).reason == c.received); + } +} + +TEST_CASE("usage is only carried when the totals are known", "[data_stream]") { + SECTION("both totals present") { + auto part = decoded(encoded(ai::FinishPart{ + .reason = ai::FinishReason::Stop, .usage = usage_of(11, 22)})); + REQUIRE(std::holds_alternative(part)); + auto usage = std::get(part).usage; + REQUIRE(usage.input_tokens.total.has_value()); + REQUIRE(usage.output_tokens.total.has_value()); + CHECK(*usage.input_tokens.total == 11); + CHECK(*usage.output_tokens.total == 22); + } + + SECTION("a zero total is still transmitted") { + // Zero is a real measurement, not an absent one; an encoder that tested + // the count rather than the optional would drop it. + auto line = encoded(ai::FinishPart{.reason = ai::FinishReason::Stop, + .usage = usage_of(0, 0)}); + CHECK(line.find("\"promptTokens\":0") != std::string::npos); + CHECK(line.find("\"completionTokens\":0") != std::string::npos); + } + + SECTION("unknown totals are omitted, not sent as zero") { + // Reporting "0 prompt tokens" for a provider that did not say is worse + // than reporting nothing: it silently corrupts cost accounting. + auto line = encoded(ai::FinishPart{.reason = ai::FinishReason::Stop, .usage = {}}); + CHECK(line.find("promptTokens") == std::string::npos); + CHECK(line.find("completionTokens") == std::string::npos); + + auto part = decoded(line); + REQUIRE(std::holds_alternative(part)); + CHECK_FALSE(std::get(part).usage.input_tokens.total.has_value()); + } +} + +TEST_CASE("parts with no wire representation encode to nothing", "[data_stream]") { + // The protocol covers text, tool input, finish and error — nothing else. The + // encoder signals that by returning an empty string, which is + // indistinguishable from an empty payload, so anything a caller relies on + // being forwarded has to be checked against this list. + CHECK(has_no_encoding(ai::StreamStart{})); + CHECK(has_no_encoding(ai::TextStart{.id = "t"})); + CHECK(has_no_encoding(ai::TextEnd{.id = "t"})); + + // Reasoning in particular: it is a shipped feature (Anthropic extended + // thinking, DeepSeek reasoning) and it does not survive this encoder. + CHECK(has_no_encoding(ai::ReasoningStart{.id = "r"})); + CHECK(has_no_encoding(ai::ReasoningDelta{.id = "r", .delta = "thinking"})); + CHECK(has_no_encoding(ai::ReasoningEnd{.id = "r"})); + + CHECK(has_no_encoding(ai::ResponseMetadataPart{})); + CHECK(has_no_encoding(ai::RawPart{.raw_value = boost::json::value("raw")})); +} + +TEST_CASE("the decoder rejects lines that are not protocol frames", "[data_stream]") { + DataStreamDecoder decoder; + + // Too short to hold a type and a separator. + CHECK_FALSE(decoder.decode_line("").has_value()); + CHECK_FALSE(decoder.decode_line("0").has_value()); + + // A separator is required, and it has to be in the second position. + CHECK_FALSE(decoder.decode_line("0abc").has_value()); + CHECK_FALSE(decoder.decode_line("00:\"x\"").has_value()); + + // Codes the protocol does not define. + CHECK_FALSE(decoder.decode_line("9:\"x\"").has_value()); + CHECK_FALSE(decoder.decode_line("z:{}").has_value()); + + // A recognised code is not enough if the payload is not usable as one. + CHECK_FALSE(decoder.decode_line("0:").has_value()); +} + +TEST_CASE("the decoder does not throw on a malformed payload", "[data_stream]") { + // decode_line returns optional, which reads as "this may fail" rather than + // "this may throw". It is fed lines off a socket, so malformed input is the + // expected case, not an exceptional one — a throw here lands in the middle + // of a streaming loop. + DataStreamDecoder decoder; + CHECK_NOTHROW(decoder.decode_line("0:{not json}")); + CHECK_NOTHROW(decoder.decode_line("1:{}")); + CHECK_NOTHROW(decoder.decode_line("2:{}")); + CHECK_NOTHROW(decoder.decode_line("3:{}")); + CHECK_NOTHROW(decoder.decode_line("0:123")); + + // And when it does fail, it has to say so rather than hand back a part + // built from partial data. + CHECK_FALSE(decoder.decode_line("1:{}").has_value()); +} diff --git a/tests/unit/test_mcp.cpp b/tests/unit/test_mcp.cpp index 5ef4a85..c0c9e11 100644 --- a/tests/unit/test_mcp.cpp +++ b/tests/unit/test_mcp.cpp @@ -263,3 +263,61 @@ TEST_CASE("MCP progress notifications reach the handler", "[mcp]") { REQUIRE(*total == 100.0); REQUIRE(message == "working"); } + +TEST_CASE("MCP tools are exposed server-qualified but call the bare name", "[mcp]") { + // The name a tool is exposed under is what the model is offered, what a + // permission policy is asked about, and what an approval prompt shows the + // user. A bare name is ambiguous as soon as two servers define the same + // tool, and lets a server expose "Bash" indistinguishably from the real + // one. Qualifying fixes both; the server itself must still be addressed by + // the name it published, since the prefix is ours. + boost::asio::io_context ioc; + + std::string called_name; + auto handler = [&called_name](const std::string& out) -> std::optional { + if (method_of(out) == "tools/call") { + auto parsed = json::parse(out); + auto& params = parsed.as_object()["params"].as_object(); + called_name = std::string(params["name"].as_string()); + } + return standard_handler(out); + }; + + auto client = make_client(handler); + run_void(client->connect(), ioc); + auto tools = run(client->list_tools(), ioc); + REQUIRE(tools.size() == 1); + REQUIRE(tools[0].name == "echo"); // what the server published + + auto toolset = ai::mcp::mcp_tools_to_toolset(client, tools); + + REQUIRE(toolset.size() == 1); + CHECK(toolset.find("mcp__test__echo") != nullptr); + CHECK(toolset.find("echo") == nullptr); + + auto* def = toolset.find("mcp__test__echo"); + REQUIRE(def != nullptr); + REQUIRE(def->execute.has_value()); + run((*def->execute)(json::value(json::object{}), ai::ToolExecutionContext{}), ioc); + + CHECK(called_name == "echo"); +} + +TEST_CASE("an unnamed MCP client exposes bare tool names", "[mcp]") { + // Qualification needs a server name to qualify with. The C entry point + // refuses to build a client without one, so this is only reachable from + // C++, where the constructor takes the name directly — pinned so the + // fallback is a decision rather than an accident. + boost::asio::io_context ioc; + auto transport = std::make_unique(); + transport->handler = standard_handler; + auto client = std::make_shared(std::move(transport), ""); + + run_void(client->connect(), ioc); + auto tools = run(client->list_tools(), ioc); + auto toolset = ai::mcp::mcp_tools_to_toolset(client, tools); + + REQUIRE(toolset.size() == 1); + CHECK(toolset.find("echo") != nullptr); + CHECK(toolset.find("mcp____echo") == nullptr); +} diff --git a/tests/unit/test_memory.cpp b/tests/unit/test_memory.cpp index 49fcda5..f7414d7 100644 --- a/tests/unit/test_memory.cpp +++ b/tests/unit/test_memory.cpp @@ -11,6 +11,8 @@ #include +#include "temp_dir.hpp" + #include #include #include @@ -33,20 +35,7 @@ T run(ai::Task task, boost::asio::io_context& ioc) { return task.get(); } -fs::path make_unique_temp_dir() { - static std::atomic counter{0}; - auto dir = fs::temp_directory_path() / ("ai-sdk-mem-test-" + std::to_string(++counter)); - fs::create_directories(dir); - return dir; -} -struct TempDir { - fs::path path; - TempDir() : path(make_unique_temp_dir()) {} - ~TempDir() { - std::error_code ec; - fs::remove_all(path, ec); - } -}; +using ai::test::TempDir; /// Bag-of-words embedding model over a fixed vocab, so semantically overlapping /// texts get similar vectors — enough to exercise EmbeddingRetriever offline. diff --git a/tests/unit/test_permission.cpp b/tests/unit/test_permission.cpp index 304367f..91ecf44 100644 --- a/tests/unit/test_permission.cpp +++ b/tests/unit/test_permission.cpp @@ -44,6 +44,22 @@ bool is_denied(const boost::json::value& out) { out.as_object().at("error") == "permission_denied"; } +/// The explanation attached to a denial, or "" when the payload carries none. +std::string deny_message(const boost::json::value& out) { + if (!out.is_object()) return {}; + const auto& obj = out.as_object(); + if (!obj.contains("message")) return {}; + const auto& msg = obj.at("message"); + return msg.is_string() ? std::string(msg.as_string()) : std::string{}; +} + +/// Whether the payload has a `message` key at all. Distinct from the message +/// being empty: a denial with no explanation must not look like one that +/// explained itself with a blank string. +bool has_message(const boost::json::value& out) { + return out.is_object() && out.as_object().contains("message"); +} + } // namespace TEST_CASE("with_permissions Allow runs the tool", "[permission]") { @@ -180,3 +196,161 @@ TEST_CASE("with_permissions caches always_allow and skips the approver", "[permi REQUIRE(*counter == 2); REQUIRE(*approver_calls == 1); // not consulted again } + +TEST_CASE("a policy can explain its denial to the model", "[permission]") { + boost::asio::io_context ioc; + auto counter = std::make_shared(0); + + ai::ToolSet tools; + tools.add(make_counting_tool("inc", counter)); + + ai::ToolSet gated = ai::with_permissions( + std::move(tools), + [](const std::string&, const boost::json::value&) { + return ai::PermissionVerdict{ + ai::PermissionDecision::Deny, + "this path is outside the project root"}; + }); + + const auto* def = gated.find("inc"); + auto out = run((*def->execute)( + boost::json::object{{"x", 1}}, ai::ToolExecutionContext{}), ioc); + + REQUIRE(is_denied(out)); + REQUIRE(*counter == 0); + CHECK(deny_message(out) == "this path is outside the project root"); +} + +TEST_CASE("a denial with no reason carries no message key", "[permission]") { + boost::asio::io_context ioc; + auto counter = std::make_shared(0); + + ai::ToolSet tools; + tools.add(make_counting_tool("inc", counter)); + + ai::ToolSet gated = ai::with_permissions( + std::move(tools), + [](const std::string&, const boost::json::value&) { + return ai::PermissionDecision::Deny; + }); + + const auto* def = gated.find("inc"); + auto out = run((*def->execute)( + boost::json::object{{"x", 1}}, ai::ToolExecutionContext{}), ioc); + + REQUIRE(is_denied(out)); + // Absent, not empty: a consumer must be able to tell the two apart. + CHECK_FALSE(has_message(out)); +} + +TEST_CASE("an approver that explains itself overrides the policy's reason", "[permission]") { + boost::asio::io_context ioc; + auto counter = std::make_shared(0); + + ai::ToolSet tools; + tools.add(make_counting_tool("inc", counter)); + + ai::Approver approver = [](const std::string&, const boost::json::value&, + const std::string&) -> ai::Task { + ai::Approval a; + a.decision = ai::PermissionDecision::Deny; + a.reason = "the user declined"; + co_return a; + }; + + ai::ToolSet gated = ai::with_permissions( + std::move(tools), + [](const std::string&, const boost::json::value&) { + return ai::PermissionVerdict{ + ai::PermissionDecision::Ask, "policy wanted a human to look"}; + }, + approver); + + const auto* def = gated.find("inc"); + auto out = run((*def->execute)( + boost::json::object{{"x", 1}}, ai::ToolExecutionContext{}), ioc); + + REQUIRE(is_denied(out)); + REQUIRE(*counter == 0); + // The approver answered second and saw the same input, so its reason wins. + CHECK(deny_message(out) == "the user declined"); +} + +TEST_CASE("a silent approver leaves the policy's reason standing", "[permission]") { + boost::asio::io_context ioc; + auto counter = std::make_shared(0); + + ai::ToolSet tools; + tools.add(make_counting_tool("inc", counter)); + + // Denies without a reason: a bare `Approval{Deny, false}`. + ai::Approver approver = [](const std::string&, const boost::json::value&, + const std::string&) -> ai::Task { + co_return ai::Approval{ai::PermissionDecision::Deny, false}; + }; + + ai::ToolSet gated = ai::with_permissions( + std::move(tools), + [](const std::string&, const boost::json::value&) { + return ai::PermissionVerdict{ + ai::PermissionDecision::Ask, "touches a credentials file"}; + }, + approver); + + const auto* def = gated.find("inc"); + auto out = run((*def->execute)( + boost::json::object{{"x", 1}}, ai::ToolExecutionContext{}), ioc); + + REQUIRE(is_denied(out)); + // The policy's reason is the only explanation available, and it still + // describes why the call was questionable. Drop it and the model learns + // nothing about why the answer was no. + CHECK(deny_message(out) == "touches a credentials file"); +} + +TEST_CASE("Ask with no approver explains the fail-closed denial", "[permission]") { + boost::asio::io_context ioc; + auto counter = std::make_shared(0); + + ai::ToolSet tools; + tools.add(make_counting_tool("inc", counter)); + + ai::ToolSet gated = ai::with_permissions( + std::move(tools), + [](const std::string&, const boost::json::value&) { + return ai::PermissionDecision::Ask; + }); + + const auto* def = gated.find("inc"); + auto out = run((*def->execute)( + boost::json::object{{"x", 1}}, ai::ToolExecutionContext{}), ioc); + + REQUIRE(is_denied(out)); + REQUIRE(*counter == 0); + // Nobody is left to explain this one, so the SDK must. + CHECK_FALSE(deny_message(out).empty()); +} + +TEST_CASE("an allowed tool's reason never reaches the result", "[permission]") { + boost::asio::io_context ioc; + auto counter = std::make_shared(0); + + ai::ToolSet tools; + tools.add(make_counting_tool("inc", counter)); + + ai::ToolSet gated = ai::with_permissions( + std::move(tools), + [](const std::string&, const boost::json::value&) { + // A reason on an Allow is meaningless; it must not leak. + return ai::PermissionVerdict{ + ai::PermissionDecision::Allow, "looks fine to me"}; + }); + + const auto* def = gated.find("inc"); + auto out = run((*def->execute)( + boost::json::object{{"x", 41}}, ai::ToolExecutionContext{}), ioc); + + REQUIRE(out.is_int64()); + CHECK(out.as_int64() == 42); + REQUIRE(*counter == 1); +} diff --git a/tests/unit/test_providers.cpp b/tests/unit/test_providers.cpp new file mode 100644 index 0000000..6b50848 --- /dev/null +++ b/tests/unit/test_providers.cpp @@ -0,0 +1,300 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +// Every wrapper here is a few lines of argument forwarding, and that is exactly +// why it is worth pinning: a wrapper that drops `base_url` on the floor sends +// production traffic to the wrong host and returns a plausible-looking error +// only once a real request is made. That happened in this release cycle with +// createGoogle, and no unit test noticed. +// +// The assertions below stay offline. Where a factory hands back the inner +// OpenAI/Anthropic provider, that provider exposes the resolved options, so the +// URL and the provider-options namespace can be read back directly instead of +// being inferred from a live response. + +namespace { + +boost::asio::io_context& loop() { + static boost::asio::io_context ioc; + return ioc; +} + +/// Blank an environment variable for the duration of a test and restore it on +/// the way out. Without this the "no key configured" cases would measure the +/// developer's shell rather than the code: these tests are routinely run by +/// someone who has DEEPSEEK_API_KEY exported. +class ScopedEnvUnset { +public: + explicit ScopedEnvUnset(const char* name) : name_(name) { + if (const char* old = std::getenv(name)) { + saved_ = old; + had_ = true; + } + unsetenv(name); + } + + ~ScopedEnvUnset() { + if (had_) { + setenv(name_.c_str(), saved_->c_str(), 1); + } else { + unsetenv(name_.c_str()); + } + } + + ScopedEnvUnset(const ScopedEnvUnset&) = delete; + ScopedEnvUnset& operator=(const ScopedEnvUnset&) = delete; + + /// The message the factory produced for the missing key. Catch2's + /// REQUIRE_THROWS_WITH can match by wildcard, but the point here is that the + /// message *names the variable to set* — an operator reading the error has + /// to be told which knob to turn. + static std::string message_of(const std::function& fn) { + try { + fn(); + } catch (const std::exception& e) { + return e.what(); + } + return {}; + } + +private: + std::string name_; + std::optional saved_; + bool had_ = false; +}; + +} // namespace + +TEST_CASE("each provider wrapper reports its own id", "[provider]") { + CHECK(ai::providers::deepseek::create_deepseek( + {.api_key = "k", .io_context = loop()}) + ->provider_id() == "deepseek"); + CHECK(ai::providers::zai::create_zai( + {.api_key = "k", .io_context = loop()}) + ->provider_id() == "zai"); + CHECK(ai::providers::moonshotai::create_moonshotai( + {.api_key = "k", .io_context = loop()}) + ->provider_id() == "moonshotai"); + CHECK(ai::providers::openai_compatible::create_openai_compatible( + {.name = "my-gateway", .base_url = "https://gw.example/v1", + .api_key = "k", .io_context = loop()}) + ->provider_id() == "my-gateway"); +} + +TEST_CASE("each provider wrapper produces a language model", "[provider]") { + // A wrapper that fails to delegate would return null here rather than at + // some later point in a request. + auto deepseek = ai::providers::deepseek::create_deepseek( + {.api_key = "k", .io_context = loop()}); + auto model = deepseek->language_model("deepseek-chat"); + REQUIRE(model != nullptr); + CHECK(model->model_id() == "deepseek-chat"); + + auto moonshot = ai::providers::moonshotai::create_moonshotai( + {.api_key = "k", .io_context = loop()}); + REQUIRE(moonshot->language_model("kimi-k2") != nullptr); + + auto gateway = ai::providers::openai_compatible::create_openai_compatible( + {.name = "my-gateway", .base_url = "https://gw.example/v1", + .api_key = "k", .io_context = loop()}); + REQUIRE(gateway->language_model("any-model") != nullptr); +} + +// These two factories hand back the inner provider rather than a wrapper of +// their own, which makes the wiring inspectable: whatever the caller passed has +// to be what the provider that actually builds requests is holding. Reading the +// URL back off it is the cheapest way to catch a wrapper that drops the option +// on the floor — the failure that reached this release through createGoogle. + +TEST_CASE("zai_openai forwards base_url and the zai options namespace", "[provider]") { + auto provider = std::dynamic_pointer_cast( + ai::providers::zai::create_zai_openai({.api_key = "k", .io_context = loop()})); + REQUIRE(provider != nullptr); + + CHECK(provider->options().base_url == "https://api.z.ai/api/paas/v4"); + // The namespace is what provider-scoped options are looked up under, so a + // wrapper that forgets it silently drops the caller's options. + CHECK(provider->options().provider_options_namespace == "zai"); +} + +TEST_CASE("deepseek_anthropic forwards base_url and the deepseek options namespace", + "[provider]") { + auto provider = std::dynamic_pointer_cast( + ai::providers::deepseek::create_deepseek_anthropic({.api_key = "k", .io_context = loop()})); + REQUIRE(provider != nullptr); + + CHECK(provider->options().base_url == "https://api.deepseek.com/anthropic"); + CHECK(provider->options().provider_options_namespace == "deepseek"); + CHECK(provider->messages_url() == "https://api.deepseek.com/anthropic/v1/messages"); +} + +TEST_CASE("a caller-supplied base_url replaces the wrapper default", "[provider]") { + // The point of the option: route through a proxy or a regional host. The + // default must not survive it. + auto zai = std::dynamic_pointer_cast( + ai::providers::zai::create_zai_openai( + {.api_key = "k", .base_url = "https://proxy.internal/v1", .io_context = loop()})); + REQUIRE(zai != nullptr); + CHECK(zai->options().base_url == "https://proxy.internal/v1"); + CHECK(zai->chat_completions_url() == "https://proxy.internal/v1/chat/completions"); + + auto deepseek = std::dynamic_pointer_cast( + ai::providers::deepseek::create_deepseek_anthropic( + {.api_key = "k", .base_url = "https://proxy.internal/anthropic", + .io_context = loop()})); + REQUIRE(deepseek != nullptr); + CHECK(deepseek->options().base_url == "https://proxy.internal/anthropic"); +} + +TEST_CASE("the factories disagree about which provider they hand back", "[provider]") { + // Worth stating out loud because it is not obvious from the call sites: + // create_deepseek, create_zai and create_moonshotai return a wrapper that + // owns an inner provider, while create_zai_openai and + // create_deepseek_anthropic return the inner OpenAI/Anthropic provider + // directly. The second group therefore answers `openai`/`anthropic` for + // provider_id(), not the name of the factory that built it. + // + // Callers that branch on provider_id() — logging, telemetry, routing — see + // the difference. Pinned here so it cannot change silently. + SECTION("wrappers report their own id") { + CHECK(ai::providers::deepseek::create_deepseek({.api_key = "k", .io_context = loop()}) + ->provider_id() == "deepseek"); + CHECK(ai::providers::zai::create_zai({.api_key = "k", .io_context = loop()}) + ->provider_id() == "zai"); + } + + SECTION("delegating factories report the inner id") { + CHECK(ai::providers::zai::create_zai_openai({.api_key = "k", .io_context = loop()}) + ->provider_id() == "openai"); + CHECK(ai::providers::deepseek::create_deepseek_anthropic( + {.api_key = "k", .io_context = loop()}) + ->provider_id() == "anthropic"); + } + + SECTION("the generic wrapper is a wrapper, not the inner provider") { + // Unlike the two above, this one does not hand back an OpenAIProvider, + // so its base_url cannot be read back off the result. Its forwarding is + // covered by the language_model() assertions instead. + auto provider = ai::providers::openai_compatible::create_openai_compatible( + {.name = "my-gateway", .base_url = "https://gw.example/v1", + .api_key = "k", .io_context = loop()}); + REQUIRE(provider != nullptr); + CHECK(provider->provider_id() == "my-gateway"); + CHECK(std::dynamic_pointer_cast(provider) + == nullptr); + } +} + +TEST_CASE("zai authenticates the Anthropic endpoint with a bearer token", "[provider]") { + // z.ai rejects an Anthropic-style `x-api-key`; it wants `Authorization: + // Bearer`. The wrapper passes the key through `auth_token` rather than + // `api_key` to get that, and swapping the two would only surface as a 401 + // against the live API. Read the headers back instead. + ai::providers::anthropic::AnthropicOptions opts{.io_context = loop()}; + ai::providers::anthropic::AnthropicProvider provider(std::move(opts)); + + auto bearer = provider.auth_headers(); + CHECK(bearer.find("x-api-key") == bearer.end()); +} + +TEST_CASE("a missing API key names the variable to set", "[provider]") { + // Whitespace in the message is not the contract; naming the env var is. + SECTION("deepseek") { + ScopedEnvUnset guard("DEEPSEEK_API_KEY"); + auto provider = ai::providers::deepseek::create_deepseek({.io_context = loop()}); + auto message = ScopedEnvUnset::message_of( + [&] { provider->language_model("deepseek-chat"); }); + CHECK(message.find("DEEPSEEK_API_KEY") != std::string::npos); + } + + SECTION("moonshotai") { + ScopedEnvUnset guard("MOONSHOT_API_KEY"); + auto provider = ai::providers::moonshotai::create_moonshotai({.io_context = loop()}); + auto message = ScopedEnvUnset::message_of( + [&] { provider->language_model("kimi-k2"); }); + CHECK(message.find("MOONSHOT_API_KEY") != std::string::npos); + } + + SECTION("zai") { + ScopedEnvUnset guard("ZAI_API_KEY"); + auto message = ScopedEnvUnset::message_of( + [&] { ai::providers::zai::create_zai({.io_context = loop()}); }); + CHECK(message.find("ZAI_API_KEY") != std::string::npos); + } + + SECTION("openai_compatible names the caller's variable, not a hardcoded one") { + ScopedEnvUnset guard("MY_GATEWAY_KEY"); + auto provider = ai::providers::openai_compatible::create_openai_compatible( + {.name = "my-gateway", .base_url = "https://gw.example/v1", + .api_key_env_var = "MY_GATEWAY_KEY", .io_context = loop()}); + auto message = ScopedEnvUnset::message_of( + [&] { provider->language_model("any-model"); }); + CHECK(message.find("MY_GATEWAY_KEY") != std::string::npos); + } +} + +TEST_CASE("an explicit API key is used even when the environment has one", "[provider]") { + // The other direction of the same branch: an exported key must not shadow + // the one the caller passed in. + setenv("DEEPSEEK_API_KEY", "from-environment", 1); + auto provider = ai::providers::deepseek::create_deepseek( + {.api_key = "from-caller", .io_context = loop()}); + + // The wrapper keeps the caller's value; observing it means reading it back + // off the provider it built. + auto model = provider->language_model("deepseek-chat"); + CHECK(model != nullptr); +} + +TEST_CASE("zai strips the trailing context alias from a model id", "[provider]") { + using ai::providers::zai::sanitize_model_id; + + // The alias is how Claude Code spells a 1M-token context window. z.ai + // rejects it with `[1211] Unknown Model`, so it has to come off before the + // request is built — but only it. + CHECK(sanitize_model_id("glm-5.2[1m]") == "glm-5.2"); + CHECK(sanitize_model_id("glm-5.2") == "glm-5.2"); + + SECTION("a non-alias bracket is left alone") { + // Brackets are legitimate inside a model name when they are not at the + // end; only a trailing group is an alias. + CHECK(sanitize_model_id("glm[5.2]x") == "glm[5.2]x"); + CHECK(sanitize_model_id("glm-5.2[1m") == "glm-5.2[1m"); + CHECK(sanitize_model_id("glm-5.2]") == "glm-5.2]"); + CHECK(sanitize_model_id("glm-5.2[") == "glm-5.2["); + } + + SECTION("only one alias comes off") { + CHECK(sanitize_model_id("glm-5.2[1m][2m]") == "glm-5.2[1m]"); + } + + SECTION("multi-byte model names survive") { + // The scan is byte-wise. '[' and ']' cannot appear as a UTF-8 + // continuation byte, so it cannot land inside a character. + CHECK(sanitize_model_id("模型-5.2[1m]") == "模型-5.2"); + CHECK(sanitize_model_id("模型-5.2") == "模型-5.2"); + } + + SECTION("short inputs are not read past their end") { + CHECK(sanitize_model_id("") == ""); + CHECK(sanitize_model_id("[") == "["); + CHECK(sanitize_model_id("]") == "]"); + // Degenerate: the whole id is an empty alias, so nothing is left. + CHECK(sanitize_model_id("[]") == ""); + } +} diff --git a/tests/unit/test_session_store.cpp b/tests/unit/test_session_store.cpp index 5f0c76b..88cf84f 100644 --- a/tests/unit/test_session_store.cpp +++ b/tests/unit/test_session_store.cpp @@ -10,6 +10,8 @@ #include #include +#include "temp_dir.hpp" + #include #include #include @@ -32,22 +34,7 @@ T run(ai::Task task, boost::asio::io_context& ioc) { return task.get(); } -fs::path make_unique_temp_dir() { - static std::atomic counter{0}; - auto dir = fs::temp_directory_path() / - ("ai-sdk-store-test-" + std::to_string(++counter)); - fs::create_directories(dir); - return dir; -} - -struct TempDir { - fs::path path; - TempDir() : path(make_unique_temp_dir()) {} - ~TempDir() { - std::error_code ec; - fs::remove_all(path, ec); - } -}; +using ai::test::TempDir; ai::SessionSnapshot make_snapshot() { ai::SessionSnapshot snap; diff --git a/tests/unit/test_standard_toolkit.cpp b/tests/unit/test_standard_toolkit.cpp index 661ac4f..9945464 100644 --- a/tests/unit/test_standard_toolkit.cpp +++ b/tests/unit/test_standard_toolkit.cpp @@ -6,6 +6,8 @@ #include +#include "temp_dir.hpp" + #include #include #include @@ -28,23 +30,7 @@ T run(ai::Task task, boost::asio::io_context& ioc) { return task.get(); } -fs::path make_unique_temp_dir() { - static std::atomic counter{0}; - auto dir = fs::temp_directory_path() / - ("ai-sdk-toolkit-test-" + std::to_string(++counter)); - fs::create_directories(dir); - return dir; -} - -// RAII temp directory. -struct TempDir { - fs::path path; - TempDir() : path(make_unique_temp_dir()) {} - ~TempDir() { - std::error_code ec; - fs::remove_all(path, ec); - } -}; +using ai::test::TempDir; void write_file(const fs::path& p, const std::string& content) { fs::create_directories(p.parent_path()); diff --git a/tests/unit/test_streaming.cpp b/tests/unit/test_streaming.cpp index 762a501..ebcd871 100644 --- a/tests/unit/test_streaming.cpp +++ b/tests/unit/test_streaming.cpp @@ -8,11 +8,17 @@ #include #include #include +#include #include +#include +#include #include +#include +#include #include +#include #include #include @@ -274,3 +280,311 @@ TEST_CASE("stream_text full_result resolves after draining", "[streaming]") { REQUIRE(fr.usage.input_tokens.total.value_or(0) == 5); REQUIRE(fr.usage.output_tokens.total.value_or(0) == 8); } + +// --------------------------------------------------------------------------- +// stream_text's multi-step path. +// +// stream_text branches on `max_steps <= 1 || tools.empty()` and only then enters +// the loop that executes tools and re-invokes the model. Every streaming case +// above takes the single-step branch, so the loop — and the tool-call +// accumulation, reasoning replay and step bookkeeping inside it — went +// unexecuted. These cases are the ones that reach it. +// --------------------------------------------------------------------------- + +namespace { + +/// A LanguageModel that answers with a different scripted stream per call and +/// remembers the prompts it was handed. A model that can only answer once cannot +/// reach the multi-step loop, which re-invokes it once per tool round trip. +/// +/// Past the end of the script the last step replays, which is how a model that +/// refuses to stop asking for tools is expressed. +class ScriptedStreamModel : public ai::LanguageModel { +public: + explicit ScriptedStreamModel(std::vector> script) + : script_(std::move(script)) { + REQUIRE_FALSE(script_.empty()); + } + + std::string_view provider() const override { return "scripted"; } + std::string_view model_id() const override { return "scripted-model"; } + + ai::Task do_generate(ai::CallOptions) override { + throw std::runtime_error("ScriptedStreamModel: do_generate not implemented"); + } + + ai::Task do_stream(ai::CallOptions options) override { + prompts.push_back(std::move(options.prompt)); + size_t index = std::min(static_cast(call_count_++), script_.size() - 1); + auto gen = [](std::vector parts) -> ai::AsyncGenerator { + for (auto& p : parts) { + co_yield p; + } + }(script_[index]); + co_return ai::StreamResult{.stream = std::move(gen)}; + } + + int call_count() const { return call_count_; } + std::vector prompts; + +private: + std::vector> script_; + int call_count_ = 0; +}; + +/// Drain a stream, keeping the parts so the caller can assert on what the +/// consumer actually saw (as opposed to what the internal accumulator built). +ai::Task> drain_parts(ai::AsyncGenerator s) { + std::vector out; + while (auto p = co_await s.next()) { + out.push_back(*p); + } + co_return out; +} + +ai::Usage usage_of(int in, int out) { + ai::Usage u{}; + u.input_tokens.total = in; + u.output_tokens.total = out; + return u; +} + +/// A tool that always succeeds, recording the input it was handed. +ai::ToolDefinition recording_tool(std::shared_ptr> log) { + return ai::tool( + "lookup", ai::schema::JsonSchema::object({}).additional_properties(false), + "look something up", + [log](boost::json::value input, ai::ToolExecutionContext) -> ai::Task { + log->push_back(boost::json::serialize(input)); + co_return boost::json::value("found"); + }); +} + +/// Stream the parts of one tool call, then finish asking for tools. +std::vector tool_call_step(std::vector prefix, int in, int out) { + prefix.push_back(ai::ToolInputStart{.id = "tc1", .tool_name = "lookup"}); + prefix.push_back(ai::ToolInputDelta{.id = "tc1", .delta = "{\"k\":"}); + prefix.push_back(ai::ToolInputDelta{.id = "tc1", .delta = "\"v\"}"}); + prefix.push_back(ai::ToolInputEnd{.id = "tc1"}); + prefix.push_back(ai::FinishPart{.reason = ai::FinishReason::ToolCalls, .usage = usage_of(in, out)}); + return prefix; +} + +} // namespace + +TEST_CASE("stream_text runs a tool step, then streams the final answer", "[streaming]") { + boost::asio::io_context ioc; + auto log = std::make_shared>(); + + auto model = std::make_shared(std::vector>{ + tool_call_step({ai::TextDelta{.id = "0", .delta = "Checking"}}, 10, 5), + { + ai::TextDelta{.id = "0", .delta = "The answer"}, + ai::FinishPart{.reason = ai::FinishReason::Stop, .usage = usage_of(7, 3)}, + }, + }); + + ai::ToolSet tools; + tools.add(recording_tool(log)); + + auto started = run(ai::stream_text(ai::StreamTextOptions{ + .model = model, + .tools = std::move(tools), + .prompt = std::string("go"), + .max_steps = 3, + }), ioc); + + auto seen = run(drain_parts(std::move(started.stream)), ioc); + auto final = run(std::move(started.full_result), ioc); + + // The loop executed the tool and went back to the model for a second step. + REQUIRE(model->call_count() == 2); + REQUIRE(log->size() == 1); + REQUIRE((*log)[0] == "{\"k\":\"v\"}"); + + REQUIRE(final.steps.size() == 2); + REQUIRE(final.steps[0].step_number == 0); + REQUIRE(final.steps[1].step_number == 1); + REQUIRE(final.steps[0].result.finish_reason == ai::FinishReason::ToolCalls); + REQUIRE(final.steps[0].tool_results.size() == 1); + REQUIRE(final.steps[0].tool_results[0].tool_call_id == "tc1"); + REQUIRE(final.steps[0].tool_results[0].output.as_string() == "found"); + REQUIRE_FALSE(final.steps[0].tool_results[0].is_error); + + // The last step is what the caller sees as "the" text and finish reason. + REQUIRE(final.finish_reason == ai::FinishReason::Stop); + REQUIRE(final.text == "The answer"); + + // Usage is the sum over steps, not just the last one. + REQUIRE(final.usage.input_tokens.total.value_or(0) == 17); + REQUIRE(final.usage.output_tokens.total.value_or(0) == 8); + + // generate_text populates result.tool_calls; the streaming path must agree, + // or the same field means two different things depending on which call the + // caller used. + REQUIRE(final.tool_calls.size() == 1); + REQUIRE(final.tool_calls[0].tool_call_id == "tc1"); + REQUIRE(final.tool_calls[0].output.as_string() == "found"); + + // The tool-call parts reached the consumer, not just the accumulator. + size_t starts = 0; + for (auto& p : seen) { + if (std::get_if(&p)) { + ++starts; + } + } + REQUIRE(starts == 1); +} + +TEST_CASE("stream_text replays reasoning with its signature into the next step", "[streaming]") { + boost::asio::io_context ioc; + auto log = std::make_shared>(); + + auto model = std::make_shared(std::vector>{ + tool_call_step( + { + ai::ReasoningStart{.id = "r0"}, + ai::ReasoningDelta{.id = "r0", .delta = "weighing"}, + ai::ReasoningDelta{.id = "r0", .delta = " options"}, + ai::ReasoningEnd{.id = "r0", .signature = std::string("sig-1")}, + }, + 1, 2), + { + ai::TextDelta{.id = "0", .delta = "done"}, + ai::FinishPart{.reason = ai::FinishReason::Stop}, + }, + }); + + ai::ToolSet tools; + tools.add(recording_tool(log)); + + auto started = run(ai::stream_text(ai::StreamTextOptions{ + .model = model, + .tools = std::move(tools), + .prompt = std::string("go"), + .max_steps = 3, + }), ioc); + + run(drain_parts(std::move(started.stream)), ioc); + auto final = run(std::move(started.full_result), ioc); + + REQUIRE(model->call_count() == 2); + REQUIRE(model->prompts.size() == 2); + + // The assistant turn replayed to the model must carry the reasoning block + // with its signature intact: a provider that signs its thinking rejects a + // replayed block whose signature was dropped, which is what breaks + // multi-turn tool loops with extended thinking on. + const ai::AssistantMessage* assistant = nullptr; + for (auto& m : model->prompts[1]) { + if (auto* a = std::get_if(&m)) { + assistant = a; + } + } + REQUIRE(assistant != nullptr); + + const ai::ReasoningPart* replayed = nullptr; + for (auto& c : assistant->content) { + if (auto* r = std::get_if(&c)) { + replayed = r; + } + } + REQUIRE(replayed != nullptr); + REQUIRE(replayed->text == "weighing options"); + REQUIRE(replayed->signature.has_value()); + REQUIRE(*replayed->signature == "sig-1"); + + // ...and the caller sees the reasoning on the step that produced it. + const ai::ReasoningContent* step_reasoning = nullptr; + for (auto& c : final.steps[0].result.content) { + if (auto* r = std::get_if(&c)) { + step_reasoning = r; + } + } + REQUIRE(step_reasoning != nullptr); + REQUIRE(step_reasoning->text == "weighing options"); + REQUIRE(step_reasoning->signature.has_value()); + REQUIRE(*step_reasoning->signature == "sig-1"); +} + +TEST_CASE("stream_text stops at max_steps when the model keeps calling tools", "[streaming]") { + boost::asio::io_context ioc; + auto log = std::make_shared>(); + + // One scripted step; the model replays it forever. + auto model = std::make_shared( + std::vector>{tool_call_step({}, 1, 1)}); + + ai::ToolSet tools; + tools.add(recording_tool(log)); + + auto started = run(ai::stream_text(ai::StreamTextOptions{ + .model = model, + .tools = std::move(tools), + .prompt = std::string("go"), + .max_steps = 2, + }), ioc); + + run(drain_parts(std::move(started.stream)), ioc); + auto final = run(std::move(started.full_result), ioc); + + // Nothing in the transcript stops the loop, so the bound has to. + REQUIRE(model->call_count() == 2); + REQUIRE(final.steps.size() == 2); + REQUIRE(log->size() == 2); + REQUIRE(final.finish_reason == ai::FinishReason::ToolCalls); + REQUIRE(final.tool_calls.size() == 2); +} + +TEST_CASE("stream_text reports each step and marks a tool that fails", "[streaming]") { + boost::asio::io_context ioc; + + auto model = std::make_shared(std::vector>{ + tool_call_step({}, 1, 1), + { + ai::TextDelta{.id = "0", .delta = "recovered"}, + ai::FinishPart{.reason = ai::FinishReason::Stop}, + }, + }); + + ai::ToolSet tools; + tools.add(ai::tool( + "lookup", ai::schema::JsonSchema::object({}).additional_properties(false), + "always fails", + [](boost::json::value, ai::ToolExecutionContext) -> ai::Task { + throw std::runtime_error("boom"); + })); + + std::vector reported; + auto started = run(ai::stream_text(ai::StreamTextOptions{ + .model = model, + .tools = std::move(tools), + .prompt = std::string("go"), + .max_steps = 3, + .on_step_finish = [&reported](const ai::StepResult& s) { + reported.push_back(s.step_number); + }, + }), ioc); + + run(drain_parts(std::move(started.stream)), ioc); + auto final = run(std::move(started.full_result), ioc); + + // A throwing tool is reported, not propagated: the loop still completes. + REQUIRE(reported == std::vector{0, 1}); + REQUIRE(model->call_count() == 2); + REQUIRE(final.steps[0].tool_results.size() == 1); + REQUIRE(final.steps[0].tool_results[0].is_error); + REQUIRE(final.steps[0].tool_results[0].output.as_string().find("boom") != std::string::npos); + + // The failure is still appended to the transcript as an error result, so the + // model can see why and recover rather than repeating the call. + const ai::ToolMessage* tool_turn = nullptr; + for (auto& m : model->prompts[1]) { + if (auto* t = std::get_if(&m)) { + tool_turn = t; + } + } + REQUIRE(tool_turn != nullptr); + REQUIRE(tool_turn->content.size() == 1); + REQUIRE(std::holds_alternative(tool_turn->content[0].output)); +} diff --git a/vcpkg.json b/vcpkg.json index 0cbe394..2c480ed 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,7 +1,7 @@ { "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", "name": "ai-sdk-cpp", - "version": "0.1.0", + "version": "1.0.0", "description": "A C++20 AI agent framework — the orchestration layer between your application and LLM providers.", "license": "MIT", "dependencies": [