feat(sdk): support AI SDK v5, v6 and v7

.github/workflows/ci.yml

@@ -33,3 +33,38 @@ jobs:
       - name: Build
         run: bun run build --dts=false
 
+  sdk-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      cells: ${{ steps.cells.outputs.cells }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: List AI SDK cells
+        id: cells
+        run: echo "cells=$(scripts/test-sdk-matrix.sh --list-json)" >> "$GITHUB_OUTPUT"
+
+  sdk-compat:
+    name: sdk (ai@${{ matrix.ai }}, zod@${{ matrix.zod }}, node ${{ matrix.node }})
+    needs: sdk-matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.sdk-matrix.outputs.cells) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+
+      - name: Test the sdk against this AI SDK and zod
+        run: scripts/test-sdk-matrix.sh "${{ matrix.ai }}" "${{ matrix.zod }}"

README.md

@@ -69,7 +69,8 @@ const result = count({
 //         description: 10,
 //         inputSchema: 15
 //       }
-//     }
+//     },
+//     unresolved: []
 //   }
 // }
 
@@ -78,6 +79,30 @@ const result = count({
 const costUSD = result.total * model.pricing.input;
 ```
 
+`ai-tokenizer/sdk` works with AI SDK v5, v6 and v7 and needs `ai` installed
+(v7 needs Node 22 or newer). Messages in any of those versions' shapes are
+counted, and tool input schemas are read with the AI SDK's own `asSchema()`, so
+zod 3, zod 4, `jsonSchema()` and (v6 and v7) Standard Schema tools all count.
+
+`count()` is synchronous. It cannot wait for a tool whose `jsonSchema` is a
+Promise (or a function that returns one), and it cannot read a schema the AI SDK
+fails to convert. It counts that part as 0 tokens, does not throw, and lists it
+in `result.tools.unresolved`:
+
+```ts
+for (const { tool, part, reason } of result.tools.unresolved) {
+  console.warn(`${tool} ${part} not counted (${reason})`);
+}
+// Fix: resolve the schema first, e.g. jsonSchema(await loadSchema()).
+```
+
+Tool schemas are counted for properties, descriptions, enums, nested objects and
+arrays of objects. Local `$ref`s (`#`, `#/definitions/X`, `#/$defs/X`) are
+followed, and a definition inside a recursive schema counts once. These are not
+walked: unions with two or more non-null variants (`anyOf`, `oneOf`), `allOf`,
+`$ref`s that point outside the schema or use an anchor (`#name`), and `$ref`
+alias chains longer than one hop.
+
 Estimate tokens using an encoding:
 
 ```ts

package.json

@@ -8,16 +8,17 @@
   },
   "devDependencies": {
     "@types/bun": "latest",
-    "ai": "^5.0.105",
+    "ai": "^7.0.162",
     "base64-js": "^1.5.1",
     "gpt-tokenizer": "^3.0.1",
     "mitata": "^1.0.34",
     "prettier": "^3.6.2",
     "tiktoken": "^1.0.22",
-    "tsdown": "^0.15.6"
+    "tsdown": "^0.15.6",
+    "zod": "^4.6.7"
   },
   "peerDependencies": {
-    "ai": "^5.0.0"
+    "ai": "^5.0.0 || ^6.0.0 || ^7.0.0"
   },
   "peerDependenciesMeta": {
     "ai": {

scripts/smoke-sdk-dist.mjs

@@ -0,0 +1,64 @@
+// Loads the built `ai-tokenizer/sdk` as CommonJS and as ESM against the
+// installed `ai` and `zod`, and fails unless both give the same counts.
+// Run after `bun run build --dts=false`.
+import assert from "node:assert/strict";
+import { createRequire } from "node:module";
+
+const require = createRequire(import.meta.url);
+const dist = new URL("../dist/", import.meta.url);
+
+const aiVersion = require("ai/package.json").version;
+const zodVersion = require("zod/package.json").version;
+
+async function smoke(format) {
+  const load = async (name) =>
+    format === "cjs"
+      ? require(new URL(`${name}.cjs`, dist).pathname)
+      : import(new URL(`${name}.js`, dist).href);
+  const { count } = await load("sdk");
+  const { Tokenizer, models } = await load("index");
+  const encoding = await load("encoding/o200k_base");
+  const ai = format === "cjs" ? require("ai") : await import("ai");
+  const { z } = format === "cjs" ? require("zod") : await import("zod");
+
+  const result = count({
+    tokenizer: new Tokenizer(encoding),
+    model: models["openai/gpt-5"],
+    messages: [
+      { role: "system", content: "Be brief." },
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "Weather in Paris?" },
+          { type: "image", image: "iVBORw0KGgo=", mediaType: "image/png" },
+        ],
+      },
+    ],
+    tools: {
+      zodTool: {
+        description: "Get the weather",
+        inputSchema: z.object({ city: z.string().describe("City name") }),
+      },
+      jsonTool: {
+        description: "Get the weather",
+        inputSchema: ai.jsonSchema({
+          type: "object",
+          properties: { city: { type: "string", description: "City name" } },
+        }),
+      },
+    },
+  });
+
+  assert.equal(result.tools.unresolved.length, 0, `${format}: unresolved tools`);
+  const { zodTool, jsonTool } = result.tools.definitions;
+  assert.ok(zodTool.inputSchema > 0, `${format}: zod schema counted`);
+  assert.equal(zodTool.inputSchema, jsonTool.inputSchema, `${format}: zod = jsonSchema`);
+  return result.total;
+}
+
+const cjs = await smoke("cjs");
+const esm = await smoke("esm");
+assert.equal(cjs, esm, "CJS and ESM totals differ");
+console.log(
+  `smoke node=${process.version} ai=${aiVersion} zod=${zodVersion} cjs=${cjs} esm=${esm}`
+);

scripts/test-sdk-matrix.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# Tests `ai-tokenizer/sdk` against one AI SDK + zod pairing, or against all of
+# them. CI runs one cell per job. Every cell does four things:
+#   1. installs the pairing in place of the dev dependencies,
+#   2. type-checks src and runs the sdk tests,
+#   3. builds dist,
+#   4. loads the CJS and ESM builds with the installed `ai` and `zod`.
+#
+#   scripts/test-sdk-matrix.sh <ai-version> <zod-version>
+#       Runs one cell in this checkout. It rewrites package.json, bun.lock and
+#       node_modules, so use it in a throwaway checkout such as a CI job.
+#   scripts/test-sdk-matrix.sh --all
+#       Runs every cell, each in a throwaway copy of the repo, on the Node
+#       that is installed here (the Node column only applies in CI).
+#   scripts/test-sdk-matrix.sh --list-json
+#       Prints the cells as a GitHub Actions matrix. CI reads this list.
+#
+# Every cell prints one line starting with "CELL".
+set -euo pipefail
+
+# ai version, zod version, Node major used in CI. Node 20 covers a v5/v6 user
+# who loads the CJS build: v7 needs Node 22.
+CELLS=(
+  "5 3.25.76 22"
+  "5 4 22"
+  "5 4 20"
+  "6 3.25.76 22"
+  "6 4 22"
+  "6 4 20"
+  "7 3.25.76 22"
+  "7 4 22"
+)
+
+installed() { node -p "require('$1/package.json').version"; }
+
+run_cell() {
+  local ai="$1" zod="$2" log
+  # Bun resolves `ai` once for the dev dependency and the peer dependency and
+  # picks the peer range (^5 || ^6 || ^7), so drop the peer range here.
+  node -e '
+    const fs = require("fs");
+    const pkg = JSON.parse(fs.readFileSync("package.json", "utf8"));
+    pkg.devDependencies.ai = process.argv[1];
+    pkg.devDependencies.zod = process.argv[2];
+    delete pkg.peerDependencies;
+    delete pkg.peerDependenciesMeta;
+    fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n");
+  ' "$ai" "$zod"
+  bun install >/dev/null
+
+  # Only the sdk files are type-checked: scripts/ and bench/ use the newest AI SDK.
+  log="$(mktemp)"
+  local typecheck=ok tests
+  printf '{"extends":"./tsconfig.json","include":["src/sdk*.ts"]}' >tsconfig.sdk-cell.json
+  bunx tsc --noEmit -p tsconfig.sdk-cell.json >"$log" 2>&1 || { cat "$log"; typecheck=FAILED; }
+  rm -f tsconfig.sdk-cell.json
+
+  bun test src/sdk.test.ts src/sdk.compat.test.ts >"$log" 2>&1 || { cat "$log"; exit 1; }
+  tests="$(grep -E '^ *[0-9]+ (pass|skip|fail)$' "$log" | tr -s ' ' | tr '\n' ' ' || true)"
+  [ -n "$tests" ] || { cat "$log"; echo "no test summary found" >&2; exit 1; }
+
+  bun run build --dts=false >/dev/null
+  local smoke
+  smoke="$(node scripts/smoke-sdk-dist.mjs)"
+
+  echo "CELL ai@$(installed ai) zod@$(installed zod) typecheck=$typecheck tests:$tests| $smoke"
+  [ "$typecheck" = ok ]
+}
+
+case "${1:-}" in
+  --list-json)
+    printf '{"include":['
+    sep=""
+    for cell in "${CELLS[@]}"; do
+      read -r ai zod node <<<"$cell"
+      printf '%s{"ai":"%s","zod":"%s","node":"%s"}' "$sep" "$ai" "$zod" "$node"
+      sep=","
+    done
+    printf ']}\n'
+    ;;
+  --all)
+    root="$(cd "$(dirname "$0")/.." && pwd)"
+    failed=0
+    for cell in "${CELLS[@]}"; do
+      read -r ai zod _node <<<"$cell"
+      tmp="$(mktemp -d)"
+      (cd "$root" && cp -r package.json bun.lock tsconfig.json tsdown.config.ts src scripts "$tmp")
+      (cd "$tmp" && bash scripts/test-sdk-matrix.sh "$ai" "$zod") || failed=1
+      rm -rf "$tmp"
+    done
+    exit "$failed"
+    ;;
+  "")
+    echo "usage: $0 <ai-version> <zod-version> | --all | --list-json" >&2
+    exit 2
+    ;;
+  *)
+    run_cell "$1" "${2:?zod version required}"
+    ;;
+esac