From 42badf1dd7750b7ec37119b11fbd2ff6b9fdb2b1 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 29 Aug 2026 22:04:04 +0300 Subject: [PATCH] Count Groq prompt cache hits instead of reporting every request as a miss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groq returns prompt_tokens_details.cached_tokens, but @ai-sdk/groq@4.0.0 parses the field in its response schema and never assigns it, so the usage object always came back as noCacheTokens = the full prompt. extractCachedTokens() in src/ai/provider.ts reads inputTokenDetails.cacheReadTokens, which was therefore always undefined, and the AI SDK OTEL exporter emitted no cached-token attribute — so Groq spans in Langfuse carried only input_tokens and output_tokens while OpenRouter spans carried the cache keys. Cache hit rate read as zero for every Groq call whether or not the prompt was actually reused. 4.0.34 adds convertGroqUsage(), which maps cached_tokens to inputTokens.cacheRead. package.json already allowed it under ^4.0; only the lockfile pinned 4.0.0. Verified with a fetch interceptor around generateText: 12 identical 28k-token prompts gave 3 raw cache hits and 3 SDK-reported hits, exact match. An end-to-end run through the same LangfuseSpanProcessor wiring the app uses produced 5 raw hits and 5 generations carrying cache_read.input_tokens in Langfuse. The lockfile also picks up the chalk entry that was missing from its workspace manifest block; chalk is already declared in package.json on main. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014xHSmzm2ZczcH2ox3WQAC3 --- CHANGELOG.md | 12 ++++++++++++ bun.lock | 11 +++++++++-- package.json | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3897bac..5a82436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 2026-08-29 + +### Changes + +- [Provider] Groq prompt cache hits are counted again. Groq reports how much of a prompt it served + from cache, but the pinned `@ai-sdk/groq` build read that number out of the response and then + dropped it, so every Groq request was recorded as a full-price miss and the cache hit rate showed + as zero no matter how much of the prompt was actually reused. Upgrading the provider carries the + number through to the run stats and to Langfuse, which records it as `cache_read.input_tokens`. + Groq's cache lives on the node that served the request, so expect an uneven rate — a hit covers + almost the whole prompt, but only some requests get one. + ## 2026-08-28 ### Changes diff --git a/bun.lock b/bun.lock index a062e6c..66eea9d 100644 --- a/bun.lock +++ b/bun.lock @@ -7,7 +7,7 @@ "dependencies": { "@ai-sdk/anthropic": "^4.0", "@ai-sdk/google": "^4.0.18", - "@ai-sdk/groq": "^4.0", + "@ai-sdk/groq": "^4.0.34", "@ai-sdk/mistral": "^4.0.13", "@ai-sdk/openai": "^4.0", "@ai-sdk/otel": "^1.0.2", @@ -29,6 +29,7 @@ "ai": "^7.0.2", "axe-core": "^4.11.1", "bash-tool": "^1.3.15", + "chalk": "^5.6.2", "cli-highlight": "^2.1.11", "codeceptjs": "4.0.0-rc.16", "commander": "^14.0.1", @@ -96,7 +97,7 @@ "@ai-sdk/google": ["@ai-sdk/google@4.0.18", "", { "dependencies": { "@ai-sdk/provider": "4.0.3", "@ai-sdk/provider-utils": "5.0.11" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-NRbXRAasXLgFLiZDTsDUiTUuQtEUDVZoqruB5Px3geltTEqOzOo2eHN5WnDp0/OgxwnrNH4olV/TVetza0PzmQ=="], - "@ai-sdk/groq": ["@ai-sdk/groq@4.0.0", "", { "dependencies": { "@ai-sdk/provider": "4.0.0", "@ai-sdk/provider-utils": "5.0.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-QjhpyudO8eGi+C/kjVzSKfVFsLcuXi4smOez8njQITrV+5wZlKJ1xvoHyQom1FlgC9PsR4Ee6icGIqogoSb1Vg=="], + "@ai-sdk/groq": ["@ai-sdk/groq@4.0.34", "", { "dependencies": { "@ai-sdk/provider": "4.0.8", "@ai-sdk/provider-utils": "5.0.33" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-k9nZLVRnw8RB0IivCwBxNMNe1E/P0cP+2ymxEZkqxZHeThxZ0/LqefHWa6wtjhPKWADL+AgnVfKLSL7TW28wNw=="], "@ai-sdk/mistral": ["@ai-sdk/mistral@4.0.13", "", { "dependencies": { "@ai-sdk/provider": "4.0.3", "@ai-sdk/provider-utils": "5.0.11" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-VYEWPEY6MhhvUKE0Huu63NJXowu2/nbbO60kE6/sdV9mLTk36PahQ7lNZFkJ2XJQ3eUy19RRWQDqYmyxKMiPlA=="], @@ -2656,6 +2657,10 @@ "@ai-sdk/google/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@5.0.11", "", { "dependencies": { "@ai-sdk/provider": "4.0.3", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-7/96wE+ZsKB35iS9ASyllrE4Ym/EolXEB7AkuJ5FI++fmS85BVTAs77890C+1Z2jwHfBKjBQSBmsliOsAh0iFQ=="], + "@ai-sdk/groq/@ai-sdk/provider": ["@ai-sdk/provider@4.0.8", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-aWO7iwhFUGf347tCwNGggggfmZigaSu7TF739IZSrWWABUp7zkb4Cr3fMqvBe5EIS7ABJJu3Cadn0g/zs1G0QQ=="], + + "@ai-sdk/groq/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@5.0.33", "", { "dependencies": { "@ai-sdk/provider": "4.0.8", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.28.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-TfjJqJmRsQyxAlb+3hGmP1o1xLUIT79yhOgtJuTF4hqsB37IC3CufGsuFhU04EeTOg7R9iptQ6Bzm0MW0n/c3g=="], + "@ai-sdk/mistral/@ai-sdk/provider": ["@ai-sdk/provider@4.0.3", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-e0CpNWJUY7OxAFAnCZkw+ri9QOHWwTs1tXP42782KFGCU07qt8NiXCrCVowyCB5dP2r5/Uls+g2oPd8kOJn9dw=="], "@ai-sdk/mistral/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@5.0.11", "", { "dependencies": { "@ai-sdk/provider": "4.0.3", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-7/96wE+ZsKB35iS9ASyllrE4Ym/EolXEB7AkuJ5FI++fmS85BVTAs77890C+1Z2jwHfBKjBQSBmsliOsAh0iFQ=="], @@ -3318,6 +3323,8 @@ "yup/type-fest": ["type-fest@2.19.0", "", {}, "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA=="], + "@ai-sdk/groq/@ai-sdk/provider-utils/undici": ["undici@7.29.0", "", {}, "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw=="], + "@ai-sdk/openai-compatible/@ai-sdk/provider-utils/eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], "@ai-sdk/openai-compatible/@ai-sdk/provider-utils/zod-to-json-schema": ["zod-to-json-schema@3.25.1", "", { "peerDependencies": { "zod": "^3.25 || ^4" } }, "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA=="], diff --git a/package.json b/package.json index f641f9a..b93c990 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "dependencies": { "@ai-sdk/anthropic": "^4.0", "@ai-sdk/google": "^4.0.18", - "@ai-sdk/groq": "^4.0", + "@ai-sdk/groq": "^4.0.34", "@ai-sdk/mistral": "^4.0.13", "@ai-sdk/openai": "^4.0", "@ai-sdk/otel": "^1.0.2",