diff --git a/src/api/providers/__tests__/openrouter.spec.ts b/src/api/providers/__tests__/openrouter.spec.ts index 1bf6669324..bb2d75acd7 100644 --- a/src/api/providers/__tests__/openrouter.spec.ts +++ b/src/api/providers/__tests__/openrouter.spec.ts @@ -23,6 +23,7 @@ import type { ModelInfo } from "@roo-code/types" import { applyOpenRouterMoonshotK3Profile } from "../fetchers/openrouter" import { OpenRouterHandler } from "../openrouter" +import { getModels } from "../fetchers/modelCache" import { Package } from "../../../shared/package" import { makeApiHandlerOptions } from "../../../test-utils/api" import { asyncStreamFrom, collectStream } from "../../../test-utils/stream" @@ -81,6 +82,34 @@ vitest.mock("../fetchers/modelCache", () => ({ description: "Claude 4.5 Sonnet", thinking: false, }, + // Not in OPEN_ROUTER_PROMPT_CACHING_MODELS: stands in for a Claude + // model released after the static allowlist was last edited. + "anthropic/claude-opus-4.8": { + maxTokens: 128000, + contextWindow: 1000000, + supportsImages: true, + supportsPromptCache: true, + inputPrice: 5, + outputPrice: 25, + cacheWritesPrice: 6.25, + cacheReadsPrice: 0.5, + description: "Claude 4.8 Opus", + thinking: false, + }, + // Not in OPEN_ROUTER_PROMPT_CACHING_MODELS and not Anthropic: the + // negative control for the widened gate. It advertises cache pricing + // so the fixture states the premise instead of relying on the + // hardcoded default info's flag. + "google/gemini-9-pro": { + maxTokens: 65536, + contextWindow: 1000000, + supportsImages: true, + supportsPromptCache: true, + inputPrice: 1.25, + outputPrice: 10, + cacheReadsPrice: 0.31, + description: "Gemini 9 Pro (hypothetical)", + }, "anthropic/claude-3.7-sonnet:thinking": { maxTokens: 128000, contextWindow: 200000, @@ -139,6 +168,32 @@ describe("OpenRouterHandler", () => { openRouterModelId: "anthropic/claude-sonnet-4", }) + /** + * Install a `chat.completions.create` stub on the vitest-mocked OpenAI class + * and hand back the mock plus a restore function. The mocked class is + * structurally incompatible with the narrow chat stub, so the double + * assertion routes through `unknown` (instead of `any`) to keep this file's + * no-explicit-any budget flat. + */ + function stubChatCompletions() { + const create = vitest + .fn() + .mockResolvedValue(asyncStreamFrom([{ id: "test-id", choices: [{ delta: { content: "test response" } }] }])) + const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } } + const originalChat = openAiPrototype.prototype.chat + openAiPrototype.prototype.chat = { completions: { create } } + + return { + create, + requestParams: () => (create.mock.calls[0] as [Record])[0], + restore: () => { + // clearAllMocks does not undo prototype assignment, so every stub + // must be removed explicitly or it leaks into later tests. + openAiPrototype.prototype.chat = originalChat + }, + } + } + beforeEach(() => { clearAllMocks() // Endpoint records default to "not fetched" (same as the real guard for @@ -485,6 +540,319 @@ describe("OpenRouterHandler", () => { ) }) + it("adds cache control for a Claude model missing from the static allowlist", async () => { + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-opus-4.8", + }), + ) + + const mockStream = asyncStreamFrom([{ id: "test-id", choices: [{ delta: { content: "test response" } }] }]) + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + const chatStub = { completions: { create: mockCreate } } + const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } } + const originalChat = openAiPrototype.prototype.chat + openAiPrototype.prototype.chat = chatStub + + try { + await handler.createMessage("test system", [{ role: "user", content: "test message" }]).next() + + const [requestParams] = mockCreate.mock.calls[0] as [Record] + expect(requestParams).toMatchObject({ + messages: expect.arrayContaining([ + expect.objectContaining({ + role: "system", + content: expect.arrayContaining([ + expect.objectContaining({ cache_control: { type: "ephemeral" } }), + ]), + }), + ]), + }) + } finally { + openAiPrototype.prototype.chat = originalChat + } + }) + + it("keeps Gemini cache breakpoints allowlist-only", async () => { + // A Gemini model absent from the allowlist must not gain Anthropic-style + // breakpoints just because its catalog record advertises cache pricing. + // `google/gemini-9-pro` is in the mocked catalog with + // `supportsPromptCache: true`, so the premise is stated by the fixture + // rather than by the hardcoded default info's flag. + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "google/gemini-9-pro", + }), + ) + const stub = stubChatCompletions() + + try { + await handler.createMessage("test system", [{ role: "user", content: "test message" }]).next() + + expect(stub.requestParams()).toMatchObject({ + messages: [{ role: "system", content: "test system" }, expect.anything()], + }) + } finally { + stub.restore() + } + }) + + it("adds no cache breakpoints when the catalog is empty and the id is not allowlisted", async () => { + // The fallback info hardcodes `supportsPromptCache: true`, so trusting + // `model.info` would hand breakpoints to any `anthropic/*` id whenever + // the catalog is empty or degraded. An id that never resolved from the + // fetched catalog is not live evidence, so the allowlist alone decides. + vitest.mocked(getModels).mockResolvedValueOnce({}) + + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-not-in-catalog", + }), + ) + const stub = stubChatCompletions() + + try { + await handler.createMessage("test system", [{ role: "user", content: "test message" }]).next() + + expect(stub.requestParams()).toMatchObject({ + messages: [{ role: "system", content: "test system" }, expect.anything()], + }) + } finally { + stub.restore() + } + }) + + it("reads the cache capability from the model record when a provider is pinned", async () => { + // getModel() substitutes the pinned endpoint's record, whose + // `supportsPromptCache` comes from that one endpoint's pricing. The gate + // must follow the model-level catalog record instead, or pinning a + // provider silently drops breakpoints for a caching Claude model. + mockGetModelEndpoints.mockResolvedValue({ + anthropic: { + maxTokens: 128000, + contextWindow: 1000000, + supportsImages: true, + supportsPromptCache: false, + inputPrice: 5, + outputPrice: 25, + description: "Claude 4.8 Opus (Anthropic endpoint)", + }, + }) + + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-opus-4.8", + openRouterSpecificProvider: providerIdentifiers.anthropic, + }), + ) + const stub = stubChatCompletions() + + try { + await handler.createMessage("test system", [{ role: "user", content: "test message" }]).next() + + // The endpoint record is what the request is built from... + expect(stub.requestParams()).toMatchObject({ model: "anthropic/claude-opus-4.8" }) + // ...but the model-level flag still turns breakpoints on. + expect(stub.requestParams()).toMatchObject({ + messages: expect.arrayContaining([ + expect.objectContaining({ + role: "system", + content: expect.arrayContaining([ + expect.objectContaining({ cache_control: { type: "ephemeral" } }), + ]), + }), + ]), + }) + } finally { + stub.restore() + } + }) + + it("adds no cache breakpoints when a pinned endpoint advertises caching the model record denies", async () => { + // Mirror of the case above: an endpoint record that reports cache + // pricing must not widen the gate for a model whose catalog record says + // no. Otherwise pinning a provider would add breakpoints the unpinned + // path withholds. + mockGetModelEndpoints.mockResolvedValue({ + anthropic: { + maxTokens: 128000, + contextWindow: 1000000, + supportsImages: true, + supportsPromptCache: true, + inputPrice: 5, + outputPrice: 25, + cacheReadsPrice: 0.5, + description: "Claude 4.8 Opus (Anthropic endpoint)", + }, + }) + vitest.mocked(getModels).mockResolvedValueOnce({ + "anthropic/claude-opus-4.8": { + maxTokens: 128000, + contextWindow: 1000000, + supportsImages: true, + supportsPromptCache: false, + inputPrice: 5, + outputPrice: 25, + description: "Claude 4.8 Opus", + }, + }) + + const handler = new OpenRouterHandler( + makeApiHandlerOptions({ + ...mockOptions, + openRouterModelId: "anthropic/claude-opus-4.8", + openRouterSpecificProvider: providerIdentifiers.anthropic, + }), + ) + const stub = stubChatCompletions() + + try { + await handler.createMessage("test system", [{ role: "user", content: "test message" }]).next() + + expect(stub.requestParams()).toMatchObject({ + messages: [{ role: "system", content: "test system" }, expect.anything()], + }) + } finally { + stub.restore() + } + }) + + it("sends the task ID as the OpenRouter cache session affinity key", async () => { + const handler = new OpenRouterHandler(mockOptions) + + const mockStream = asyncStreamFrom([{ id: "test-id", choices: [{ delta: { content: "test response" } }] }]) + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + const chatStub = { completions: { create: mockCreate } } + const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } } + const originalChat = openAiPrototype.prototype.chat + openAiPrototype.prototype.chat = chatStub + + try { + await handler + .createMessage("test system", [{ role: "user", content: "test message" }], { taskId: "task-123" }) + .next() + + const [requestParams] = mockCreate.mock.calls[0] as [Record] + expect(requestParams).toMatchObject({ session_id: "task-123" }) + } finally { + openAiPrototype.prototype.chat = originalChat + } + }) + + it("omits session_id when no task ID is available", async () => { + const handler = new OpenRouterHandler(mockOptions) + + const mockStream = asyncStreamFrom([{ id: "test-id", choices: [{ delta: { content: "test response" } }] }]) + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + const chatStub = { completions: { create: mockCreate } } + const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } } + const originalChat = openAiPrototype.prototype.chat + openAiPrototype.prototype.chat = chatStub + + try { + await handler.createMessage("test system", [{ role: "user", content: "test message" }]).next() + + const [requestParams] = mockCreate.mock.calls[0] as [Record] + expect(requestParams).not.toHaveProperty("session_id") + } finally { + openAiPrototype.prototype.chat = originalChat + } + }) + + it("truncates an over-long task ID to OpenRouter's 256-character session_id limit", async () => { + const handler = new OpenRouterHandler(mockOptions) + + const mockStream = asyncStreamFrom([{ id: "test-id", choices: [{ delta: { content: "test response" } }] }]) + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + const chatStub = { completions: { create: mockCreate } } + const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } } + const originalChat = openAiPrototype.prototype.chat + openAiPrototype.prototype.chat = chatStub + + try { + await handler + .createMessage("test system", [{ role: "user", content: "test message" }], { + taskId: "a".repeat(300), + }) + .next() + + const [requestParams] = mockCreate.mock.calls[0] as [Record] + expect(requestParams.session_id).toBe("a".repeat(256)) + } finally { + openAiPrototype.prototype.chat = originalChat + } + }) + + it("maps OpenRouter cache read and write usage tokens", async () => { + const handler = new OpenRouterHandler(mockOptions) + + const mockStream = asyncStreamFrom([ + { + id: "test-id", + choices: [{ delta: {} }], + usage: { + prompt_tokens: 10, + completion_tokens: 2, + prompt_tokens_details: { cached_tokens: 6, cache_write_tokens: 4 }, + }, + }, + ]) + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + const chatStub = { completions: { create: mockCreate } } + const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } } + const originalChat = openAiPrototype.prototype.chat + openAiPrototype.prototype.chat = chatStub + + try { + const chunks = await collectStream( + handler.createMessage("test system", [{ role: "user", content: "test message" }]), + ) + + expect(chunks).toContainEqual({ + type: "usage", + inputTokens: 10, + outputTokens: 2, + cacheReadTokens: 6, + cacheWriteTokens: 4, + totalCost: 0, + }) + } finally { + openAiPrototype.prototype.chat = originalChat + } + }) + + it("leaves cache token fields unset when OpenRouter reports no cache usage", async () => { + const handler = new OpenRouterHandler(mockOptions) + + const mockStream = asyncStreamFrom([ + { + id: "test-id", + choices: [{ delta: {} }], + usage: { prompt_tokens: 10, completion_tokens: 2 }, + }, + ]) + const mockCreate = vitest.fn().mockResolvedValue(mockStream) + const chatStub = { completions: { create: mockCreate } } + const openAiPrototype = OpenAI as unknown as { prototype: { chat?: unknown } } + const originalChat = openAiPrototype.prototype.chat + openAiPrototype.prototype.chat = chatStub + + try { + const chunks = await collectStream( + handler.createMessage("test system", [{ role: "user", content: "test message" }]), + ) + + const usageChunk = chunks.find((chunk) => chunk.type === "usage") + expect(usageChunk).toMatchObject({ cacheReadTokens: undefined, cacheWriteTokens: undefined }) + } finally { + openAiPrototype.prototype.chat = originalChat + } + }) + it("handles API errors and captures telemetry", async () => { const handler = new OpenRouterHandler(mockOptions) const mockStream = asyncStreamFrom([{ error: { message: "API Error", code: 500 } }]) diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts index 36e09aa883..caf009b47c 100644 --- a/src/api/providers/openrouter.ts +++ b/src/api/providers/openrouter.ts @@ -49,10 +49,16 @@ import { applyRouterToolPreferences } from "./utils/router-tool-preferences" type OpenRouterChatCompletionParams = OpenAI.Chat.ChatCompletionCreateParams & { transforms?: string[] include_reasoning?: boolean + // https://openrouter.ai/docs/guides/best-practices/prompt-caching#using-session_id-for-sticky-sessions + session_id?: string // https://openrouter.ai/docs/use-cases/reasoning-tokens reasoning?: OpenRouterReasoningParams } +// OpenRouter rejects a `session_id` longer than this. +// https://openrouter.ai/docs/guides/best-practices/prompt-caching#using-session_id-for-sticky-sessions +const OPENROUTER_MAX_SESSION_ID_LENGTH = 256 + // Zod schema for OpenRouter error response structure (for caught exceptions) const OpenRouterErrorResponseSchema = z.object({ error: z @@ -125,6 +131,8 @@ interface CompletionUsage { prompt_tokens?: number prompt_tokens_details?: { cached_tokens?: number + // https://openrouter.ai/docs/guides/best-practices/prompt-caching#usage-object-fields + cache_write_tokens?: number } total_tokens?: number cost?: number @@ -219,6 +227,10 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH let { id: modelId, maxTokens, temperature, topP, reasoning } = model + // OpenRouter caps `session_id` at 256 characters. Task IDs are UUIDs and + // far shorter, so this only guards against an over-long caller value. + const sessionId = metadata?.taskId?.slice(0, OPENROUTER_MAX_SESSION_ID_LENGTH) + // Reset reasoning_details accumulator for this request this.currentReasoningDetails = [] @@ -302,9 +314,28 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH }) } - // https://openrouter.ai/docs/features/prompt-caching - // TODO: Add a `promptCacheStratey` field to `ModelInfo`. - if (OPEN_ROUTER_PROMPT_CACHING_MODELS.has(modelId)) { + // https://openrouter.ai/docs/guides/best-practices/prompt-caching + // TODO: Add a `promptCacheStrategy` field to `ModelInfo`. + // The allowlist is a static snapshot, so a Claude model released after it + // was last edited got no breakpoints even though OpenRouter advertises + // cache pricing for it. Trust the live capability flag for Claude and keep + // the allowlist as the fallback; Gemini and every other provider keep + // allowlist-only behavior because breakpoint placement is not + // interchangeable between them. + // + // Only the model-level catalog record carries a live flag, so read it from + // `this.models` instead of `model.info`. `model.info` is not a safe source: + // an empty catalog falls back to `openRouterDefaultModelInfo`, which + // hardcodes `supportsPromptCache: true`, and a pinned provider substitutes + // that endpoint's record, whose flag comes from one endpoint's pricing. + // Either would hand breakpoints to every `anthropic/*` id, so an id that + // did not resolve from the fetched catalog counts as no live evidence. + const catalogInfo = Object.hasOwn(this.models, modelId) ? this.models[modelId] : undefined + const supportsCacheBreakpoints = + OPEN_ROUTER_PROMPT_CACHING_MODELS.has(modelId) || + (modelId.startsWith("anthropic/") && catalogInfo?.supportsPromptCache === true) + + if (supportsCacheBreakpoints) { if (modelId.startsWith("google")) { addGeminiCacheBreakpoints(systemPrompt, openAiMessages) } else { @@ -315,6 +346,14 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH // https://openrouter.ai/docs/transforms const completionParams: OpenRouterChatCompletionParams = { model: modelId, + // OpenRouter derives its sticky-routing key from the opening messages + // unless `session_id` is sent, and without it stickiness only starts + // after a cache hit is observed. A task ID gives every turn of the task + // one routing key from the first request, so turns stay on the endpoint + // holding the warm cache. OpenRouter does not use sticky routing when a + // manual `provider.order` is set, so pinning a specific provider limits + // the benefit to that provider's own cache. + ...(sessionId ? { session_id: sessionId } : {}), ...(maxTokens && maxTokens > 0 && { max_tokens: maxTokens }), temperature, top_p: topP, @@ -540,6 +579,7 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH inputTokens: lastUsage.prompt_tokens || 0, outputTokens: lastUsage.completion_tokens || 0, cacheReadTokens: lastUsage.prompt_tokens_details?.cached_tokens, + cacheWriteTokens: lastUsage.prompt_tokens_details?.cache_write_tokens, reasoningTokens: lastUsage.completion_tokens_details?.reasoning_tokens, totalCost: (lastUsage.cost_details?.upstream_inference_cost || 0) + (lastUsage.cost || 0), }