Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions src/adapters/bedrock/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,11 +120,12 @@ export class BedrockAdapter implements LlmAdapter<BedrockAuthSpec> {
commandInput.inferenceConfig = inferenceConfig;
}

if (req.cacheKey) {
const caching = req.cacheKey ? cachePointSupport(req.model) : undefined;
if (caching) {
if (commandInput.system) {
(commandInput.system as unknown[]).push({ cachePoint: { type: 'default' } });
}
if (tools) {
if (tools && caching.tools) {
(tools.tools as unknown[]).push({ cachePoint: { type: 'default' } });
}
// Third cache point after the final message's content so the ENTIRE
Expand Down Expand Up @@ -450,6 +451,22 @@ function buildWireMessages(req: PromptRequest): unknown[] {
return out;
}

/**
* Which Converse fields accept an explicit `cachePoint` for this model, or
* undefined if the model doesn't support explicit prompt caching at all.
* Per the Bedrock prompt-caching docs, Anthropic Claude models accept cache
* points in system, messages and tools; Amazon Nova only in system and
* messages (a cachePoint in toolConfig.tools is rejected as an extraneous
* key). Other models get none — they rely on implicit caching, if any.
* Matched by substring so cross-region profile IDs (`us.`, `global.`, ...)
* resolve to the underlying model.
*/
function cachePointSupport(model: string): { tools: boolean } | undefined {
if (model.includes('anthropic.claude')) return { tools: true };
if (model.includes('amazon.nova')) return { tools: false };
return undefined;
}

/**
* Replace the last wire message with a copy whose content array gains a
* trailing `{cachePoint: {type: 'default'}}` block. COPY, never mutate: the
Expand Down
76 changes: 76 additions & 0 deletions test/adapters/bedrock/bedrock.cache-key.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -341,3 +341,79 @@ describe('Bedrock adapter — history cache point and cache-token usage', () =>
});
});
});

// ---------------------------------------------------------------------------
// Per-model cache point support. Bedrock rejects a cachePoint in a field the
// model doesn't support (Nova: "#/toolConfig/tools/N: extraneous key
// [cachePoint] is not permitted"), failing the whole prompt.
// ---------------------------------------------------------------------------

describe('Bedrock adapter — cachePoint gated by model support', () => {
beforeEach(() => {
_resetRegistryForTests();
registerAdapter(bedrockFactory);
bedrockMock.reset();
});

afterEach(() => {
_resetRegistryForTests();
bedrockMock.reset();
});

async function sendWithCacheKey(model: string): Promise<Record<string, unknown>> {
bedrockMock.on(ConverseStreamCommand).resolvesOnce({
stream: mockStream([{ messageStop: { stopReason: 'end_turn' } }]) as never,
});
const adapter = await buildAdapter();
await drain(adapter, {
model,
system: 'You are a helpful assistant.',
messages: [{ role: 'user', content: 'hello' }],
cacheKey: 'session-abc',
tools: [
{
name: 'lookup_order',
description: 'Find an order by ID',
parameters: { type: 'object', properties: { id: { type: 'string' } } },
},
],
});
return bedrockMock.commandCalls(ConverseStreamCommand)[0]!.args[0]
.input as unknown as Record<string, unknown>;
}

const hasCachePoint = (arr: unknown[]): boolean =>
arr.some((el) => (el as Record<string, unknown>).cachePoint !== undefined);

it.each(['amazon.nova-lite-v1:0', 'us.amazon.nova-pro-v1:0'])(
'%s: cachePoint on system and messages, but NOT on toolConfig.tools',
async (model) => {
const input = await sendWithCacheKey(model);
const tools = (input.toolConfig as { tools: unknown[] }).tools;
expect(tools).toHaveLength(1);
expect(hasCachePoint(tools)).toBe(false);
const system = input.system as unknown[];
expect(system[system.length - 1]).toEqual(CACHE_POINT);
const messages = input.messages as Array<{ content: unknown[] }>;
expect(messages[0]!.content[messages[0]!.content.length - 1]).toEqual(CACHE_POINT);
},
);

it('cross-region Claude profile: cachePoint on tools, system and messages', async () => {
const input = await sendWithCacheKey('us.anthropic.claude-sonnet-4-5-20250929-v1:0');
const tools = (input.toolConfig as { tools: unknown[] }).tools;
expect(tools[tools.length - 1]).toEqual(CACHE_POINT);
const system = input.system as unknown[];
expect(system[system.length - 1]).toEqual(CACHE_POINT);
const messages = input.messages as Array<{ content: unknown[] }>;
expect(messages[0]!.content[messages[0]!.content.length - 1]).toEqual(CACHE_POINT);
});

it('model without explicit caching (Llama): no cachePoint anywhere', async () => {
const input = await sendWithCacheKey('meta.llama3-70b-instruct-v1:0');
expect(hasCachePoint((input.toolConfig as { tools: unknown[] }).tools)).toBe(false);
expect(hasCachePoint(input.system as unknown[])).toBe(false);
const messages = input.messages as Array<{ content: unknown[] }>;
expect(hasCachePoint(messages[0]!.content)).toBe(false);
});
});
Loading