From a02415a3ebd51f0ecf79b09793bb0aa2d02aa99d Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Sun, 6 Sep 2026 17:00:12 -0500 Subject: [PATCH] fix(ai): restrict Bedrock cache checkpoints to Claude (#47660) --- packages/ai/src/protocols/bedrock-converse.ts | 7 +- .../ai/src/protocols/utils/bedrock-cache.ts | 34 ++++- .../provider/bedrock-converse-cache.test.ts | 119 ++++++++++++++++++ .../ai/test/provider/bedrock-converse.test.ts | 4 +- 4 files changed, 151 insertions(+), 13 deletions(-) create mode 100644 packages/ai/test/provider/bedrock-converse-cache.test.ts diff --git a/packages/ai/src/protocols/bedrock-converse.ts b/packages/ai/src/protocols/bedrock-converse.ts index 2b9bce1e397..1bb29128544 100644 --- a/packages/ai/src/protocols/bedrock-converse.ts +++ b/packages/ai/src/protocols/bedrock-converse.ts @@ -415,10 +415,7 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* ( // System prompts share the cache-point convention: emit the text block, then // optionally a positional `cachePoint` marker. -const lowerSystem = ( - breakpoints: BedrockCache.Breakpoints, - system: ReadonlyArray, -) => { +const lowerSystem = (breakpoints: BedrockCache.Breakpoints, system: ReadonlyArray) => { const content = system .filter((part) => part.text.length > 0) .flatMap((part) => textWithCache(breakpoints, part.text, part.cache)) @@ -431,7 +428,7 @@ const fromRequest = Effect.fn("BedrockConverse.fromRequest")(function* (request: const generation = request.generation // Bedrock-Claude shares Anthropic's 4-breakpoint cap. Spend the budget in // tools → system → messages order to favour the highest-impact prefixes. - const breakpoints = BedrockCache.breakpoints() + const breakpoints = BedrockCache.breakpoints(request.model.id) const toolConfig = (() => { if (flattened.tools.length === 0) return undefined return { diff --git a/packages/ai/src/protocols/utils/bedrock-cache.ts b/packages/ai/src/protocols/utils/bedrock-cache.ts index db7ff46a0b2..136731919a3 100644 --- a/packages/ai/src/protocols/utils/bedrock-cache.ts +++ b/packages/ai/src/protocols/utils/bedrock-cache.ts @@ -1,6 +1,6 @@ import { Schema } from "effect" import type { CacheHint } from "../../schema/index.js" -import { newBreakpoints, ttlBucket, type Breakpoints } from "./cache.js" +import { newBreakpoints, ttlBucket } from "./cache.js" // Bedrock cache markers are positional: emit a `cachePoint` block immediately // after the content the caller wants treated as a cacheable prefix. Bedrock @@ -13,24 +13,46 @@ export const CachePointBlock = Schema.Struct({ }) export type CachePointBlock = Schema.Schema.Type -// Callers pass a shared counter through every `block()` call site so the -// four-breakpoint budget is respected across `system`, `messages`, and `tools`. +const LEGACY_CLAUDE = ["anthropic.claude-instant", "anthropic.claude-v1", "anthropic.claude-v2", "anthropic.claude-3-"] + +// These legacy Claude releases support explicit caching, but only for five minutes. +const CLAUDE_5M = [ + "anthropic.claude-3-5-sonnet-20241022-v2:0", + "anthropic.claude-3-5-haiku-20241022-v1:0", + "anthropic.claude-3-7-sonnet-20250219-v1:0", + "anthropic.claude-sonnet-4-20250514-v1:0", + "anthropic.claude-opus-4-20250514-v1:0", + "anthropic.claude-opus-4-1-20250805-v1:0", +] + +// Callers share the four-breakpoint budget across system, messages, and tools. export const BEDROCK_BREAKPOINT_CAP = 4 -export type { Breakpoints } from "./cache.js" -export const breakpoints = () => newBreakpoints(BEDROCK_BREAKPOINT_CAP) +export const breakpoints = (modelID: string) => { + // Substring matching also handles regional prefixes and model-bearing ARNs. + const short = CLAUDE_5M.some((id) => modelID.includes(id)) + return { + ...newBreakpoints(BEDROCK_BREAKPOINT_CAP), + // Assume modern Claude releases retain caching support; older generations need an explicit exception. + // Other model families use implicit caching where available. + supported: modelID.includes("anthropic.claude-") && (short || !LEGACY_CLAUDE.some((id) => modelID.includes(id))), + ttl1h: !short, + } +} +export type Breakpoints = ReturnType const DEFAULT_5M: CachePointBlock = { cachePoint: { type: "default" } } const DEFAULT_1H: CachePointBlock = { cachePoint: { type: "default", ttl: "1h" } } export const block = (breakpoints: Breakpoints, cache: CacheHint | undefined): CachePointBlock | undefined => { + if (!breakpoints.supported) return undefined if (cache?.type !== "ephemeral" && cache?.type !== "persistent") return undefined if (breakpoints.remaining <= 0) { breakpoints.dropped += 1 return undefined } breakpoints.remaining -= 1 - return ttlBucket(cache.ttlSeconds) === "1h" ? DEFAULT_1H : DEFAULT_5M + return breakpoints.ttl1h && ttlBucket(cache.ttlSeconds) === "1h" ? DEFAULT_1H : DEFAULT_5M } export * as BedrockCache from "./bedrock-cache.js" diff --git a/packages/ai/test/provider/bedrock-converse-cache.test.ts b/packages/ai/test/provider/bedrock-converse-cache.test.ts new file mode 100644 index 00000000000..29d9709470f --- /dev/null +++ b/packages/ai/test/provider/bedrock-converse-cache.test.ts @@ -0,0 +1,119 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { CacheHint, LLM, Message, ToolCallPart } from "../../src/index.js" +import { AmazonBedrock } from "../../src/providers.js" +import { compileRequest } from "../../src/route/client.js" +import { it } from "../lib/effect.js" + +const bedrock = AmazonBedrock.configure({ apiKey: "fixture" }) + +describe("Bedrock Converse cache policy", () => { + for (const id of [ + "deepseek.r1-v1:0", + "meta.llama3-3-70b-instruct-v1:0", + "mistral.mistral-large-2402-v1:0", + "qwen.qwen3-coder-480b-a35b-v1:0", + "openai.gpt-oss-120b-1:0", + "cohere.command-r-v1:0", + "anthropic.claude-instant-v1", + "anthropic.claude-v1", + "anthropic.claude-v2", + "anthropic.claude-v2:1", + "anthropic.claude-3-haiku-20240307-v1:0", + "anthropic.claude-3-sonnet-20240229-v1:0", + "anthropic.claude-3-opus-20240229-v1:0", + "anthropic.claude-3-5-sonnet-20240620-v1:0", + "amazon.nova-lite-v1:0", + "global.amazon.nova-2-lite-v1:0", + "custom-model", + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", + ]) { + for (const policy of [undefined, "auto", "none", { tools: true, system: true, messages: { tail: 3 } }] as const) { + it.effect(`omits checkpoints for ${id} (${JSON.stringify(policy)})`, () => + Effect.gen(function* () { + // Exercise both automatic placement and manual hints at every lowering site. + const cache = policy === "none" ? new CacheHint({ type: "ephemeral", ttlSeconds: 3600 }) : undefined + const prepared = yield* compileRequest( + LLM.request({ + model: bedrock.model(id), + cache: policy, + system: [{ type: "text", text: "System prefix", cache }], + tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object" }, cache }], + messages: [ + Message.user([{ type: "text", text: "Question", cache }]), + Message.system([{ type: "text", text: "Update", cache }]), + Message.assistant([ + { type: "text", text: "Answer", cache }, + { type: "reasoning", text: "Unsigned reasoning", cache }, + ToolCallPart.make({ id: "call_1", name: "lookup", input: {} }), + ]), + Message.tool({ id: "call_1", name: "lookup", result: "Result", cache }), + ], + }), + ) + + expect(JSON.stringify(prepared.body)).not.toContain("cachePoint") + expect(prepared.body).toMatchObject({ + modelId: id, + system: [{ text: "System prefix" }], + toolConfig: { tools: [{ toolSpec: { name: "lookup" } }] }, + messages: [ + { role: "user", content: [{ text: "Question" }, { text: "\nUpdate\n" }] }, + { + role: "assistant", + content: [{ text: "Answer" }, { text: "Unsigned reasoning" }, { toolUse: { name: "lookup" } }], + }, + { role: "user", content: [{ toolResult: { content: [{ json: "Result" }] } }] }, + ], + }) + }), + ) + } + } + + for (const [id, ttl] of [ + ["anthropic.claude-3-5-sonnet-20241022-v2:0", undefined], + ["us.anthropic.claude-3-5-haiku-20241022-v1:0", undefined], + ["eu.anthropic.claude-3-7-sonnet-20250219-v1:0", undefined], + ["apac.anthropic.claude-sonnet-4-20250514-v1:0", undefined], + ["anthropic.claude-opus-4-20250514-v1:0", undefined], + ["anthropic.claude-opus-4-1-20250805-v1:0", undefined], + ["anthropic.claude-sonnet-4-5-20250929-v1:0", "1h"], + ["global.anthropic.claude-sonnet-99", "1h"], + ["anthropic.claude-new-family-99", "1h"], + ["arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-sonnet-4-6", "1h"], + ["arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude-sonnet-4-6", "1h"], + ] as const) { + for (const ttlSeconds of [undefined, 3600]) { + it.effect(`preserves Claude checkpoints for ${id} (TTL: ${ttlSeconds ?? "default"})`, () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: bedrock.model(id), + system: [ + { type: "text", text: "Agent" }, + { type: "text", text: "Project" }, + ], + tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object" } }], + prompt: "Question", + cache: + ttlSeconds === undefined ? undefined : { tools: true, system: true, messages: { tail: 1 }, ttlSeconds }, + }), + ) + const marker = { + cachePoint: ttlSeconds === undefined || ttl === undefined ? { type: "default" } : { type: "default", ttl }, + } + + expect(prepared.body).toMatchObject({ + modelId: id, + toolConfig: { tools: [{ toolSpec: { name: "lookup" } }, marker] }, + system: [{ text: "Agent" }, marker, { text: "Project" }, marker], + messages: [{ role: "user", content: [{ text: "Question" }, marker] }], + }) + if (ttlSeconds === undefined || ttl === undefined) + expect(JSON.stringify(prepared.body)).not.toContain('"ttl"') + }), + ) + } + } +}) diff --git a/packages/ai/test/provider/bedrock-converse.test.ts b/packages/ai/test/provider/bedrock-converse.test.ts index 05891cf32df..eb9e4a19fd9 100644 --- a/packages/ai/test/provider/bedrock-converse.test.ts +++ b/packages/ai/test/provider/bedrock-converse.test.ts @@ -136,7 +136,7 @@ const captureHeaders = (target: LanguageModel) => const model = AmazonBedrock.configure({ baseURL: "https://bedrock-runtime.test", apiKey: "test-bearer", -}).model("anthropic.claude-3-5-sonnet-20240620-v1:0") +}).model("anthropic.claude-sonnet-4-5-20250929-v1:0") const baseRequest = LLM.request({ id: "req_1", @@ -155,7 +155,7 @@ describe("Bedrock Converse route", () => { const prepared = yield* compileRequest(baseRequest) expect(prepared.body).toEqual({ - modelId: "anthropic.claude-3-5-sonnet-20240620-v1:0", + modelId: "anthropic.claude-sonnet-4-5-20250929-v1:0", system: [{ text: "You are concise." }], messages: [{ role: "user", content: [{ text: "Say hello." }] }], inferenceConfig: { maxTokens: 64, temperature: 0 },