diff --git a/packages/ai/src/protocols/bedrock-converse.ts b/packages/ai/src/protocols/bedrock-converse.ts index 52a411e966e..9e31442b03e 100644 --- a/packages/ai/src/protocols/bedrock-converse.ts +++ b/packages/ai/src/protocols/bedrock-converse.ts @@ -319,6 +319,9 @@ const lowerToolResult = Effect.fn("BedrockConverse.lowerToolResult")(function* ( } satisfies BedrockToolResultBlock }) +// Keep Claude and Nova tool-result images inline; put other models' images beside the result. +const keepToolImagesInline = (id: string) => id.includes("anthropic.claude-") || id.includes("amazon.nova-") + const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* ( request: LLMRequest, breakpoints: BedrockCache.Breakpoints, @@ -328,8 +331,19 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* ( // Mistral can reject replay IDs even when they satisfy Converse's broader ID syntax. const normalizeID = request.model.id.includes("mistral.") ? MistralToolID.normalizer(request) : (id: string) => id const providerMetadataKey = request.model.route.providerMetadataKey ?? String(request.model.provider) + const hoistImages = !keepToolImagesInline(request.model.id) + // Bedrock expects parallel tool results before any images hoisted beside them. + const pendingImages: BedrockMedia.ImageBlock[] = [] + const flushImages = () => { + if (pendingImages.length === 0) return + const previous = messages.at(-1) + if (previous?.role === "user") + messages[messages.length - 1] = { role: "user", content: [...previous.content, ...pendingImages] } + pendingImages.length = 0 + } for (const message of request.messages) { + if (message.role !== "tool") flushImages() if (message.role === "system") { const part = yield* ProviderShared.wrappedSystemUpdate("Bedrock Converse", message) const content = textWithCache(breakpoints, part.text, part.cache) @@ -403,7 +417,22 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* ( for (const part of message.content) { if (!ProviderShared.supportsContent(part, ["tool-result"])) return yield* ProviderShared.unsupportedContent("Bedrock Converse", "tool", ["tool-result"]) - content.push(yield* lowerToolResult(part, documentNames, normalizeID)) + const result = yield* lowerToolResult(part, documentNames, normalizeID) + const images: BedrockMedia.ImageBlock[] = hoistImages + ? result.toolResult.content.filter((item) => "image" in item) + : [] + const nonImageContent = result.toolResult.content.filter((item) => !("image" in item)) + content.push( + images.length === 0 + ? result + : { + toolResult: { + ...result.toolResult, + content: nonImageContent.length > 0 ? nonImageContent : [{ text: "See attached image." }], + }, + }, + ) + pendingImages.push(...images) const cachePoint = BedrockCache.block(breakpoints, part.cache) if (cachePoint) content.push(cachePoint) } @@ -413,6 +442,7 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* ( else messages.push({ role: "user", content }) } + flushImages() return messages }) diff --git a/packages/ai/test/provider/bedrock-converse.test.ts b/packages/ai/test/provider/bedrock-converse.test.ts index 7c8f9ea732c..9a6471fa378 100644 --- a/packages/ai/test/provider/bedrock-converse.test.ts +++ b/packages/ai/test/provider/bedrock-converse.test.ts @@ -648,6 +648,177 @@ describe("Bedrock Converse route", () => { }), ) + it.effect("hoists tool-result images beside the result for Bedrock GPT models", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: AmazonBedrock.configure({ baseURL: "https://bedrock-runtime.test", apiKey: "test-bearer" }).model( + "global.openai.gpt-6-sol", + ), + messages: [ + Message.user("What is in this image?"), + Message.assistant([ToolCallPart.make({ id: "tool_1", name: "read", input: {} })]), + Message.tool({ + id: "tool_1", + name: "read", + result: { + type: "content", + value: [ + { type: "text", text: "Image loaded." }, + { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }, + { type: "file", uri: "data:application/pdf;base64,QkI=", mime: "application/pdf", name: "note.pdf" }, + ], + }, + }), + ], + cache: "none", + }), + ) + + expect(prepared.body.messages[2]).toEqual({ + role: "user", + content: [ + { + toolResult: { + toolUseId: "tool_1", + content: [ + { text: "Image loaded." }, + { text: 'Attached file "note.pdf" has document label "note".' }, + { document: { format: "pdf", name: "note", source: { bytes: "QkI=" } } }, + ], + status: "success", + }, + }, + { image: { format: "png", source: { bytes: "AAAA" } } }, + ], + }) + }), + ) + + it.effect("hoists tool-result images for other Bedrock model families", () => + Effect.gen(function* () { + for (const id of [ + "qwen.qwen3-vl-235b-a22b", + "global.xai.grok-4.7", + "global.moonshotai.kimi-k3", + "us.meta.llama4-scout-17b-instruct-v1:0", + ]) { + const prepared = yield* compileRequest( + LLM.request({ + model: AmazonBedrock.configure({ baseURL: "https://bedrock-runtime.test", apiKey: "test-bearer" }).model( + id, + ), + messages: [ + Message.assistant([ToolCallPart.make({ id: "tool_1", name: "read", input: {} })]), + Message.tool({ + id: "tool_1", + name: "read", + result: { + type: "content", + value: [{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }], + }, + }), + ], + cache: "none", + }), + ) + expect(prepared.body.messages[1]).toEqual({ + role: "user", + content: [ + { toolResult: { toolUseId: "tool_1", content: [{ text: "See attached image." }], status: "success" } }, + { image: { format: "png", source: { bytes: "AAAA" } } }, + ], + }) + } + }), + ) + ;["global.anthropic.claude-sonnet-4-5-20250929-v1:0", "us.amazon.nova-pro-v1:0"].forEach((id) => { + it.effect(`keeps ${id} tool images inside the result`, () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: AmazonBedrock.configure({ baseURL: "https://bedrock-runtime.test", apiKey: "test-bearer" }).model( + id, + ), + messages: [ + Message.assistant([ToolCallPart.make({ id: "tool_1", name: "read", input: {} })]), + Message.tool({ + id: "tool_1", + name: "read", + result: { + type: "content", + value: [{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }], + }, + }), + ], + cache: "none", + }), + ) + + expect(prepared.body.messages[1]).toEqual({ + role: "user", + content: [ + { + toolResult: { + toolUseId: "tool_1", + content: [{ image: { format: "png", source: { bytes: "AAAA" } } }], + status: "success", + }, + }, + ], + }) + }), + ) + }) + + it.effect("keeps parallel tool results before hoisted images and gives image-only results text", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: AmazonBedrock.configure({ baseURL: "https://bedrock-runtime.test", apiKey: "test-bearer" }).model( + "global.openai.gpt-6-sol", + ), + messages: [ + Message.assistant([ + ToolCallPart.make({ id: "tool_1", name: "first", input: {} }), + ToolCallPart.make({ id: "tool_2", name: "second", input: {} }), + ]), + Message.tool({ + id: "tool_1", + name: "first", + result: { + type: "content", + value: [{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }], + }, + }), + Message.tool({ + id: "tool_2", + name: "second", + result: { + type: "content", + value: [ + { type: "text", text: "Second image." }, + { type: "file", uri: "data:image/jpeg;base64,BBBB", mime: "image/jpeg" }, + ], + }, + }), + ], + cache: "none", + }), + ) + + expect(prepared.body.messages[1]).toEqual({ + role: "user", + content: [ + { toolResult: { toolUseId: "tool_1", content: [{ text: "See attached image." }], status: "success" } }, + { toolResult: { toolUseId: "tool_2", content: [{ text: "Second image." }], status: "success" } }, + { image: { format: "png", source: { bytes: "AAAA" } } }, + { image: { format: "jpeg", source: { bytes: "BBBB" } } }, + ], + }) + }), + ) + it.effect("decodes text-delta + messageStop + metadata usage from binary event stream", () => Effect.gen(function* () { const body = eventStreamBody(