diff --git a/packages/core/src/aisdk.ts b/packages/core/src/aisdk.ts index fd273a948d2..51765799248 100644 --- a/packages/core/src/aisdk.ts +++ b/packages/core/src/aisdk.ts @@ -508,8 +508,23 @@ function toolOutput(result: ToolResultValue) { case "text": case "error": return { type: "text" as const, value: messageValue(result.value) } + case "content": + return { + type: "content" as const, + value: result.value.map((item) => { + if (item.type === "text") return { type: "text" as const, text: item.text } + const data = /^data:[^;,]+(?:;[^,]*)*;base64,(.*)$/s.exec(item.uri)?.[1] + const image = item.mime.toLowerCase().startsWith("image/") + if (data !== undefined) + return image + ? { type: "image-data" as const, data, mediaType: item.mime } + : { type: "file-data" as const, data, mediaType: item.mime, filename: item.name } + return image ? { type: "image-url" as const, url: item.uri } : { type: "file-url" as const, url: item.uri } + }), + } + case "json": + return { type: "json" as const, value: jsonValue(result.value) } } - return { type: "json" as const, value: jsonValue(result.value) } } function tool(input: ToolDefinition): LanguageModelV3FunctionTool { diff --git a/packages/core/test/aisdk.test.ts b/packages/core/test/aisdk.test.ts index 6fbb1539db6..9decce67b81 100644 --- a/packages/core/test/aisdk.test.ts +++ b/packages/core/test/aisdk.test.ts @@ -275,6 +275,73 @@ it.effect("projects replay metadata onto AI SDK prompt parts", () => }), ) +it.effect("preserves tool result content in AI SDK prompts", () => + Effect.gen(function* () { + const aisdk = yield* AISDK.Service + yield* aisdk.hook.sdk((event) => { + event.sdk = { languageModel: () => ({ provider: event.model.providerID }) } + }) + + const resolved = yield* aisdk.model(model("test-ai-sdk")) + const prepared = yield* compileRequest( + LLM.request({ + model: resolved, + messages: [ + Message.tool({ + id: "call_1", + name: "read", + result: { + type: "content", + value: [ + { type: "text", text: "attachments" }, + { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "pixel.png" }, + { + type: "file", + uri: "data:application/pdf;charset=utf-8;base64,JVBERg==", + mime: "application/pdf", + name: "document.pdf", + }, + { type: "file", uri: "data:audio/mpeg;base64,SUQz", mime: "audio/mpeg", name: "clip.mp3" }, + { type: "file", uri: "https://example.com/pixel.png", mime: "image/png" }, + { type: "file", uri: "https://example.com/document.pdf", mime: "application/pdf" }, + ], + }, + }), + ], + }), + ) + + expect(prepared.body.prompt).toEqual([ + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call_1", + toolName: "read", + output: { + type: "content", + value: [ + { type: "text", text: "attachments" }, + { type: "image-data", data: "AAAA", mediaType: "image/png" }, + { + type: "file-data", + data: "JVBERg==", + mediaType: "application/pdf", + filename: "document.pdf", + }, + { type: "file-data", data: "SUQz", mediaType: "audio/mpeg", filename: "clip.mp3" }, + { type: "image-url", url: "https://example.com/pixel.png" }, + { type: "file-url", url: "https://example.com/document.pdf" }, + ], + }, + }, + ], + }, + ]) + }), +) + it.effect("emits malformed AI SDK tool input without executing it", () => Effect.gen(function* () { const aisdk = yield* AISDK.Service