diff --git a/packages/core/src/aisdk.ts b/packages/core/src/aisdk.ts index 22a7d2d1b09..841ca6a5172 100644 --- a/packages/core/src/aisdk.ts +++ b/packages/core/src/aisdk.ts @@ -482,8 +482,7 @@ function toolMessage(input: LLMRequest["messages"][number]) { const value = part.result.value.filter((item) => { if (item.type !== "file") return true if (!item.mime.startsWith("image/") && item.mime !== "application/pdf") return true - const data = /^data:[^;,]+(?:;[^,]*)*;base64,(.*)$/s.exec(item.uri)?.[1] ?? item.uri - media.push({ type: "file", mediaType: item.mime, data, filename: item.name }) + media.push({ type: "file", mediaType: item.mime, data: fileData(item.uri), filename: item.name }) return false }) return toolResultPart({ @@ -507,7 +506,7 @@ function text(part: ContentPart) { function userPart(part: ContentPart): UserContent { if (part.type === "text") return [{ type: "text", text: part.text }] if (part.type === "media") - return [{ type: "file", mediaType: part.mediaType, data: part.data, filename: part.filename }] + return [{ type: "file", mediaType: part.mediaType, data: fileData(part.data), filename: part.filename }] return [] } @@ -516,7 +515,7 @@ function assistantPart(part: ContentPart): AssistantContent { case "text": return [{ type: "text", text: part.text, providerOptions: metadataProviderOptions(part.providerMetadata) }] case "media": - return [{ type: "file", mediaType: part.mediaType, data: part.data, filename: part.filename }] + return [{ type: "file", mediaType: part.mediaType, data: fileData(part.data), filename: part.filename }] case "reasoning": return [{ type: "reasoning", text: part.text, providerOptions: metadataProviderOptions(part.providerMetadata) }] case "tool-call": @@ -535,6 +534,15 @@ function assistantPart(part: ContentPart): AssistantContent { } } +function fileData(data: Extract["data"]) { + if (typeof data !== "string") return data + const base64 = /^data:[^;,]+(?:;[^,]*)*;base64,(.*)$/s.exec(data)?.[1] + if (base64 !== undefined) return base64 + if (!URL.canParse(data)) return data + const url = new URL(data) + return url.protocol === "http:" || url.protocol === "https:" ? url : data +} + function toolResultPart(part: ContentPart): ToolResultContent[] { if (part.type !== "tool-result") return [] return [ diff --git a/packages/core/test/aisdk.test.ts b/packages/core/test/aisdk.test.ts index 0de60087a29..436a56533ae 100644 --- a/packages/core/test/aisdk.test.ts +++ b/packages/core/test/aisdk.test.ts @@ -375,7 +375,100 @@ it.effect("projects replay metadata onto AI SDK prompt parts", () => }), ) -it.effect("moves a tool image through the real Mistral provider as a user message", () => +it.effect("normalizes file data across AI SDK prompt parts", () => + Effect.gen(function* () { + const aisdk = yield* AISDK.Service + yield* aisdk.hook.sdk((event) => { + event.sdk = { languageModel: () => ({ provider: event.model.providerID }) } + }) + + const resolved = yield* aisdk.model(model("opaque-provider")) + const bytes = new Uint8Array([0, 1, 2, 3]) + const prepared = yield* compileRequest( + LLM.request({ + model: resolved, + messages: [ + Message.user([ + { type: "media", mediaType: "image/png", data: bytes, filename: "bytes.png" }, + { type: "media", mediaType: "image/png", data: "AAAA", filename: "base64.png" }, + { + type: "media", + mediaType: "image/png", + data: "data:image/png;charset=utf-8;base64,AQID", + filename: "inline.png", + }, + { type: "media", mediaType: "image/png", data: "https://example.com/image.png" }, + { type: "media", mediaType: "image/png", data: "s3://bucket/image.png" }, + ]), + Message.assistant({ + type: "media", + mediaType: "application/pdf", + data: "http://example.com/document.pdf", + filename: "document.pdf", + }), + Message.tool({ + id: "call_1", + name: "screenshot", + result: { + type: "content", + value: [{ type: "file", uri: "data:image/png;base64,BAUG", mime: "image/png", name: "tool.png" }], + }, + }), + ], + }), + ) + + expect(prepared.body.prompt).toEqual([ + { + role: "user", + content: [ + { type: "file", mediaType: "image/png", data: bytes, filename: "bytes.png" }, + { type: "file", mediaType: "image/png", data: "AAAA", filename: "base64.png" }, + { type: "file", mediaType: "image/png", data: "AQID", filename: "inline.png" }, + { + type: "file", + mediaType: "image/png", + data: new URL("https://example.com/image.png"), + filename: undefined, + }, + { type: "file", mediaType: "image/png", data: "s3://bucket/image.png", filename: undefined }, + ], + }, + { + role: "assistant", + content: [ + { + type: "file", + mediaType: "application/pdf", + data: new URL("http://example.com/document.pdf"), + filename: "document.pdf", + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call_1", + toolName: "screenshot", + output: { type: "text", value: "Media attached in the following user message." }, + providerOptions: undefined, + }, + ], + }, + { + role: "user", + content: [ + { type: "text", text: "Attached media from tool result:" }, + { type: "file", mediaType: "image/png", data: "BAUG", filename: "tool.png" }, + ], + }, + ]) + }), +) + +it.effect("normalizes user and tool media through the real Mistral provider", () => Effect.gen(function* () { const aisdk = yield* AISDK.Service let body: { messages?: unknown[] } | undefined @@ -415,7 +508,14 @@ it.effect("moves a tool image through the real Mistral provider as a user messag LLM.request({ model: resolved, messages: [ - Message.user("Inspect the screenshot."), + Message.user([ + { type: "text", text: "Inspect the attachments." }, + { type: "media", mediaType: "image/png", data: new Uint8Array([0, 1, 2, 3]) }, + { type: "media", mediaType: "image/png", data: "AQID" }, + { type: "media", mediaType: "image/png", data: "data:image/png;base64,BAUG" }, + { type: "media", mediaType: "image/png", data: "http://example.com/image.png" }, + { type: "media", mediaType: "application/pdf", data: "https://example.com/document.pdf" }, + ]), Message.assistant({ type: "tool-call", id: "call_1", name: "screenshot", input: {} }), Message.tool({ type: "tool-result", @@ -426,6 +526,12 @@ it.effect("moves a tool image through the real Mistral provider as a user messag value: [ { type: "text", text: "Screenshot captured" }, { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "screen.png" }, + { + type: "file", + uri: "https://example.com/tool-document.pdf", + mime: "application/pdf", + name: "tool-document.pdf", + }, ], }, }), @@ -434,7 +540,17 @@ it.effect("moves a tool image through the real Mistral provider as a user messag ).pipe(Effect.provide(client)) expect(body?.messages).toEqual([ - { role: "user", content: [{ type: "text", text: "Inspect the screenshot." }] }, + { + role: "user", + content: [ + { type: "text", text: "Inspect the attachments." }, + { type: "image_url", image_url: "data:image/png;base64,AAECAw==" }, + { type: "image_url", image_url: "data:image/png;base64,AQID" }, + { type: "image_url", image_url: "data:image/png;base64,BAUG" }, + { type: "image_url", image_url: "http://example.com/image.png" }, + { type: "document_url", document_url: "https://example.com/document.pdf" }, + ], + }, { role: "assistant", content: "", @@ -457,6 +573,7 @@ it.effect("moves a tool image through the real Mistral provider as a user messag content: [ { type: "text", text: "Attached media from tool result:" }, { type: "image_url", image_url: "data:image/png;base64,AAAA" }, + { type: "document_url", document_url: "https://example.com/tool-document.pdf" }, ], }, ])