diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index 1dcc0ef69a2..ca115c78c3e 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -130,6 +130,10 @@ const OpenAIChatUserContent = Schema.Union([ type: Schema.Literal("image_url"), image_url: Schema.Struct({ url: Schema.String }), }), + Schema.Struct({ + type: Schema.Literal("file"), + file: Schema.Struct({ filename: Schema.String, file_data: Schema.String }), + }), ]) const OpenAIChatMessage = Schema.Union([ @@ -360,6 +364,15 @@ const lowerToolCall = (part: ToolCallPart, options: LoweringOptions): OpenAIChat }) const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart) { + // Chat Completions accepts PDFs, and no other documents, as inline `file` parts; file URLs are not supported. + if (part.media.mediaType.toLowerCase() === "application/pdf") + return { + type: "file" as const, + file: { + filename: part.filename ?? "document.pdf", + file_data: (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl, + }, + } if (part.media.kind !== "image") return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.media.mediaType}`) const url = @@ -489,7 +502,7 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* ( options: LoweringOptions, ) { const messages: OpenAIChatMessage[] = [] - const images: Array> = [] + const attachments: Array> = [] for (const part of message.content) { if (!ProviderShared.supportsContent(part, ["tool-result"])) return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"]) @@ -511,9 +524,9 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* ( cache_control: options.cacheControl?.(part.cache), }) const files = content.filter((item) => item.type === "file") - images.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item))))) + attachments.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item))))) } - return { messages, images } + return { messages, attachments } }) const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* ( @@ -574,21 +587,21 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request: if (requireAssistantAfterTool && messages.at(-1)?.role === "tool") messages.push({ role: "assistant", content: "Done." }) } - const pendingImages: Array> = [] - const flushImages = () => { - if (pendingImages.length === 0) return + const pendingAttachments: Array> = [] + const flushAttachments = () => { + if (pendingAttachments.length === 0) return bridgeTools() - messages.push({ role: "user", content: pendingImages.splice(0) }) + messages.push({ role: "user", content: pendingAttachments.splice(0) }) } for (const message of request.messages) { if (message.role === "user") bridgeTools() if (message.role === "system") { const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message) - if (pendingImages.length > 0) { + if (pendingAttachments.length > 0) { messages.push({ role: "user", content: [ - ...pendingImages.splice(0), + ...pendingAttachments.splice(0), { type: "text", text: part.text, cache_control: options.cacheControl?.(part.cache) }, ], }) @@ -632,13 +645,13 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request: if (message.role === "tool") { const lowered = yield* lowerToolMessages(message, lowering) messages.push(...lowered.messages) - pendingImages.push(...lowered.images) + pendingAttachments.push(...lowered.attachments) continue } - flushImages() + flushAttachments() messages.push(...(yield* lowerMessage(message, reasoningField, requireReasoning, lowering))) } - flushImages() + flushAttachments() return messages }) diff --git a/packages/ai/test/provider/openai-chat.test.ts b/packages/ai/test/provider/openai-chat.test.ts index 453b37af6a7..99445904878 100644 --- a/packages/ai/test/provider/openai-chat.test.ts +++ b/packages/ai/test/provider/openai-chat.test.ts @@ -791,6 +791,89 @@ describe("OpenAI Chat route", () => { }), ) + it.effect("lowers inline PDFs as file parts", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model, + messages: [ + Message.user([ + { type: "text", text: "Summarize these." }, + { type: "media", media: Media.base64("JVBERi0=", "application/pdf"), filename: "report.pdf" }, + { type: "media", media: Media.fromDataUrl("data:application/pdf;base64,JVBERi0=") }, + ]), + ], + }), + ) + expect(prepared.body.messages).toEqual([ + { + role: "user", + content: [ + { type: "text", text: "Summarize these." }, + { type: "file", file: { filename: "report.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } }, + { type: "file", file: { filename: "document.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } }, + ], + }, + ]) + }), + ) + + it.effect("moves PDFs from tool results into a follow-up user message", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model, + messages: [ + Message.user("Read the report."), + Message.assistant([ToolCallPart.make({ id: "call_pdf", name: "read", input: {} })]), + Message.tool({ + id: "call_pdf", + name: "read", + resultType: "content", + result: [ + { type: "text", text: "PDF read successfully" }, + { + type: "file", + mime: "application/pdf", + uri: "data:application/pdf;base64,JVBERi0=", + name: "report.pdf", + }, + ], + }), + ], + }), + ) + expect(prepared.body.messages).toContainEqual({ + role: "tool", + tool_call_id: "call_pdf", + content: "PDF read successfully", + }) + expect(prepared.body.messages.at(-1)).toEqual({ + role: "user", + content: [ + { type: "file", file: { filename: "report.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } }, + ], + }) + }), + ) + + it.effect("requires inline data for PDF files", () => + Effect.gen(function* () { + const error = yield* compileRequest( + LLM.request({ + model, + messages: [ + Message.user({ + type: "media", + media: Media.url("https://example.com/report.pdf", { mediaType: "application/pdf" }), + }), + ], + }), + ).pipe(Effect.flip) + expect(error.message).toContain("OpenAI Chat requires inline media") + }), + ) + it.effect("prepares raw and data URL image media as vision input", () => Effect.gen(function* () { const prepared = yield* compileRequest( diff --git a/packages/ai/test/provider/zai.test.ts b/packages/ai/test/provider/zai.test.ts index f9f41ee2ac2..118380851dc 100644 --- a/packages/ai/test/provider/zai.test.ts +++ b/packages/ai/test/provider/zai.test.ts @@ -158,6 +158,38 @@ it.effect("ZAI lowers effort using the selected native API without inventing thi }), ) +it.effect("ZAI Coding Chat sends PDFs from tool results as file parts", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model: ZAICodingPlan.configure({ apiKey: "fixture" }).chat("glm-5.3-flash"), + messages: [ + Message.user("Read the report."), + Message.assistant({ type: "tool-call", id: "call_pdf", name: "read", input: {} }), + Message.tool({ + id: "call_pdf", + name: "read", + resultType: "content", + result: [ + { + type: "file", + mime: "application/pdf", + uri: "data:application/pdf;base64,JVBERi0=", + name: "report.pdf", + }, + ], + }), + ], + }), + ) + + expect(prepared.body.messages.at(-1)).toEqual({ + role: "user", + content: [{ type: "file", file: { filename: "report.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } }], + }) + }), +) + it.effect("ZAI rejects invalid typed thinking options before execution", () => Effect.gen(function* () { const model = ZAI.configure({ apiKey: "fixture" })