mirrors/opencode
1
0
Fork 0
mirror of https://github.com/anomalyco/opencode.git synced 2026-11-14 03:56:55 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(ai): send PDFs as file parts on OpenAI Chat routes (#50933)

Browse Source
This commit is contained in:
Aiden Cline
2026-09-23 11:27:33 -05:00
committed by GitHub
Unverified
parent 0332a26be6
commit f2d9ebb886
3 changed files with 140 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/src/protocols/openai-chat.ts
+25 -12
View File
@@ -130,6 +130,10 @@ const OpenAIChatUserContent = Schema.Union([
type: Schema.Literal("image_url"),
image_url: Schema.Struct({ url: Schema.String }),
}),
Schema.Struct({
type: Schema.Literal("file"),
file: Schema.Struct({ filename: Schema.String, file_data: Schema.String }),
}),
])
const OpenAIChatMessage = Schema.Union([
@@ -360,6 +364,15 @@ const lowerToolCall = (part: ToolCallPart, options: LoweringOptions): OpenAIChat
})
const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart) {
// Chat Completions accepts PDFs, and no other documents, as inline `file` parts; file URLs are not supported.
if (part.media.mediaType.toLowerCase() === "application/pdf")
return {
type: "file" as const,
file: {
filename: part.filename ?? "document.pdf",
file_data: (yield* ProviderShared.requireInlineMedia("OpenAI Chat", part.media)).dataUrl,
},
}
if (part.media.kind !== "image")
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.media.mediaType}`)
const url =
@@ -489,7 +502,7 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (
options: LoweringOptions,
) {
const messages: OpenAIChatMessage[] = []
const images: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const attachments: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["tool-result"]))
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"])
@@ -511,9 +524,9 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (
cache_control: options.cacheControl?.(part.cache),
})
const files = content.filter((item) => item.type === "file")
images.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))))
attachments.push(...(yield* Effect.forEach(files, (item) => lowerMedia(ProviderShared.toolFileMedia(item)))))
}
return { messages, images }
return { messages, attachments }
})
const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (
@@ -574,21 +587,21 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
if (requireAssistantAfterTool && messages.at(-1)?.role === "tool")
messages.push({ role: "assistant", content: "Done." })
}
const pendingImages: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const flushImages = () => {
if (pendingImages.length === 0) return
const pendingAttachments: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const flushAttachments = () => {
if (pendingAttachments.length === 0) return
bridgeTools()
messages.push({ role: "user", content: pendingImages.splice(0) })
messages.push({ role: "user", content: pendingAttachments.splice(0) })
}
for (const message of request.messages) {
if (message.role === "user") bridgeTools()
if (message.role === "system") {
const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message)
if (pendingImages.length > 0) {
if (pendingAttachments.length > 0) {
messages.push({
role: "user",
content: [
...pendingImages.splice(0),
...pendingAttachments.splice(0),
{ type: "text", text: part.text, cache_control: options.cacheControl?.(part.cache) },
],
})
@@ -632,13 +645,13 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
if (message.role === "tool") {
const lowered = yield* lowerToolMessages(message, lowering)
messages.push(...lowered.messages)
pendingImages.push(...lowered.images)
pendingAttachments.push(...lowered.attachments)
continue
}
flushImages()
flushAttachments()
messages.push(...(yield* lowerMessage(message, reasoningField, requireReasoning, lowering)))
}
flushImages()
flushAttachments()
return messages
})
packages/ai/test/provider/openai-chat.test.ts
+83
View File
@@ -791,6 +791,89 @@ describe("OpenAI Chat route", () => {
}),
)
it.effect("lowers inline PDFs as file parts", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user([
{ type: "text", text: "Summarize these." },
{ type: "media", media: Media.base64("JVBERi0=", "application/pdf"), filename: "report.pdf" },
{ type: "media", media: Media.fromDataUrl("data:application/pdf;base64,JVBERi0=") },
]),
],
}),
)
expect(prepared.body.messages).toEqual([
{
role: "user",
content: [
{ type: "text", text: "Summarize these." },
{ type: "file", file: { filename: "report.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } },
{ type: "file", file: { filename: "document.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } },
],
},
])
}),
)
it.effect("moves PDFs from tool results into a follow-up user message", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user("Read the report."),
Message.assistant([ToolCallPart.make({ id: "call_pdf", name: "read", input: {} })]),
Message.tool({
id: "call_pdf",
name: "read",
resultType: "content",
result: [
{ type: "text", text: "PDF read successfully" },
{
type: "file",
mime: "application/pdf",
uri: "data:application/pdf;base64,JVBERi0=",
name: "report.pdf",
},
],
}),
],
}),
)
expect(prepared.body.messages).toContainEqual({
role: "tool",
tool_call_id: "call_pdf",
content: "PDF read successfully",
})
expect(prepared.body.messages.at(-1)).toEqual({
role: "user",
content: [
{ type: "file", file: { filename: "report.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } },
],
})
}),
)
it.effect("requires inline data for PDF files", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user({
type: "media",
media: Media.url("https://example.com/report.pdf", { mediaType: "application/pdf" }),
}),
],
}),
).pipe(Effect.flip)
expect(error.message).toContain("OpenAI Chat requires inline media")
}),
)
it.effect("prepares raw and data URL image media as vision input", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
packages/ai/test/provider/zai.test.ts
+32
View File
@@ -158,6 +158,38 @@ it.effect("ZAI lowers effort using the selected native API without inventing thi
}),
)
it.effect("ZAI Coding Chat sends PDFs from tool results as file parts", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model: ZAICodingPlan.configure({ apiKey: "fixture" }).chat("glm-5.3-flash"),
messages: [
Message.user("Read the report."),
Message.assistant({ type: "tool-call", id: "call_pdf", name: "read", input: {} }),
Message.tool({
id: "call_pdf",
name: "read",
resultType: "content",
result: [
{
type: "file",
mime: "application/pdf",
uri: "data:application/pdf;base64,JVBERi0=",
name: "report.pdf",
},
],
}),
],
}),
)
expect(prepared.body.messages.at(-1)).toEqual({
role: "user",
content: [{ type: "file", file: { filename: "report.pdf", file_data: "data:application/pdf;base64,JVBERi0=" } }],
})
}),
)
it.effect("ZAI rejects invalid typed thinking options before execution", () =>
Effect.gen(function* () {
const model = ZAI.configure({ apiKey: "fixture" })