From ddd9e103fe082dcb24aaf8682d7e79eb9b0dce50 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:15:31 -0500 Subject: [PATCH] fix(ai): support Responses video inputs (#58615) --- packages/ai/src/protocols/meta-responses.ts | 34 +++++- packages/ai/src/protocols/open-responses.ts | 36 ++---- packages/ai/src/protocols/xai-responses.ts | 7 ++ .../ai/test/provider/openai-responses.test.ts | 106 ++++++++++++++---- 4 files changed, 133 insertions(+), 50 deletions(-) diff --git a/packages/ai/src/protocols/meta-responses.ts b/packages/ai/src/protocols/meta-responses.ts index 33c88caa8c4..eb9898976b3 100644 --- a/packages/ai/src/protocols/meta-responses.ts +++ b/packages/ai/src/protocols/meta-responses.ts @@ -61,7 +61,7 @@ const adapter = { const fromRequest = Effect.fn("MetaResponses.fromRequest")(function* (request: LLMRequest) { const key = request.model.route.providerMetadataKey ?? String(request.model.provider) - return yield* OpenResponses.fromRequestWithAdapter( + const body = yield* OpenResponses.fromRequestWithAdapter( LLMRequest.update(request, { messages: request.messages.map((message) => Message.make({ @@ -89,8 +89,40 @@ const fromRequest = Effect.fn("MetaResponses.fromRequest")(function* (request: L }), adapter, ) + // Meta cannot see video nested in tool results; attach it after the complete result batch. + const input: InputItem[] = [] + const videos: OpenResponses.MediaInput[] = [] + const flushVideos = () => { + if (videos.length > 0) input.push({ type: "message", role: "user", content: videos.splice(0) }) + } + for (const item of body.input) { + if (!isFunctionCallOutput(item)) { + flushVideos() + input.push(item) + continue + } + if (typeof item.output === "string") { + input.push(item) + continue + } + const content = item.output.filter((part) => part.type !== "input_video") + videos.push(...item.output.filter((part) => part.type === "input_video")) + input.push({ + ...item, + output: + content.length === 0 && item.output.length > 0 ? "Video attached in the following user message." : content, + }) + } + flushVideos() + return { ...body, input } }) +type InputItem = OpenResponses.OpenResponsesBody["input"][number] + +// Hosted replay items have an open `type`, so a plain tag check does not narrow the union. +const isFunctionCallOutput = (item: InputItem): item is Extract => + item.type === "function_call_output" + const HOSTED_TOOLS = { web_search_call: { name: "web_search", input: (item) => item.action ?? {} }, image_generation_call: { diff --git a/packages/ai/src/protocols/open-responses.ts b/packages/ai/src/protocols/open-responses.ts index a57cfa7bb96..560d4ca8b06 100644 --- a/packages/ai/src/protocols/open-responses.ts +++ b/packages/ai/src/protocols/open-responses.ts @@ -60,7 +60,7 @@ const OpenResponsesInputVideo = Schema.Struct({ type: Schema.tag("input_video"), video_url: Schema.String, }) -const MediaInput = Schema.Union([OpenResponsesInputImage, OpenResponsesInputFile]) +const MediaInput = Schema.Union([OpenResponsesInputImage, OpenResponsesInputFile, OpenResponsesInputVideo]) export type MediaInput = Schema.Schema.Type const OpenResponsesInputContent = Schema.Union([OpenResponsesInputText, MediaInput]) @@ -513,12 +513,7 @@ const lowerReasoning = (part: ReasoningPart, providerMetadataKey: string): OpenR const decodeImageDetail = ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenResponsesInputImage.fields.detail)) const decodeMessageMetadata = ProviderShared.validateWith(Schema.decodeUnknownEffect(MessageMetadata)) -const lowerMedia = Effect.fnUntraced(function* ( - part: MediaPart, - request: LLMRequest, - adapter: ProviderAdapter, - target: "message" | "tool-result", -) { +const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMRequest, adapter: ProviderAdapter) { const media = part.media.inline() const providerMedia = adapter.lowerMedia?.({ part, media, request }) if (providerMedia) return providerMedia @@ -528,8 +523,7 @@ const lowerMedia = Effect.fnUntraced(function* ( const url = ProviderShared.mediaUrl(part.media) const location = url ?? (yield* ProviderShared.requireInlineMedia(adapter.name, part.media)).dataUrl if (part.media.kind !== "image") { - if (target === "tool-result" && part.media.kind === "video") - return { type: "input_video" as const, video_url: location } + if (part.media.kind === "video") return { type: "input_video" as const, video_url: location } return { type: "input_file" as const, filename: part.filename ?? (mime === "application/pdf" ? "document.pdf" : "file"), @@ -550,17 +544,10 @@ const lowerUserContent = Effect.fnUntraced(function* ( adapter: ProviderAdapter, ) { if (part.type === "text") return { type: "input_text" as const, text: part.text } - if (part.type === "media") return yield* lowerMessageMedia(part, request, adapter) + if (part.type === "media") return yield* lowerMedia(part, request, adapter) return yield* ProviderShared.unsupportedContent(adapter.name, "user", ["text", "media"]) }) -const lowerMessageMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMRequest, adapter: ProviderAdapter) { - const lowered = yield* lowerMedia(part, request, adapter, "message") - if (lowered.type === "input_video") - return yield* ProviderShared.invalidRequest(`${adapter.name} user messages do not support input_video`) - return lowered -}) - // Tool results may carry structured text, images, and files. Keep media as provider-native // content instead of JSON-stringifying base64 into a prompt string. const lowerToolResultContentItem = Effect.fnUntraced(function* ( @@ -569,16 +556,7 @@ const lowerToolResultContentItem = Effect.fnUntraced(function* ( adapter: ProviderAdapter, ) { if (item.type === "text") return { type: "input_text" as const, text: item.text } - return yield* lowerMedia(ProviderShared.toolFileMedia(item), request, adapter, "tool-result") -}) - -const lowerHostedToolResultContentItem = Effect.fnUntraced(function* ( - item: Content, - request: LLMRequest, - adapter: ProviderAdapter, -) { - if (item.type === "text") return { type: "input_text" as const, text: item.text } - return yield* lowerMessageMedia(ProviderShared.toolFileMedia(item), request, adapter) + return yield* lowerMedia(ProviderShared.toolFileMedia(item), request, adapter) }) const lowerToolResultOutput = Effect.fnUntraced(function* ( @@ -719,7 +697,7 @@ const lowerMessages = Effect.fnUntraced(function* ( input.push({ type: "message", role: "user", - content: yield* Effect.forEach(content, (item) => lowerHostedToolResultContentItem(item, request, adapter)), + content: yield* Effect.forEach(content, (item) => lowerToolResultContentItem(item, request, adapter)), }) continue } @@ -729,7 +707,7 @@ const lowerMessages = Effect.fnUntraced(function* ( input.push({ type: "message", role: "user", - content: [yield* lowerMessageMedia(part, request, adapter)], + content: [yield* lowerMedia(part, request, adapter)], }) continue } diff --git a/packages/ai/src/protocols/xai-responses.ts b/packages/ai/src/protocols/xai-responses.ts index 130a9fd23fc..94eb7e670d6 100644 --- a/packages/ai/src/protocols/xai-responses.ts +++ b/packages/ai/src/protocols/xai-responses.ts @@ -34,6 +34,13 @@ const XAIResponsesHostedToolItem = Schema.Union([ const adapter = { id: ADAPTER, name: NAME, + lowerMedia: ({ part, media }) => { + if (part.media.kind !== "video") return undefined + const filename = part.filename ?? "file" + const url = ProviderShared.mediaUrl(part.media) + if (url) return { type: "input_file", filename, file_url: url } + return media && { type: "input_file", filename, file_data: media.dataUrl } + }, restoreHostedToolItem: (item: unknown) => (Schema.is(XAIResponsesHostedToolItem)(item) ? item : undefined), } satisfies OpenResponses.ProviderAdapter diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index 1a9031c8859..4f474d12db8 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -30,6 +30,8 @@ import { compileRequest } from "../../src/route/client.js" import * as Azure from "../../src/providers/azure.js" import * as OpenAI from "../../src/providers/openai.js" import * as XAI from "../../src/providers/xai.js" +import { Meta } from "../../src/providers/meta.js" +import { OpenAICompatibleResponses } from "../../src/protocols/openai-compatible-responses.js" import * as OpenAIResponses from "../../src/protocols/openai-responses.js" import { OpenResponses } from "../../src/protocols/open-responses.js" import { OpenResponsesContinuation } from "../../src/protocols/open-responses-continuation.js" @@ -1863,29 +1865,93 @@ describe("OpenAI Responses route", () => { }), ) - it.effect("lowers tool-result videos as input_video", () => + it.effect("lowers Responses video by provider and preserves ordered tool results", () => Effect.gen(function* () { - const prepared = yield* compileRequest( - LLM.request({ - model, - messages: [ - Message.assistant([ToolCallPart.make({ id: "call_1", name: "record", input: {} })]), - Message.tool({ - id: "call_1", - name: "record", - resultType: "content", - result: [ - { type: "file", uri: "data:video/mp4;base64,AAECAw==", mime: "video/mp4" }, - { type: "file", uri: "https://example.com/demo.mp4", mime: "video/mp4" }, - ], - }), - ], - }), + const videoResult = Message.tool({ + id: "call_video", + name: "record", + resultType: "content", + result: [{ type: "file", uri: "https://example.com/clip.mp4", mime: "video/mp4", name: "clip.mp4" }], + }) + const messages = [ + Message.user({ type: "media", media: Media.base64("AAECAw==", "video/mp4"), filename: "inline.mp4" }), + Message.assistant([ToolCallPart.make({ id: "call_video", name: "record", input: {} })]), + videoResult, + ] + const expectedInput = (user: object, tool: object) => [ + { type: "message", role: "user", content: [user] }, + { type: "function_call", call_id: "call_video", name: "record", arguments: "{}" }, + { type: "function_call_output", call_id: "call_video", output: [tool] }, + ] + const custom = OpenAICompatibleResponses.route + .with({ provider: "custom", endpoint: { baseURL: "https://provider.example/v1" } }) + .model({ id: "video-model" }) + expect( + (yield* compileRequest(LLM.request({ model: custom, messages }))).body.input, + "unknown provider uses input_video for user and tool video", + ).toMatchObject( + expectedInput( + { type: "input_video", video_url: "data:video/mp4;base64,AAECAw==" }, + { type: "input_video", video_url: "https://example.com/clip.mp4" }, + ), + ) + expect( + (yield* compileRequest(LLM.request({ model: xaiModel, messages }))).body.input, + "xAI uses input_file for user and tool video", + ).toMatchObject( + expectedInput( + { type: "input_file", filename: "inline.mp4", file_data: "data:video/mp4;base64,AAECAw==" }, + { type: "input_file", filename: "clip.mp4", file_url: "https://example.com/clip.mp4" }, + ), ) - expect(expectToolOutput(prepared.body).output).toEqual([ - { type: "input_video", video_url: "data:video/mp4;base64,AAECAw==" }, - { type: "input_video", video_url: "https://example.com/demo.mp4" }, + const meta = Meta.configure({ apiKey: "test" }).responses("muse-spark-1.3") + const metaRequest = LLM.request({ + model: meta, + messages: [ + Message.assistant([ + ToolCallPart.make({ id: "call_video", name: "record", input: {} }), + ToolCallPart.make({ id: "call_text", name: "read", input: {} }), + ]), + videoResult, + Message.tool({ id: "call_text", name: "read", resultType: "text", result: "plain result" }), + ], + }) + const metaInput = [ + { type: "function_call", call_id: "call_video", name: "record", arguments: "{}" }, + { type: "function_call", call_id: "call_text", name: "read", arguments: "{}" }, + { + type: "function_call_output", + call_id: "call_video", + output: "Video attached in the following user message.", + }, + { type: "function_call_output", call_id: "call_text", output: "plain result" }, + { + type: "message", + role: "user", + content: [{ type: "input_video", video_url: "https://example.com/clip.mp4" }], + }, + ] + expect((yield* compileRequest(metaRequest)).body.input, "Meta flushes videos at end of history").toMatchObject( + metaInput, + ) + const continued = yield* compileRequest( + LLMRequest.update(metaRequest, { messages: [...metaRequest.messages, Message.user("continue")] }), + ) + expect(continued.body.input, "Meta flushes videos before the next user message").toMatchObject([ + ...metaInput, + { type: "message", role: "user", content: [{ type: "input_text", text: "continue" }] }, + ]) + const nested = yield* compileRequest(LLMRequest.update(metaRequest, { model: custom })) + expect(nested.body.input, "the shared protocol keeps tool video nested").toMatchObject([ + { type: "function_call", call_id: "call_video", name: "record", arguments: "{}" }, + { type: "function_call", call_id: "call_text", name: "read", arguments: "{}" }, + { + type: "function_call_output", + call_id: "call_video", + output: [{ type: "input_video", video_url: "https://example.com/clip.mp4" }], + }, + { type: "function_call_output", call_id: "call_text", output: "plain result" }, ]) }), )