fix(ai): support Responses video inputs (#58615)

packages/ai/src/protocols/meta-responses.ts

@@ -61,7 +61,7 @@ const adapter = {
 
 const fromRequest = Effect.fn("MetaResponses.fromRequest")(function* (request: LLMRequest) {
   const key = request.model.route.providerMetadataKey ?? String(request.model.provider)
-  return yield* OpenResponses.fromRequestWithAdapter(
+  const body = yield* OpenResponses.fromRequestWithAdapter(
     LLMRequest.update(request, {
       messages: request.messages.map((message) =>
         Message.make({
@@ -89,8 +89,40 @@ const fromRequest = Effect.fn("MetaResponses.fromRequest")(function* (request: L
     }),
     adapter,
   )
+  // Meta cannot see video nested in tool results; attach it after the complete result batch.
+  const input: InputItem[] = []
+  const videos: OpenResponses.MediaInput[] = []
+  const flushVideos = () => {
+    if (videos.length > 0) input.push({ type: "message", role: "user", content: videos.splice(0) })
+  }
+  for (const item of body.input) {
+    if (!isFunctionCallOutput(item)) {
+      flushVideos()
+      input.push(item)
+      continue
+    }
+    if (typeof item.output === "string") {
+      input.push(item)
+      continue
+    }
+    const content = item.output.filter((part) => part.type !== "input_video")
+    videos.push(...item.output.filter((part) => part.type === "input_video"))
+    input.push({
+      ...item,
+      output:
+        content.length === 0 && item.output.length > 0 ? "Video attached in the following user message." : content,
+    })
+  }
+  flushVideos()
+  return { ...body, input }
 })
 
+type InputItem = OpenResponses.OpenResponsesBody["input"][number]
+
+// Hosted replay items have an open `type`, so a plain tag check does not narrow the union.
+const isFunctionCallOutput = (item: InputItem): item is Extract<InputItem, { type: "function_call_output" }> =>
+  item.type === "function_call_output"
+
 const HOSTED_TOOLS = {
   web_search_call: { name: "web_search", input: (item) => item.action ?? {} },
   image_generation_call: {

packages/ai/src/protocols/open-responses.ts

@@ -60,7 +60,7 @@ const OpenResponsesInputVideo = Schema.Struct({
   type: Schema.tag("input_video"),
   video_url: Schema.String,
 })
-const MediaInput = Schema.Union([OpenResponsesInputImage, OpenResponsesInputFile])
+const MediaInput = Schema.Union([OpenResponsesInputImage, OpenResponsesInputFile, OpenResponsesInputVideo])
 export type MediaInput = Schema.Schema.Type<typeof MediaInput>
 const OpenResponsesInputContent = Schema.Union([OpenResponsesInputText, MediaInput])
 
@@ -513,12 +513,7 @@ const lowerReasoning = (part: ReasoningPart, providerMetadataKey: string): OpenR
 const decodeImageDetail = ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenResponsesInputImage.fields.detail))
 const decodeMessageMetadata = ProviderShared.validateWith(Schema.decodeUnknownEffect(MessageMetadata))
 
-const lowerMedia = Effect.fnUntraced(function* (
-  part: MediaPart,
-  request: LLMRequest,
-  adapter: ProviderAdapter,
-  target: "message" | "tool-result",
-) {
+const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMRequest, adapter: ProviderAdapter) {
   const media = part.media.inline()
   const providerMedia = adapter.lowerMedia?.({ part, media, request })
   if (providerMedia) return providerMedia
@@ -528,8 +523,7 @@ const lowerMedia = Effect.fnUntraced(function* (
   const url = ProviderShared.mediaUrl(part.media)
   const location = url ?? (yield* ProviderShared.requireInlineMedia(adapter.name, part.media)).dataUrl
   if (part.media.kind !== "image") {
-    if (target === "tool-result" && part.media.kind === "video")
-      return { type: "input_video" as const, video_url: location }
+    if (part.media.kind === "video") return { type: "input_video" as const, video_url: location }
     return {
       type: "input_file" as const,
       filename: part.filename ?? (mime === "application/pdf" ? "document.pdf" : "file"),
@@ -550,17 +544,10 @@ const lowerUserContent = Effect.fnUntraced(function* (
   adapter: ProviderAdapter,
 ) {
   if (part.type === "text") return { type: "input_text" as const, text: part.text }
-  if (part.type === "media") return yield* lowerMessageMedia(part, request, adapter)
+  if (part.type === "media") return yield* lowerMedia(part, request, adapter)
   return yield* ProviderShared.unsupportedContent(adapter.name, "user", ["text", "media"])
 })
 
-const lowerMessageMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMRequest, adapter: ProviderAdapter) {
-  const lowered = yield* lowerMedia(part, request, adapter, "message")
-  if (lowered.type === "input_video")
-    return yield* ProviderShared.invalidRequest(`${adapter.name} user messages do not support input_video`)
-  return lowered
-})
-
 // Tool results may carry structured text, images, and files. Keep media as provider-native
 // content instead of JSON-stringifying base64 into a prompt string.
 const lowerToolResultContentItem = Effect.fnUntraced(function* (
@@ -569,16 +556,7 @@ const lowerToolResultContentItem = Effect.fnUntraced(function* (
   adapter: ProviderAdapter,
 ) {
   if (item.type === "text") return { type: "input_text" as const, text: item.text }
-  return yield* lowerMedia(ProviderShared.toolFileMedia(item), request, adapter, "tool-result")
-})
-
-const lowerHostedToolResultContentItem = Effect.fnUntraced(function* (
-  item: Content,
-  request: LLMRequest,
-  adapter: ProviderAdapter,
-) {
-  if (item.type === "text") return { type: "input_text" as const, text: item.text }
-  return yield* lowerMessageMedia(ProviderShared.toolFileMedia(item), request, adapter)
+  return yield* lowerMedia(ProviderShared.toolFileMedia(item), request, adapter)
 })
 
 const lowerToolResultOutput = Effect.fnUntraced(function* (
@@ -719,7 +697,7 @@ const lowerMessages = Effect.fnUntraced(function* (
           input.push({
             type: "message",
             role: "user",
-            content: yield* Effect.forEach(content, (item) => lowerHostedToolResultContentItem(item, request, adapter)),
+            content: yield* Effect.forEach(content, (item) => lowerToolResultContentItem(item, request, adapter)),
           })
           continue
         }
@@ -729,7 +707,7 @@ const lowerMessages = Effect.fnUntraced(function* (
           input.push({
             type: "message",
             role: "user",
-            content: [yield* lowerMessageMedia(part, request, adapter)],
+            content: [yield* lowerMedia(part, request, adapter)],
           })
           continue
         }

packages/ai/src/protocols/xai-responses.ts

@@ -34,6 +34,13 @@ const XAIResponsesHostedToolItem = Schema.Union([
 const adapter = {
   id: ADAPTER,
   name: NAME,
+  lowerMedia: ({ part, media }) => {
+    if (part.media.kind !== "video") return undefined
+    const filename = part.filename ?? "file"
+    const url = ProviderShared.mediaUrl(part.media)
+    if (url) return { type: "input_file", filename, file_url: url }
+    return media && { type: "input_file", filename, file_data: media.dataUrl }
+  },
   restoreHostedToolItem: (item: unknown) => (Schema.is(XAIResponsesHostedToolItem)(item) ? item : undefined),
 } satisfies OpenResponses.ProviderAdapter
 

packages/ai/test/provider/openai-responses.test.ts

@@ -30,6 +30,8 @@ import { compileRequest } from "../../src/route/client.js"
 import * as Azure from "../../src/providers/azure.js"
 import * as OpenAI from "../../src/providers/openai.js"
 import * as XAI from "../../src/providers/xai.js"
+import { Meta } from "../../src/providers/meta.js"
+import { OpenAICompatibleResponses } from "../../src/protocols/openai-compatible-responses.js"
 import * as OpenAIResponses from "../../src/protocols/openai-responses.js"
 import { OpenResponses } from "../../src/protocols/open-responses.js"
 import { OpenResponsesContinuation } from "../../src/protocols/open-responses-continuation.js"
@@ -1863,29 +1865,93 @@ describe("OpenAI Responses route", () => {
     }),
   )
 
-  it.effect("lowers tool-result videos as input_video", () =>
+  it.effect("lowers Responses video by provider and preserves ordered tool results", () =>
     Effect.gen(function* () {
-      const prepared = yield* compileRequest(
-        LLM.request({
-          model,
-          messages: [
-            Message.assistant([ToolCallPart.make({ id: "call_1", name: "record", input: {} })]),
-            Message.tool({
-              id: "call_1",
-              name: "record",
-              resultType: "content",
-              result: [
-                { type: "file", uri: "data:video/mp4;base64,AAECAw==", mime: "video/mp4" },
-                { type: "file", uri: "https://example.com/demo.mp4", mime: "video/mp4" },
-              ],
-            }),
-          ],
-        }),
+      const videoResult = Message.tool({
+        id: "call_video",
+        name: "record",
+        resultType: "content",
+        result: [{ type: "file", uri: "https://example.com/clip.mp4", mime: "video/mp4", name: "clip.mp4" }],
+      })
+      const messages = [
+        Message.user({ type: "media", media: Media.base64("AAECAw==", "video/mp4"), filename: "inline.mp4" }),
+        Message.assistant([ToolCallPart.make({ id: "call_video", name: "record", input: {} })]),
+        videoResult,
+      ]
+      const expectedInput = (user: object, tool: object) => [
+        { type: "message", role: "user", content: [user] },
+        { type: "function_call", call_id: "call_video", name: "record", arguments: "{}" },
+        { type: "function_call_output", call_id: "call_video", output: [tool] },
+      ]
+      const custom = OpenAICompatibleResponses.route
+        .with({ provider: "custom", endpoint: { baseURL: "https://provider.example/v1" } })
+        .model({ id: "video-model" })
+      expect(
+        (yield* compileRequest(LLM.request({ model: custom, messages }))).body.input,
+        "unknown provider uses input_video for user and tool video",
+      ).toMatchObject(
+        expectedInput(
+          { type: "input_video", video_url: "data:video/mp4;base64,AAECAw==" },
+          { type: "input_video", video_url: "https://example.com/clip.mp4" },
+        ),
+      )
+      expect(
+        (yield* compileRequest(LLM.request({ model: xaiModel, messages }))).body.input,
+        "xAI uses input_file for user and tool video",
+      ).toMatchObject(
+        expectedInput(
+          { type: "input_file", filename: "inline.mp4", file_data: "data:video/mp4;base64,AAECAw==" },
+          { type: "input_file", filename: "clip.mp4", file_url: "https://example.com/clip.mp4" },
+        ),
       )
 
-      expect(expectToolOutput(prepared.body).output).toEqual([
-        { type: "input_video", video_url: "data:video/mp4;base64,AAECAw==" },
-        { type: "input_video", video_url: "https://example.com/demo.mp4" },
+      const meta = Meta.configure({ apiKey: "test" }).responses("muse-spark-1.3")
+      const metaRequest = LLM.request({
+        model: meta,
+        messages: [
+          Message.assistant([
+            ToolCallPart.make({ id: "call_video", name: "record", input: {} }),
+            ToolCallPart.make({ id: "call_text", name: "read", input: {} }),
+          ]),
+          videoResult,
+          Message.tool({ id: "call_text", name: "read", resultType: "text", result: "plain result" }),
+        ],
+      })
+      const metaInput = [
+        { type: "function_call", call_id: "call_video", name: "record", arguments: "{}" },
+        { type: "function_call", call_id: "call_text", name: "read", arguments: "{}" },
+        {
+          type: "function_call_output",
+          call_id: "call_video",
+          output: "Video attached in the following user message.",
+        },
+        { type: "function_call_output", call_id: "call_text", output: "plain result" },
+        {
+          type: "message",
+          role: "user",
+          content: [{ type: "input_video", video_url: "https://example.com/clip.mp4" }],
+        },
+      ]
+      expect((yield* compileRequest(metaRequest)).body.input, "Meta flushes videos at end of history").toMatchObject(
+        metaInput,
+      )
+      const continued = yield* compileRequest(
+        LLMRequest.update(metaRequest, { messages: [...metaRequest.messages, Message.user("continue")] }),
+      )
+      expect(continued.body.input, "Meta flushes videos before the next user message").toMatchObject([
+        ...metaInput,
+        { type: "message", role: "user", content: [{ type: "input_text", text: "continue" }] },
+      ])
+      const nested = yield* compileRequest(LLMRequest.update(metaRequest, { model: custom }))
+      expect(nested.body.input, "the shared protocol keeps tool video nested").toMatchObject([
+        { type: "function_call", call_id: "call_video", name: "record", arguments: "{}" },
+        { type: "function_call", call_id: "call_text", name: "read", arguments: "{}" },
+        {
+          type: "function_call_output",
+          call_id: "call_video",
+          output: [{ type: "input_video", video_url: "https://example.com/clip.mp4" }],
+        },
+        { type: "function_call_output", call_id: "call_text", output: "plain result" },
       ])
     }),
   )