fix(ai): support Messages video inputs (#58614)

packages/ai/src/protocols/anthropic-messages.ts

@@ -117,6 +117,13 @@ const AnthropicImageBlock = Schema.Struct({
 })
 type AnthropicImageBlock = Schema.Schema.Type<typeof AnthropicImageBlock>
 
+const AnthropicVideoBlock = Schema.Struct({
+  type: Schema.tag("video"),
+  source: AnthropicImageSource,
+  cache_control: Schema.optional(AnthropicCacheControl),
+})
+type AnthropicVideoBlock = Schema.Schema.Type<typeof AnthropicVideoBlock>
+
 // SDK: Base64PDFSource:209 {type:"base64", media_type:"application/pdf", data}, PlainTextSource:2716 {type:"text", media_type:"text/plain", data},
 // SDK: URLPDFSource:3823 {type:"url", url}, FileDocumentSource:2344 {type:"file", file_id}, ContentBlockSource:2266 {type:"content", content}
 // SDK: DocumentBlockParam:2297 {source: 5-way union, cache_control, citations, context, title}
@@ -206,7 +213,12 @@ type AnthropicServerToolResultBlock = Schema.Schema.Type<typeof AnthropicServerT
 // Anthropic accepts either a plain string or an ordered array of text, image, and
 // document blocks inside `tool_result.content`. The array form keeps media as native
 // model input instead of JSON-stringifying base64 into prompt text.
-const AnthropicToolResultContent = Schema.Union([AnthropicTextBlock, AnthropicImageBlock, AnthropicDocumentBlock])
+const AnthropicToolResultContent = Schema.Union([
+  AnthropicTextBlock,
+  AnthropicImageBlock,
+  AnthropicDocumentBlock,
+  AnthropicVideoBlock,
+])
 
 const AnthropicToolResultBlock = Schema.Struct({
   type: Schema.tag("tool_result"),
@@ -220,6 +232,7 @@ const AnthropicUserBlock = Schema.Union([
   AnthropicTextBlock,
   AnthropicImageBlock,
   AnthropicDocumentBlock,
+  AnthropicVideoBlock,
   AnthropicToolResultBlock,
 ])
 type AnthropicUserBlock = Schema.Schema.Type<typeof AnthropicUserBlock>
@@ -671,9 +684,26 @@ const citationsFromMetadata = (metadata: MediaPart["metadata"]): AnthropicDocume
 
 const isHttpUrl = (value: string) => /^https?:\/\//i.test(value.trim())
 
-const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, breakpoints?: Cache.Breakpoints) {
+const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMRequest, breakpoints?: Cache.Breakpoints) {
   const mime = part.media.mediaType.toLowerCase()
   const cacheControlValue = breakpoints ? cacheControl(breakpoints, part.cache) : undefined
+  if (mime.startsWith("video/")) {
+    const inline = part.media.inline()
+    const source = inline
+      ? { type: "base64" as const, media_type: mime, data: inline.base64 }
+      : yield* ProviderShared.mediaReference(part.media, request.model.provider, "Messages video").pipe(
+          Effect.map((reference) =>
+            reference.type === "ref" && !reference.value.includes("://")
+              ? { type: "file" as const, file_id: reference.value }
+              : { type: "url" as const, url: reference.value },
+          ),
+        )
+    return {
+      type: "video" as const,
+      source,
+      ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }),
+    } satisfies AnthropicVideoBlock
+  }
   const fileId = fileIdFromMetadata(part.metadata)
 
   // SDK file sources: FileImageSource:2350 / FileDocumentSource:2344 {type:"file", file_id}
@@ -782,18 +812,18 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, breakpoints?: C
 
 // Tool results may carry structured text, images, and documents. Keep media as provider-native
 // content instead of JSON-stringifying base64 into a prompt string.
-const lowerToolResultContentItem = Effect.fnUntraced(function* (item: Tool.Content) {
+const lowerToolResultContentItem = Effect.fnUntraced(function* (item: Tool.Content, request: LLMRequest) {
   if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock
-  return yield* lowerMedia(ProviderShared.toolFileMedia(item))
+  return yield* lowerMedia(ProviderShared.toolFileMedia(item), request)
 })
 
-const lowerToolResultContent = Effect.fnUntraced(function* (part: ToolResultPart) {
+const lowerToolResultContent = Effect.fnUntraced(function* (part: ToolResultPart, request: LLMRequest) {
   // Text / json / error results stay as a string for backward compatibility
   // with existing cassettes and provider expectations.
   if (part.result.type !== "content") return ProviderShared.toolResultText(part)
   // Preserve the narrowed array element type when compiled through a consumer package.
   const content: ReadonlyArray<Tool.Content> = part.result.value
-  return yield* Effect.forEach(content, lowerToolResultContentItem)
+  return yield* Effect.forEach(content, (item) => lowerToolResultContentItem(item, request))
 })
 
 const requireThinkingSignature = (request: LLMRequest) => {
@@ -907,7 +937,7 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest, breakpoi
           continue
         }
         if (part.type === "media") {
-          content.push(yield* lowerMedia(part, breakpoints))
+          content.push(yield* lowerMedia(part, request, breakpoints))
           continue
         }
         return yield* ProviderShared.unsupportedContent("Anthropic Messages", "user", ["text", "media"])
@@ -983,7 +1013,7 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest, breakpoi
       content.push({
         type: "tool_result",
         tool_use_id: scrubToolCallID(part.id),
-        content: yield* lowerToolResultContent(part),
+        content: yield* lowerToolResultContent(part, request),
         is_error: part.result.type === "error" ? true : undefined,
         cache_control: cacheControl(breakpoints, part.cache),
       })

packages/ai/src/protocols/meta-messages.ts

@@ -30,6 +30,16 @@ const fromRequest = Effect.fn("MetaMessages.fromRequest")(function* (request: LL
   const body = yield* AnthropicMessages.protocol.body.from(projected.request)
   return {
     ...body,
+    messages: body.messages.map((message) => {
+      if (message.role !== "user") return message
+      const videos: Array<(typeof message.content)[number]> = []
+      const content = message.content.map((block) => {
+        if (block.type !== "tool_result" || typeof block.content === "string") return block
+        videos.push(...block.content.filter((item) => item.type === "video"))
+        return { ...block, content: block.content.filter((item) => item.type !== "video") }
+      })
+      return videos.length === 0 ? message : { ...message, content: [...content, ...videos] }
+    }),
     tools:
       body.tools === undefined
         ? undefined

packages/ai/src/protocols/shared.ts

@@ -270,14 +270,14 @@ export const mediaReference = (
 }
 
 /**
- * Lift a tool-result file into a `MediaPart`. Tool files carry either a data URL, an `http(s)` URL, or raw base64 in
+ * Lift a tool-result file into a `MediaPart`. Tool files carry either a data URL, a URI, or raw base64 in
  * `uri`; the declared `mime` wins over any data-URL prefix so tool authors control the type the model sees.
  */
 export const toolFileMedia = (item: Tool.FileContent): MediaPart => {
   const parsed = Media.parseDataUrl(item.uri)
   const asset = parsed
     ? Media.from({ ...parsed.source, mediaType: item.mime })
-    : /^https?:\/\//.test(item.uri)
+    : item.uri.includes("://")
       ? Media.url(item.uri, { mediaType: item.mime })
       : Media.base64(item.uri, item.mime)
   return Message.media(asset, { filename: item.name })

packages/ai/src/route/client.ts

@@ -564,6 +564,20 @@ const prepareRequest = (request: LLMRequest) => {
 }
 
 const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest, options?: StreamOptions) {
+  const references = request.messages.flatMap((message) =>
+    message.content.flatMap((part) => {
+      if (part.type === "media") {
+        const source = part.media.source
+        return source.type === "url" ? [source.url] : source.type === "ref" ? [source.id] : []
+      }
+      if (part.type !== "tool-result" || part.result.type !== "content") return []
+      return part.result.value.flatMap((item) => (item.type === "file" ? [item.uri] : []))
+    }),
+  )
+  if (references.some((reference) => /^\s*file:/i.test(reference)))
+    return yield* ProviderShared.invalidRequest(
+      "Local file references must be read with Media.file(path) before sending",
+    )
   const resolved = prepareRequest(request)
   const route = resolved.model.route
 

packages/ai/test/media.test.ts

@@ -3,7 +3,9 @@ import { NodeFileSystem } from "@effect/platform-node"
 import { Effect, Ref, Schema } from "effect"
 import { FileSystem } from "effect"
 import { HttpClientRequest } from "effect/http"
-import { AIError, Media, Message } from "../src/index.js"
+import { AIError, LLM, Media, Message, ToolCallPart } from "../src/index.js"
+import { OpenAI } from "../src/providers.js"
+import { compileRequest } from "../src/route/client.js"
 import { it } from "./lib/effect.js"
 import { dynamicResponse, scriptedResponses } from "./lib/http.js"
 
@@ -38,6 +40,44 @@ describe("Media", () => {
     }),
   )
 
+  it.effect("rejects local file references as request errors without breaking construction or decoding", () =>
+    Effect.gen(function* () {
+      const model = OpenAI.configure({ apiKey: "test" }).responses("video-model")
+      const restored = yield* Schema.decodeUnknownEffect(Media.AssetSchema)({
+        source: { type: "url", url: " FILE:/C:/clips/video.mp4", mediaType: "video/mp4" },
+      })
+      for (const asset of [
+        Media.url("file:///Users/example/clip.mp4", { mediaType: "video/mp4" }),
+        Media.ref("openai", "file:///Users/example/clip.mp4", "video/mp4"),
+        restored,
+      ]) {
+        const error = yield* compileRequest(LLM.request({ model, prompt: [Message.media(asset)] })).pipe(Effect.flip)
+        expect(error.reason._tag).toBe("InvalidRequest")
+        expect(error.message).toContain("Media.file(path)")
+      }
+      const error = yield* compileRequest(
+        LLM.request({
+          model,
+          messages: [
+            Message.assistant([ToolCallPart.make({ id: "read", name: "read", input: {} })]),
+            Message.tool({
+              id: "read",
+              name: "read",
+              resultType: "content",
+              result: [{ type: "file", uri: "file:///Users/example/clip.mp4", mime: "video/mp4" }],
+            }),
+          ],
+        }),
+      ).pipe(Effect.flip)
+      expect(error.reason._tag).toBe("InvalidRequest")
+      expect(error.message).toContain("Media.file(path)")
+      expect(Media.ref("google", "gs://bucket/clip.mp4").source).toMatchObject({ id: "gs://bucket/clip.mp4" })
+      expect(Media.url("https://example.test/clip.mp4").source).toMatchObject({
+        url: "https://example.test/clip.mp4",
+      })
+    }),
+  )
+
   it.effect("parses data URLs and rejects malformed ones", () =>
     Effect.sync(() => {
       const asset = Media.fromDataUrl("data:image/jpeg;base64,/9j/")

packages/ai/test/provider/anthropic-messages.test.ts

@@ -14,8 +14,8 @@ import {
 } from "../../src/index.js"
 import { Auth, Endpoint, LLMClient, Route } from "../../src/route.js"
 import { compileRequest } from "../../src/route/client.js"
-import * as AnthropicMessages from "../../src/protocols/anthropic-messages.js"
-import { GoogleVertexMessages } from "../../src/providers.js"
+import { AnthropicMessages } from "../../src/protocols/anthropic-messages.js"
+import { GoogleVertexMessages, Meta } from "../../src/providers.js"
 import { continuationRequest, nativeAnthropicMessagesContinuation } from "../continuation-scenarios.js"
 import { it } from "../lib/effect.js"
 import { dynamicResponse, fixedResponse } from "../lib/http.js"
@@ -860,6 +860,107 @@ describe("Anthropic Messages route", () => {
     }),
   )
 
+  const video = { type: "video", source: { type: "base64", media_type: "video/mp4", data: "AAECAw==" } }
+  const videoFile = { type: "file" as const, uri: "data:video/mp4;base64,AAECAw==", mime: "video/mp4" }
+  const customModel = AnthropicMessages.route.with({ provider: "custom" }).model({ id: "video-model" })
+
+  it.effect("serializes Messages video inputs", () =>
+    Effect.gen(function* () {
+      const user = LLM.request({
+        model: customModel,
+        prompt: [
+          Message.media(Media.base64("AAECAw==", "video/mp4"), { cache: new CacheHint({ type: "ephemeral" }) }),
+          Message.media(Media.ref("custom", "file_clip", "video/mp4")),
+        ],
+        cache: "none",
+      })
+      const prompt = yield* compileRequest(user)
+      expect(prompt.body.messages).toEqual([
+        {
+          role: "user",
+          content: [
+            { ...video, cache_control: { type: "ephemeral" } },
+            { type: "video", source: { type: "file", file_id: "file_clip" } },
+          ],
+        },
+      ])
+
+      const nested = yield* compileRequest(
+        LLMRequest.update(user, {
+          messages: [
+            Message.assistant([ToolCallPart.make({ id: "clip", name: "read", input: {} })]),
+            Message.tool({
+              id: "clip",
+              name: "read",
+              resultType: "content",
+              result: [videoFile, { type: "file", uri: "storage://clip", mime: "video/mp4" }],
+            }),
+          ],
+        }),
+      )
+      expect(expectToolResult(nested.body).content).toEqual([
+        video,
+        { type: "video", source: { type: "url", url: "storage://clip" } },
+      ])
+
+      const foreign = yield* compileRequest(LLMRequest.update(user, { model })).pipe(Effect.flip)
+      expect(foreign.message).toContain("got custom:file_clip")
+
+      const native = yield* compileRequest(
+        LLM.request({
+          model: AnthropicMessages.route.model({ id: "video-model" }),
+          prompt: Message.media(Media.base64("AAECAw==", "video/mp4")),
+          cache: "none",
+        }),
+      )
+      expect(native.body.messages).toEqual([{ role: "user", content: [video] }])
+    }),
+  )
+
+  it.effect("moves Meta tool result videos after consecutive results", () =>
+    Effect.gen(function* () {
+      const meta = LLM.request({
+        model: Meta.configure({ apiKey: "test" }).messages("video-model"),
+        messages: [
+          Message.assistant([
+            ToolCallPart.make({ id: "first", name: "read", input: {} }),
+            ToolCallPart.make({ id: "second", name: "read", input: {} }),
+          ]),
+          Message.tool({ id: "first", name: "read", resultType: "content", result: [videoFile] }),
+          Message.tool({ id: "second", name: "read", resultType: "content", result: [videoFile] }),
+          Message.assistant("Observed."),
+        ],
+        cache: "none",
+      })
+      const hoisted = yield* compileRequest(meta)
+      expect(hoisted.body.messages).toMatchObject([
+        {
+          role: "assistant",
+          content: [
+            { type: "tool_use", id: "first" },
+            { type: "tool_use", id: "second" },
+          ],
+        },
+        {
+          role: "user",
+          content: [
+            { type: "tool_result", tool_use_id: "first", content: [] },
+            { type: "tool_result", tool_use_id: "second", content: [] },
+            video,
+            video,
+          ],
+        },
+        { role: "assistant", content: [{ type: "text", text: "Observed." }] },
+      ])
+
+      const generic = yield* compileRequest(LLMRequest.update(meta, { model: customModel }))
+      expect(generic.body.messages[1]?.content).toEqual([
+        { type: "tool_result", tool_use_id: "first", content: [video] },
+        { type: "tool_result", tool_use_id: "second", content: [video] },
+      ])
+    }),
+  )
+
   it.effect("prepares the composed native continuation request", () =>
     Effect.gen(function* () {
       const prepared = yield* compileRequest(