diff --git a/packages/ai/src/protocols/anthropic-messages.ts b/packages/ai/src/protocols/anthropic-messages.ts index 43462baf821..7ee20881571 100644 --- a/packages/ai/src/protocols/anthropic-messages.ts +++ b/packages/ai/src/protocols/anthropic-messages.ts @@ -117,6 +117,13 @@ const AnthropicImageBlock = Schema.Struct({ }) type AnthropicImageBlock = Schema.Schema.Type +const AnthropicVideoBlock = Schema.Struct({ + type: Schema.tag("video"), + source: AnthropicImageSource, + cache_control: Schema.optional(AnthropicCacheControl), +}) +type AnthropicVideoBlock = Schema.Schema.Type + // SDK: Base64PDFSource:209 {type:"base64", media_type:"application/pdf", data}, PlainTextSource:2716 {type:"text", media_type:"text/plain", data}, // SDK: URLPDFSource:3823 {type:"url", url}, FileDocumentSource:2344 {type:"file", file_id}, ContentBlockSource:2266 {type:"content", content} // SDK: DocumentBlockParam:2297 {source: 5-way union, cache_control, citations, context, title} @@ -206,7 +213,12 @@ type AnthropicServerToolResultBlock = Schema.Schema.Type @@ -671,9 +684,26 @@ const citationsFromMetadata = (metadata: MediaPart["metadata"]): AnthropicDocume const isHttpUrl = (value: string) => /^https?:\/\//i.test(value.trim()) -const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, breakpoints?: Cache.Breakpoints) { +const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMRequest, breakpoints?: Cache.Breakpoints) { const mime = part.media.mediaType.toLowerCase() const cacheControlValue = breakpoints ? cacheControl(breakpoints, part.cache) : undefined + if (mime.startsWith("video/")) { + const inline = part.media.inline() + const source = inline + ? { type: "base64" as const, media_type: mime, data: inline.base64 } + : yield* ProviderShared.mediaReference(part.media, request.model.provider, "Messages video").pipe( + Effect.map((reference) => + reference.type === "ref" && !reference.value.includes("://") + ? { type: "file" as const, file_id: reference.value } + : { type: "url" as const, url: reference.value }, + ), + ) + return { + type: "video" as const, + source, + ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), + } satisfies AnthropicVideoBlock + } const fileId = fileIdFromMetadata(part.metadata) // SDK file sources: FileImageSource:2350 / FileDocumentSource:2344 {type:"file", file_id} @@ -782,18 +812,18 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, breakpoints?: C // Tool results may carry structured text, images, and documents. Keep media as provider-native // content instead of JSON-stringifying base64 into a prompt string. -const lowerToolResultContentItem = Effect.fnUntraced(function* (item: Tool.Content) { +const lowerToolResultContentItem = Effect.fnUntraced(function* (item: Tool.Content, request: LLMRequest) { if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock - return yield* lowerMedia(ProviderShared.toolFileMedia(item)) + return yield* lowerMedia(ProviderShared.toolFileMedia(item), request) }) -const lowerToolResultContent = Effect.fnUntraced(function* (part: ToolResultPart) { +const lowerToolResultContent = Effect.fnUntraced(function* (part: ToolResultPart, request: LLMRequest) { // Text / json / error results stay as a string for backward compatibility // with existing cassettes and provider expectations. if (part.result.type !== "content") return ProviderShared.toolResultText(part) // Preserve the narrowed array element type when compiled through a consumer package. const content: ReadonlyArray = part.result.value - return yield* Effect.forEach(content, lowerToolResultContentItem) + return yield* Effect.forEach(content, (item) => lowerToolResultContentItem(item, request)) }) const requireThinkingSignature = (request: LLMRequest) => { @@ -907,7 +937,7 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest, breakpoi continue } if (part.type === "media") { - content.push(yield* lowerMedia(part, breakpoints)) + content.push(yield* lowerMedia(part, request, breakpoints)) continue } return yield* ProviderShared.unsupportedContent("Anthropic Messages", "user", ["text", "media"]) @@ -983,7 +1013,7 @@ const lowerMessages = Effect.fnUntraced(function* (request: LLMRequest, breakpoi content.push({ type: "tool_result", tool_use_id: scrubToolCallID(part.id), - content: yield* lowerToolResultContent(part), + content: yield* lowerToolResultContent(part, request), is_error: part.result.type === "error" ? true : undefined, cache_control: cacheControl(breakpoints, part.cache), }) diff --git a/packages/ai/src/protocols/meta-messages.ts b/packages/ai/src/protocols/meta-messages.ts index 0b9f6bd3243..867c887c0f7 100644 --- a/packages/ai/src/protocols/meta-messages.ts +++ b/packages/ai/src/protocols/meta-messages.ts @@ -30,6 +30,16 @@ const fromRequest = Effect.fn("MetaMessages.fromRequest")(function* (request: LL const body = yield* AnthropicMessages.protocol.body.from(projected.request) return { ...body, + messages: body.messages.map((message) => { + if (message.role !== "user") return message + const videos: Array<(typeof message.content)[number]> = [] + const content = message.content.map((block) => { + if (block.type !== "tool_result" || typeof block.content === "string") return block + videos.push(...block.content.filter((item) => item.type === "video")) + return { ...block, content: block.content.filter((item) => item.type !== "video") } + }) + return videos.length === 0 ? message : { ...message, content: [...content, ...videos] } + }), tools: body.tools === undefined ? undefined diff --git a/packages/ai/src/protocols/shared.ts b/packages/ai/src/protocols/shared.ts index 80d7022555b..7f0d8667d81 100644 --- a/packages/ai/src/protocols/shared.ts +++ b/packages/ai/src/protocols/shared.ts @@ -270,14 +270,14 @@ export const mediaReference = ( } /** - * Lift a tool-result file into a `MediaPart`. Tool files carry either a data URL, an `http(s)` URL, or raw base64 in + * Lift a tool-result file into a `MediaPart`. Tool files carry either a data URL, a URI, or raw base64 in * `uri`; the declared `mime` wins over any data-URL prefix so tool authors control the type the model sees. */ export const toolFileMedia = (item: Tool.FileContent): MediaPart => { const parsed = Media.parseDataUrl(item.uri) const asset = parsed ? Media.from({ ...parsed.source, mediaType: item.mime }) - : /^https?:\/\//.test(item.uri) + : item.uri.includes("://") ? Media.url(item.uri, { mediaType: item.mime }) : Media.base64(item.uri, item.mime) return Message.media(asset, { filename: item.name }) diff --git a/packages/ai/src/route/client.ts b/packages/ai/src/route/client.ts index c8d319018ef..967645433fd 100644 --- a/packages/ai/src/route/client.ts +++ b/packages/ai/src/route/client.ts @@ -564,6 +564,20 @@ const prepareRequest = (request: LLMRequest) => { } const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest, options?: StreamOptions) { + const references = request.messages.flatMap((message) => + message.content.flatMap((part) => { + if (part.type === "media") { + const source = part.media.source + return source.type === "url" ? [source.url] : source.type === "ref" ? [source.id] : [] + } + if (part.type !== "tool-result" || part.result.type !== "content") return [] + return part.result.value.flatMap((item) => (item.type === "file" ? [item.uri] : [])) + }), + ) + if (references.some((reference) => /^\s*file:/i.test(reference))) + return yield* ProviderShared.invalidRequest( + "Local file references must be read with Media.file(path) before sending", + ) const resolved = prepareRequest(request) const route = resolved.model.route diff --git a/packages/ai/test/media.test.ts b/packages/ai/test/media.test.ts index 5f90b566282..df1727ca7ed 100644 --- a/packages/ai/test/media.test.ts +++ b/packages/ai/test/media.test.ts @@ -3,7 +3,9 @@ import { NodeFileSystem } from "@effect/platform-node" import { Effect, Ref, Schema } from "effect" import { FileSystem } from "effect" import { HttpClientRequest } from "effect/http" -import { AIError, Media, Message } from "../src/index.js" +import { AIError, LLM, Media, Message, ToolCallPart } from "../src/index.js" +import { OpenAI } from "../src/providers.js" +import { compileRequest } from "../src/route/client.js" import { it } from "./lib/effect.js" import { dynamicResponse, scriptedResponses } from "./lib/http.js" @@ -38,6 +40,44 @@ describe("Media", () => { }), ) + it.effect("rejects local file references as request errors without breaking construction or decoding", () => + Effect.gen(function* () { + const model = OpenAI.configure({ apiKey: "test" }).responses("video-model") + const restored = yield* Schema.decodeUnknownEffect(Media.AssetSchema)({ + source: { type: "url", url: " FILE:/C:/clips/video.mp4", mediaType: "video/mp4" }, + }) + for (const asset of [ + Media.url("file:///Users/example/clip.mp4", { mediaType: "video/mp4" }), + Media.ref("openai", "file:///Users/example/clip.mp4", "video/mp4"), + restored, + ]) { + const error = yield* compileRequest(LLM.request({ model, prompt: [Message.media(asset)] })).pipe(Effect.flip) + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("Media.file(path)") + } + const error = yield* compileRequest( + LLM.request({ + model, + messages: [ + Message.assistant([ToolCallPart.make({ id: "read", name: "read", input: {} })]), + Message.tool({ + id: "read", + name: "read", + resultType: "content", + result: [{ type: "file", uri: "file:///Users/example/clip.mp4", mime: "video/mp4" }], + }), + ], + }), + ).pipe(Effect.flip) + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("Media.file(path)") + expect(Media.ref("google", "gs://bucket/clip.mp4").source).toMatchObject({ id: "gs://bucket/clip.mp4" }) + expect(Media.url("https://example.test/clip.mp4").source).toMatchObject({ + url: "https://example.test/clip.mp4", + }) + }), + ) + it.effect("parses data URLs and rejects malformed ones", () => Effect.sync(() => { const asset = Media.fromDataUrl("data:image/jpeg;base64,/9j/") diff --git a/packages/ai/test/provider/anthropic-messages.test.ts b/packages/ai/test/provider/anthropic-messages.test.ts index 7aebb0d974c..8c8685f899f 100644 --- a/packages/ai/test/provider/anthropic-messages.test.ts +++ b/packages/ai/test/provider/anthropic-messages.test.ts @@ -14,8 +14,8 @@ import { } from "../../src/index.js" import { Auth, Endpoint, LLMClient, Route } from "../../src/route.js" import { compileRequest } from "../../src/route/client.js" -import * as AnthropicMessages from "../../src/protocols/anthropic-messages.js" -import { GoogleVertexMessages } from "../../src/providers.js" +import { AnthropicMessages } from "../../src/protocols/anthropic-messages.js" +import { GoogleVertexMessages, Meta } from "../../src/providers.js" import { continuationRequest, nativeAnthropicMessagesContinuation } from "../continuation-scenarios.js" import { it } from "../lib/effect.js" import { dynamicResponse, fixedResponse } from "../lib/http.js" @@ -860,6 +860,107 @@ describe("Anthropic Messages route", () => { }), ) + const video = { type: "video", source: { type: "base64", media_type: "video/mp4", data: "AAECAw==" } } + const videoFile = { type: "file" as const, uri: "data:video/mp4;base64,AAECAw==", mime: "video/mp4" } + const customModel = AnthropicMessages.route.with({ provider: "custom" }).model({ id: "video-model" }) + + it.effect("serializes Messages video inputs", () => + Effect.gen(function* () { + const user = LLM.request({ + model: customModel, + prompt: [ + Message.media(Media.base64("AAECAw==", "video/mp4"), { cache: new CacheHint({ type: "ephemeral" }) }), + Message.media(Media.ref("custom", "file_clip", "video/mp4")), + ], + cache: "none", + }) + const prompt = yield* compileRequest(user) + expect(prompt.body.messages).toEqual([ + { + role: "user", + content: [ + { ...video, cache_control: { type: "ephemeral" } }, + { type: "video", source: { type: "file", file_id: "file_clip" } }, + ], + }, + ]) + + const nested = yield* compileRequest( + LLMRequest.update(user, { + messages: [ + Message.assistant([ToolCallPart.make({ id: "clip", name: "read", input: {} })]), + Message.tool({ + id: "clip", + name: "read", + resultType: "content", + result: [videoFile, { type: "file", uri: "storage://clip", mime: "video/mp4" }], + }), + ], + }), + ) + expect(expectToolResult(nested.body).content).toEqual([ + video, + { type: "video", source: { type: "url", url: "storage://clip" } }, + ]) + + const foreign = yield* compileRequest(LLMRequest.update(user, { model })).pipe(Effect.flip) + expect(foreign.message).toContain("got custom:file_clip") + + const native = yield* compileRequest( + LLM.request({ + model: AnthropicMessages.route.model({ id: "video-model" }), + prompt: Message.media(Media.base64("AAECAw==", "video/mp4")), + cache: "none", + }), + ) + expect(native.body.messages).toEqual([{ role: "user", content: [video] }]) + }), + ) + + it.effect("moves Meta tool result videos after consecutive results", () => + Effect.gen(function* () { + const meta = LLM.request({ + model: Meta.configure({ apiKey: "test" }).messages("video-model"), + messages: [ + Message.assistant([ + ToolCallPart.make({ id: "first", name: "read", input: {} }), + ToolCallPart.make({ id: "second", name: "read", input: {} }), + ]), + Message.tool({ id: "first", name: "read", resultType: "content", result: [videoFile] }), + Message.tool({ id: "second", name: "read", resultType: "content", result: [videoFile] }), + Message.assistant("Observed."), + ], + cache: "none", + }) + const hoisted = yield* compileRequest(meta) + expect(hoisted.body.messages).toMatchObject([ + { + role: "assistant", + content: [ + { type: "tool_use", id: "first" }, + { type: "tool_use", id: "second" }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "first", content: [] }, + { type: "tool_result", tool_use_id: "second", content: [] }, + video, + video, + ], + }, + { role: "assistant", content: [{ type: "text", text: "Observed." }] }, + ]) + + const generic = yield* compileRequest(LLMRequest.update(meta, { model: customModel })) + expect(generic.body.messages[1]?.content).toEqual([ + { type: "tool_result", tool_use_id: "first", content: [video] }, + { type: "tool_result", tool_use_id: "second", content: [video] }, + ]) + }), + ) + it.effect("prepares the composed native continuation request", () => Effect.gen(function* () { const prepared = yield* compileRequest(