From 52c7faa07368ffa781c0605fc71ab3eb43053fcf Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:16:40 -0600 Subject: [PATCH] refactor(ai): read Anthropic media options from provider metadata (#61019) --- .../ai/src/protocols/anthropic-messages.ts | 95 +++++-------------- packages/ai/src/schema/messages.ts | 1 - .../test/provider/anthropic-messages.test.ts | 59 ++++++++++++ packages/core/test/session-native.test.ts | 3 +- 4 files changed, 85 insertions(+), 73 deletions(-) diff --git a/packages/ai/src/protocols/anthropic-messages.ts b/packages/ai/src/protocols/anthropic-messages.ts index 062b644a8c5..980bad1232e 100644 --- a/packages/ai/src/protocols/anthropic-messages.ts +++ b/packages/ai/src/protocols/anthropic-messages.ts @@ -625,59 +625,15 @@ const lowerServerToolResult = Effect.fnUntraced(function* (part: ToolResultPart, } satisfies AnthropicServerToolResultBlock }) -const fileIdFromMetadata = (metadata: MediaPart["metadata"]): string | undefined => { - if (!metadata) return undefined - const anthropic = metadata.anthropic - if (ProviderShared.isRecord(anthropic)) { - if (typeof anthropic.file_id === "string") return anthropic.file_id - if (typeof anthropic.fileId === "string") return anthropic.fileId - } - if (typeof metadata.file_id === "string") return metadata.file_id - if (typeof metadata.fileId === "string") return metadata.fileId - return undefined -} - -const transformationsFromMetadata = ( - metadata: MediaPart["metadata"], -): AnthropicImageBlock["transformations"] | undefined => { - if (!metadata) return undefined - const anthropic = ProviderShared.isRecord(metadata.anthropic) ? metadata.anthropic : undefined - const raw = anthropic?.transformations ?? metadata.transformations - if (ProviderShared.isRecord(raw)) { - const value = raw.oversized_image - if (value === "downsize" || value === "error") return { oversized_image: value } - } - if (anthropic && (anthropic.oversized_image === "downsize" || anthropic.oversized_image === "error")) - return { oversized_image: anthropic.oversized_image } - return undefined -} - -const documentTitleFromPart = (part: MediaPart): string | undefined => { - if (part.metadata !== undefined) { - const anthropic = part.metadata.anthropic - if (ProviderShared.isRecord(anthropic) && typeof anthropic.title === "string") return anthropic.title - if (typeof part.metadata.title === "string") return part.metadata.title - } - if (part.filename && part.filename.length > 0) return part.filename - return undefined -} - -const documentContextFromMetadata = (metadata: MediaPart["metadata"]): string | undefined => { - if (!metadata) return undefined - const anthropic = ProviderShared.isRecord(metadata.anthropic) ? metadata.anthropic : undefined - if (anthropic && typeof anthropic.context === "string") return anthropic.context - if (typeof metadata.context === "string") return metadata.context - return undefined -} - -const citationsFromMetadata = (metadata: MediaPart["metadata"]): AnthropicDocumentBlock["citations"] | undefined => { - if (!metadata) return undefined - const raw = ProviderShared.isRecord(metadata.anthropic) - ? (metadata.anthropic.citations ?? metadata.citations) - : metadata.citations - if (ProviderShared.isRecord(raw) && typeof raw.enabled === "boolean") return { enabled: raw.enabled } - return undefined -} +// Per-part Anthropic block fields, read from `providerMetadata[providerMetadataKey]` with Anthropic's wire names. +const MediaOptions = Schema.Struct({ + file_id: Schema.optional(Schema.String), + title: Schema.optional(Schema.String), + context: Schema.optional(Schema.String), + citations: Schema.optional(Schema.Struct({ enabled: Schema.Boolean })), + transformations: Schema.optional(AnthropicImageTransformations), +}) +const decodeMediaOptions = ProviderShared.validateWith(Schema.decodeUnknownEffect(MediaOptions)) const isHttpUrl = (value: string) => /^https?:\/\//i.test(value.trim()) @@ -701,34 +657,31 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMReq ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), } satisfies AnthropicVideoBlock } - const fileId = fileIdFromMetadata(part.metadata) - const image = (source: AnthropicImageBlock["source"]): AnthropicImageBlock => { - const transformations = transformationsFromMetadata(part.metadata) - return { - type: "image", - source, - ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), - ...(transformations === undefined ? {} : { transformations }), - } - } + const options = yield* decodeMediaOptions( + part.providerMetadata?.[request.model.route.providerMetadataKey ?? request.model.provider] ?? {}, + ) + const image = (source: AnthropicImageBlock["source"]): AnthropicImageBlock => ({ + type: "image", + source, + ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), + ...(options.transformations === undefined ? {} : { transformations: options.transformations }), + }) const document = (source: AnthropicDocumentBlock["source"]): AnthropicDocumentBlock => { - const title = documentTitleFromPart(part) - const context = documentContextFromMetadata(part.metadata) - const citations = citationsFromMetadata(part.metadata) + const title = options.title ?? (part.filename ? part.filename : undefined) return { type: "document", source, ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }), ...(title === undefined ? {} : { title }), - ...(context === undefined ? {} : { context }), - ...(citations === undefined ? {} : { citations }), + ...(options.context === undefined ? {} : { context: options.context }), + ...(options.citations === undefined ? {} : { citations: options.citations }), } } // SDK file sources: FileImageSource:2350 / FileDocumentSource:2344 {type:"file", file_id} - if (fileId) { - if (mime.startsWith("image/")) return image({ type: "file", file_id: fileId }) - return document({ type: "file", file_id: fileId }) + if (options.file_id) { + if (mime.startsWith("image/")) return image({ type: "file", file_id: options.file_id }) + return document({ type: "file", file_id: options.file_id }) } const rawString = ProviderShared.mediaUrl(part.media)?.trim() diff --git a/packages/ai/src/schema/messages.ts b/packages/ai/src/schema/messages.ts index 9908785338a..31097162c8b 100644 --- a/packages/ai/src/schema/messages.ts +++ b/packages/ai/src/schema/messages.ts @@ -50,7 +50,6 @@ export const MediaPart = Schema.Struct({ media: Media.AssetSchema, filename: Schema.optional(Schema.String), cache: Schema.optional(CacheHint), - metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), providerMetadata: Schema.optional(ProviderMetadata), }).annotate({ identifier: "LLM.Content.Media" }) export type MediaPart = Schema.Schema.Type diff --git a/packages/ai/test/provider/anthropic-messages.test.ts b/packages/ai/test/provider/anthropic-messages.test.ts index 3d0bdcacae4..a1135d94080 100644 --- a/packages/ai/test/provider/anthropic-messages.test.ts +++ b/packages/ai/test/provider/anthropic-messages.test.ts @@ -858,6 +858,65 @@ describe("Anthropic Messages route", () => { }), ) + it.effect("lowers per-part Anthropic media options from provider metadata", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + ai.request({ + model, + prompt: [ + Message.media(Media.base64("AQID", "image/png"), { + providerMetadata: { + anthropic: { file_id: "file_img", transformations: { oversized_image: "downsize" } }, + }, + }), + Message.media(Media.url("https://example.com/report.pdf", { mediaType: "application/pdf" }), { + filename: "report.pdf", + providerMetadata: { anthropic: { context: "Q3", citations: { enabled: true } } }, + }), + Message.media(Media.base64("aGk=", "text/plain"), { + filename: "notes.txt", + providerMetadata: { anthropic: { title: "Notes" } }, + }), + ], + cache: "none", + }), + ) + + expect(prepared.body.messages[0]?.content).toEqual([ + { + type: "image", + source: { type: "file", file_id: "file_img" }, + transformations: { oversized_image: "downsize" }, + }, + { + type: "document", + source: { type: "url", url: "https://example.com/report.pdf" }, + title: "report.pdf", + context: "Q3", + citations: { enabled: true }, + }, + { type: "document", source: { type: "text", media_type: "text/plain", data: "hi" }, title: "Notes" }, + ]) + }), + ) + + it.effect("rejects invalid per-part Anthropic media options", () => + Effect.gen(function* () { + const error = yield* compileRequest( + ai.request({ + model, + prompt: [ + Message.media(Media.base64("AQID", "image/png"), { + providerMetadata: { anthropic: { transformations: { oversized_image: "shrink" } } }, + }), + ], + }), + ).pipe(Effect.flip) + + expect(error.message).toContain("oversized_image") + }), + ) + const video = { type: "video", source: { type: "base64", media_type: "video/mp4", data: "AAECAw==" } } const videoFile = { type: "file" as const, uri: "data:video/mp4;base64,AAECAw==", mime: "video/mp4" } const customModel = AnthropicMessages.route.with({ provider: "custom" }).model({ id: "video-model" }) diff --git a/packages/core/test/session-native.test.ts b/packages/core/test/session-native.test.ts index 940436ecfb3..78dd4788b42 100644 --- a/packages/core/test/session-native.test.ts +++ b/packages/core/test/session-native.test.ts @@ -72,11 +72,12 @@ test("reads pre-2.0.15 checkpoint media without rewriting the stored payload", a media: { source: { type: "base64", data: "AQID", mediaType: "image/png" } }, filename: "attachment.png", cache: { type: "ephemeral" }, - metadata: { attachment: "original" }, providerMetadata: { openai: { detail: "high" } }, }, { type: "media", media: { source: { type: "base64", data: "BAUG", mediaType: "image/jpeg" } } }, ]) + // The stored `metadata` key is no longer part of `MediaPart`, so decoding drops it. + expect(decoded[0]?.content[1]).not.toHaveProperty("metadata") expect(decoded[1]?.content).toMatchObject(legacyMedia[1]?.content ?? []) expect(JSON.stringify(legacyMedia)).toBe(before) })