refactor(ai): read Anthropic media options from provider metadata (#61019)

packages/ai/src/protocols/anthropic-messages.ts

@@ -625,59 +625,15 @@ const lowerServerToolResult = Effect.fnUntraced(function* (part: ToolResultPart,
   } satisfies AnthropicServerToolResultBlock
 })
 
-const fileIdFromMetadata = (metadata: MediaPart["metadata"]): string | undefined => {
-  if (!metadata) return undefined
-  const anthropic = metadata.anthropic
-  if (ProviderShared.isRecord(anthropic)) {
-    if (typeof anthropic.file_id === "string") return anthropic.file_id
-    if (typeof anthropic.fileId === "string") return anthropic.fileId
-  }
-  if (typeof metadata.file_id === "string") return metadata.file_id
-  if (typeof metadata.fileId === "string") return metadata.fileId
-  return undefined
-}
-
-const transformationsFromMetadata = (
-  metadata: MediaPart["metadata"],
-): AnthropicImageBlock["transformations"] | undefined => {
-  if (!metadata) return undefined
-  const anthropic = ProviderShared.isRecord(metadata.anthropic) ? metadata.anthropic : undefined
-  const raw = anthropic?.transformations ?? metadata.transformations
-  if (ProviderShared.isRecord(raw)) {
-    const value = raw.oversized_image
-    if (value === "downsize" || value === "error") return { oversized_image: value }
-  }
-  if (anthropic && (anthropic.oversized_image === "downsize" || anthropic.oversized_image === "error"))
-    return { oversized_image: anthropic.oversized_image }
-  return undefined
-}
-
-const documentTitleFromPart = (part: MediaPart): string | undefined => {
-  if (part.metadata !== undefined) {
-    const anthropic = part.metadata.anthropic
-    if (ProviderShared.isRecord(anthropic) && typeof anthropic.title === "string") return anthropic.title
-    if (typeof part.metadata.title === "string") return part.metadata.title
-  }
-  if (part.filename && part.filename.length > 0) return part.filename
-  return undefined
-}
-
-const documentContextFromMetadata = (metadata: MediaPart["metadata"]): string | undefined => {
-  if (!metadata) return undefined
-  const anthropic = ProviderShared.isRecord(metadata.anthropic) ? metadata.anthropic : undefined
-  if (anthropic && typeof anthropic.context === "string") return anthropic.context
-  if (typeof metadata.context === "string") return metadata.context
-  return undefined
-}
-
-const citationsFromMetadata = (metadata: MediaPart["metadata"]): AnthropicDocumentBlock["citations"] | undefined => {
-  if (!metadata) return undefined
-  const raw = ProviderShared.isRecord(metadata.anthropic)
-    ? (metadata.anthropic.citations ?? metadata.citations)
-    : metadata.citations
-  if (ProviderShared.isRecord(raw) && typeof raw.enabled === "boolean") return { enabled: raw.enabled }
-  return undefined
-}
+// Per-part Anthropic block fields, read from `providerMetadata[providerMetadataKey]` with Anthropic's wire names.
+const MediaOptions = Schema.Struct({
+  file_id: Schema.optional(Schema.String),
+  title: Schema.optional(Schema.String),
+  context: Schema.optional(Schema.String),
+  citations: Schema.optional(Schema.Struct({ enabled: Schema.Boolean })),
+  transformations: Schema.optional(AnthropicImageTransformations),
+})
+const decodeMediaOptions = ProviderShared.validateWith(Schema.decodeUnknownEffect(MediaOptions))
 
 const isHttpUrl = (value: string) => /^https?:\/\//i.test(value.trim())
 
@@ -701,34 +657,31 @@ const lowerMedia = Effect.fnUntraced(function* (part: MediaPart, request: LLMReq
       ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }),
     } satisfies AnthropicVideoBlock
   }
-  const fileId = fileIdFromMetadata(part.metadata)
-  const image = (source: AnthropicImageBlock["source"]): AnthropicImageBlock => {
-    const transformations = transformationsFromMetadata(part.metadata)
-    return {
-      type: "image",
-      source,
-      ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }),
-      ...(transformations === undefined ? {} : { transformations }),
-    }
-  }
+  const options = yield* decodeMediaOptions(
+    part.providerMetadata?.[request.model.route.providerMetadataKey ?? request.model.provider] ?? {},
+  )
+  const image = (source: AnthropicImageBlock["source"]): AnthropicImageBlock => ({
+    type: "image",
+    source,
+    ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }),
+    ...(options.transformations === undefined ? {} : { transformations: options.transformations }),
+  })
   const document = (source: AnthropicDocumentBlock["source"]): AnthropicDocumentBlock => {
-    const title = documentTitleFromPart(part)
-    const context = documentContextFromMetadata(part.metadata)
-    const citations = citationsFromMetadata(part.metadata)
+    const title = options.title ?? (part.filename ? part.filename : undefined)
     return {
       type: "document",
       source,
       ...(cacheControlValue === undefined ? {} : { cache_control: cacheControlValue }),
       ...(title === undefined ? {} : { title }),
-      ...(context === undefined ? {} : { context }),
-      ...(citations === undefined ? {} : { citations }),
+      ...(options.context === undefined ? {} : { context: options.context }),
+      ...(options.citations === undefined ? {} : { citations: options.citations }),
     }
   }
 
   // SDK file sources: FileImageSource:2350 / FileDocumentSource:2344 {type:"file", file_id}
-  if (fileId) {
-    if (mime.startsWith("image/")) return image({ type: "file", file_id: fileId })
-    return document({ type: "file", file_id: fileId })
+  if (options.file_id) {
+    if (mime.startsWith("image/")) return image({ type: "file", file_id: options.file_id })
+    return document({ type: "file", file_id: options.file_id })
   }
 
   const rawString = ProviderShared.mediaUrl(part.media)?.trim()

packages/ai/src/schema/messages.ts

@@ -50,7 +50,6 @@ export const MediaPart = Schema.Struct({
   media: Media.AssetSchema,
   filename: Schema.optional(Schema.String),
   cache: Schema.optional(CacheHint),
-  metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
   providerMetadata: Schema.optional(ProviderMetadata),
 }).annotate({ identifier: "LLM.Content.Media" })
 export type MediaPart = Schema.Schema.Type<typeof MediaPart>

packages/ai/test/provider/anthropic-messages.test.ts

@@ -858,6 +858,65 @@ describe("Anthropic Messages route", () => {
     }),
   )
 
+  it.effect("lowers per-part Anthropic media options from provider metadata", () =>
+    Effect.gen(function* () {
+      const prepared = yield* compileRequest(
+        ai.request({
+          model,
+          prompt: [
+            Message.media(Media.base64("AQID", "image/png"), {
+              providerMetadata: {
+                anthropic: { file_id: "file_img", transformations: { oversized_image: "downsize" } },
+              },
+            }),
+            Message.media(Media.url("https://example.com/report.pdf", { mediaType: "application/pdf" }), {
+              filename: "report.pdf",
+              providerMetadata: { anthropic: { context: "Q3", citations: { enabled: true } } },
+            }),
+            Message.media(Media.base64("aGk=", "text/plain"), {
+              filename: "notes.txt",
+              providerMetadata: { anthropic: { title: "Notes" } },
+            }),
+          ],
+          cache: "none",
+        }),
+      )
+
+      expect(prepared.body.messages[0]?.content).toEqual([
+        {
+          type: "image",
+          source: { type: "file", file_id: "file_img" },
+          transformations: { oversized_image: "downsize" },
+        },
+        {
+          type: "document",
+          source: { type: "url", url: "https://example.com/report.pdf" },
+          title: "report.pdf",
+          context: "Q3",
+          citations: { enabled: true },
+        },
+        { type: "document", source: { type: "text", media_type: "text/plain", data: "hi" }, title: "Notes" },
+      ])
+    }),
+  )
+
+  it.effect("rejects invalid per-part Anthropic media options", () =>
+    Effect.gen(function* () {
+      const error = yield* compileRequest(
+        ai.request({
+          model,
+          prompt: [
+            Message.media(Media.base64("AQID", "image/png"), {
+              providerMetadata: { anthropic: { transformations: { oversized_image: "shrink" } } },
+            }),
+          ],
+        }),
+      ).pipe(Effect.flip)
+
+      expect(error.message).toContain("oversized_image")
+    }),
+  )
+
   const video = { type: "video", source: { type: "base64", media_type: "video/mp4", data: "AAECAw==" } }
   const videoFile = { type: "file" as const, uri: "data:video/mp4;base64,AAECAw==", mime: "video/mp4" }
   const customModel = AnthropicMessages.route.with({ provider: "custom" }).model({ id: "video-model" })

packages/core/test/session-native.test.ts

@@ -72,11 +72,12 @@ test("reads pre-2.0.15 checkpoint media without rewriting the stored payload", a
       media: { source: { type: "base64", data: "AQID", mediaType: "image/png" } },
       filename: "attachment.png",
       cache: { type: "ephemeral" },
-      metadata: { attachment: "original" },
       providerMetadata: { openai: { detail: "high" } },
     },
     { type: "media", media: { source: { type: "base64", data: "BAUG", mediaType: "image/jpeg" } } },
   ])
+  // The stored `metadata` key is no longer part of `MediaPart`, so decoding drops it.
+  expect(decoded[0]?.content[1]).not.toHaveProperty("metadata")
   expect(decoded[1]?.content).toMatchObject(legacyMedia[1]?.content ?? [])
   expect(JSON.stringify(legacyMedia)).toBe(before)
 })