fix(ai): preserve responses image detail (#46429)

packages/ai/src/protocols/open-responses.ts

@@ -39,6 +39,7 @@ const OpenResponsesInputText = Schema.Struct({
 const OpenResponsesInputImage = Schema.Struct({
   type: Schema.tag("input_image"),
   image_url: Schema.String,
+  detail: Schema.optional(Schema.String),
 })
 const OpenResponsesInputFile = Schema.Struct({
   type: Schema.tag("input_file"),
@@ -504,7 +505,13 @@ const lowerMedia = Effect.fn("OpenResponses.lowerMedia")(function* (
       ...(url ? { file_url: url } : { file_data: media.dataUrl }),
     }
   }
-  return { type: "input_image" as const, image_url: url ?? media.dataUrl }
+  return {
+    type: "input_image" as const,
+    image_url: url ?? media.dataUrl,
+    detail: yield* ProviderShared.validateWith(Schema.decodeUnknownEffect(OpenResponsesInputImage.fields.detail))(
+      part.providerMetadata?.[request.model.route.providerMetadataKey ?? "openresponses"]?.detail,
+    ),
+  }
 })
 
 const lowerUserContent = Effect.fnUntraced(function* (

packages/ai/src/schema/messages.ts

@@ -52,6 +52,7 @@ export const MediaPart = Schema.Struct({
   filename: Schema.optional(Schema.String),
   cache: Schema.optional(CacheHint),
   metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
+  providerMetadata: Schema.optional(ProviderMetadata),
 }).annotate({ identifier: "LLM.Content.Media" })
 export type MediaPart = Schema.Schema.Type<typeof MediaPart>
 

packages/ai/test/provider/image-detail.test.ts

@@ -0,0 +1,60 @@
+import { expect } from "bun:test"
+import { Effect, Schema } from "effect"
+import { LLM, Message } from "../../src/index.js"
+import { OpenAI, Azure, XAI } from "../../src/providers.js"
+import { compileRequest } from "../../src/route/client.js"
+import { it } from "../lib/effect.js"
+
+for (const model of [
+  OpenAI.configure({ apiKey: "test" }).responses("fixture"),
+  Azure.configure({ apiKey: "test", resourceName: "test" }).responses("fixture"),
+  XAI.configure({ apiKey: "test" }).responses("fixture"),
+]) {
+  it.effect(`${model.provider} preserves image detail through message serialization and lowering`, () =>
+    Effect.gen(function* () {
+      const details = [undefined, "low", "high", "auto"]
+      const message = Message.user(
+        details.map((detail) => ({
+          type: "media",
+          mediaType: "image/png",
+          data: "https://example.com/image.png",
+          providerMetadata:
+            detail === undefined ? undefined : { [model.route.providerMetadataKey ?? model.provider]: { detail } },
+        })),
+      )
+      const codec = Schema.fromJsonString(Message)
+      const prepared = yield* compileRequest(
+        LLM.request({
+          model,
+          messages: [Schema.decodeSync(codec)(Schema.encodeSync(codec)(message))],
+        }),
+      )
+      expect(prepared.body.input[0].content).toEqual(
+        details.map((detail) => ({
+          type: "input_image",
+          image_url: "https://example.com/image.png",
+          detail,
+        })),
+      )
+    }),
+  )
+}
+
+it.effect("rejects malformed image detail instead of silently discarding it", () =>
+  Effect.gen(function* () {
+    const error = yield* compileRequest(
+      LLM.request({
+        model: OpenAI.configure({ apiKey: "test" }).responses("fixture"),
+        messages: [
+          Message.user({
+            type: "media",
+            mediaType: "image/png",
+            data: "https://example.com/image.png",
+            providerMetadata: { openai: { detail: 42 } },
+          }),
+        ],
+      }),
+    ).pipe(Effect.flip)
+    expect(error.reason._tag).toBe("InvalidRequest")
+  }),
+)