diff --git a/packages/ai/src/protocols/openai-images.ts b/packages/ai/src/protocols/openai-images.ts index a38add77761..c096f535dbb 100644 --- a/packages/ai/src/protocols/openai-images.ts +++ b/packages/ai/src/protocols/openai-images.ts @@ -13,7 +13,7 @@ import { Framing } from "../route/framing.js" import { MediaProtocol } from "../route/media-protocol.js" import { MediaRoute } from "../route/media.js" import { AIError, mergeJsonRecords, ProviderID, type MediaUsage, type OpenString } from "../schema/index.js" -import { ProviderShared } from "./shared.js" +import { optionalNull, ProviderShared } from "./shared.js" import { ImageSizing } from "./utils/image-sizing.js" import { MediaInput } from "./utils/media-input.js" @@ -64,7 +64,8 @@ const Settings = { } const OpenAIImageResponse = Schema.Struct({ - data: Schema.Array(Schema.Struct({ b64_json: Schema.String })), + // `response_format` picks `b64_json` or `url`; DALLĀ·E and many compatible hosts default to `url`. + data: Schema.Array(Schema.Struct({ b64_json: optionalNull(Schema.String), url: optionalNull(Schema.String) })), ...Settings, usage: Schema.optional(Usage), }) @@ -307,7 +308,7 @@ const onDocument = Effect.fnUntraced(function* (frame: Exclude) { const format = decoded.output_format const mediaType = format === undefined ? undefined : `image/${format}` const images = yield* Effect.forEach(decoded.data, (item, index) => - MediaInput.decodedAsset(invalid, `${route.name} result ${index}`, item.b64_json, mediaType, { + MediaInput.imageOutput(invalid, `${route.name} result ${index}`, item, mediaType, { info: info(format, decoded.size), }), ) diff --git a/packages/ai/src/protocols/utils/media-input.ts b/packages/ai/src/protocols/utils/media-input.ts index 26b35ead62f..4d8692b69cf 100644 --- a/packages/ai/src/protocols/utils/media-input.ts +++ b/packages/ai/src/protocols/utils/media-input.ts @@ -53,7 +53,10 @@ export const decodedAsset = ( Effect.map((bytes) => Media.bytes(bytes, mediaType, options)), ) -/** One image of an OpenAI-shaped `data` array, which carries either `b64_json` or a `url`. */ +/** + * One image of an OpenAI-shaped `data` array, which carries `b64_json`, a `url`, or both. `b64_json` wins; a `data:` + * URL decodes inline with its own media type, and any other URL stays a URL asset for the caller to download. + */ export const imageOutput = ( invalid: (message: string, cause?: unknown) => AIError, label: string, @@ -62,6 +65,10 @@ export const imageOutput = ( options?: Media.AssetOptions, ) => { if (item.b64_json) return decodedAsset(invalid, label, item.b64_json, mediaType, options) + if (item.url?.startsWith("data:")) { + const asset = Media.parseDataUrl(item.url, options) + return asset === undefined ? Effect.fail(invalid(`${label} has an invalid data URL`)) : Effect.succeed(asset) + } if (item.url) return Effect.succeed(Media.url(item.url, { ...options, mediaType })) return Effect.fail(invalid(`${label} has neither image data nor a URL`)) } diff --git a/packages/ai/test/image.test.ts b/packages/ai/test/image.test.ts index b46d6dad397..a3547f8b9a6 100644 --- a/packages/ai/test/image.test.ts +++ b/packages/ai/test/image.test.ts @@ -169,6 +169,48 @@ describe("Image", () => { }), ) + describe("OpenAI-style url results", () => { + const generate = (data: unknown) => + Image.generate({ + model: OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" }).image("dall-e-3"), + prompt: "A lighthouse", + }).pipe(Effect.provide(layer((input) => Effect.succeed(json(input, { data }))))) + + it.effect("keeps an http url as a URL asset", () => + Effect.gen(function* () { + const response = yield* generate([{ url: "https://cdn.image.test/result.png" }]) + expect(response.image.source).toEqual({ type: "url", url: "https://cdn.image.test/result.png" }) + }), + ) + + it.effect("decodes a data url inline with its media type", () => + Effect.gen(function* () { + const response = yield* generate([{ url: "data:image/png;base64,iVBORw==" }]) + expect(response.image.source).toEqual({ type: "base64", data: "iVBORw==", mediaType: "image/png" }) + expect(response.image.mediaType).toBe("image/png") + }), + ) + + it.effect("prefers b64_json when url is null", () => + Effect.gen(function* () { + const response = yield* generate([{ b64_json: "/9j/4AAQ", url: null }]) + expect(response.image.source).toEqual({ + type: "bytes", + data: Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]), + mediaType: "image/jpeg", + }) + }), + ) + + it.effect("rejects an item with neither b64_json nor url", () => + Effect.gen(function* () { + const error = yield* generate([{}]).pipe(Effect.flip) + expect(error.reason._tag).toBe("InvalidProviderOutput") + expect(error.message).toContain("has neither image data nor a URL") + }), + ) + }) + it.effect("classifies an OpenAI stream moderation error event as a content policy rejection", () => Effect.gen(function* () { const error = yield* Image.stream({