diff --git a/packages/ai/README.md b/packages/ai/README.md index fc59bba24f3..0b2bdc47937 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -621,8 +621,7 @@ persist the bytes promptly if they must remain available. ### Partial images OpenAI's GPT image models stream previews. `Image.stream` sends `stream: true` with `partialImages` (0–3, default 2) -and emits `image-partial` events before each final `image`; `Image.generate` keeps the plain JSON request. -`dall-e-*` models do not stream and fail typed: +and emits `image-partial` events before each final `image`; `Image.generate` keeps the plain JSON request: ```ts import { Stream } from "effect" diff --git a/packages/ai/src/protocols/openai-images.ts b/packages/ai/src/protocols/openai-images.ts index 2aaa5c0518c..f7a96de9e5b 100644 --- a/packages/ai/src/protocols/openai-images.ts +++ b/packages/ai/src/protocols/openai-images.ts @@ -48,15 +48,17 @@ const Usage = Schema.Struct({ output_tokens_details: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), }) -const OpenAIImageResponse = Schema.Struct({ - data: Schema.Array( - Schema.Struct({ - b64_json: Schema.optional(Schema.String), - url: Schema.optional(Schema.String), - revised_prompt: Schema.optional(Schema.String), - }), - ), +/** What the provider actually rendered; it can differ from the request when `auto` or a default applied. */ +const Settings = { output_format: Schema.optional(Schema.String), + size: Schema.optional(Schema.String), + quality: Schema.optional(Schema.String), + background: Schema.optional(Schema.String), +} + +const OpenAIImageResponse = Schema.Struct({ + data: Schema.Array(Schema.Struct({ b64_json: Schema.String })), + ...Settings, usage: Schema.optional(Usage), }) @@ -69,11 +71,13 @@ const StreamEvent = Schema.Union([ type: Schema.Literals(["image_generation.partial_image", "image_edit.partial_image"]), b64_json: Schema.String, partial_image_index: Schema.Number, + ...Settings, output_format: Schema.String, }), Schema.Struct({ type: Schema.Literals(["image_generation.completed", "image_edit.completed"]), b64_json: Schema.String, + ...Settings, output_format: Schema.String, usage: Schema.optional(Usage), }), @@ -92,6 +96,9 @@ type Frame = string | { readonly document: string; readonly requested: string | interface State { readonly completed: number readonly format?: string + readonly size?: string + readonly quality?: string + readonly background?: string readonly usage?: MediaUsage } @@ -110,10 +117,6 @@ const nativeOptions = (options: OpenAIImageOptions | undefined) => { const streamOptions = (request: MediaProtocol.Addressed) => { if (request.mode !== "stream") return Effect.succeed(undefined) - if (request.model.id.startsWith("dall-e")) - return Effect.fail( - route.unsupported("media.stream", `${request.model.id} does not stream; use Image.generate or a GPT image model`), - ) if (request.n !== undefined && request.n > 1) return Effect.fail( route.unsupported("media.n", `${route.name} streams one image; use Image.generate for n=${request.n}`), @@ -194,21 +197,34 @@ const usage = (value: Schema.Schema.Type | undefined): MediaUsage details: { openai: value }, } -const eventImage = (frame: string, label: string, data: string, format: string) => +/** `size` echoes the rendered `WIDTHxHEIGHT`; `auto` or any other value leaves the dimensions unknown. */ +const info = (format: string, size: string | undefined): Media.Info => { + const match = size?.match(/^(\d+)x(\d+)$/) + return match ? { format, width: Number(match[1]), height: Number(match[2]) } : { format } +} + +const eventImage = (frame: string, label: string, data: string, format: string, size: string | undefined) => MediaInput.decodedAsset((message, cause) => route.frameError(message, frame, cause), label, data, `image/${format}`, { - info: { format }, + info: info(format, size), }) const onEvent = Effect.fn("OpenAIImages.onEvent")(function* (state: State, frame: string) { const event = yield* decodeEvent(frame) const format = event.output_format if ("partial_image_index" in event) { - const image = yield* eventImage(frame, `${route.name} partial image`, event.b64_json, format) + const image = yield* eventImage(frame, `${route.name} partial image`, event.b64_json, format, event.size) return [state, [ImagePartialEvent.make({ index: event.partial_image_index, image })]] as const } - const image = yield* eventImage(frame, `${route.name} result ${state.completed}`, event.b64_json, format) + const image = yield* eventImage(frame, `${route.name} result ${state.completed}`, event.b64_json, format, event.size) return [ - { ...state, completed: state.completed + 1, format, usage: usage(event.usage) }, + { + completed: state.completed + 1, + format, + size: event.size, + quality: event.quality, + background: event.background, + usage: usage(event.usage), + }, [ImageOutputEvent.make({ index: state.completed, image })], ] as const }) @@ -219,16 +235,20 @@ const onDocument = Effect.fn("OpenAIImages.onDocument")(function* (frame: Exclud Effect.mapError((cause) => invalid(`${route.name} returned an invalid response`, cause)), ) const format = decoded.output_format ?? frame.requested ?? "png" - const mediaType = `image/${format}` const images = yield* Effect.forEach(decoded.data, (item, index) => - MediaInput.imageOutput(invalid, `${route.name} result ${index}`, item, mediaType, { - info: { format }, - providerMetadata: - item.revised_prompt === undefined ? undefined : { openai: { revisedPrompt: item.revised_prompt } }, + MediaInput.decodedAsset(invalid, `${route.name} result ${index}`, item.b64_json, `image/${format}`, { + info: info(format, decoded.size), }), ) if (images.length === 0) return yield* invalid(`${route.name} returned no images`) - const state: State = { completed: images.length, format, usage: usage(decoded.usage) } + const state: State = { + completed: images.length, + format, + size: decoded.size, + quality: decoded.quality, + background: decoded.background, + usage: usage(decoded.usage), + } return [state, images.map((image, index) => ImageOutputEvent.make({ index, image }))] as const }) @@ -237,7 +257,17 @@ const step = (state: State, frame: Frame) => (typeof frame === "string" ? onEven const finish = (state: State) => { if (state.completed === 0) return Effect.fail(route.incomplete()) return Effect.succeed([ - ImageFinishEvent.make({ usage: state.usage, providerMetadata: { openai: { outputFormat: state.format } } }), + ImageFinishEvent.make({ + usage: state.usage, + providerMetadata: { + openai: { + outputFormat: state.format, + size: state.size, + quality: state.quality, + background: state.background, + }, + }, + }), ]) } diff --git a/packages/ai/src/protocols/xai-images.ts b/packages/ai/src/protocols/xai-images.ts index 33a0dbc5442..01593ffcf37 100644 --- a/packages/ai/src/protocols/xai-images.ts +++ b/packages/ai/src/protocols/xai-images.ts @@ -101,7 +101,8 @@ const decodeResponse = Effect.fn("XAIImages.decodeResponse")(function* ( ) if (images.length === 0) return yield* output.invalid(`${route.name} returned no images`) const usage = ProviderShared.isRecord(decoded.usage) ? decoded.usage : undefined - // xAI reports image counts rather than tokens, seconds, or credits; the raw record stays in provider metadata. + // xAI reports a USD cost (`cost_in_usd_ticks`) rather than tokens, seconds, or credits; the raw record stays in + // provider metadata. return new ImageResponse({ images, providerMetadata: usage === undefined ? undefined : { xai: { usage } }, diff --git a/packages/ai/src/protocols/zai-images.ts b/packages/ai/src/protocols/zai-images.ts index 0d57db01e2e..736e650361f 100644 --- a/packages/ai/src/protocols/zai-images.ts +++ b/packages/ai/src/protocols/zai-images.ts @@ -1,7 +1,6 @@ -import { Effect, Schema } from "effect" +import { Duration, Effect, Schema } from "effect" import type { HttpClientResponse } from "effect/unstable/http" import { ImageModel, ImageResponse, type ImageRequestFor } from "../image.js" -import { Media } from "../media.js" import { MediaProtocol } from "../route/media-protocol.js" import { MediaRoute } from "../route/media.js" import { mergeJsonRecords, type OpenString } from "../schema/index.js" @@ -9,6 +8,7 @@ import { mergeJsonRecords, type OpenString } from "../schema/index.js" const route = MediaProtocol.identity({ id: "zai-images", name: "Z.ai Images", provider: "zai" }) export const DEFAULT_BASE_URL = "https://api.z.ai/api/paas/v4" export const PATH = "/images/generations" +const OUTPUT_RETENTION = Duration.days(30) // --------------------------------------------------------------------------- // 1. Public model input @@ -76,7 +76,7 @@ const decodeResponse = Effect.fn("ZAIImages.decodeResponse")(function* ( const filters = decoded.content_filter ?? [] return new ImageResponse({ // Z.ai returns only URLs and no content type; the media type resolves when the asset is materialized. - images: decoded.data.map((item) => Media.url(item.url)), + images: yield* Effect.forEach(decoded.data, (item) => MediaProtocol.expiringUrl(item.url, OUTPUT_RETENTION)), // Z.ai reports applied content filters alongside a successful result; surface them instead of dropping them. notices: filters.length === 0 diff --git a/packages/ai/test/image.test.ts b/packages/ai/test/image.test.ts index 149edfb2f1e..b10e2c97487 100644 --- a/packages/ai/test/image.test.ts +++ b/packages/ai/test/image.test.ts @@ -78,8 +78,11 @@ describe("Image", () => { mediaType: "image/webp", }) expect(yield* response.image.bytes()).toEqual(Uint8Array.from([1, 2, 3])) - expect(response.image.providerMetadata).toEqual({ openai: { revisedPrompt: "A precise robot" } }) + expect(response.image.info).toEqual({ format: "webp", width: 2048, height: 2048 }) expect(response.usage).toMatchObject({ type: "tokens", total: 12 }) + expect(response.providerMetadata).toEqual({ + openai: { outputFormat: "webp", size: "2048x2048", quality: "high", background: "opaque" }, + }) }).pipe( Effect.provide( ImageClient.layer.pipe( @@ -107,8 +110,11 @@ describe("Image", () => { }) return input.respond( JSON.stringify({ - data: [{ b64_json: "AQID", revised_prompt: "A precise robot" }, { b64_json: "BAUG" }], + data: [{ b64_json: "AQID" }, { b64_json: "BAUG" }], output_format: "webp", + size: "2048x2048", + quality: "high", + background: "opaque", usage: { input_tokens: 4, output_tokens: 8, total_tokens: 12 }, }), { headers: { "content-type": "application/json" } }, @@ -144,6 +150,7 @@ describe("Image", () => { ), ) expect(response.image.source).toEqual({ type: "bytes", data: Uint8Array.from([1, 2, 3]), mediaType: "image/png" }) + expect(response.image.info).toEqual({ format: "png" }) }), ) @@ -736,7 +743,6 @@ describe("Image", () => { prompt, size: "512x512", }), - Stream.runCollect(Image.stream({ model: openai.image("dall-e-3"), prompt })), Stream.runCollect(Image.stream({ model: openai.image("gpt-image-2"), prompt, n: 2 })), Image.start({ model: replicate, prompt, seed: 7 }), Image.start({ @@ -753,7 +759,6 @@ describe("Image", () => { ["UnsupportedOperation", "media.n"], ["UnsupportedOperation", "media.aspectRatio"], ["UnsupportedOperation", "media.size"], - ["UnsupportedOperation", "media.stream"], ["UnsupportedOperation", "media.n"], ["UnsupportedOperation", "media.seed"], ["InvalidRequest", false], diff --git a/packages/ai/test/provider/openai-images.recorded.test.ts b/packages/ai/test/provider/openai-images.recorded.test.ts index 07925b84f58..27e3665b0c6 100644 --- a/packages/ai/test/provider/openai-images.recorded.test.ts +++ b/packages/ai/test/provider/openai-images.recorded.test.ts @@ -29,7 +29,11 @@ describe("OpenAI Images recorded", () => { expect(response.images).toHaveLength(1) expect(response.image.mediaType).toBe("image/jpeg") + expect(response.image.info).toEqual({ format: "jpeg", width: 1024, height: 1024 }) expect((yield* response.image.bytes()).length).toBeGreaterThan(0) + expect(response.providerMetadata).toEqual({ + openai: { outputFormat: "jpeg", size: "1024x1024", quality: "low", background: "opaque" }, + }) }), ) @@ -76,8 +80,13 @@ describe("OpenAI Images recorded", () => { expect(events.map((event) => event.type)).toEqual(["image-partial", "image", "finish"]) const image = events.find(ImageEvent.is.image) expect(image?.image.mediaType).toBe("image/jpeg") + expect(image?.image.info).toEqual({ format: "jpeg", width: 1024, height: 1024 }) expect(dimensions(yield* image!.image.bytes())).toEqual({ width: 1024, height: 1024 }) - expect(events.find(ImageEvent.is.finish)?.usage).toMatchObject({ type: "tokens" }) + const finish = events.find(ImageEvent.is.finish) + expect(finish?.usage).toMatchObject({ type: "tokens" }) + expect(finish?.providerMetadata).toEqual({ + openai: { outputFormat: "jpeg", size: "1024x1024", quality: "low", background: "opaque" }, + }) }), ) }) diff --git a/packages/ai/test/provider/zai-images.test.ts b/packages/ai/test/provider/zai-images.test.ts index ccc7794961d..c33d3fe483d 100644 --- a/packages/ai/test/provider/zai-images.test.ts +++ b/packages/ai/test/provider/zai-images.test.ts @@ -32,7 +32,12 @@ describe("Z.ai Images", () => { expect(response.images).toHaveLength(1) expect(response.image.mediaType).toBe("application/octet-stream") - expect(response.image.source).toEqual({ type: "url", url: "https://cdn.z.ai/generated.png" }) + // Z.ai documents that output URLs expire 30 days after generation; the test clock starts at 0. + expect(response.image.source).toEqual({ + type: "url", + url: "https://cdn.z.ai/generated.png", + expiresAt: 30 * 24 * 60 * 60 * 1000, + }) expect(response.notices).toEqual([ { type: "moderated",