fix(ai): send image sizes the way OpenRouter, Vercel Grok and Google expect (#59927)

This commit is contained in:
Aiden Cline
2026-10-28 21:21:04 -05:00
committed by GitHub
Unverified
parent aaa0d7c8f6
commit 78c31ea68c
12 changed files with 310 additions and 35 deletions
+5 -2
View File
@@ -701,13 +701,14 @@ table before swapping only the `model`:
| Provider | `n` | `aspect` | `resolution` | `images` | Moved to `providerOptions` |
| --------------------- | --- | -------------------- | --------------------- | -------------------------------- | -------------------------------------------------- |
| OpenAI | ✓¹ | nearest size | nearest size | ✓ | `size`, `outputFormat`, `mask` |
| Google (Gemini) | 1 | ✓ | ✓ (`imageSize`) | ✓ (no public URLs) | `seed` |
| Google (Gemini) | 1 | ✓ | ✓ (`0.5K` as `512`) | ✓ (no public URLs) | `seed` |
| xAI | ✓ | ✓ | `1k`/`2k` | ✓ | none |
| Meta | ✓ | ✓ (hint) | ignored | ✓ | `size`, `outputFormat` |
| Z.ai | ✗ | width × height | width × height | ✗ | `size` |
| Alibaba | ✓ | width × height | width × height | ✓ | `size`, `seed` |
| MiniMax | ✓ | preset or w × h⁴ | preset or w × h⁴ | 1 (portrait subject) | `aspect_ratio`, `width`, `height`, `seed` |
| Vercel AI Gateway | ✓ | as OpenAI³ | as OpenAI³ | per model | as OpenAI |
| OpenRouter | ✓ | ✓ (`aspect_ratio`) | tier⁵ | per model | as OpenAI |
| Cloudflare Workers AI | 1 | width × height² | width × height² | ✗ | `width`, `height`, `seed` |
✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. "width × height"
@@ -717,8 +718,10 @@ fixed sizes. `providerOptions.size` replaces `aspect` and `resolution`, so setti
forwards `outputFormat` but does not apply it, so the returned bytes keep the upstream model's own format. ² Workers AI
`@cf/black-forest-labs/flux-1-schnell` takes no size, and the provider rejects it. ³ Non-OpenAI gateway models
(`bfl/flux-2-pro`) get a computed width × height, which the gateway passes through; models with a fixed list, such as
Recraft, reject sizes outside it. ⁴ MiniMax sends its `aspect_ratio` preset for a preset aspect at about `1K` and an
Recraft, reject sizes outside it. Grok (`spacexai/*`, `xai/*`) ignores `size`, so it gets `providerOptions.xai`
`aspect_ratio` and `1k`/`2k` instead. ⁴ MiniMax sends its `aspect_ratio` preset for a preset aspect at about `1K` and an
exact width × height (512–2048 px, steps of 8) otherwise; `image-01-live` takes only the presets, minus `21:9`.
⁵ OpenRouter takes `512`, `768`, `1K`, `1.5K`, `2K` or `4K` rather than pixels; other values are left out.
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
+3 -1
View File
@@ -83,7 +83,9 @@ const GoogleImageResponse = Schema.Struct({
const generationConfig = (request: Request) => {
const { thinkingLevel, includeThoughts, ...native } = request.providerOptions ?? {}
const imageConfig = { aspectRatio: request.aspect, imageSize: request.resolution }
// Google names the half tier `512` and rejects `0.5K`.
const imageSize = request.resolution?.toUpperCase() === "0.5K" ? "512" : request.resolution?.toUpperCase()
const imageConfig = { aspectRatio: request.aspect, imageSize }
const thinkingConfig = { thinkingLevel, includeThoughts }
return (
mergeJsonRecords(
+36 -19
View File
@@ -174,10 +174,14 @@ const sizes = (model: string): ImageSizing.Limits | ReadonlyArray<ImageSizing.Si
return { multiple: 16 }
}
const size = (request: Request) => {
/** The body fields `aspect` and `resolution` lower to; gateways that read other fields than `size` pass their own. */
export type OpenAIImageSizing = (request: Request) => Record<string, unknown> | undefined
const pixelSize: OpenAIImageSizing = (request) => {
if (!ImageSizing.isRequested(request)) return undefined
const options = sizes(request.model.id)
return "multiple" in options ? ImageSizing.fit(request, options) : ImageSizing.nearest(request, options)
const size = "multiple" in options ? ImageSizing.fit(request, options) : ImageSizing.nearest(request, options)
return { size: `${size.width}x${size.height}` }
}
const streamOptions = (request: MediaProtocol.Addressed<Request>) => {
@@ -198,7 +202,10 @@ const reference = (asset: Media.Asset, provider: ProviderID) =>
Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })),
)
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (
request: MediaProtocol.Addressed<Request>,
sizing: OpenAIImageSizing,
) {
const images = request.images ?? []
const mask = request.providerOptions?.mask
if (mask !== undefined && images.length === 0)
@@ -207,11 +214,10 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Me
return yield* ProviderShared.invalidRequest(
"OpenAI providerOptions.size replaces aspect and resolution; set one or the other",
)
const chosen = size(request)
const fields = mergeJsonRecords(
{
n: request.n,
size: chosen === undefined ? undefined : `${chosen.width}x${chosen.height}`,
...sizing(request),
...(yield* streamOptions(request)),
},
nativeOptions(request.providerOptions),
@@ -353,29 +359,40 @@ const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) =
// 7. Protocol and route
// ---------------------------------------------------------------------------
export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>(route, {
body: { from: fromRequest },
frames: (bytes, context) =>
context.request.mode === "stream"
? Framing.sse.frame(bytes)
: Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
initial: () => ({ completed: 0 }),
step,
finish,
})
const makeProtocol = (sizing: OpenAIImageSizing) =>
MediaProtocol.stream<Request, ImageEvent, Frame, State>(route, {
body: { from: (request) => fromRequest(request, sizing) },
frames: (bytes, context) =>
context.request.mode === "stream"
? Framing.sse.frame(bytes)
: Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
initial: () => ({ completed: 0 }),
step,
finish,
})
/** `provider` attributes models, refs, and metadata to an OpenAI-compatible host; it defaults to `openai`. */
export const model = (input: MediaRoute.ModelInput & { readonly provider?: string }) =>
ImageModel.fromRoute<OpenAIImageOptions, Frame, State>(
export const protocol = makeProtocol(pixelSize)
/**
* `provider` attributes models, refs, and metadata to an OpenAI-compatible host; it defaults to `openai`. `sizing`
* replaces the `size` that `aspect` and `resolution` lower to.
*/
export const model = (
input: MediaRoute.ModelInput & { readonly provider?: string; readonly sizing?: OpenAIImageSizing },
) => {
const base = input.sizing === undefined ? protocol : makeProtocol(input.sizing)
return ImageModel.fromRoute<OpenAIImageOptions, Frame, State>(
{
protocol: input.provider === undefined ? protocol : { ...protocol, provider: ProviderID.make(input.provider) },
protocol: input.provider === undefined ? base : { ...base, provider: ProviderID.make(input.provider) },
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
},
input,
)
}
export const OpenAIImages = {
protocol,
model,
pixelSize,
} as const
+2 -1
View File
@@ -53,7 +53,7 @@ const nativeOptions = (options: XAIImageOptions | undefined) => {
}
/** xAI renders `1k` or `2k`; other tiers use the nearest one. */
const resolution = (request: Request) => {
export const resolution = (request: { readonly resolution?: string }) => {
if (request.resolution === undefined) return undefined
return (ImageSizing.tier(request.resolution) ?? 1) < 1.5 ? "1k" : "2k"
}
@@ -133,4 +133,5 @@ export const model = (input: MediaRoute.ModelInput) =>
export const XAIImages = {
protocol,
model,
resolution,
} as const
+17 -2
View File
@@ -6,7 +6,7 @@ import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
import { ProviderID, type ModelID, type OpenString, type ProviderOptions } from "../schema/index.js"
import type { ProviderPackage } from "../provider-package.js"
import { SystemOne } from "../protocols/system-one.js"
import { OpenAIImages } from "../protocols/openai-images.js"
import { OpenAIImages, type OpenAIImageSizing } from "../protocols/openai-images.js"
import { MediaRoute } from "../route/media.js"
import { OpenAIChat } from "../protocols/openai-chat.js"
import { OpenResponses } from "../protocols/open-responses.js"
@@ -176,6 +176,21 @@ const bodyOptions = (body: OpenAIChat.OpenAIChatBody, input: unknown, maxTokens:
}
}
const IMAGE_RESOLUTIONS = new Set(["512", "768", "1K", "1.5K", "2K", "4K"])
// A pixel `size` renders at the upstream's own tier (Gemini 2K comes back as 4K) and FLUX 3 rejects it, so sizing goes
// out as an aspect ratio and a resolution tier. Values OpenRouter doesn't list are left out.
const imageSizing: OpenAIImageSizing = (request) => {
const resolution = request.resolution
?.toUpperCase()
.replace(/^0\.5K$/, "512")
.replace(/^0\.75K$/, "768")
return {
aspect_ratio: request.aspect,
resolution: resolution !== undefined && IMAGE_RESOLUTIONS.has(resolution) ? resolution : undefined,
}
}
export const route = Route.make({
id: ADAPTER,
provider: id,
@@ -213,7 +228,7 @@ export const configure = (input: Config = {}) => {
})
const media = MediaRoute.deployment({ ...input, baseURL: input.baseURL ?? baseURL }, auth)
const image = (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
OpenAIImages.model({ ...media, id: modelID, provider: id, providerOptions })
OpenAIImages.model({ ...media, id: modelID, provider: id, providerOptions, sizing: imageSizing })
const decision = (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
decisionRoute.model<OpenRouterDecisionOptions>({
id: modelID,
+11 -2
View File
@@ -3,7 +3,8 @@ import { AnthropicMessages } from "../protocols/anthropic-messages.js"
import { OpenAIChat } from "../protocols/openai-chat.js"
import { OpenResponses } from "../protocols/open-responses.js"
import { OpenAIDecisions } from "../protocols/openai-decisions.js"
import { OpenAIImages } from "../protocols/openai-images.js"
import { OpenAIImages, type OpenAIImageSizing } from "../protocols/openai-images.js"
import { XAIImages } from "../protocols/xai-images.js"
import { ProviderShared } from "../protocols/shared.js"
import { gatewayProtocol } from "../protocols/utils/gateway-protocol.js"
import type { ProviderPackage } from "../provider-package.js"
@@ -211,6 +212,14 @@ const chatRoute = route({
export const routes = [messagesRoute, responsesRoute, chatRoute]
// The Gateway passes `size` through to most image models, but Grok ignores it and renders 1024x1024; it only reads
// sizing from `providerOptions.xai`.
const imageSizing: OpenAIImageSizing = (request) => {
if (!/^(spacexai|xai)\//.test(request.model.id)) return OpenAIImages.pixelSize(request)
if (request.aspect === undefined && request.resolution === undefined) return undefined
return { providerOptions: { xai: { aspect_ratio: request.aspect, resolution: XAIImages.resolution(request) } } }
}
export const configure = (input: Config = {}) => {
const { apiKey: _apiKey, auth: _auth, baseURL: endpoint, ...defaults } = input
const configured = {
@@ -238,7 +247,7 @@ export const configure = (input: Config = {}) => {
}
const media = MediaRoute.deployment({ ...input, baseURL: configured.endpoint.baseURL }, configured.auth)
const image = (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
OpenAIImages.model({ ...media, id: modelID, provider: id, providerOptions })
OpenAIImages.model({ ...media, id: modelID, provider: id, providerOptions, sizing: imageSizing })
const decisionRoute = OpenAIDecisions.route.with({
provider: id,
endpoint: { baseURL: configured.endpoint.baseURL },
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+69 -8
View File
@@ -2,14 +2,7 @@ import { describe, expect } from "bun:test"
import { Effect, Layer, Stream } from "effect"
import { HttpClientRequest } from "effect/http"
import { Image, ImageClient, Media } from "../src/effect.js"
import {
Google,
OpenAI,
OpenAICompatible,
VercelAIGateway,
XAI,
ZAI,
} from "../src/providers.js"
import { Google, OpenAI, OpenAICompatible, OpenRouter, VercelAIGateway, XAI, ZAI } from "../src/providers.js"
import { it } from "./lib/effect.js"
import { dynamicResponse, json } from "./lib/http.js"
@@ -351,6 +344,74 @@ describe("Image", () => {
),
)
describe("gateway and Google sizing", () => {
const sent = (expected: Record<string, unknown>, response: unknown = { data: [{ b64_json: "AQID" }] }) =>
layer((input) => {
expect(JSON.parse(input.text)).toEqual(expected)
return Effect.succeed(json(input, response))
})
const prompt = "A lighthouse"
const openrouter = OpenRouter.configure({ apiKey: "test", baseURL: "https://openrouter.test/v1" })
it.effect("sends OpenRouter an aspect ratio and resolution tier instead of pixels", () =>
Image.generate({
model: openrouter.image("google/gemini-3.1-flash-image"),
prompt,
aspect: "16:9",
resolution: "0.5K",
}).pipe(
Effect.provide(
sent({ model: "google/gemini-3.1-flash-image", prompt, aspect_ratio: "16:9", resolution: "512" }),
),
),
)
it.effect("leaves out a resolution OpenRouter doesn't list", () =>
Image.generate({ model: openrouter.image("x/y"), prompt, aspect: "1:1", resolution: "8K" }).pipe(
Effect.provide(sent({ model: "x/y", prompt, aspect_ratio: "1:1" })),
),
)
for (const modelID of ["spacexai/grok-imagine-image", "xai/grok-imagine-image"])
it.effect(`sizes Vercel AI Gateway ${modelID} through xAI provider options`, () =>
Image.generate({
model: VercelAIGateway.configure({ apiKey: "test", baseURL: "https://gateway.test/v1" }).image(modelID),
prompt,
aspect: "16:9",
resolution: "2K",
}).pipe(
Effect.provide(
sent({ model: modelID, prompt, providerOptions: { xai: { aspect_ratio: "16:9", resolution: "2k" } } }),
),
),
)
it.effect("names Google's half tier 512", () =>
Image.generate({
model: Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).image(
"gemini-3.1-flash-image",
),
prompt,
aspect: "1:1",
resolution: "0.5K",
}).pipe(
Effect.provide(
layer((input) => {
expect(JSON.parse(input.text).generationConfig.imageConfig).toEqual({
aspectRatio: "1:1",
imageSize: "512",
})
return Effect.succeed(
json(input, {
candidates: [{ content: { parts: [{ inlineData: { mimeType: "image/png", data: "AQID" } }] } }],
}),
)
}),
),
),
)
})
it.effect("preserves native snake_case and unknown request options", () =>
Image.generate({
model: OpenAI.configure({
@@ -0,0 +1,51 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Image } from "../../src/effect.js"
import { OpenRouter } from "../../src/providers.js"
import { dimensions } from "../lib/image.js"
import { recordedTests } from "../recorded-test.js"
const openrouter = OpenRouter.configure({ apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" })
const recorded = recordedTests({
prefix: "openrouter-images",
provider: "openrouter",
protocol: "openai-images",
requires: ["OPENROUTER_API_KEY"],
})
// Replay matches the recorded request body, so these also pin `aspect_ratio` and `resolution` going out instead of a
// pixel `size`.
describe("OpenRouter Images recorded", () => {
recorded.effect(
"renders Gemini at the requested 2K tier",
() =>
Effect.gen(function* () {
const response = yield* Image.generate({
model: openrouter.image("google/gemini-3.1-flash-image"),
prompt: "A simple flat black circle centered on a plain white background.",
aspect: "16:9",
resolution: "2K",
})
expect(dimensions(yield* response.image.bytes())).toEqual({ width: 2752, height: 1536 })
}),
{ timeout: 120_000 },
)
recorded.effect(
"renders FLUX 3 from an aspect ratio and tier",
() =>
Effect.gen(function* () {
const response = yield* Image.generate({
model: openrouter.image("black-forest-labs/flux-3-image-klein"),
prompt: "A simple flat black circle centered on a plain white background.",
aspect: "16:9",
resolution: "1K",
})
expect(dimensions(yield* response.image.bytes())).toEqual({ width: 1360, height: 768 })
}),
{ timeout: 120_000 },
)
})
@@ -2,6 +2,7 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Image } from "../../src/effect.js"
import { VercelAIGateway } from "../../src/providers.js"
import { dimensions } from "../lib/image.js"
import { recordedTests } from "../recorded-test.js"
const model = VercelAIGateway.configure({
@@ -32,4 +33,23 @@ describe("Vercel AI Gateway Images recorded", () => {
expect(response.providerMetadata).toHaveProperty("vercel-ai-gateway")
}),
)
// Grok ignores `size` behind the Gateway; replay matches the recorded `providerOptions.xai` body.
recorded.effect(
"sizes Grok through xAI provider options",
() =>
Effect.gen(function* () {
const response = yield* Image.generate({
model: VercelAIGateway.configure({ apiKey: process.env.AI_GATEWAY_API_KEY ?? "fixture" }).image(
"spacexai/grok-imagine-image",
),
prompt: "A simple flat black circle centered on a plain white background.",
aspect: "16:9",
resolution: "1K",
})
expect(dimensions(yield* response.image.bytes())).toEqual({ width: 1280, height: 720 })
}),
{ timeout: 120_000 },
)
})