mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 13:05:14 +03:00
fix(ai): send image sizes the way OpenRouter, Vercel Grok and Google expect (#59927)
This commit is contained in:
committed by
GitHub
Unverified
parent
aaa0d7c8f6
commit
78c31ea68c
@@ -701,13 +701,14 @@ table before swapping only the `model`:
|
||||
| Provider | `n` | `aspect` | `resolution` | `images` | Moved to `providerOptions` |
|
||||
| --------------------- | --- | -------------------- | --------------------- | -------------------------------- | -------------------------------------------------- |
|
||||
| OpenAI | ✓¹ | nearest size | nearest size | ✓ | `size`, `outputFormat`, `mask` |
|
||||
| Google (Gemini) | 1 | ✓ | ✓ (`imageSize`) | ✓ (no public URLs) | `seed` |
|
||||
| Google (Gemini) | 1 | ✓ | ✓ (`0.5K` as `512`) | ✓ (no public URLs) | `seed` |
|
||||
| xAI | ✓ | ✓ | `1k`/`2k` | ✓ | none |
|
||||
| Meta | ✓ | ✓ (hint) | ignored | ✓ | `size`, `outputFormat` |
|
||||
| Z.ai | ✗ | width × height | width × height | ✗ | `size` |
|
||||
| Alibaba | ✓ | width × height | width × height | ✓ | `size`, `seed` |
|
||||
| MiniMax | ✓ | preset or w × h⁴ | preset or w × h⁴ | 1 (portrait subject) | `aspect_ratio`, `width`, `height`, `seed` |
|
||||
| Vercel AI Gateway | ✓ | as OpenAI³ | as OpenAI³ | per model | as OpenAI |
|
||||
| OpenRouter | ✓ | ✓ (`aspect_ratio`) | tier⁵ | per model | as OpenAI |
|
||||
| Cloudflare Workers AI | 1 | width × height² | width × height² | ✗ | `width`, `height`, `seed` |
|
||||
|
||||
✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. "width × height"
|
||||
@@ -717,8 +718,10 @@ fixed sizes. `providerOptions.size` replaces `aspect` and `resolution`, so setti
|
||||
forwards `outputFormat` but does not apply it, so the returned bytes keep the upstream model's own format. ² Workers AI
|
||||
`@cf/black-forest-labs/flux-1-schnell` takes no size, and the provider rejects it. ³ Non-OpenAI gateway models
|
||||
(`bfl/flux-2-pro`) get a computed width × height, which the gateway passes through; models with a fixed list, such as
|
||||
Recraft, reject sizes outside it. ⁴ MiniMax sends its `aspect_ratio` preset for a preset aspect at about `1K` and an
|
||||
Recraft, reject sizes outside it. Grok (`spacexai/*`, `xai/*`) ignores `size`, so it gets `providerOptions.xai`
|
||||
`aspect_ratio` and `1k`/`2k` instead. ⁴ MiniMax sends its `aspect_ratio` preset for a preset aspect at about `1K` and an
|
||||
exact width × height (512–2048 px, steps of 8) otherwise; `image-01-live` takes only the presets, minus `21:9`.
|
||||
⁵ OpenRouter takes `512`, `768`, `1K`, `1.5K`, `2K` or `4K` rather than pixels; other values are left out.
|
||||
|
||||
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
|
||||
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
|
||||
|
||||
@@ -83,7 +83,9 @@ const GoogleImageResponse = Schema.Struct({
|
||||
|
||||
const generationConfig = (request: Request) => {
|
||||
const { thinkingLevel, includeThoughts, ...native } = request.providerOptions ?? {}
|
||||
const imageConfig = { aspectRatio: request.aspect, imageSize: request.resolution }
|
||||
// Google names the half tier `512` and rejects `0.5K`.
|
||||
const imageSize = request.resolution?.toUpperCase() === "0.5K" ? "512" : request.resolution?.toUpperCase()
|
||||
const imageConfig = { aspectRatio: request.aspect, imageSize }
|
||||
const thinkingConfig = { thinkingLevel, includeThoughts }
|
||||
return (
|
||||
mergeJsonRecords(
|
||||
|
||||
@@ -174,10 +174,14 @@ const sizes = (model: string): ImageSizing.Limits | ReadonlyArray<ImageSizing.Si
|
||||
return { multiple: 16 }
|
||||
}
|
||||
|
||||
const size = (request: Request) => {
|
||||
/** The body fields `aspect` and `resolution` lower to; gateways that read other fields than `size` pass their own. */
|
||||
export type OpenAIImageSizing = (request: Request) => Record<string, unknown> | undefined
|
||||
|
||||
const pixelSize: OpenAIImageSizing = (request) => {
|
||||
if (!ImageSizing.isRequested(request)) return undefined
|
||||
const options = sizes(request.model.id)
|
||||
return "multiple" in options ? ImageSizing.fit(request, options) : ImageSizing.nearest(request, options)
|
||||
const size = "multiple" in options ? ImageSizing.fit(request, options) : ImageSizing.nearest(request, options)
|
||||
return { size: `${size.width}x${size.height}` }
|
||||
}
|
||||
|
||||
const streamOptions = (request: MediaProtocol.Addressed<Request>) => {
|
||||
@@ -198,7 +202,10 @@ const reference = (asset: Media.Asset, provider: ProviderID) =>
|
||||
Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })),
|
||||
)
|
||||
|
||||
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
|
||||
const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (
|
||||
request: MediaProtocol.Addressed<Request>,
|
||||
sizing: OpenAIImageSizing,
|
||||
) {
|
||||
const images = request.images ?? []
|
||||
const mask = request.providerOptions?.mask
|
||||
if (mask !== undefined && images.length === 0)
|
||||
@@ -207,11 +214,10 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Me
|
||||
return yield* ProviderShared.invalidRequest(
|
||||
"OpenAI providerOptions.size replaces aspect and resolution; set one or the other",
|
||||
)
|
||||
const chosen = size(request)
|
||||
const fields = mergeJsonRecords(
|
||||
{
|
||||
n: request.n,
|
||||
size: chosen === undefined ? undefined : `${chosen.width}x${chosen.height}`,
|
||||
...sizing(request),
|
||||
...(yield* streamOptions(request)),
|
||||
},
|
||||
nativeOptions(request.providerOptions),
|
||||
@@ -353,29 +359,40 @@ const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) =
|
||||
// 7. Protocol and route
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>(route, {
|
||||
body: { from: fromRequest },
|
||||
frames: (bytes, context) =>
|
||||
context.request.mode === "stream"
|
||||
? Framing.sse.frame(bytes)
|
||||
: Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
|
||||
initial: () => ({ completed: 0 }),
|
||||
step,
|
||||
finish,
|
||||
})
|
||||
const makeProtocol = (sizing: OpenAIImageSizing) =>
|
||||
MediaProtocol.stream<Request, ImageEvent, Frame, State>(route, {
|
||||
body: { from: (request) => fromRequest(request, sizing) },
|
||||
frames: (bytes, context) =>
|
||||
context.request.mode === "stream"
|
||||
? Framing.sse.frame(bytes)
|
||||
: Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
|
||||
initial: () => ({ completed: 0 }),
|
||||
step,
|
||||
finish,
|
||||
})
|
||||
|
||||
/** `provider` attributes models, refs, and metadata to an OpenAI-compatible host; it defaults to `openai`. */
|
||||
export const model = (input: MediaRoute.ModelInput & { readonly provider?: string }) =>
|
||||
ImageModel.fromRoute<OpenAIImageOptions, Frame, State>(
|
||||
export const protocol = makeProtocol(pixelSize)
|
||||
|
||||
/**
|
||||
* `provider` attributes models, refs, and metadata to an OpenAI-compatible host; it defaults to `openai`. `sizing`
|
||||
* replaces the `size` that `aspect` and `resolution` lower to.
|
||||
*/
|
||||
export const model = (
|
||||
input: MediaRoute.ModelInput & { readonly provider?: string; readonly sizing?: OpenAIImageSizing },
|
||||
) => {
|
||||
const base = input.sizing === undefined ? protocol : makeProtocol(input.sizing)
|
||||
return ImageModel.fromRoute<OpenAIImageOptions, Frame, State>(
|
||||
{
|
||||
protocol: input.provider === undefined ? protocol : { ...protocol, provider: ProviderID.make(input.provider) },
|
||||
protocol: input.provider === undefined ? base : { ...base, provider: ProviderID.make(input.provider) },
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
|
||||
},
|
||||
input,
|
||||
)
|
||||
}
|
||||
|
||||
export const OpenAIImages = {
|
||||
protocol,
|
||||
model,
|
||||
pixelSize,
|
||||
} as const
|
||||
|
||||
@@ -53,7 +53,7 @@ const nativeOptions = (options: XAIImageOptions | undefined) => {
|
||||
}
|
||||
|
||||
/** xAI renders `1k` or `2k`; other tiers use the nearest one. */
|
||||
const resolution = (request: Request) => {
|
||||
export const resolution = (request: { readonly resolution?: string }) => {
|
||||
if (request.resolution === undefined) return undefined
|
||||
return (ImageSizing.tier(request.resolution) ?? 1) < 1.5 ? "1k" : "2k"
|
||||
}
|
||||
@@ -133,4 +133,5 @@ export const model = (input: MediaRoute.ModelInput) =>
|
||||
export const XAIImages = {
|
||||
protocol,
|
||||
model,
|
||||
resolution,
|
||||
} as const
|
||||
|
||||
@@ -6,7 +6,7 @@ import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import { ProviderID, type ModelID, type OpenString, type ProviderOptions } from "../schema/index.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
import { SystemOne } from "../protocols/system-one.js"
|
||||
import { OpenAIImages } from "../protocols/openai-images.js"
|
||||
import { OpenAIImages, type OpenAIImageSizing } from "../protocols/openai-images.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { OpenAIChat } from "../protocols/openai-chat.js"
|
||||
import { OpenResponses } from "../protocols/open-responses.js"
|
||||
@@ -176,6 +176,21 @@ const bodyOptions = (body: OpenAIChat.OpenAIChatBody, input: unknown, maxTokens:
|
||||
}
|
||||
}
|
||||
|
||||
const IMAGE_RESOLUTIONS = new Set(["512", "768", "1K", "1.5K", "2K", "4K"])
|
||||
|
||||
// A pixel `size` renders at the upstream's own tier (Gemini 2K comes back as 4K) and FLUX 3 rejects it, so sizing goes
|
||||
// out as an aspect ratio and a resolution tier. Values OpenRouter doesn't list are left out.
|
||||
const imageSizing: OpenAIImageSizing = (request) => {
|
||||
const resolution = request.resolution
|
||||
?.toUpperCase()
|
||||
.replace(/^0\.5K$/, "512")
|
||||
.replace(/^0\.75K$/, "768")
|
||||
return {
|
||||
aspect_ratio: request.aspect,
|
||||
resolution: resolution !== undefined && IMAGE_RESOLUTIONS.has(resolution) ? resolution : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
export const route = Route.make({
|
||||
id: ADAPTER,
|
||||
provider: id,
|
||||
@@ -213,7 +228,7 @@ export const configure = (input: Config = {}) => {
|
||||
})
|
||||
const media = MediaRoute.deployment({ ...input, baseURL: input.baseURL ?? baseURL }, auth)
|
||||
const image = (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
|
||||
OpenAIImages.model({ ...media, id: modelID, provider: id, providerOptions })
|
||||
OpenAIImages.model({ ...media, id: modelID, provider: id, providerOptions, sizing: imageSizing })
|
||||
const decision = (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
|
||||
decisionRoute.model<OpenRouterDecisionOptions>({
|
||||
id: modelID,
|
||||
|
||||
@@ -3,7 +3,8 @@ import { AnthropicMessages } from "../protocols/anthropic-messages.js"
|
||||
import { OpenAIChat } from "../protocols/openai-chat.js"
|
||||
import { OpenResponses } from "../protocols/open-responses.js"
|
||||
import { OpenAIDecisions } from "../protocols/openai-decisions.js"
|
||||
import { OpenAIImages } from "../protocols/openai-images.js"
|
||||
import { OpenAIImages, type OpenAIImageSizing } from "../protocols/openai-images.js"
|
||||
import { XAIImages } from "../protocols/xai-images.js"
|
||||
import { ProviderShared } from "../protocols/shared.js"
|
||||
import { gatewayProtocol } from "../protocols/utils/gateway-protocol.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
@@ -211,6 +212,14 @@ const chatRoute = route({
|
||||
|
||||
export const routes = [messagesRoute, responsesRoute, chatRoute]
|
||||
|
||||
// The Gateway passes `size` through to most image models, but Grok ignores it and renders 1024x1024; it only reads
|
||||
// sizing from `providerOptions.xai`.
|
||||
const imageSizing: OpenAIImageSizing = (request) => {
|
||||
if (!/^(spacexai|xai)\//.test(request.model.id)) return OpenAIImages.pixelSize(request)
|
||||
if (request.aspect === undefined && request.resolution === undefined) return undefined
|
||||
return { providerOptions: { xai: { aspect_ratio: request.aspect, resolution: XAIImages.resolution(request) } } }
|
||||
}
|
||||
|
||||
export const configure = (input: Config = {}) => {
|
||||
const { apiKey: _apiKey, auth: _auth, baseURL: endpoint, ...defaults } = input
|
||||
const configured = {
|
||||
@@ -238,7 +247,7 @@ export const configure = (input: Config = {}) => {
|
||||
}
|
||||
const media = MediaRoute.deployment({ ...input, baseURL: configured.endpoint.baseURL }, configured.auth)
|
||||
const image = (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
|
||||
OpenAIImages.model({ ...media, id: modelID, provider: id, providerOptions })
|
||||
OpenAIImages.model({ ...media, id: modelID, provider: id, providerOptions, sizing: imageSizing })
|
||||
const decisionRoute = OpenAIDecisions.route.with({
|
||||
provider: id,
|
||||
endpoint: { baseURL: configured.endpoint.baseURL },
|
||||
|
||||
+32
File diff suppressed because one or more lines are too long
Vendored
+32
File diff suppressed because one or more lines are too long
+32
File diff suppressed because one or more lines are too long
@@ -2,14 +2,7 @@ import { describe, expect } from "bun:test"
|
||||
import { Effect, Layer, Stream } from "effect"
|
||||
import { HttpClientRequest } from "effect/http"
|
||||
import { Image, ImageClient, Media } from "../src/effect.js"
|
||||
import {
|
||||
Google,
|
||||
OpenAI,
|
||||
OpenAICompatible,
|
||||
VercelAIGateway,
|
||||
XAI,
|
||||
ZAI,
|
||||
} from "../src/providers.js"
|
||||
import { Google, OpenAI, OpenAICompatible, OpenRouter, VercelAIGateway, XAI, ZAI } from "../src/providers.js"
|
||||
import { it } from "./lib/effect.js"
|
||||
import { dynamicResponse, json } from "./lib/http.js"
|
||||
|
||||
@@ -351,6 +344,74 @@ describe("Image", () => {
|
||||
),
|
||||
)
|
||||
|
||||
describe("gateway and Google sizing", () => {
|
||||
const sent = (expected: Record<string, unknown>, response: unknown = { data: [{ b64_json: "AQID" }] }) =>
|
||||
layer((input) => {
|
||||
expect(JSON.parse(input.text)).toEqual(expected)
|
||||
return Effect.succeed(json(input, response))
|
||||
})
|
||||
const prompt = "A lighthouse"
|
||||
const openrouter = OpenRouter.configure({ apiKey: "test", baseURL: "https://openrouter.test/v1" })
|
||||
|
||||
it.effect("sends OpenRouter an aspect ratio and resolution tier instead of pixels", () =>
|
||||
Image.generate({
|
||||
model: openrouter.image("google/gemini-3.1-flash-image"),
|
||||
prompt,
|
||||
aspect: "16:9",
|
||||
resolution: "0.5K",
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
sent({ model: "google/gemini-3.1-flash-image", prompt, aspect_ratio: "16:9", resolution: "512" }),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("leaves out a resolution OpenRouter doesn't list", () =>
|
||||
Image.generate({ model: openrouter.image("x/y"), prompt, aspect: "1:1", resolution: "8K" }).pipe(
|
||||
Effect.provide(sent({ model: "x/y", prompt, aspect_ratio: "1:1" })),
|
||||
),
|
||||
)
|
||||
|
||||
for (const modelID of ["spacexai/grok-imagine-image", "xai/grok-imagine-image"])
|
||||
it.effect(`sizes Vercel AI Gateway ${modelID} through xAI provider options`, () =>
|
||||
Image.generate({
|
||||
model: VercelAIGateway.configure({ apiKey: "test", baseURL: "https://gateway.test/v1" }).image(modelID),
|
||||
prompt,
|
||||
aspect: "16:9",
|
||||
resolution: "2K",
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
sent({ model: modelID, prompt, providerOptions: { xai: { aspect_ratio: "16:9", resolution: "2k" } } }),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("names Google's half tier 512", () =>
|
||||
Image.generate({
|
||||
model: Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).image(
|
||||
"gemini-3.1-flash-image",
|
||||
),
|
||||
prompt,
|
||||
aspect: "1:1",
|
||||
resolution: "0.5K",
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
layer((input) => {
|
||||
expect(JSON.parse(input.text).generationConfig.imageConfig).toEqual({
|
||||
aspectRatio: "1:1",
|
||||
imageSize: "512",
|
||||
})
|
||||
return Effect.succeed(
|
||||
json(input, {
|
||||
candidates: [{ content: { parts: [{ inlineData: { mimeType: "image/png", data: "AQID" } }] } }],
|
||||
}),
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
it.effect("preserves native snake_case and unknown request options", () =>
|
||||
Image.generate({
|
||||
model: OpenAI.configure({
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { Image } from "../../src/effect.js"
|
||||
import { OpenRouter } from "../../src/providers.js"
|
||||
import { dimensions } from "../lib/image.js"
|
||||
import { recordedTests } from "../recorded-test.js"
|
||||
|
||||
const openrouter = OpenRouter.configure({ apiKey: process.env.OPENROUTER_API_KEY ?? "fixture" })
|
||||
|
||||
const recorded = recordedTests({
|
||||
prefix: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
protocol: "openai-images",
|
||||
requires: ["OPENROUTER_API_KEY"],
|
||||
})
|
||||
|
||||
// Replay matches the recorded request body, so these also pin `aspect_ratio` and `resolution` going out instead of a
|
||||
// pixel `size`.
|
||||
describe("OpenRouter Images recorded", () => {
|
||||
recorded.effect(
|
||||
"renders Gemini at the requested 2K tier",
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: openrouter.image("google/gemini-3.1-flash-image"),
|
||||
prompt: "A simple flat black circle centered on a plain white background.",
|
||||
aspect: "16:9",
|
||||
resolution: "2K",
|
||||
})
|
||||
|
||||
expect(dimensions(yield* response.image.bytes())).toEqual({ width: 2752, height: 1536 })
|
||||
}),
|
||||
{ timeout: 120_000 },
|
||||
)
|
||||
|
||||
recorded.effect(
|
||||
"renders FLUX 3 from an aspect ratio and tier",
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: openrouter.image("black-forest-labs/flux-3-image-klein"),
|
||||
prompt: "A simple flat black circle centered on a plain white background.",
|
||||
aspect: "16:9",
|
||||
resolution: "1K",
|
||||
})
|
||||
|
||||
expect(dimensions(yield* response.image.bytes())).toEqual({ width: 1360, height: 768 })
|
||||
}),
|
||||
{ timeout: 120_000 },
|
||||
)
|
||||
})
|
||||
@@ -2,6 +2,7 @@ import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { Image } from "../../src/effect.js"
|
||||
import { VercelAIGateway } from "../../src/providers.js"
|
||||
import { dimensions } from "../lib/image.js"
|
||||
import { recordedTests } from "../recorded-test.js"
|
||||
|
||||
const model = VercelAIGateway.configure({
|
||||
@@ -32,4 +33,23 @@ describe("Vercel AI Gateway Images recorded", () => {
|
||||
expect(response.providerMetadata).toHaveProperty("vercel-ai-gateway")
|
||||
}),
|
||||
)
|
||||
|
||||
// Grok ignores `size` behind the Gateway; replay matches the recorded `providerOptions.xai` body.
|
||||
recorded.effect(
|
||||
"sizes Grok through xAI provider options",
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model: VercelAIGateway.configure({ apiKey: process.env.AI_GATEWAY_API_KEY ?? "fixture" }).image(
|
||||
"spacexai/grok-imagine-image",
|
||||
),
|
||||
prompt: "A simple flat black circle centered on a plain white background.",
|
||||
aspect: "16:9",
|
||||
resolution: "1K",
|
||||
})
|
||||
|
||||
expect(dimensions(yield* response.image.bytes())).toEqual({ width: 1280, height: 720 })
|
||||
}),
|
||||
{ timeout: 120_000 },
|
||||
)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user