feat(ai): add Vercel AI Gateway image generation (#58469)

This commit is contained in:
Aiden Cline
2026-10-23 10:45:35 -05:00
committed by GitHub
Unverified
parent e3e83be566
commit 23cd3109b4
6 changed files with 99 additions and 17 deletions
+3 -1
View File
@@ -575,9 +575,11 @@ call rather than being dropped, so check this table before swapping only the `mo
| Replicate | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ (use `providerOptions`) | ✗ |
| Stability `image` | 1 | ✗ | ✓ | ✓ | ✓ | 1 (not on `core`) | ✗ |
| Stability `upscale()` | ✗ | ✗ | ✗ | ✓ | ✓ | exactly 1 (required) | ✗ |
| Vercel AI Gateway | ✓ | per model | ✗ | ✗ | ignored | per model | per model |
✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. ¹ `Image.stream` on OpenAI generates one image. fal
rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model.
rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model. Vercel AI Gateway
forwards `format` but does not apply it, so the returned bytes keep the upstream model's own format.
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
+21 -15
View File
@@ -11,7 +11,7 @@ import { Media } from "../media.js"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type MediaUsage, type OpenString } from "../schema/index.js"
import { mergeJsonRecords, ProviderID, type MediaUsage, type OpenString } from "../schema/index.js"
import { ProviderShared } from "./shared.js"
import { MediaInput } from "./utils/media-input.js"
@@ -99,7 +99,7 @@ interface State {
readonly size?: string
readonly quality?: string
readonly background?: string
readonly usage?: MediaUsage
readonly usage?: Schema.Schema.Type<typeof Usage>
}
// ---------------------------------------------------------------------------
@@ -128,8 +128,8 @@ const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
const isInline = (asset: Media.Asset) => asset.source.type === "bytes" || asset.source.type === "base64"
const reference = (asset: Media.Asset) =>
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
const reference = (asset: Media.Asset, provider: ProviderID) =>
ProviderShared.mediaReference(asset, provider, route.name).pipe(
Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })),
)
@@ -161,8 +161,8 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Me
return MediaProtocol.multipart(form)
}
const references = yield* Effect.forEach(images, reference)
const maskReference = mask === undefined ? undefined : yield* reference(mask)
const references = yield* Effect.forEach(images, (image) => reference(image, request.model.provider))
const maskReference = mask === undefined ? undefined : yield* reference(mask, request.model.provider)
return MediaProtocol.json(
mergeJsonRecords(
{
@@ -180,7 +180,7 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Me
// 6. Stream parsing
// ---------------------------------------------------------------------------
const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage | undefined =>
const usage = (value: Schema.Schema.Type<typeof Usage> | undefined, provider: ProviderID): MediaUsage | undefined =>
value === undefined
? undefined
: {
@@ -188,7 +188,7 @@ const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage
input: value.input_tokens,
output: value.output_tokens,
total: value.total_tokens,
details: { openai: value },
details: { [provider]: value },
}
/** `size` echoes the rendered `WIDTHxHEIGHT`; `auto` or any other value leaves the dimensions unknown. */
@@ -217,7 +217,7 @@ const onEvent = Effect.fnUntraced(function* (state: State, frame: string) {
size: event.size,
quality: event.quality,
background: event.background,
usage: usage(event.usage),
usage: event.usage,
},
[ImageOutputEvent.make({ index: state.completed, image })],
] as const
@@ -243,20 +243,21 @@ const onDocument = Effect.fnUntraced(function* (frame: Exclude<Frame, string>) {
size: decoded.size,
quality: decoded.quality,
background: decoded.background,
usage: usage(decoded.usage),
usage: decoded.usage,
}
return [state, images.map((image, index) => ImageOutputEvent.make({ index, image }))] as const
})
const step = (state: State, frame: Frame) => (typeof frame === "string" ? onEvent(state, frame) : onDocument(frame))
const finish = (state: State) => {
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
if (state.completed === 0) return Effect.fail(route.incomplete())
const provider = context.request.model.provider
return Effect.succeed([
ImageFinishEvent.make({
usage: state.usage,
usage: usage(state.usage, provider),
providerMetadata: {
openai: {
[provider]: {
outputFormat: state.format,
size: state.size,
quality: state.quality,
@@ -283,9 +284,14 @@ export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>(
finish,
})
export const model = (input: MediaRoute.ModelInput) =>
/** `provider` attributes models, refs, and metadata to an OpenAI-compatible host; it defaults to `openai`. */
export const model = (input: MediaRoute.ModelInput & { readonly provider?: string }) =>
ImageModel.fromRoute<OpenAIImageOptions, Frame, State>(
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH) },
{
protocol: input.provider === undefined ? protocol : { ...protocol, provider: ProviderID.make(input.provider) },
baseURL: DEFAULT_BASE_URL,
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
},
input,
)
@@ -11,6 +11,7 @@ import {
import { AnthropicMessages } from "../protocols/anthropic-messages.js"
import { OpenAIChat } from "../protocols/openai-chat.js"
import { OpenResponses } from "../protocols/open-responses.js"
import { OpenAIImages } from "../protocols/openai-images.js"
import { JsonObject, optionalNull, ProviderShared } from "../protocols/shared.js"
import { gatewayProtocol } from "../protocols/utils/gateway-protocol.js"
import type { ProviderPackage } from "../provider-package.js"
@@ -20,6 +21,7 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
import { Endpoint } from "../route/endpoint.js"
import { RequestExecutor } from "../route/executor.js"
import { Framing } from "../route/framing.js"
import { MediaRoute } from "../route/media.js"
import type { Protocol } from "../route/protocol.js"
import {
AIError,
@@ -350,7 +352,9 @@ export const configure = (input: Options = {}) => {
}),
},
})
return { id, model, messages, responses, chat, experimental: { evaluation }, configure }
const media = MediaRoute.deployment({ ...input, baseURL: configured.endpoint.baseURL }, configured.auth)
const image = (modelID: string | ModelID) => OpenAIImages.model({ ...media, id: modelID, provider: id })
return { id, model, messages, responses, chat, image, experimental: { evaluation }, configure }
}
export const provider = configure()
@@ -358,6 +362,7 @@ export const experimental = provider.experimental
export const messages = provider.messages
export const responses = provider.responses
export const chat = provider.chat
export const image = provider.image
export const model: ProviderPackage.Definition<Settings, ProviderOptionsInput>["model"] = (modelID, settings) =>
fromSettings(settings).model(modelID)
+2
View File
@@ -183,6 +183,8 @@ describe("public exports", () => {
expect(Alibaba.configure({ region: "ap-southeast-1", apiKey: "fixture" }).image("qwen-image-2.0").route.id).toBe(
"alibaba-images",
)
expect(VercelAIGateway.configure({ apiKey: "fixture" }).image("meta/muse-image-1.0").route.id).toBe("openai-images")
expect(VercelAIGateway.provider.image).toBe(VercelAIGateway.image)
expect(XAI.configure({ apiKey: "fixture" }).video("grok-imagine-video-1.5").route.id).toBe("xai-video")
expect(XAI.provider.video).toBe(XAI.video)
expect(Google.configure({ apiKey: "fixture" }).video("veo-3.1-generate-preview").route.id).toBe("google-video")
File diff suppressed because one or more lines are too long
@@ -0,0 +1,35 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { Image } from "../../src/index.js"
import { VercelAIGateway } from "../../src/providers.js"
import { recordedTests } from "../recorded-test.js"
const model = VercelAIGateway.configure({
apiKey: process.env.AI_GATEWAY_API_KEY ?? "fixture",
}).image("meta/muse-image-1.0")
const recorded = recordedTests({
prefix: "vercel-ai-gateway-images",
provider: "vercel-ai-gateway",
protocol: "openai-images",
requires: ["AI_GATEWAY_API_KEY"],
})
describe("Vercel AI Gateway Images recorded", () => {
recorded.effect("generates an image", () =>
Effect.gen(function* () {
const response = yield* Image.generate({
model,
prompt: "A simple flat black circle centered on a plain white background.",
format: "webp",
})
expect(response.images).toHaveLength(1)
expect(response.image.mediaType).toBe("image/webp")
expect(new TextDecoder().decode((yield* response.image.bytes()).subarray(8, 12))).toBe("WEBP")
expect(model.provider).toBe("vercel-ai-gateway")
expect(response.usage?.details).toHaveProperty("vercel-ai-gateway")
expect(response.providerMetadata).toHaveProperty("vercel-ai-gateway")
}),
)
})