mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-19 08:39:02 +03:00
feat(ai): add Vercel AI Gateway image generation (#58469)
This commit is contained in:
committed by
GitHub
Unverified
parent
e3e83be566
commit
23cd3109b4
@@ -575,9 +575,11 @@ call rather than being dropped, so check this table before swapping only the `mo
|
||||
| Replicate | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ (use `providerOptions`) | ✗ |
|
||||
| Stability `image` | 1 | ✗ | ✓ | ✓ | ✓ | 1 (not on `core`) | ✗ |
|
||||
| Stability `upscale()` | ✗ | ✗ | ✗ | ✓ | ✓ | exactly 1 (required) | ✗ |
|
||||
| Vercel AI Gateway | ✓ | per model | ✗ | ✗ | ignored | per model | per model |
|
||||
|
||||
✓ lowers natively; ✗ fails whenever the field is set (including `n: 1`); `1` means `n > 1` fails. ¹ `Image.stream` on OpenAI generates one image. fal
|
||||
rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model.
|
||||
rejects `size` and `aspectRatio` together; which one a fal or BFL model takes depends on the model. Vercel AI Gateway
|
||||
forwards `format` but does not apply it, so the returned bytes keep the upstream model's own format.
|
||||
|
||||
`Media.Asset` is the one asset type shared by image requests, image responses, LLM messages, and tool results.
|
||||
`asset.source` is the serializable `Media.Source` (`bytes`, `base64`, `url`, or `ref`); `asset.bytes()`,
|
||||
|
||||
@@ -11,7 +11,7 @@ import { Media } from "../media.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type MediaUsage, type OpenString } from "../schema/index.js"
|
||||
import { mergeJsonRecords, ProviderID, type MediaUsage, type OpenString } from "../schema/index.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
@@ -99,7 +99,7 @@ interface State {
|
||||
readonly size?: string
|
||||
readonly quality?: string
|
||||
readonly background?: string
|
||||
readonly usage?: MediaUsage
|
||||
readonly usage?: Schema.Schema.Type<typeof Usage>
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -128,8 +128,8 @@ const isEdit = (request: Request) => (request.images?.length ?? 0) > 0
|
||||
|
||||
const isInline = (asset: Media.Asset) => asset.source.type === "bytes" || asset.source.type === "base64"
|
||||
|
||||
const reference = (asset: Media.Asset) =>
|
||||
ProviderShared.mediaReference(asset, route.provider, route.name).pipe(
|
||||
const reference = (asset: Media.Asset, provider: ProviderID) =>
|
||||
ProviderShared.mediaReference(asset, provider, route.name).pipe(
|
||||
Effect.map((item) => (item.type === "ref" ? { file_id: item.value } : { image_url: item.value })),
|
||||
)
|
||||
|
||||
@@ -161,8 +161,8 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Me
|
||||
return MediaProtocol.multipart(form)
|
||||
}
|
||||
|
||||
const references = yield* Effect.forEach(images, reference)
|
||||
const maskReference = mask === undefined ? undefined : yield* reference(mask)
|
||||
const references = yield* Effect.forEach(images, (image) => reference(image, request.model.provider))
|
||||
const maskReference = mask === undefined ? undefined : yield* reference(mask, request.model.provider)
|
||||
return MediaProtocol.json(
|
||||
mergeJsonRecords(
|
||||
{
|
||||
@@ -180,7 +180,7 @@ const fromRequest = Effect.fn("OpenAIImages.fromRequest")(function* (request: Me
|
||||
// 6. Stream parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage | undefined =>
|
||||
const usage = (value: Schema.Schema.Type<typeof Usage> | undefined, provider: ProviderID): MediaUsage | undefined =>
|
||||
value === undefined
|
||||
? undefined
|
||||
: {
|
||||
@@ -188,7 +188,7 @@ const usage = (value: Schema.Schema.Type<typeof Usage> | undefined): MediaUsage
|
||||
input: value.input_tokens,
|
||||
output: value.output_tokens,
|
||||
total: value.total_tokens,
|
||||
details: { openai: value },
|
||||
details: { [provider]: value },
|
||||
}
|
||||
|
||||
/** `size` echoes the rendered `WIDTHxHEIGHT`; `auto` or any other value leaves the dimensions unknown. */
|
||||
@@ -217,7 +217,7 @@ const onEvent = Effect.fnUntraced(function* (state: State, frame: string) {
|
||||
size: event.size,
|
||||
quality: event.quality,
|
||||
background: event.background,
|
||||
usage: usage(event.usage),
|
||||
usage: event.usage,
|
||||
},
|
||||
[ImageOutputEvent.make({ index: state.completed, image })],
|
||||
] as const
|
||||
@@ -243,20 +243,21 @@ const onDocument = Effect.fnUntraced(function* (frame: Exclude<Frame, string>) {
|
||||
size: decoded.size,
|
||||
quality: decoded.quality,
|
||||
background: decoded.background,
|
||||
usage: usage(decoded.usage),
|
||||
usage: decoded.usage,
|
||||
}
|
||||
return [state, images.map((image, index) => ImageOutputEvent.make({ index, image }))] as const
|
||||
})
|
||||
|
||||
const step = (state: State, frame: Frame) => (typeof frame === "string" ? onEvent(state, frame) : onDocument(frame))
|
||||
|
||||
const finish = (state: State) => {
|
||||
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
|
||||
if (state.completed === 0) return Effect.fail(route.incomplete())
|
||||
const provider = context.request.model.provider
|
||||
return Effect.succeed([
|
||||
ImageFinishEvent.make({
|
||||
usage: state.usage,
|
||||
usage: usage(state.usage, provider),
|
||||
providerMetadata: {
|
||||
openai: {
|
||||
[provider]: {
|
||||
outputFormat: state.format,
|
||||
size: state.size,
|
||||
quality: state.quality,
|
||||
@@ -283,9 +284,14 @@ export const protocol = MediaProtocol.stream<Request, ImageEvent, Frame, State>(
|
||||
finish,
|
||||
})
|
||||
|
||||
export const model = (input: MediaRoute.ModelInput) =>
|
||||
/** `provider` attributes models, refs, and metadata to an OpenAI-compatible host; it defaults to `openai`. */
|
||||
export const model = (input: MediaRoute.ModelInput & { readonly provider?: string }) =>
|
||||
ImageModel.fromRoute<OpenAIImageOptions, Frame, State>(
|
||||
{ protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH) },
|
||||
{
|
||||
protocol: input.provider === undefined ? protocol : { ...protocol, provider: ProviderID.make(input.provider) },
|
||||
baseURL: DEFAULT_BASE_URL,
|
||||
path: ({ request }) => (isEdit(request) ? EDIT_PATH : PATH),
|
||||
},
|
||||
input,
|
||||
)
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
import { AnthropicMessages } from "../protocols/anthropic-messages.js"
|
||||
import { OpenAIChat } from "../protocols/openai-chat.js"
|
||||
import { OpenResponses } from "../protocols/open-responses.js"
|
||||
import { OpenAIImages } from "../protocols/openai-images.js"
|
||||
import { JsonObject, optionalNull, ProviderShared } from "../protocols/shared.js"
|
||||
import { gatewayProtocol } from "../protocols/utils/gateway-protocol.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
@@ -20,6 +21,7 @@ import { Route, type RouteDefaultsInput } from "../route/client.js"
|
||||
import { Endpoint } from "../route/endpoint.js"
|
||||
import { RequestExecutor } from "../route/executor.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import type { Protocol } from "../route/protocol.js"
|
||||
import {
|
||||
AIError,
|
||||
@@ -350,7 +352,9 @@ export const configure = (input: Options = {}) => {
|
||||
}),
|
||||
},
|
||||
})
|
||||
return { id, model, messages, responses, chat, experimental: { evaluation }, configure }
|
||||
const media = MediaRoute.deployment({ ...input, baseURL: configured.endpoint.baseURL }, configured.auth)
|
||||
const image = (modelID: string | ModelID) => OpenAIImages.model({ ...media, id: modelID, provider: id })
|
||||
return { id, model, messages, responses, chat, image, experimental: { evaluation }, configure }
|
||||
}
|
||||
|
||||
export const provider = configure()
|
||||
@@ -358,6 +362,7 @@ export const experimental = provider.experimental
|
||||
export const messages = provider.messages
|
||||
export const responses = provider.responses
|
||||
export const chat = provider.chat
|
||||
export const image = provider.image
|
||||
|
||||
export const model: ProviderPackage.Definition<Settings, ProviderOptionsInput>["model"] = (modelID, settings) =>
|
||||
fromSettings(settings).model(modelID)
|
||||
|
||||
@@ -183,6 +183,8 @@ describe("public exports", () => {
|
||||
expect(Alibaba.configure({ region: "ap-southeast-1", apiKey: "fixture" }).image("qwen-image-2.0").route.id).toBe(
|
||||
"alibaba-images",
|
||||
)
|
||||
expect(VercelAIGateway.configure({ apiKey: "fixture" }).image("meta/muse-image-1.0").route.id).toBe("openai-images")
|
||||
expect(VercelAIGateway.provider.image).toBe(VercelAIGateway.image)
|
||||
expect(XAI.configure({ apiKey: "fixture" }).video("grok-imagine-video-1.5").route.id).toBe("xai-video")
|
||||
expect(XAI.provider.video).toBe(XAI.video)
|
||||
expect(Google.configure({ apiKey: "fixture" }).video("veo-3.1-generate-preview").route.id).toBe("google-video")
|
||||
|
||||
+32
File diff suppressed because one or more lines are too long
@@ -0,0 +1,35 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { Effect } from "effect"
|
||||
import { Image } from "../../src/index.js"
|
||||
import { VercelAIGateway } from "../../src/providers.js"
|
||||
import { recordedTests } from "../recorded-test.js"
|
||||
|
||||
const model = VercelAIGateway.configure({
|
||||
apiKey: process.env.AI_GATEWAY_API_KEY ?? "fixture",
|
||||
}).image("meta/muse-image-1.0")
|
||||
|
||||
const recorded = recordedTests({
|
||||
prefix: "vercel-ai-gateway-images",
|
||||
provider: "vercel-ai-gateway",
|
||||
protocol: "openai-images",
|
||||
requires: ["AI_GATEWAY_API_KEY"],
|
||||
})
|
||||
|
||||
describe("Vercel AI Gateway Images recorded", () => {
|
||||
recorded.effect("generates an image", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* Image.generate({
|
||||
model,
|
||||
prompt: "A simple flat black circle centered on a plain white background.",
|
||||
format: "webp",
|
||||
})
|
||||
|
||||
expect(response.images).toHaveLength(1)
|
||||
expect(response.image.mediaType).toBe("image/webp")
|
||||
expect(new TextDecoder().decode((yield* response.image.bytes()).subarray(8, 12))).toBe("WEBP")
|
||||
expect(model.provider).toBe("vercel-ai-gateway")
|
||||
expect(response.usage?.details).toHaveProperty("vercel-ai-gateway")
|
||||
expect(response.providerMetadata).toHaveProperty("vercel-ai-gateway")
|
||||
}),
|
||||
)
|
||||
})
|
||||
Reference in New Issue
Block a user