mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 21:15:57 +03:00
fix(ai): send input images to Cloudflare Workers AI image models (#59485)
This commit is contained in:
committed by
GitHub
Unverified
parent
6b1bc48148
commit
c504b9f289
@@ -5,6 +5,7 @@ import { Media } from "../media.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import type { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords } from "../schema/index.js"
|
||||
import { ProviderShared } from "./shared.js"
|
||||
import { ImageSizing } from "./utils/image-sizing.js"
|
||||
import { MediaInput } from "./utils/media-input.js"
|
||||
|
||||
@@ -60,10 +61,23 @@ const fromRequest = Effect.fn("CloudflareWorkersAIImages.fromRequest")(function*
|
||||
...(yield* ImageSizing.exact(route, request, { multiple: 16, minEdge: 256, maxEdge: 2048 })),
|
||||
}
|
||||
const native = mergeJsonRecords(request.providerOptions, request.http?.body)
|
||||
if (!request.model.id.startsWith(MULTIPART_MODEL_PREFIX))
|
||||
return MediaProtocol.json(mergeJsonRecords(fields, native) ?? {})
|
||||
const images = request.images ?? []
|
||||
if (!request.model.id.startsWith(MULTIPART_MODEL_PREFIX)) {
|
||||
if (images.length > 1)
|
||||
return yield* route.unsupported("media.images", `${request.model.id} takes one input image as image_b64`)
|
||||
const image = images[0]?.inline()
|
||||
if (images[0] !== undefined && image === undefined) return yield* ProviderShared.inlineRequired(route.id, images[0])
|
||||
return MediaProtocol.json(mergeJsonRecords({ ...fields, image_b64: image?.base64 }, native) ?? {})
|
||||
}
|
||||
const body = new FormData()
|
||||
MediaInput.appendFields(body, fields, { overlay: native, reserved: RESERVED_FORM_FIELDS })
|
||||
yield* Effect.forEach(images, (image, index) =>
|
||||
MediaInput.inlineBytes(route.id, image).pipe(
|
||||
Effect.map((bytes) =>
|
||||
body.append(`input_image_${index}`, MediaInput.blob(bytes, image.mediaType), `image_${index}`),
|
||||
),
|
||||
),
|
||||
)
|
||||
return MediaProtocol.multipart(body)
|
||||
})
|
||||
|
||||
@@ -94,7 +108,6 @@ const decodeResponse = Effect.fn("CloudflareWorkersAIImages.decodeResponse")(fun
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const protocol = MediaProtocol.inline<Request, ImageResponse>(route, {
|
||||
unsupported: ["images"],
|
||||
body: { from: fromRequest },
|
||||
response: { decode: decodeResponse },
|
||||
})
|
||||
|
||||
@@ -84,13 +84,64 @@ describe("Cloudflare Workers AI Images", () => {
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("sends FLUX.2 input images as numbered multipart files", () =>
|
||||
Image.generate({
|
||||
model: workers.image("@cf/black-forest-labs/flux-2-klein-4b"),
|
||||
prompt: "Put the cat from image 1 in image 0",
|
||||
images: [Media.bytes(new Uint8Array([1, 2]), "image/png"), Media.base64("AwQ=", "image/jpeg")],
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) =>
|
||||
Effect.gen(function* () {
|
||||
const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie)
|
||||
const form = yield* Effect.promise(() => request.formData())
|
||||
const bytes = yield* Effect.forEach(["input_image_0", "input_image_1"], (name) =>
|
||||
Effect.promise(async () => [...new Uint8Array(await (form.get(name) as Blob).arrayBuffer())]),
|
||||
)
|
||||
expect(form.get("prompt")).toBe("Put the cat from image 1 in image 0")
|
||||
expect(bytes).toEqual([
|
||||
[1, 2],
|
||||
[3, 4],
|
||||
])
|
||||
return input.respond(JSON.stringify({ result: { image: "iVBORw0KGgo=" } }), {
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("sends the first input image of a JSON model as image_b64", () =>
|
||||
Image.generate({
|
||||
model: workers.image("@cf/lykon/dreamshaper-8-lcm"),
|
||||
prompt: "Make it green",
|
||||
images: [Media.bytes(new Uint8Array([1, 2, 3]), "image/png")],
|
||||
}).pipe(
|
||||
Effect.provide(
|
||||
ImageClient.layer.pipe(
|
||||
Layer.provide(
|
||||
dynamicResponse((input) => {
|
||||
expect(JSON.parse(input.text)).toEqual({ prompt: "Make it green", image_b64: "AQID" })
|
||||
return Effect.succeed(input.respond(new Uint8Array([0x89, 0x50, 0x4e, 0x47])))
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
it.effect("fails typed for fields the Run API image models cannot take", () =>
|
||||
Effect.gen(function* () {
|
||||
const model = workers.image("@cf/black-forest-labs/flux-2-dev")
|
||||
const image = Media.bytes(new Uint8Array([1]), "image/png")
|
||||
const errors = yield* Effect.forEach(
|
||||
[
|
||||
{ model, prompt: "test", n: 2 },
|
||||
{ model, prompt: "test", images: [Media.bytes(new Uint8Array([1]), "image/png")] },
|
||||
{ model: workers.image("@cf/black-forest-labs/flux-2-dev"), prompt: "test", n: 2 },
|
||||
{ model: workers.image("@cf/lykon/dreamshaper-8-lcm"), prompt: "test", images: [image, image] },
|
||||
],
|
||||
(request) => Image.generate(request).pipe(Effect.flip),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user