mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
refactor(ai): ignore video sound on models that can't switch it (#63726)
This commit is contained in:
committed by
GitHub
Unverified
parent
ba77ff90d3
commit
f6e0c2d083
@@ -1087,7 +1087,9 @@ const program = Effect.gen(function* () {
|
||||
|
||||
The shared fields are `prompt`, `start`, `end`, `references`, `duration`, `aspect`, `resolution` and `sound`;
|
||||
everything else is provider-specific and goes under `providerOptions`, typed from the selected model. A request only
|
||||
fails before sending when its protocol has no way to express it, such as a reference video on xAI. Values the provider
|
||||
fails before sending when its protocol has no way to express it, such as a reference video on xAI. `sound` is the
|
||||
exception: on models whose sound can't be switched (MiniMax H3 and Gemini Omni always have it, Hailuo never does) it
|
||||
isn't sent, and the model's own behavior applies. Values the provider
|
||||
may reject, like an unsupported duration or too many references, go to the provider, and its error comes back typed.
|
||||
|
||||
`start` and `end` are images. They take a URL, data URL, base64 string or bytes, or an asset built with
|
||||
@@ -1131,11 +1133,11 @@ Provider notes:
|
||||
`/v1`, and a bare host (`https://api.minimax.io`) works as is. On H3, `start`, `end` and `references` become `content[]` items with `first_frame`, `last_frame` and `reference_image` /
|
||||
`reference_video` / `reference_audio` roles, MiniMax file refs become `mm_file://` URLs, `aspect: "auto"` becomes
|
||||
`ratio: "adaptive"`, and `resolution` is uppercased (`480P`, `2K`). H3 requires `resolution` and `duration`, plus a
|
||||
`ratio` for text-only prompts; nothing is filled in, so MiniMax rejects a request without them. H3 always has sound,
|
||||
so `sound: false` fails before sending. On Hailuo, `start` and `end` map to `first_frame_image` and
|
||||
`ratio` for text-only prompts; nothing is filled in, so MiniMax rejects a request without them. H3 always has sound
|
||||
and has no field for it, so `sound` isn't sent. On Hailuo, `start` and `end` map to `first_frame_image` and
|
||||
`last_frame_image`, image references to `subject_reference`, and the result looks up the file's download URL, which
|
||||
expires after one hour. Hailuo is silent and has no aspect field, so `sound: true`, an explicit `aspect`, and video
|
||||
or audio references fail before sending. Cancel works on H3 (only while queued); Hailuo has no cancel. Native fields
|
||||
expires after one hour. Hailuo is always silent, so `sound` isn't sent; it has no aspect field, so an explicit
|
||||
`aspect` and video or audio references fail before sending. Cancel works on H3 (only while queued); Hailuo has no cancel. Native fields
|
||||
such as `extra.prompt_expansion_mode` (H3), `prompt_optimizer` (Hailuo) and `callback_url` go under `providerOptions`.
|
||||
- **Google** (`Google.video("gemini-omni-1.2-flash")`) starts a background Gemini Omni interaction. Omni has no
|
||||
role field for media, so `start`, `end` and `references` go into `input` in that order (images, plus video
|
||||
@@ -1143,7 +1145,7 @@ Provider notes:
|
||||
`reference_to_video` for references only; with both, the prompt decides. `duration` becomes
|
||||
`response_format.duration` (`"3s"`), and `aspect` and `resolution` go to `response_format` as is (resolution
|
||||
lowercased); `aspect: "auto"` leaves the ratio out. It rejects `end` without `start` (a lone image reads as the
|
||||
first frame) and `sound: false` (Omni always generates audio). Videos come back as inline bytes. `cancel` posts to
|
||||
first frame). Omni always generates audio and has no field for it, so `sound` isn't sent. Videos come back as inline bytes. `cancel` posts to
|
||||
`/interactions/{id}/cancel`. `providerOptions` takes native Interactions fields, such as `previous_interaction_id`
|
||||
to edit or extend a generated video, `generation_config.video_config.task`, `service_tier` and `webhook_config`.
|
||||
|
||||
|
||||
@@ -129,11 +129,6 @@ const task = (request: Request) => {
|
||||
const fromRequest = Effect.fn("GoogleInteractionsVideo.fromRequest")(function* (request: Request) {
|
||||
if (request.end !== undefined && request.start === undefined)
|
||||
return yield* route.unsupported("video.end", `${route.name} reads a lone image as the start frame; send start too`)
|
||||
if (request.sound === false)
|
||||
return yield* route.unsupported(
|
||||
"video.sound",
|
||||
`${route.name} always generates audio and has no option to turn it off`,
|
||||
)
|
||||
const media = yield* Effect.forEach(
|
||||
[request.start, request.end, ...(request.references ?? [])].filter(
|
||||
(asset): asset is Media.Asset => asset !== undefined,
|
||||
|
||||
@@ -177,9 +177,7 @@ const reference = (asset: Media.Asset) =>
|
||||
const resolution = (request: Request) => request.resolution?.toUpperCase()
|
||||
|
||||
const fromRequest = Effect.fn("MiniMaxVideo.fromRequest")(function* (request: Request) {
|
||||
// H3 always generates a soundtrack and has no field to turn it off.
|
||||
if (request.sound === false)
|
||||
return yield* route.unsupported("video.sound", `${route.name} always generates sound and can't be silenced`)
|
||||
// H3 always generates a soundtrack and has no field for `sound`, so it isn't sent.
|
||||
const content = [
|
||||
{ type: "text", text: request.prompt },
|
||||
...(request.start === undefined ? [] : [yield* item(request.start, "first_frame")]),
|
||||
@@ -213,8 +211,7 @@ const hailuoFromRequest = Effect.fn("MiniMaxHailuoVideo.fromRequest")(function*
|
||||
`${hailuoRoute.name} takes only image (subject) references`,
|
||||
)
|
||||
// Hailuo output is always silent and its shape follows the model or the first frame, with no field for either.
|
||||
if (request.sound === true)
|
||||
return yield* hailuoRoute.unsupported("video.sound", `${hailuoRoute.name} generates silent video only`)
|
||||
// `sound` has nothing to send, so it isn't; an aspect ratio would change the shape the caller asked for.
|
||||
if (request.aspect !== undefined && request.aspect !== "auto")
|
||||
return yield* hailuoRoute.unsupported("video.aspect", `${hailuoRoute.name} has no aspect ratio input`)
|
||||
const subjects = yield* Effect.forEach(references, (asset) =>
|
||||
|
||||
@@ -97,7 +97,9 @@ describe("Generation", () => {
|
||||
const generation = new Generation(route, "t", { id: "gen_1", status: "queued" })
|
||||
|
||||
const fiber = yield* Effect.forkChild(
|
||||
generation.events({ poll: { interval: "1 second", timeout: "5 seconds" } }).pipe(Stream.runCollect, Effect.flip),
|
||||
generation
|
||||
.events({ poll: { interval: "1 second", timeout: "5 seconds" } })
|
||||
.pipe(Stream.runCollect, Effect.flip),
|
||||
)
|
||||
yield* TestClock.adjust("6 seconds")
|
||||
const error = yield* Fiber.join(fiber)
|
||||
|
||||
@@ -101,7 +101,9 @@ describe("ai.video.start / Google Interactions", () => {
|
||||
model,
|
||||
prompt: "Morph",
|
||||
start: Media.image(Media.fromDataUrl("data:image/png;base64,AQID")),
|
||||
end: Media.image(Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/last", "image/jpeg")),
|
||||
end: Media.image(
|
||||
Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/last", "image/jpeg"),
|
||||
),
|
||||
})
|
||||
expect(body.input).toEqual([
|
||||
{ type: "image", data: "AQID", mime_type: "image/png" },
|
||||
@@ -147,9 +149,15 @@ describe("ai.video.start / Google Interactions", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("sends nothing for sound, since Omni always generates audio and has no field for it", () =>
|
||||
Effect.gen(function* () {
|
||||
const silent = yield* sent({ model, prompt: "x", sound: false })
|
||||
expect(JSON.stringify(silent.body)).not.toContain("audio")
|
||||
}),
|
||||
)
|
||||
|
||||
for (const rejected of [
|
||||
{ name: "an end frame without a start frame", input: { end: "data:image/png;base64,AQID" }, operation: "end" },
|
||||
{ name: "silent video", input: { sound: false }, operation: "sound" },
|
||||
])
|
||||
it.effect(`rejects ${rejected.name}, which Omni has no way to send, before sending`, () =>
|
||||
Effect.gen(function* () {
|
||||
|
||||
@@ -136,12 +136,19 @@ describe("MiniMax Video / H3 start", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects silent output, which H3 can't produce, before sending", () =>
|
||||
it.effect("sends nothing for sound, which neither API has a field for", () =>
|
||||
Effect.gen(function* () {
|
||||
const error = yield* ai.video.start({ model: h3, prompt: "x", sound: false }).pipe(Effect.flip)
|
||||
expect(error.reason._tag).toBe("UnsupportedOperation")
|
||||
expect(error.message).toContain("always generates sound")
|
||||
}).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))),
|
||||
const requests = yield* sent([
|
||||
{ model: h3, sound: false },
|
||||
{ model: h3, sound: true },
|
||||
{ model: hailuo, sound: true },
|
||||
])
|
||||
expect(requests.map((request) => "generate_audio" in request.body || "sound" in request.body)).toEqual([
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("returns MiniMax's HTTP errors typed", () =>
|
||||
@@ -242,7 +249,6 @@ describe("MiniMax Video / Hailuo start", () => {
|
||||
request: { references: [Media.url("https://example.test/v.mp3", { mediaType: "audio/mpeg" })] },
|
||||
tag: "UnsupportedOperation",
|
||||
},
|
||||
{ name: "sound", request: { sound: true }, tag: "UnsupportedOperation" },
|
||||
{ name: "an aspect ratio", request: { aspect: "16:9" as const }, tag: "UnsupportedOperation" },
|
||||
{ name: "a MiniMax file reference", request: { start: Media.ref("minimax", "file_1") }, tag: "InvalidRequest" },
|
||||
]) {
|
||||
|
||||
Reference in New Issue
Block a user