refactor(ai): ignore video sound on models that can't switch it (#63726)

This commit is contained in:
Aiden Cline
2026-11-13 09:21:28 -06:00
committed by GitHub
Unverified
parent ba77ff90d3
commit f6e0c2d083
6 changed files with 35 additions and 25 deletions
+8 -6
View File
@@ -1087,7 +1087,9 @@ const program = Effect.gen(function* () {
The shared fields are `prompt`, `start`, `end`, `references`, `duration`, `aspect`, `resolution` and `sound`;
everything else is provider-specific and goes under `providerOptions`, typed from the selected model. A request only
fails before sending when its protocol has no way to express it, such as a reference video on xAI. Values the provider
fails before sending when its protocol has no way to express it, such as a reference video on xAI. `sound` is the
exception: on models whose sound can't be switched (MiniMax H3 and Gemini Omni always have it, Hailuo never does) it
isn't sent, and the model's own behavior applies. Values the provider
may reject, like an unsupported duration or too many references, go to the provider, and its error comes back typed.
`start` and `end` are images. They take a URL, data URL, base64 string or bytes, or an asset built with
@@ -1131,11 +1133,11 @@ Provider notes:
`/v1`, and a bare host (`https://api.minimax.io`) works as is. On H3, `start`, `end` and `references` become `content[]` items with `first_frame`, `last_frame` and `reference_image` /
`reference_video` / `reference_audio` roles, MiniMax file refs become `mm_file://` URLs, `aspect: "auto"` becomes
`ratio: "adaptive"`, and `resolution` is uppercased (`480P`, `2K`). H3 requires `resolution` and `duration`, plus a
`ratio` for text-only prompts; nothing is filled in, so MiniMax rejects a request without them. H3 always has sound,
so `sound: false` fails before sending. On Hailuo, `start` and `end` map to `first_frame_image` and
`ratio` for text-only prompts; nothing is filled in, so MiniMax rejects a request without them. H3 always has sound
and has no field for it, so `sound` isn't sent. On Hailuo, `start` and `end` map to `first_frame_image` and
`last_frame_image`, image references to `subject_reference`, and the result looks up the file's download URL, which
expires after one hour. Hailuo is silent and has no aspect field, so `sound: true`, an explicit `aspect`, and video
or audio references fail before sending. Cancel works on H3 (only while queued); Hailuo has no cancel. Native fields
expires after one hour. Hailuo is always silent, so `sound` isn't sent; it has no aspect field, so an explicit
`aspect` and video or audio references fail before sending. Cancel works on H3 (only while queued); Hailuo has no cancel. Native fields
such as `extra.prompt_expansion_mode` (H3), `prompt_optimizer` (Hailuo) and `callback_url` go under `providerOptions`.
- **Google** (`Google.video("gemini-omni-1.2-flash")`) starts a background Gemini Omni interaction. Omni has no
role field for media, so `start`, `end` and `references` go into `input` in that order (images, plus video
@@ -1143,7 +1145,7 @@ Provider notes:
`reference_to_video` for references only; with both, the prompt decides. `duration` becomes
`response_format.duration` (`"3s"`), and `aspect` and `resolution` go to `response_format` as is (resolution
lowercased); `aspect: "auto"` leaves the ratio out. It rejects `end` without `start` (a lone image reads as the
first frame) and `sound: false` (Omni always generates audio). Videos come back as inline bytes. `cancel` posts to
first frame). Omni always generates audio and has no field for it, so `sound` isn't sent. Videos come back as inline bytes. `cancel` posts to
`/interactions/{id}/cancel`. `providerOptions` takes native Interactions fields, such as `previous_interaction_id`
to edit or extend a generated video, `generation_config.video_config.task`, `service_tier` and `webhook_config`.
@@ -129,11 +129,6 @@ const task = (request: Request) => {
const fromRequest = Effect.fn("GoogleInteractionsVideo.fromRequest")(function* (request: Request) {
if (request.end !== undefined && request.start === undefined)
return yield* route.unsupported("video.end", `${route.name} reads a lone image as the start frame; send start too`)
if (request.sound === false)
return yield* route.unsupported(
"video.sound",
`${route.name} always generates audio and has no option to turn it off`,
)
const media = yield* Effect.forEach(
[request.start, request.end, ...(request.references ?? [])].filter(
(asset): asset is Media.Asset => asset !== undefined,
+2 -5
View File
@@ -177,9 +177,7 @@ const reference = (asset: Media.Asset) =>
const resolution = (request: Request) => request.resolution?.toUpperCase()
const fromRequest = Effect.fn("MiniMaxVideo.fromRequest")(function* (request: Request) {
// H3 always generates a soundtrack and has no field to turn it off.
if (request.sound === false)
return yield* route.unsupported("video.sound", `${route.name} always generates sound and can't be silenced`)
// H3 always generates a soundtrack and has no field for `sound`, so it isn't sent.
const content = [
{ type: "text", text: request.prompt },
...(request.start === undefined ? [] : [yield* item(request.start, "first_frame")]),
@@ -213,8 +211,7 @@ const hailuoFromRequest = Effect.fn("MiniMaxHailuoVideo.fromRequest")(function*
`${hailuoRoute.name} takes only image (subject) references`,
)
// Hailuo output is always silent and its shape follows the model or the first frame, with no field for either.
if (request.sound === true)
return yield* hailuoRoute.unsupported("video.sound", `${hailuoRoute.name} generates silent video only`)
// `sound` has nothing to send, so it isn't; an aspect ratio would change the shape the caller asked for.
if (request.aspect !== undefined && request.aspect !== "auto")
return yield* hailuoRoute.unsupported("video.aspect", `${hailuoRoute.name} has no aspect ratio input`)
const subjects = yield* Effect.forEach(references, (asset) =>
+3 -1
View File
@@ -97,7 +97,9 @@ describe("Generation", () => {
const generation = new Generation(route, "t", { id: "gen_1", status: "queued" })
const fiber = yield* Effect.forkChild(
generation.events({ poll: { interval: "1 second", timeout: "5 seconds" } }).pipe(Stream.runCollect, Effect.flip),
generation
.events({ poll: { interval: "1 second", timeout: "5 seconds" } })
.pipe(Stream.runCollect, Effect.flip),
)
yield* TestClock.adjust("6 seconds")
const error = yield* Fiber.join(fiber)
@@ -101,7 +101,9 @@ describe("ai.video.start / Google Interactions", () => {
model,
prompt: "Morph",
start: Media.image(Media.fromDataUrl("data:image/png;base64,AQID")),
end: Media.image(Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/last", "image/jpeg")),
end: Media.image(
Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/last", "image/jpeg"),
),
})
expect(body.input).toEqual([
{ type: "image", data: "AQID", mime_type: "image/png" },
@@ -147,9 +149,15 @@ describe("ai.video.start / Google Interactions", () => {
}),
)
it.effect("sends nothing for sound, since Omni always generates audio and has no field for it", () =>
Effect.gen(function* () {
const silent = yield* sent({ model, prompt: "x", sound: false })
expect(JSON.stringify(silent.body)).not.toContain("audio")
}),
)
for (const rejected of [
{ name: "an end frame without a start frame", input: { end: "data:image/png;base64,AQID" }, operation: "end" },
{ name: "silent video", input: { sound: false }, operation: "sound" },
])
it.effect(`rejects ${rejected.name}, which Omni has no way to send, before sending`, () =>
Effect.gen(function* () {
+12 -6
View File
@@ -136,12 +136,19 @@ describe("MiniMax Video / H3 start", () => {
}),
)
it.effect("rejects silent output, which H3 can't produce, before sending", () =>
it.effect("sends nothing for sound, which neither API has a field for", () =>
Effect.gen(function* () {
const error = yield* ai.video.start({ model: h3, prompt: "x", sound: false }).pipe(Effect.flip)
expect(error.reason._tag).toBe("UnsupportedOperation")
expect(error.message).toContain("always generates sound")
}).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))),
const requests = yield* sent([
{ model: h3, sound: false },
{ model: h3, sound: true },
{ model: hailuo, sound: true },
])
expect(requests.map((request) => "generate_audio" in request.body || "sound" in request.body)).toEqual([
false,
false,
false,
])
}),
)
it.effect("returns MiniMax's HTTP errors typed", () =>
@@ -242,7 +249,6 @@ describe("MiniMax Video / Hailuo start", () => {
request: { references: [Media.url("https://example.test/v.mp3", { mediaType: "audio/mpeg" })] },
tag: "UnsupportedOperation",
},
{ name: "sound", request: { sound: true }, tag: "UnsupportedOperation" },
{ name: "an aspect ratio", request: { aspect: "16:9" as const }, tag: "UnsupportedOperation" },
{ name: "a MiniMax file reference", request: { start: Media.ref("minimax", "file_1") }, tag: "InvalidRequest" },
]) {