From f6e0c2d0833ffe772035f221fed923e1cd5fb408 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Fri, 13 Nov 2026 09:21:28 -0600 Subject: [PATCH] refactor(ai): ignore video sound on models that can't switch it (#63726) --- packages/ai/README.md | 14 ++++++++------ .../src/protocols/google-interactions-video.ts | 5 ----- packages/ai/src/protocols/minimax-video.ts | 7 ++----- packages/ai/test/generation.test.ts | 4 +++- .../ai/test/google-interactions-video.test.ts | 12 ++++++++++-- packages/ai/test/minimax-video.test.ts | 18 ++++++++++++------ 6 files changed, 35 insertions(+), 25 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index db73e71af7c..8231abb1a63 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1087,7 +1087,9 @@ const program = Effect.gen(function* () { The shared fields are `prompt`, `start`, `end`, `references`, `duration`, `aspect`, `resolution` and `sound`; everything else is provider-specific and goes under `providerOptions`, typed from the selected model. A request only -fails before sending when its protocol has no way to express it, such as a reference video on xAI. Values the provider +fails before sending when its protocol has no way to express it, such as a reference video on xAI. `sound` is the +exception: on models whose sound can't be switched (MiniMax H3 and Gemini Omni always have it, Hailuo never does) it +isn't sent, and the model's own behavior applies. Values the provider may reject, like an unsupported duration or too many references, go to the provider, and its error comes back typed. `start` and `end` are images. They take a URL, data URL, base64 string or bytes, or an asset built with @@ -1131,11 +1133,11 @@ Provider notes: `/v1`, and a bare host (`https://api.minimax.io`) works as is. On H3, `start`, `end` and `references` become `content[]` items with `first_frame`, `last_frame` and `reference_image` / `reference_video` / `reference_audio` roles, MiniMax file refs become `mm_file://` URLs, `aspect: "auto"` becomes `ratio: "adaptive"`, and `resolution` is uppercased (`480P`, `2K`). H3 requires `resolution` and `duration`, plus a - `ratio` for text-only prompts; nothing is filled in, so MiniMax rejects a request without them. H3 always has sound, - so `sound: false` fails before sending. On Hailuo, `start` and `end` map to `first_frame_image` and + `ratio` for text-only prompts; nothing is filled in, so MiniMax rejects a request without them. H3 always has sound + and has no field for it, so `sound` isn't sent. On Hailuo, `start` and `end` map to `first_frame_image` and `last_frame_image`, image references to `subject_reference`, and the result looks up the file's download URL, which - expires after one hour. Hailuo is silent and has no aspect field, so `sound: true`, an explicit `aspect`, and video - or audio references fail before sending. Cancel works on H3 (only while queued); Hailuo has no cancel. Native fields + expires after one hour. Hailuo is always silent, so `sound` isn't sent; it has no aspect field, so an explicit + `aspect` and video or audio references fail before sending. Cancel works on H3 (only while queued); Hailuo has no cancel. Native fields such as `extra.prompt_expansion_mode` (H3), `prompt_optimizer` (Hailuo) and `callback_url` go under `providerOptions`. - **Google** (`Google.video("gemini-omni-1.2-flash")`) starts a background Gemini Omni interaction. Omni has no role field for media, so `start`, `end` and `references` go into `input` in that order (images, plus video @@ -1143,7 +1145,7 @@ Provider notes: `reference_to_video` for references only; with both, the prompt decides. `duration` becomes `response_format.duration` (`"3s"`), and `aspect` and `resolution` go to `response_format` as is (resolution lowercased); `aspect: "auto"` leaves the ratio out. It rejects `end` without `start` (a lone image reads as the - first frame) and `sound: false` (Omni always generates audio). Videos come back as inline bytes. `cancel` posts to + first frame). Omni always generates audio and has no field for it, so `sound` isn't sent. Videos come back as inline bytes. `cancel` posts to `/interactions/{id}/cancel`. `providerOptions` takes native Interactions fields, such as `previous_interaction_id` to edit or extend a generated video, `generation_config.video_config.task`, `service_tier` and `webhook_config`. diff --git a/packages/ai/src/protocols/google-interactions-video.ts b/packages/ai/src/protocols/google-interactions-video.ts index 79c81433c5e..0f61c6a597c 100644 --- a/packages/ai/src/protocols/google-interactions-video.ts +++ b/packages/ai/src/protocols/google-interactions-video.ts @@ -129,11 +129,6 @@ const task = (request: Request) => { const fromRequest = Effect.fn("GoogleInteractionsVideo.fromRequest")(function* (request: Request) { if (request.end !== undefined && request.start === undefined) return yield* route.unsupported("video.end", `${route.name} reads a lone image as the start frame; send start too`) - if (request.sound === false) - return yield* route.unsupported( - "video.sound", - `${route.name} always generates audio and has no option to turn it off`, - ) const media = yield* Effect.forEach( [request.start, request.end, ...(request.references ?? [])].filter( (asset): asset is Media.Asset => asset !== undefined, diff --git a/packages/ai/src/protocols/minimax-video.ts b/packages/ai/src/protocols/minimax-video.ts index fe45f999608..aab982336fe 100644 --- a/packages/ai/src/protocols/minimax-video.ts +++ b/packages/ai/src/protocols/minimax-video.ts @@ -177,9 +177,7 @@ const reference = (asset: Media.Asset) => const resolution = (request: Request) => request.resolution?.toUpperCase() const fromRequest = Effect.fn("MiniMaxVideo.fromRequest")(function* (request: Request) { - // H3 always generates a soundtrack and has no field to turn it off. - if (request.sound === false) - return yield* route.unsupported("video.sound", `${route.name} always generates sound and can't be silenced`) + // H3 always generates a soundtrack and has no field for `sound`, so it isn't sent. const content = [ { type: "text", text: request.prompt }, ...(request.start === undefined ? [] : [yield* item(request.start, "first_frame")]), @@ -213,8 +211,7 @@ const hailuoFromRequest = Effect.fn("MiniMaxHailuoVideo.fromRequest")(function* `${hailuoRoute.name} takes only image (subject) references`, ) // Hailuo output is always silent and its shape follows the model or the first frame, with no field for either. - if (request.sound === true) - return yield* hailuoRoute.unsupported("video.sound", `${hailuoRoute.name} generates silent video only`) + // `sound` has nothing to send, so it isn't; an aspect ratio would change the shape the caller asked for. if (request.aspect !== undefined && request.aspect !== "auto") return yield* hailuoRoute.unsupported("video.aspect", `${hailuoRoute.name} has no aspect ratio input`) const subjects = yield* Effect.forEach(references, (asset) => diff --git a/packages/ai/test/generation.test.ts b/packages/ai/test/generation.test.ts index 90eda9f3a63..e89222573b3 100644 --- a/packages/ai/test/generation.test.ts +++ b/packages/ai/test/generation.test.ts @@ -97,7 +97,9 @@ describe("Generation", () => { const generation = new Generation(route, "t", { id: "gen_1", status: "queued" }) const fiber = yield* Effect.forkChild( - generation.events({ poll: { interval: "1 second", timeout: "5 seconds" } }).pipe(Stream.runCollect, Effect.flip), + generation + .events({ poll: { interval: "1 second", timeout: "5 seconds" } }) + .pipe(Stream.runCollect, Effect.flip), ) yield* TestClock.adjust("6 seconds") const error = yield* Fiber.join(fiber) diff --git a/packages/ai/test/google-interactions-video.test.ts b/packages/ai/test/google-interactions-video.test.ts index 4aabcbaf4f0..a57253f7f7f 100644 --- a/packages/ai/test/google-interactions-video.test.ts +++ b/packages/ai/test/google-interactions-video.test.ts @@ -101,7 +101,9 @@ describe("ai.video.start / Google Interactions", () => { model, prompt: "Morph", start: Media.image(Media.fromDataUrl("data:image/png;base64,AQID")), - end: Media.image(Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/last", "image/jpeg")), + end: Media.image( + Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/last", "image/jpeg"), + ), }) expect(body.input).toEqual([ { type: "image", data: "AQID", mime_type: "image/png" }, @@ -147,9 +149,15 @@ describe("ai.video.start / Google Interactions", () => { }), ) + it.effect("sends nothing for sound, since Omni always generates audio and has no field for it", () => + Effect.gen(function* () { + const silent = yield* sent({ model, prompt: "x", sound: false }) + expect(JSON.stringify(silent.body)).not.toContain("audio") + }), + ) + for (const rejected of [ { name: "an end frame without a start frame", input: { end: "data:image/png;base64,AQID" }, operation: "end" }, - { name: "silent video", input: { sound: false }, operation: "sound" }, ]) it.effect(`rejects ${rejected.name}, which Omni has no way to send, before sending`, () => Effect.gen(function* () { diff --git a/packages/ai/test/minimax-video.test.ts b/packages/ai/test/minimax-video.test.ts index 7434dcd7648..514458e913c 100644 --- a/packages/ai/test/minimax-video.test.ts +++ b/packages/ai/test/minimax-video.test.ts @@ -136,12 +136,19 @@ describe("MiniMax Video / H3 start", () => { }), ) - it.effect("rejects silent output, which H3 can't produce, before sending", () => + it.effect("sends nothing for sound, which neither API has a field for", () => Effect.gen(function* () { - const error = yield* ai.video.start({ model: h3, prompt: "x", sound: false }).pipe(Effect.flip) - expect(error.reason._tag).toBe("UnsupportedOperation") - expect(error.message).toContain("always generates sound") - }).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))), + const requests = yield* sent([ + { model: h3, sound: false }, + { model: h3, sound: true }, + { model: hailuo, sound: true }, + ]) + expect(requests.map((request) => "generate_audio" in request.body || "sound" in request.body)).toEqual([ + false, + false, + false, + ]) + }), ) it.effect("returns MiniMax's HTTP errors typed", () => @@ -242,7 +249,6 @@ describe("MiniMax Video / Hailuo start", () => { request: { references: [Media.url("https://example.test/v.mp3", { mediaType: "audio/mpeg" })] }, tag: "UnsupportedOperation", }, - { name: "sound", request: { sound: true }, tag: "UnsupportedOperation" }, { name: "an aspect ratio", request: { aspect: "16:9" as const }, tag: "UnsupportedOperation" }, { name: "a MiniMax file reference", request: { start: Media.ref("minimax", "file_1") }, tag: "InvalidRequest" }, ]) {