fix(ai): describe speech assets in the format actually sent (#51377)

packages/ai/src/protocols/deepgram-speech.ts

@@ -89,24 +89,32 @@ const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request:
 // 6. Stream parsing
 // ---------------------------------------------------------------------------
 
-const HEADERLESS_ENCODINGS: Readonly<Record<string, SpeechStream.PcmEncoding>> = {
-  linear16: "pcm_s16le",
-  mulaw: "pcm_mulaw",
-  alaw: "pcm_alaw",
+/** Deepgram wraps raw encodings in WAV unless `container` is `none`, and defaults their sample rate per encoding. */
+const HEADERLESS_ENCODINGS: Readonly<
+  Record<string, { readonly encoding: SpeechStream.PcmEncoding; readonly sampleRate: number }>
+> = {
+  linear16: { encoding: "pcm_s16le", sampleRate: 24000 },
+  mulaw: { encoding: "pcm_mulaw", sampleRate: 8000 },
+  alaw: { encoding: "pcm_alaw", sampleRate: 8000 },
 }
 
 const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
   const headers = context.http.headers
   const mediaType = headers["content-type"]
   const format = audioFormat(context.request)
-  const encoding = HEADERLESS_ENCODINGS[format.encoding ?? ""]
+  const headerless = HEADERLESS_ENCODINGS[format.encoding ?? ""]
+  const container = format.container ?? (headerless === undefined ? undefined : "wav")
   const requestID = headers["dg-request-id"]
   const modelName = headers["dg-model-name"]
   return SpeechStream.finish(route, state, {
-    ...(format.container === "none" && encoding !== undefined
-      ? SpeechStream.pcm(encoding, SpeechStream.sampleRate(mediaType), mediaType)
+    ...(container === "none" && headerless !== undefined
+      ? SpeechStream.pcm(
+          headerless.encoding,
+          SpeechStream.sampleRate(mediaType) ?? context.request.providerOptions?.sampleRate ?? headerless.sampleRate,
+          mediaType,
+        )
       : // Deepgram's default encoding is MP3; WAV is a container around any encoding.
-        { mediaType, info: { format: format.container === "wav" ? "wav" : (format.encoding ?? "mp3") } }),
+        { mediaType, info: { format: container === "wav" ? "wav" : (format.encoding ?? "mp3") } }),
     usage: SpeechStream.headerUsage("characters", headers["dg-char-count"]),
     providerMetadata:
       requestID === undefined && modelName === undefined

packages/ai/src/protocols/openai-speech.ts

@@ -60,10 +60,17 @@ interface State extends SpeechStream.Audio {
 // `sse` is not supported for `tts-1` or `tts-1-hd`; those models stream the raw audio body instead.
 const supportsSse = (model: string) => !/^tts-1(-hd)?(-|$)/.test(model)
 
+const FORMATS = new Set(["mp3", "opus", "aac", "flac", "wav", "pcm"])
+
 const fromRequest = Effect.fn("OpenAISpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
   // Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing.
   if (request.timestamps === true)
     return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`)
+  if (request.format !== undefined && !FORMATS.has(request.format))
+    return yield* route.unsupported(
+      "media.format",
+      `${route.name} supports the mp3, opus, aac, flac, wav, and pcm formats, not "${request.format}"`,
+    )
   return MediaProtocol.json(
     mergeJsonRecords(
       {
@@ -112,7 +119,9 @@ const onEvent = Effect.fn("OpenAISpeech.onEvent")(function* (state: State, frame
 
 const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
   if (isSse(context.body) && !state.done) return Effect.fail(route.incomplete())
-  const format = context.request.format ?? "mp3"
+  // The sent body reflects `providerOptions` and `http.body` overrides of `format`.
+  const sent = context.body.type === "json" ? context.body.value.response_format : undefined
+  const format = typeof sent === "string" ? sent : "mp3"
   return SpeechStream.finish(route, state, {
     ...(format === "pcm" ? SpeechStream.pcm("pcm_s16le", PCM_SAMPLE_RATE) : SpeechStream.container(format)),
     usage: state.usage,

packages/ai/test/speech.test.ts

@@ -60,6 +60,42 @@ describe("Speech", () => {
     }),
   )
 
+  it.effect("describes OpenAI audio in the format the request body actually asked for", () =>
+    Effect.gen(function* () {
+      const [pcm, wav] = yield* Effect.all([
+        Speech.generate({ model: openai, text: "Hi", format: "mp3", providerOptions: { response_format: "pcm" } }),
+        Speech.generate({ model: openai, text: "Hi", http: { body: { response_format: "wav" } } }),
+      ]).pipe(Effect.provide(respond("\u0001\u0002", "application/octet-stream")))
+      expect(pcm.audio.mediaType).toBe("audio/pcm")
+      expect(pcm.audio.info).toEqual({ format: "pcm", encoding: "pcm_s16le", sampleRate: 24000, channels: 1 })
+      expect(wav.audio.mediaType).toBe("audio/wav")
+      expect(wav.audio.info?.format).toBe("wav")
+    }),
+  )
+
+  it.effect("describes Deepgram raw encodings in their default WAV container", () =>
+    Effect.gen(function* () {
+      const response = yield* Speech.generate({ model: deepgram, text: "Hi", providerOptions: { encoding: "mulaw" } })
+      expect(response.audio.mediaType).toBe("audio/wav")
+      expect(response.audio.info?.format).toBe("wav")
+    }).pipe(Effect.provide(respond("RIFF....WAVEfmt ", "audio/wav"))),
+  )
+
+  it.effect("always gives headerless Deepgram PCM a sample rate", () =>
+    Effect.gen(function* () {
+      const [requested, defaulted] = yield* Effect.all([
+        Speech.generate({
+          model: deepgram,
+          text: "Hi",
+          providerOptions: { encoding: "mulaw", container: "none", sampleRate: 16000 },
+        }),
+        Speech.generate({ model: deepgram, text: "Hi", providerOptions: { encoding: "alaw", container: "none" } }),
+      ]).pipe(Effect.provide(respond("\u0001\u0002", "audio/basic")))
+      expect(requested.audio.info).toEqual({ format: "pcm", encoding: "pcm_mulaw", sampleRate: 16000, channels: 1 })
+      expect(defaulted.audio.info).toEqual({ format: "pcm", encoding: "pcm_alaw", sampleRate: 8000, channels: 1 })
+    }),
+  )
+
   it.effect("rejects raw PCM for Gemini 3.8 unary requests before sending", () =>
     Effect.gen(function* () {
       const errors = yield* Effect.all(
@@ -76,6 +112,7 @@ describe("Speech", () => {
       const errors = yield* Effect.all(
         [
           Speech.generate({ model: openai, text: "Hi", timestamps: true }),
+          Speech.generate({ model: openai, text: "Hi", format: "ogg" }),
           Speech.generate({ model: google, text: "Hi", format: "mp3" }),
           Speech.generate({ model: google, text: "Hi", instructions: "Warm." }),
           collect(Speech.stream({ model: elevenlabs, text: "Hi", voice, format: "wav" })),
@@ -87,13 +124,15 @@ describe("Speech", () => {
         [
           ["UnsupportedOperation", "media.timestamps"],
           ["UnsupportedOperation", "media.format"],
+          ["UnsupportedOperation", "media.format"],
           ["UnsupportedOperation", "media.instructions"],
           ["UnsupportedOperation", "media.format"],
           ["UnsupportedOperation", "media.format"],
           ["UnsupportedOperation", "media.voice"],
         ],
       )
-      expect(errors[1].reason).toMatchObject({ provider: "google", route: "google-speech" })
+      expect(errors[1].reason).toMatchObject({ provider: "openai", route: "openai-speech" })
+      expect(errors[2].reason).toMatchObject({ provider: "google", route: "google-speech" })
     }).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))),
   )