diff --git a/packages/ai/src/protocols/deepgram-speech.ts b/packages/ai/src/protocols/deepgram-speech.ts index b8d3390fc72..eb505487168 100644 --- a/packages/ai/src/protocols/deepgram-speech.ts +++ b/packages/ai/src/protocols/deepgram-speech.ts @@ -89,24 +89,32 @@ const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request: // 6. Stream parsing // --------------------------------------------------------------------------- -const HEADERLESS_ENCODINGS: Readonly> = { - linear16: "pcm_s16le", - mulaw: "pcm_mulaw", - alaw: "pcm_alaw", +/** Deepgram wraps raw encodings in WAV unless `container` is `none`, and defaults their sample rate per encoding. */ +const HEADERLESS_ENCODINGS: Readonly< + Record +> = { + linear16: { encoding: "pcm_s16le", sampleRate: 24000 }, + mulaw: { encoding: "pcm_mulaw", sampleRate: 8000 }, + alaw: { encoding: "pcm_alaw", sampleRate: 8000 }, } const finish = (state: State, context: MediaProtocol.ResponseContext) => { const headers = context.http.headers const mediaType = headers["content-type"] const format = audioFormat(context.request) - const encoding = HEADERLESS_ENCODINGS[format.encoding ?? ""] + const headerless = HEADERLESS_ENCODINGS[format.encoding ?? ""] + const container = format.container ?? (headerless === undefined ? undefined : "wav") const requestID = headers["dg-request-id"] const modelName = headers["dg-model-name"] return SpeechStream.finish(route, state, { - ...(format.container === "none" && encoding !== undefined - ? SpeechStream.pcm(encoding, SpeechStream.sampleRate(mediaType), mediaType) + ...(container === "none" && headerless !== undefined + ? SpeechStream.pcm( + headerless.encoding, + SpeechStream.sampleRate(mediaType) ?? context.request.providerOptions?.sampleRate ?? headerless.sampleRate, + mediaType, + ) : // Deepgram's default encoding is MP3; WAV is a container around any encoding. - { mediaType, info: { format: format.container === "wav" ? "wav" : (format.encoding ?? "mp3") } }), + { mediaType, info: { format: container === "wav" ? "wav" : (format.encoding ?? "mp3") } }), usage: SpeechStream.headerUsage("characters", headers["dg-char-count"]), providerMetadata: requestID === undefined && modelName === undefined diff --git a/packages/ai/src/protocols/openai-speech.ts b/packages/ai/src/protocols/openai-speech.ts index 50ea4cbd21b..45e88af604a 100644 --- a/packages/ai/src/protocols/openai-speech.ts +++ b/packages/ai/src/protocols/openai-speech.ts @@ -60,10 +60,17 @@ interface State extends SpeechStream.Audio { // `sse` is not supported for `tts-1` or `tts-1-hd`; those models stream the raw audio body instead. const supportsSse = (model: string) => !/^tts-1(-hd)?(-|$)/.test(model) +const FORMATS = new Set(["mp3", "opus", "aac", "flac", "wav", "pcm"]) + const fromRequest = Effect.fn("OpenAISpeech.fromRequest")(function* (request: MediaProtocol.Addressed) { // Not in `unsupported`: that list would also reject `timestamps: false`, which asks for nothing. if (request.timestamps === true) return yield* route.unsupported("media.timestamps", `${route.name} does not return timestamps`) + if (request.format !== undefined && !FORMATS.has(request.format)) + return yield* route.unsupported( + "media.format", + `${route.name} supports the mp3, opus, aac, flac, wav, and pcm formats, not "${request.format}"`, + ) return MediaProtocol.json( mergeJsonRecords( { @@ -112,7 +119,9 @@ const onEvent = Effect.fn("OpenAISpeech.onEvent")(function* (state: State, frame const finish = (state: State, context: MediaProtocol.ResponseContext) => { if (isSse(context.body) && !state.done) return Effect.fail(route.incomplete()) - const format = context.request.format ?? "mp3" + // The sent body reflects `providerOptions` and `http.body` overrides of `format`. + const sent = context.body.type === "json" ? context.body.value.response_format : undefined + const format = typeof sent === "string" ? sent : "mp3" return SpeechStream.finish(route, state, { ...(format === "pcm" ? SpeechStream.pcm("pcm_s16le", PCM_SAMPLE_RATE) : SpeechStream.container(format)), usage: state.usage, diff --git a/packages/ai/test/speech.test.ts b/packages/ai/test/speech.test.ts index d5731b97a95..78205c8e4a7 100644 --- a/packages/ai/test/speech.test.ts +++ b/packages/ai/test/speech.test.ts @@ -60,6 +60,42 @@ describe("Speech", () => { }), ) + it.effect("describes OpenAI audio in the format the request body actually asked for", () => + Effect.gen(function* () { + const [pcm, wav] = yield* Effect.all([ + Speech.generate({ model: openai, text: "Hi", format: "mp3", providerOptions: { response_format: "pcm" } }), + Speech.generate({ model: openai, text: "Hi", http: { body: { response_format: "wav" } } }), + ]).pipe(Effect.provide(respond("\u0001\u0002", "application/octet-stream"))) + expect(pcm.audio.mediaType).toBe("audio/pcm") + expect(pcm.audio.info).toEqual({ format: "pcm", encoding: "pcm_s16le", sampleRate: 24000, channels: 1 }) + expect(wav.audio.mediaType).toBe("audio/wav") + expect(wav.audio.info?.format).toBe("wav") + }), + ) + + it.effect("describes Deepgram raw encodings in their default WAV container", () => + Effect.gen(function* () { + const response = yield* Speech.generate({ model: deepgram, text: "Hi", providerOptions: { encoding: "mulaw" } }) + expect(response.audio.mediaType).toBe("audio/wav") + expect(response.audio.info?.format).toBe("wav") + }).pipe(Effect.provide(respond("RIFF....WAVEfmt ", "audio/wav"))), + ) + + it.effect("always gives headerless Deepgram PCM a sample rate", () => + Effect.gen(function* () { + const [requested, defaulted] = yield* Effect.all([ + Speech.generate({ + model: deepgram, + text: "Hi", + providerOptions: { encoding: "mulaw", container: "none", sampleRate: 16000 }, + }), + Speech.generate({ model: deepgram, text: "Hi", providerOptions: { encoding: "alaw", container: "none" } }), + ]).pipe(Effect.provide(respond("\u0001\u0002", "audio/basic"))) + expect(requested.audio.info).toEqual({ format: "pcm", encoding: "pcm_mulaw", sampleRate: 16000, channels: 1 }) + expect(defaulted.audio.info).toEqual({ format: "pcm", encoding: "pcm_alaw", sampleRate: 8000, channels: 1 }) + }), + ) + it.effect("rejects raw PCM for Gemini 3.8 unary requests before sending", () => Effect.gen(function* () { const errors = yield* Effect.all( @@ -76,6 +112,7 @@ describe("Speech", () => { const errors = yield* Effect.all( [ Speech.generate({ model: openai, text: "Hi", timestamps: true }), + Speech.generate({ model: openai, text: "Hi", format: "ogg" }), Speech.generate({ model: google, text: "Hi", format: "mp3" }), Speech.generate({ model: google, text: "Hi", instructions: "Warm." }), collect(Speech.stream({ model: elevenlabs, text: "Hi", voice, format: "wav" })), @@ -87,13 +124,15 @@ describe("Speech", () => { [ ["UnsupportedOperation", "media.timestamps"], ["UnsupportedOperation", "media.format"], + ["UnsupportedOperation", "media.format"], ["UnsupportedOperation", "media.instructions"], ["UnsupportedOperation", "media.format"], ["UnsupportedOperation", "media.format"], ["UnsupportedOperation", "media.voice"], ], ) - expect(errors[1].reason).toMatchObject({ provider: "google", route: "google-speech" }) + expect(errors[1].reason).toMatchObject({ provider: "openai", route: "openai-speech" }) + expect(errors[2].reason).toMatchObject({ provider: "google", route: "google-speech" }) }).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))), )