From f954688fbb4c476052ee2db74d675b70a22bd819 Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Fri, 25 Sep 2026 22:51:22 +0530 Subject: [PATCH] fix(ai): harden OpenAI transcription stream parsing (#51378) --- packages/ai/docs/media-design.md | 2 +- .../ai/src/protocols/openai-transcription.ts | 33 ++++++++-- .../openai-transcription.recorded.test.ts | 7 ++- packages/ai/test/transcription.test.ts | 63 ++++++++++++++++++- 4 files changed, 96 insertions(+), 9 deletions(-) diff --git a/packages/ai/docs/media-design.md b/packages/ai/docs/media-design.md index b28510600d4..b3c56a1db21 100644 --- a/packages/ai/docs/media-design.md +++ b/packages/ai/docs/media-design.md @@ -329,7 +329,7 @@ Settled rules: | Provider | Kind | Audio input | `timestamps` | `diarize` | Unsupported | Usage | |---|---|---|---|---|---|---| -| OpenAI | stream (`stream: true` in `stream` mode) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model; streaming on `whisper-1` | `tokens` or `seconds` | +| OpenAI | stream (`stream: true` in `stream` mode; `whisper-1` ignores `stream`, so it emits only `finish`) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model | `tokens` or `seconds` | | Gemini | stream (`generateContent` / `streamGenerateContent`) | `inlineData` or Gemini Files `fileData` | `audioTranscriptionConfig.wordTimestamp` | `audioTranscriptionConfig.diarization` | `prompt`, `speakers` | `tokens` | | Deepgram | inline | raw body, or JSON `{ url }` | words always; `segment` → `utterances` | `diarize_model=latest` + `utterances` | `prompt`, `speakers` | `seconds` (`metadata.duration`) | | AssemblyAI | queued (upload → submit → poll) | `/v2/upload` then `audio_url`, or a URL | words always; `segment` → `speaker_labels` | `speaker_labels` | — | `seconds` (`audio_duration`) | diff --git a/packages/ai/src/protocols/openai-transcription.ts b/packages/ai/src/protocols/openai-transcription.ts index 38fb348a46c..9d267842ae5 100644 --- a/packages/ai/src/protocols/openai-transcription.ts +++ b/packages/ai/src/protocols/openai-transcription.ts @@ -1,8 +1,9 @@ import { Effect, Schema, Stream } from "effect" +import { classifyProviderFailure } from "../provider-error.js" import { Framing } from "../route/framing.js" import { MediaProtocol } from "../route/media-protocol.js" import { MediaRoute } from "../route/media.js" -import { mergeJsonRecords, type MediaUsage } from "../schema/index.js" +import { AIError, mergeJsonRecords, type MediaUsage } from "../schema/index.js" import { TranscriptionFinishEvent, TranscriptionModel, @@ -59,6 +60,9 @@ const Usage = Schema.Union([ input_tokens: Schema.optional(Schema.Number), output_tokens: Schema.optional(Schema.Number), total_tokens: Schema.optional(Schema.Number), + input_token_details: Schema.optional( + Schema.Struct({ audio_tokens: Schema.optional(Schema.Number), text_tokens: Schema.optional(Schema.Number) }), + ), }), Schema.Struct({ type: Schema.Literal("duration"), seconds: Schema.Number }), ]) @@ -75,14 +79,23 @@ const transcriptFields = { usage: Schema.optional(Usage), } +/** OpenAI may add stream event types; frames outside `EVENT_TYPES` are ignored. */ +const EventType = Schema.Struct({ type: Schema.String }) const Event = Schema.Union([ Schema.Struct({ type: Schema.Literal("transcript.text.delta"), delta: Schema.String }), Schema.Struct({ type: Schema.Literal("transcript.text.segment"), ...Segment.fields }), Schema.Struct({ type: Schema.Literal("transcript.text.done"), ...transcriptFields }), + Schema.Struct({ + type: Schema.Literal("error"), + message: Schema.optional(Schema.String), + error: Schema.optional(Schema.Struct({ message: Schema.optional(Schema.String) })), + }), ]) +const EVENT_TYPES = new Set(["transcript.text.delta", "transcript.text.segment", "transcript.text.done", "error"]) const Transcript = Schema.Struct(transcriptFields) type Transcript = Schema.Schema.Type +const decodeEventType = route.decodeFrame(EventType) const decodeEvent = route.decodeFrame(Event) const decodeTranscript = route.decodeFrame(Transcript) @@ -118,10 +131,12 @@ const capabilities = (model: string): Capabilities => { return TRANSCRIBE } +/** whisper-1 ignores `stream`, so its `stream` mode sends a plain request and emits only `finish`. */ +const streamsEvents = (request: MediaProtocol.Addressed) => + request.mode === "stream" && capabilities(request.model.id).stream + const validate = (request: MediaProtocol.Addressed, model: Capabilities) => { const id = request.model.id - if (request.mode === "stream" && !model.stream) - return Effect.fail(route.unsupported("media.stream", `${id} does not stream; use Transcription.generate`)) if (request.diarize === true && !model.diarize) return Effect.fail(route.unsupported("media.diarize", `${id} does not diarize; use gpt-4o-transcribe-diarize`)) if (request.prompt !== undefined && model.diarize) @@ -173,7 +188,7 @@ const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (requ timestamp_granularities: responseFormat === "verbose_json" ? [request.timestamps] : undefined, // Diarizing audio longer than 30 seconds requires a chunking strategy. chunking_strategy: model.diarize ? "auto" : undefined, - stream: request.mode === "stream" ? true : undefined, + stream: streamsEvents(request) ? true : undefined, }, { overlay: mergeJsonRecords(request.providerOptions, request.http?.body), @@ -196,7 +211,15 @@ const segment = (value: Schema.Schema.Type): TranscriptionSegmen }) const onEvent = Effect.fn("OpenAITranscription.onEvent")(function* (state: State, frame: string) { + if (!EVENT_TYPES.has((yield* decodeEventType(frame)).type)) return [state, []] as const const event = yield* decodeEvent(frame) + if (event.type === "error") + return yield* new AIError({ + reason: classifyProviderFailure({ + message: `${route.name} stream failed: ${event.message ?? event.error?.message ?? "unknown error"}`, + rawBody: frame, + }), + }) if (event.type === "transcript.text.done") return [{ ...state, transcript: event }, []] as const if (event.type === "transcript.text.delta") return [state, event.delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta: event.delta })]] as const @@ -246,7 +269,7 @@ export const protocol = MediaProtocol.stream - context.request.mode === "stream" + streamsEvents(context.request) ? Framing.sse.frame(bytes) : Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))), initial: () => ({ segments: [] }), diff --git a/packages/ai/test/provider/openai-transcription.recorded.test.ts b/packages/ai/test/provider/openai-transcription.recorded.test.ts index e8ce460bf79..906f2148b91 100644 --- a/packages/ai/test/provider/openai-transcription.recorded.test.ts +++ b/packages/ai/test/provider/openai-transcription.recorded.test.ts @@ -44,7 +44,12 @@ describe("OpenAI Transcription recorded", () => { expect(deltas.length).toBeGreaterThan(1) expect(deltas.join("")).toBe(finish.text) expect(finish.text).toMatch(TRANSCRIPT) - expect(finish.usage).toMatchObject({ type: "tokens", input: expect.any(Number), output: expect.any(Number) }) + expect(finish.usage).toMatchObject({ + type: "tokens", + input: expect.any(Number), + output: expect.any(Number), + details: { openai: { input_token_details: { audio_tokens: expect.any(Number) } } }, + }) }), ) diff --git a/packages/ai/test/transcription.test.ts b/packages/ai/test/transcription.test.ts index e74ed34cf84..1efdd9a79e5 100644 --- a/packages/ai/test/transcription.test.ts +++ b/packages/ai/test/transcription.test.ts @@ -25,7 +25,6 @@ describe("Transcription", () => { Effect.gen(function* () { const errors = yield* Effect.all( [ - Stream.runCollect(Transcription.stream({ model: openai.transcription("whisper-1"), audio })), Transcription.generate({ model: openai.transcription("gpt-4o-mini-transcribe"), audio, diarize: true }), Transcription.generate({ model: openai.transcription("gpt-4o-mini-transcribe"), audio, timestamps: "word" }), Transcription.generate({ model: openai.transcription("gpt-4o-transcribe-diarize"), audio, prompt: "Names" }), @@ -53,7 +52,6 @@ describe("Transcription", () => { ) expect(errors.map((error) => [error.reason._tag, "operation" in error.reason && error.reason.operation])).toEqual( [ - ["UnsupportedOperation", "media.stream"], ["UnsupportedOperation", "media.diarize"], ["UnsupportedOperation", "media.timestamps"], ["UnsupportedOperation", "media.prompt"], @@ -70,6 +68,67 @@ describe("Transcription", () => { }).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))), ) + it.effect("ignores unknown OpenAI stream events and fails on an error event with the frame", () => + Effect.gen(function* () { + const sse = (...frames: ReadonlyArray) => frames.map((frame) => `data: ${frame}\n\n`).join("") + const failure = `{"type":"error","error":{"type":"server_error","code":"server_error","message":"The server had an error"}}` + const bodies = [ + sse( + `{"type":"transcript.text.delta","delta":"Hi"}`, + `{"type":"transcript.text.future","payload":1}`, + `{"type":"transcript.text.done","text":"Hi"}`, + "[DONE]", + ), + sse(`{"type":"transcript.text.delta","delta":"Hi"}`, failure), + ] + const model = openai.transcription("gpt-4o-mini-transcribe") + const program = Effect.gen(function* () { + const events = Array.from(yield* Stream.runCollect(Transcription.stream({ model, audio }))) + const error = yield* Stream.runCollect(Transcription.stream({ model, audio })).pipe(Effect.flip) + return { events, error } + }) + const { events, error } = yield* program.pipe( + Effect.provide( + layer((input) => + Effect.sync(() => + input.respond(bodies.shift() ?? "", { headers: { "content-type": "text/event-stream" } }), + ), + ), + ), + ) + + expect(events.map((event) => event.type)).toEqual(["text-delta", "finish"]) + expect(error.reason).toMatchObject({ _tag: "ProviderInternal", body: failure }) + expect(error.message).toContain("The server had an error") + }), + ) + + it.effect("streams whisper-1 as a single finish from a plain request", () => + Effect.gen(function* () { + const bodies: Array = [] + const events = Array.from( + yield* Stream.runCollect(Transcription.stream({ model: openai.transcription("whisper-1"), audio })).pipe( + Effect.provide( + layer((input) => + Effect.sync(() => { + bodies.push(input.text) + return input.respond( + JSON.stringify({ text: "Hello there.", usage: { type: "duration", seconds: 2 } }), + { headers: { "content-type": "application/json" } }, + ) + }), + ), + ), + ), + ) + + expect(bodies[0]).not.toContain('name="stream"') + expect(events).toEqual([ + expect.objectContaining({ type: "finish", text: "Hello there.", usage: { type: "seconds", seconds: 2 } }), + ]) + }), + ) + it.effect( "uploads inline audio to AssemblyAI, resumes polling from a persisted token, and surfaces failed transcripts", () =>