fix(ai): harden OpenAI transcription stream parsing (#51378)

This commit is contained in:
Shoubhit Dash
2026-09-25 22:51:22 +05:30
committed by GitHub
Unverified
parent 1463dabde9
commit f954688fbb
4 changed files with 96 additions and 9 deletions
+1 -1
View File
@@ -329,7 +329,7 @@ Settled rules:
| Provider | Kind | Audio input | `timestamps` | `diarize` | Unsupported | Usage |
|---|---|---|---|---|---|---|
| OpenAI | stream (`stream: true` in `stream` mode) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model; streaming on `whisper-1` | `tokens` or `seconds` |
| OpenAI | stream (`stream: true` in `stream` mode; `whisper-1` ignores `stream`, so it emits only `finish`) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model | `tokens` or `seconds` |
| Gemini | stream (`generateContent` / `streamGenerateContent`) | `inlineData` or Gemini Files `fileData` | `audioTranscriptionConfig.wordTimestamp` | `audioTranscriptionConfig.diarization` | `prompt`, `speakers` | `tokens` |
| Deepgram | inline | raw body, or JSON `{ url }` | words always; `segment` → `utterances` | `diarize_model=latest` + `utterances` | `prompt`, `speakers` | `seconds` (`metadata.duration`) |
| AssemblyAI | queued (upload → submit → poll) | `/v2/upload` then `audio_url`, or a URL | words always; `segment` → `speaker_labels` | `speaker_labels` | — | `seconds` (`audio_duration`) |
@@ -1,8 +1,9 @@
import { Effect, Schema, Stream } from "effect"
import { classifyProviderFailure } from "../provider-error.js"
import { Framing } from "../route/framing.js"
import { MediaProtocol } from "../route/media-protocol.js"
import { MediaRoute } from "../route/media.js"
import { mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import { AIError, mergeJsonRecords, type MediaUsage } from "../schema/index.js"
import {
TranscriptionFinishEvent,
TranscriptionModel,
@@ -59,6 +60,9 @@ const Usage = Schema.Union([
input_tokens: Schema.optional(Schema.Number),
output_tokens: Schema.optional(Schema.Number),
total_tokens: Schema.optional(Schema.Number),
input_token_details: Schema.optional(
Schema.Struct({ audio_tokens: Schema.optional(Schema.Number), text_tokens: Schema.optional(Schema.Number) }),
),
}),
Schema.Struct({ type: Schema.Literal("duration"), seconds: Schema.Number }),
])
@@ -75,14 +79,23 @@ const transcriptFields = {
usage: Schema.optional(Usage),
}
/** OpenAI may add stream event types; frames outside `EVENT_TYPES` are ignored. */
const EventType = Schema.Struct({ type: Schema.String })
const Event = Schema.Union([
Schema.Struct({ type: Schema.Literal("transcript.text.delta"), delta: Schema.String }),
Schema.Struct({ type: Schema.Literal("transcript.text.segment"), ...Segment.fields }),
Schema.Struct({ type: Schema.Literal("transcript.text.done"), ...transcriptFields }),
Schema.Struct({
type: Schema.Literal("error"),
message: Schema.optional(Schema.String),
error: Schema.optional(Schema.Struct({ message: Schema.optional(Schema.String) })),
}),
])
const EVENT_TYPES = new Set(["transcript.text.delta", "transcript.text.segment", "transcript.text.done", "error"])
const Transcript = Schema.Struct(transcriptFields)
type Transcript = Schema.Schema.Type<typeof Transcript>
const decodeEventType = route.decodeFrame(EventType)
const decodeEvent = route.decodeFrame(Event)
const decodeTranscript = route.decodeFrame(Transcript)
@@ -118,10 +131,12 @@ const capabilities = (model: string): Capabilities => {
return TRANSCRIBE
}
/** whisper-1 ignores `stream`, so its `stream` mode sends a plain request and emits only `finish`. */
const streamsEvents = (request: MediaProtocol.Addressed<Request>) =>
request.mode === "stream" && capabilities(request.model.id).stream
const validate = (request: MediaProtocol.Addressed<Request>, model: Capabilities) => {
const id = request.model.id
if (request.mode === "stream" && !model.stream)
return Effect.fail(route.unsupported("media.stream", `${id} does not stream; use Transcription.generate`))
if (request.diarize === true && !model.diarize)
return Effect.fail(route.unsupported("media.diarize", `${id} does not diarize; use gpt-4o-transcribe-diarize`))
if (request.prompt !== undefined && model.diarize)
@@ -173,7 +188,7 @@ const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (requ
timestamp_granularities: responseFormat === "verbose_json" ? [request.timestamps] : undefined,
// Diarizing audio longer than 30 seconds requires a chunking strategy.
chunking_strategy: model.diarize ? "auto" : undefined,
stream: request.mode === "stream" ? true : undefined,
stream: streamsEvents(request) ? true : undefined,
},
{
overlay: mergeJsonRecords(request.providerOptions, request.http?.body),
@@ -196,7 +211,15 @@ const segment = (value: Schema.Schema.Type<typeof Segment>): TranscriptionSegmen
})
const onEvent = Effect.fn("OpenAITranscription.onEvent")(function* (state: State, frame: string) {
if (!EVENT_TYPES.has((yield* decodeEventType(frame)).type)) return [state, []] as const
const event = yield* decodeEvent(frame)
if (event.type === "error")
return yield* new AIError({
reason: classifyProviderFailure({
message: `${route.name} stream failed: ${event.message ?? event.error?.message ?? "unknown error"}`,
rawBody: frame,
}),
})
if (event.type === "transcript.text.done") return [{ ...state, transcript: event }, []] as const
if (event.type === "transcript.text.delta")
return [state, event.delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta: event.delta })]] as const
@@ -246,7 +269,7 @@ export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, Frame,
unsupported: ["speakers"],
body: { from: fromRequest },
frames: (bytes, context) =>
context.request.mode === "stream"
streamsEvents(context.request)
? Framing.sse.frame(bytes)
: Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
initial: () => ({ segments: [] }),
@@ -44,7 +44,12 @@ describe("OpenAI Transcription recorded", () => {
expect(deltas.length).toBeGreaterThan(1)
expect(deltas.join("")).toBe(finish.text)
expect(finish.text).toMatch(TRANSCRIPT)
expect(finish.usage).toMatchObject({ type: "tokens", input: expect.any(Number), output: expect.any(Number) })
expect(finish.usage).toMatchObject({
type: "tokens",
input: expect.any(Number),
output: expect.any(Number),
details: { openai: { input_token_details: { audio_tokens: expect.any(Number) } } },
})
}),
)
+61 -2
View File
@@ -25,7 +25,6 @@ describe("Transcription", () => {
Effect.gen(function* () {
const errors = yield* Effect.all(
[
Stream.runCollect(Transcription.stream({ model: openai.transcription("whisper-1"), audio })),
Transcription.generate({ model: openai.transcription("gpt-4o-mini-transcribe"), audio, diarize: true }),
Transcription.generate({ model: openai.transcription("gpt-4o-mini-transcribe"), audio, timestamps: "word" }),
Transcription.generate({ model: openai.transcription("gpt-4o-transcribe-diarize"), audio, prompt: "Names" }),
@@ -53,7 +52,6 @@ describe("Transcription", () => {
)
expect(errors.map((error) => [error.reason._tag, "operation" in error.reason && error.reason.operation])).toEqual(
[
["UnsupportedOperation", "media.stream"],
["UnsupportedOperation", "media.diarize"],
["UnsupportedOperation", "media.timestamps"],
["UnsupportedOperation", "media.prompt"],
@@ -70,6 +68,67 @@ describe("Transcription", () => {
}).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))),
)
it.effect("ignores unknown OpenAI stream events and fails on an error event with the frame", () =>
Effect.gen(function* () {
const sse = (...frames: ReadonlyArray<string>) => frames.map((frame) => `data: ${frame}\n\n`).join("")
const failure = `{"type":"error","error":{"type":"server_error","code":"server_error","message":"The server had an error"}}`
const bodies = [
sse(
`{"type":"transcript.text.delta","delta":"Hi"}`,
`{"type":"transcript.text.future","payload":1}`,
`{"type":"transcript.text.done","text":"Hi"}`,
"[DONE]",
),
sse(`{"type":"transcript.text.delta","delta":"Hi"}`, failure),
]
const model = openai.transcription("gpt-4o-mini-transcribe")
const program = Effect.gen(function* () {
const events = Array.from(yield* Stream.runCollect(Transcription.stream({ model, audio })))
const error = yield* Stream.runCollect(Transcription.stream({ model, audio })).pipe(Effect.flip)
return { events, error }
})
const { events, error } = yield* program.pipe(
Effect.provide(
layer((input) =>
Effect.sync(() =>
input.respond(bodies.shift() ?? "", { headers: { "content-type": "text/event-stream" } }),
),
),
),
)
expect(events.map((event) => event.type)).toEqual(["text-delta", "finish"])
expect(error.reason).toMatchObject({ _tag: "ProviderInternal", body: failure })
expect(error.message).toContain("The server had an error")
}),
)
it.effect("streams whisper-1 as a single finish from a plain request", () =>
Effect.gen(function* () {
const bodies: Array<string> = []
const events = Array.from(
yield* Stream.runCollect(Transcription.stream({ model: openai.transcription("whisper-1"), audio })).pipe(
Effect.provide(
layer((input) =>
Effect.sync(() => {
bodies.push(input.text)
return input.respond(
JSON.stringify({ text: "Hello there.", usage: { type: "duration", seconds: 2 } }),
{ headers: { "content-type": "application/json" } },
)
}),
),
),
),
)
expect(bodies[0]).not.toContain('name="stream"')
expect(events).toEqual([
expect.objectContaining({ type: "finish", text: "Hello there.", usage: { type: "seconds", seconds: 2 } }),
])
}),
)
it.effect(
"uploads inline audio to AssemblyAI, resumes polling from a persisted token, and surfaces failed transcripts",
() =>