mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 21:15:57 +03:00
fix(ai): harden OpenAI transcription stream parsing (#51378)
This commit is contained in:
committed by
GitHub
Unverified
parent
1463dabde9
commit
f954688fbb
@@ -329,7 +329,7 @@ Settled rules:
|
||||
|
||||
| Provider | Kind | Audio input | `timestamps` | `diarize` | Unsupported | Usage |
|
||||
|---|---|---|---|---|---|---|
|
||||
| OpenAI | stream (`stream: true` in `stream` mode) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model; streaming on `whisper-1` | `tokens` or `seconds` |
|
||||
| OpenAI | stream (`stream: true` in `stream` mode; `whisper-1` ignores `stream`, so it emits only `finish`) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model | `tokens` or `seconds` |
|
||||
| Gemini | stream (`generateContent` / `streamGenerateContent`) | `inlineData` or Gemini Files `fileData` | `audioTranscriptionConfig.wordTimestamp` | `audioTranscriptionConfig.diarization` | `prompt`, `speakers` | `tokens` |
|
||||
| Deepgram | inline | raw body, or JSON `{ url }` | words always; `segment` → `utterances` | `diarize_model=latest` + `utterances` | `prompt`, `speakers` | `seconds` (`metadata.duration`) |
|
||||
| AssemblyAI | queued (upload → submit → poll) | `/v2/upload` then `audio_url`, or a URL | words always; `segment` → `speaker_labels` | `speaker_labels` | — | `seconds` (`audio_duration`) |
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { Effect, Schema, Stream } from "effect"
|
||||
import { classifyProviderFailure } from "../provider-error.js"
|
||||
import { Framing } from "../route/framing.js"
|
||||
import { MediaProtocol } from "../route/media-protocol.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { mergeJsonRecords, type MediaUsage } from "../schema/index.js"
|
||||
import { AIError, mergeJsonRecords, type MediaUsage } from "../schema/index.js"
|
||||
import {
|
||||
TranscriptionFinishEvent,
|
||||
TranscriptionModel,
|
||||
@@ -59,6 +60,9 @@ const Usage = Schema.Union([
|
||||
input_tokens: Schema.optional(Schema.Number),
|
||||
output_tokens: Schema.optional(Schema.Number),
|
||||
total_tokens: Schema.optional(Schema.Number),
|
||||
input_token_details: Schema.optional(
|
||||
Schema.Struct({ audio_tokens: Schema.optional(Schema.Number), text_tokens: Schema.optional(Schema.Number) }),
|
||||
),
|
||||
}),
|
||||
Schema.Struct({ type: Schema.Literal("duration"), seconds: Schema.Number }),
|
||||
])
|
||||
@@ -75,14 +79,23 @@ const transcriptFields = {
|
||||
usage: Schema.optional(Usage),
|
||||
}
|
||||
|
||||
/** OpenAI may add stream event types; frames outside `EVENT_TYPES` are ignored. */
|
||||
const EventType = Schema.Struct({ type: Schema.String })
|
||||
const Event = Schema.Union([
|
||||
Schema.Struct({ type: Schema.Literal("transcript.text.delta"), delta: Schema.String }),
|
||||
Schema.Struct({ type: Schema.Literal("transcript.text.segment"), ...Segment.fields }),
|
||||
Schema.Struct({ type: Schema.Literal("transcript.text.done"), ...transcriptFields }),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("error"),
|
||||
message: Schema.optional(Schema.String),
|
||||
error: Schema.optional(Schema.Struct({ message: Schema.optional(Schema.String) })),
|
||||
}),
|
||||
])
|
||||
const EVENT_TYPES = new Set(["transcript.text.delta", "transcript.text.segment", "transcript.text.done", "error"])
|
||||
const Transcript = Schema.Struct(transcriptFields)
|
||||
type Transcript = Schema.Schema.Type<typeof Transcript>
|
||||
|
||||
const decodeEventType = route.decodeFrame(EventType)
|
||||
const decodeEvent = route.decodeFrame(Event)
|
||||
const decodeTranscript = route.decodeFrame(Transcript)
|
||||
|
||||
@@ -118,10 +131,12 @@ const capabilities = (model: string): Capabilities => {
|
||||
return TRANSCRIBE
|
||||
}
|
||||
|
||||
/** whisper-1 ignores `stream`, so its `stream` mode sends a plain request and emits only `finish`. */
|
||||
const streamsEvents = (request: MediaProtocol.Addressed<Request>) =>
|
||||
request.mode === "stream" && capabilities(request.model.id).stream
|
||||
|
||||
const validate = (request: MediaProtocol.Addressed<Request>, model: Capabilities) => {
|
||||
const id = request.model.id
|
||||
if (request.mode === "stream" && !model.stream)
|
||||
return Effect.fail(route.unsupported("media.stream", `${id} does not stream; use Transcription.generate`))
|
||||
if (request.diarize === true && !model.diarize)
|
||||
return Effect.fail(route.unsupported("media.diarize", `${id} does not diarize; use gpt-4o-transcribe-diarize`))
|
||||
if (request.prompt !== undefined && model.diarize)
|
||||
@@ -173,7 +188,7 @@ const fromRequest = Effect.fn("OpenAITranscription.fromRequest")(function* (requ
|
||||
timestamp_granularities: responseFormat === "verbose_json" ? [request.timestamps] : undefined,
|
||||
// Diarizing audio longer than 30 seconds requires a chunking strategy.
|
||||
chunking_strategy: model.diarize ? "auto" : undefined,
|
||||
stream: request.mode === "stream" ? true : undefined,
|
||||
stream: streamsEvents(request) ? true : undefined,
|
||||
},
|
||||
{
|
||||
overlay: mergeJsonRecords(request.providerOptions, request.http?.body),
|
||||
@@ -196,7 +211,15 @@ const segment = (value: Schema.Schema.Type<typeof Segment>): TranscriptionSegmen
|
||||
})
|
||||
|
||||
const onEvent = Effect.fn("OpenAITranscription.onEvent")(function* (state: State, frame: string) {
|
||||
if (!EVENT_TYPES.has((yield* decodeEventType(frame)).type)) return [state, []] as const
|
||||
const event = yield* decodeEvent(frame)
|
||||
if (event.type === "error")
|
||||
return yield* new AIError({
|
||||
reason: classifyProviderFailure({
|
||||
message: `${route.name} stream failed: ${event.message ?? event.error?.message ?? "unknown error"}`,
|
||||
rawBody: frame,
|
||||
}),
|
||||
})
|
||||
if (event.type === "transcript.text.done") return [{ ...state, transcript: event }, []] as const
|
||||
if (event.type === "transcript.text.delta")
|
||||
return [state, event.delta.length === 0 ? [] : [TranscriptionTextDeltaEvent.make({ delta: event.delta })]] as const
|
||||
@@ -246,7 +269,7 @@ export const protocol = MediaProtocol.stream<Request, TranscriptionEvent, Frame,
|
||||
unsupported: ["speakers"],
|
||||
body: { from: fromRequest },
|
||||
frames: (bytes, context) =>
|
||||
context.request.mode === "stream"
|
||||
streamsEvents(context.request)
|
||||
? Framing.sse.frame(bytes)
|
||||
: Framing.document.frame(bytes).pipe(Stream.map((document) => ({ document }))),
|
||||
initial: () => ({ segments: [] }),
|
||||
|
||||
@@ -44,7 +44,12 @@ describe("OpenAI Transcription recorded", () => {
|
||||
expect(deltas.length).toBeGreaterThan(1)
|
||||
expect(deltas.join("")).toBe(finish.text)
|
||||
expect(finish.text).toMatch(TRANSCRIPT)
|
||||
expect(finish.usage).toMatchObject({ type: "tokens", input: expect.any(Number), output: expect.any(Number) })
|
||||
expect(finish.usage).toMatchObject({
|
||||
type: "tokens",
|
||||
input: expect.any(Number),
|
||||
output: expect.any(Number),
|
||||
details: { openai: { input_token_details: { audio_tokens: expect.any(Number) } } },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ describe("Transcription", () => {
|
||||
Effect.gen(function* () {
|
||||
const errors = yield* Effect.all(
|
||||
[
|
||||
Stream.runCollect(Transcription.stream({ model: openai.transcription("whisper-1"), audio })),
|
||||
Transcription.generate({ model: openai.transcription("gpt-4o-mini-transcribe"), audio, diarize: true }),
|
||||
Transcription.generate({ model: openai.transcription("gpt-4o-mini-transcribe"), audio, timestamps: "word" }),
|
||||
Transcription.generate({ model: openai.transcription("gpt-4o-transcribe-diarize"), audio, prompt: "Names" }),
|
||||
@@ -53,7 +52,6 @@ describe("Transcription", () => {
|
||||
)
|
||||
expect(errors.map((error) => [error.reason._tag, "operation" in error.reason && error.reason.operation])).toEqual(
|
||||
[
|
||||
["UnsupportedOperation", "media.stream"],
|
||||
["UnsupportedOperation", "media.diarize"],
|
||||
["UnsupportedOperation", "media.timestamps"],
|
||||
["UnsupportedOperation", "media.prompt"],
|
||||
@@ -70,6 +68,67 @@ describe("Transcription", () => {
|
||||
}).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))),
|
||||
)
|
||||
|
||||
it.effect("ignores unknown OpenAI stream events and fails on an error event with the frame", () =>
|
||||
Effect.gen(function* () {
|
||||
const sse = (...frames: ReadonlyArray<string>) => frames.map((frame) => `data: ${frame}\n\n`).join("")
|
||||
const failure = `{"type":"error","error":{"type":"server_error","code":"server_error","message":"The server had an error"}}`
|
||||
const bodies = [
|
||||
sse(
|
||||
`{"type":"transcript.text.delta","delta":"Hi"}`,
|
||||
`{"type":"transcript.text.future","payload":1}`,
|
||||
`{"type":"transcript.text.done","text":"Hi"}`,
|
||||
"[DONE]",
|
||||
),
|
||||
sse(`{"type":"transcript.text.delta","delta":"Hi"}`, failure),
|
||||
]
|
||||
const model = openai.transcription("gpt-4o-mini-transcribe")
|
||||
const program = Effect.gen(function* () {
|
||||
const events = Array.from(yield* Stream.runCollect(Transcription.stream({ model, audio })))
|
||||
const error = yield* Stream.runCollect(Transcription.stream({ model, audio })).pipe(Effect.flip)
|
||||
return { events, error }
|
||||
})
|
||||
const { events, error } = yield* program.pipe(
|
||||
Effect.provide(
|
||||
layer((input) =>
|
||||
Effect.sync(() =>
|
||||
input.respond(bodies.shift() ?? "", { headers: { "content-type": "text/event-stream" } }),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual(["text-delta", "finish"])
|
||||
expect(error.reason).toMatchObject({ _tag: "ProviderInternal", body: failure })
|
||||
expect(error.message).toContain("The server had an error")
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("streams whisper-1 as a single finish from a plain request", () =>
|
||||
Effect.gen(function* () {
|
||||
const bodies: Array<string> = []
|
||||
const events = Array.from(
|
||||
yield* Stream.runCollect(Transcription.stream({ model: openai.transcription("whisper-1"), audio })).pipe(
|
||||
Effect.provide(
|
||||
layer((input) =>
|
||||
Effect.sync(() => {
|
||||
bodies.push(input.text)
|
||||
return input.respond(
|
||||
JSON.stringify({ text: "Hello there.", usage: { type: "duration", seconds: 2 } }),
|
||||
{ headers: { "content-type": "application/json" } },
|
||||
)
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
expect(bodies[0]).not.toContain('name="stream"')
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({ type: "finish", text: "Hello there.", usage: { type: "seconds", seconds: 2 } }),
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect(
|
||||
"uploads inline audio to AssemblyAI, resumes polling from a persisted token, and surfaces failed transcripts",
|
||||
() =>
|
||||
|
||||
Reference in New Issue
Block a user