From 73a460e995c43f8adff93bcd20812de35c198034 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 4 Nov 2026 12:23:33 -0600 Subject: [PATCH] fix(ai): keep Gemini text after inline images in order --- packages/ai/src/protocols/gemini.ts | 101 ++++++++++------------- packages/ai/test/provider/gemini.test.ts | 40 +++++++++ 2 files changed, 84 insertions(+), 57 deletions(-) diff --git a/packages/ai/src/protocols/gemini.ts b/packages/ai/src/protocols/gemini.ts index 22389c0e654..d348fd00238 100644 --- a/packages/ai/src/protocols/gemini.ts +++ b/packages/ai/src/protocols/gemini.ts @@ -266,15 +266,21 @@ interface ParserState { /** Usage of the earlier requests of a response Gemini continued across requests. */ readonly continuedUsage?: ReadonlyArray readonly lifecycle: Lifecycle.State - readonly reasoningSignature?: string - readonly textSignature?: string - readonly reasoningId?: string - readonly textId?: string + /** At most one text or reasoning block is open; it stays open across continued requests of one response. */ + readonly active: ActiveBlock readonly nextReasoningId: number readonly nextTextId: number readonly seenCallIds?: ReadonlySet } +type OpenBlock = { + readonly _tag: "text" | "reasoning" + readonly id: string + /** Text emits it with the next delta, else at block end; reasoning emits the latest at block end. */ + readonly signature?: string +} +type ActiveBlock = { readonly _tag: "none" } | OpenBlock + // ============================================================================= // Request Lowering // ============================================================================= @@ -593,21 +599,7 @@ const finish = (state: ParserState): ReadonlyArray => { if (finishReason === undefined && state.usage === undefined && state.continuedUsage === undefined) return [] const events: LLMEvent[] = [] - let lifecycle = state.lifecycle - if (state.reasoningId !== undefined) - lifecycle = Lifecycle.reasoningEnd( - lifecycle, - events, - state.reasoningId, - signatureMetadata(state.providerMetadataKey, state.reasoningSignature), - ) - if (state.textId !== undefined) - lifecycle = Lifecycle.textEnd( - lifecycle, - events, - state.textId, - signatureMetadata(state.providerMetadataKey, state.textSignature), - ) + const lifecycle = endBlock(state.lifecycle, events, state.active, state.providerMetadataKey) Lifecycle.finish(lifecycle, events, { reason: { normalized: @@ -626,6 +618,13 @@ const finish = (state: ParserState): ReadonlyArray => { return events } +const endBlock = (lifecycle: Lifecycle.State, events: LLMEvent[], active: ActiveBlock, metadataKey: string) => { + if (active._tag === "none") return lifecycle + const metadata = signatureMetadata(metadataKey, active.signature) + if (active._tag === "text") return Lifecycle.textEnd(lifecycle, events, active.id, metadata) + return Lifecycle.reasoningEnd(lifecycle, events, active.id, metadata) +} + const step = (state: ParserState, event: GeminiEvent | NextRequest) => { // Usage is cumulative within one request, so the next request of the same response counts from zero. if (event instanceof NextRequest) @@ -675,26 +674,24 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => { const events: LLMEvent[] = [] let hasToolCalls = nextState.hasToolCalls let lifecycle = nextState.lifecycle - let reasoningSignature = nextState.reasoningSignature - let textSignature = nextState.textSignature - let reasoningId = nextState.reasoningId - let textId = nextState.textId + let active = nextState.active let nextReasoningId = nextState.nextReasoningId let nextTextId = nextState.nextTextId // Supplier ids must be tracked across chunks of the same response, not just within one event's parts. const seenCallIds = new Set(nextState.seenCallIds) const signed = (signature: string | undefined) => signatureMetadata(state.providerMetadataKey, signature) - const endReasoning = () => { - if (reasoningId === undefined) return - lifecycle = Lifecycle.reasoningEnd(lifecycle, events, reasoningId, signed(reasoningSignature)) - reasoningId = undefined - reasoningSignature = undefined + const close = () => { + lifecycle = endBlock(lifecycle, events, active, state.providerMetadataKey) + active = { _tag: "none" } } - const endText = () => { - if (textId === undefined) return - lifecycle = Lifecycle.textEnd(lifecycle, events, textId, signed(textSignature)) - textId = undefined - textSignature = undefined + // Entering a different kind closes the open block first, so blocks never interleave. + const enter = (_tag: OpenBlock["_tag"], signature: string | undefined) => { + if (active._tag !== _tag) { + close() + active = { _tag, id: _tag === "text" ? `text-${nextTextId++}` : `reasoning-${nextReasoningId++}` } + } + active = { ...active, signature: signature ?? active.signature } + return active } for (const input of candidate.content.parts ?? []) { @@ -713,12 +710,9 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => { ) const part = decoded.value const signature = "thoughtSignature" in part && part.thoughtSignature ? part.thoughtSignature : undefined - // Gemini attaches replay signatures to thought parts, visible text, or function calls; - // each block kind must retain the signature attached to its own parts. - if (signature !== undefined && "thought" in part && part.thought) reasoningSignature = signature - else if (signature !== undefined && "text" in part) textSignature = signature // Image-capable Gemini models return generated images as inline data parts; surface them as first-class output. if ("inlineData" in part) { + close() lifecycle = Lifecycle.stepStart(lifecycle, events) events.push( LLMEvent.media({ @@ -728,23 +722,19 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => { ) continue } - if ("text" in part && part.text.length > 0) { + // Gemini attaches replay signatures to thought parts, visible text, or function calls; each block + // keeps the signatures of its own parts, including a trailing empty part that only carries one. + if ("text" in part && (part.text.length > 0 || signature !== undefined)) { if (part.thought) { - endText() - if (reasoningId === undefined) { - reasoningId = `reasoning-${nextReasoningId}` - nextReasoningId += 1 - } - lifecycle = Lifecycle.reasoningDelta(lifecycle, events, reasoningId, part.text, signed(signature)) + const block = enter("reasoning", signature) + if (part.text.length > 0) + lifecycle = Lifecycle.reasoningDelta(lifecycle, events, block.id, part.text, signed(signature)) continue } - endReasoning() - if (textId === undefined) { - textId = `text-${nextTextId}` - nextTextId += 1 - } - lifecycle = Lifecycle.textDelta(lifecycle, events, textId, part.text, signed(textSignature)) - textSignature = undefined + const block = enter("text", signature) + if (part.text.length === 0) continue + lifecycle = Lifecycle.textDelta(lifecycle, events, block.id, part.text, signed(block.signature)) + active = { _tag: "text", id: block.id } continue } @@ -758,8 +748,7 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => { const duplicate = supplied !== undefined && seenCallIds.has(supplied) if (supplied !== undefined) seenCallIds.add(supplied) const id = supplied !== undefined && !duplicate ? supplied : `tool_${crypto.randomUUID().replaceAll("-", "")}` - endReasoning() - endText() + close() lifecycle = Lifecycle.stepStart(lifecycle, events) events.push( LLMEvent.toolCall({ @@ -778,10 +767,7 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => { ...nextState, hasToolCalls, lifecycle, - reasoningSignature, - textSignature, - reasoningId, - textId, + active, nextReasoningId, nextTextId, seenCallIds, @@ -904,6 +890,7 @@ export const protocol = Protocol.make({ providerMetadataKey: request.model.route.providerMetadataKey ?? request.model.provider, hasToolCalls: false, lifecycle: Lifecycle.initial(), + active: { _tag: "none" }, nextReasoningId: 0, nextTextId: 0, }), diff --git a/packages/ai/test/provider/gemini.test.ts b/packages/ai/test/provider/gemini.test.ts index 9776d57a551..500a721c773 100644 --- a/packages/ai/test/provider/gemini.test.ts +++ b/packages/ai/test/provider/gemini.test.ts @@ -1208,6 +1208,46 @@ describe("Gemini route", () => { }), ) + it.effect("starts a new text block after an inline image", () => + Effect.gen(function* () { + const body = sseEvents( + { candidates: [{ content: { role: "model", parts: [{ text: "A" }] } }] }, + { + candidates: [ + { + content: { role: "model", parts: [{ inlineData: { mimeType: "image/png", data: "AAECAw==" } }] }, + }, + ], + }, + { candidates: [{ content: { role: "model", parts: [{ text: "B" }] }, finishReason: "STOP" }] }, + ) + const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body))) + + expect( + response.events.flatMap((event) => + event.type === "text-start" || event.type === "text-delta" || event.type === "text-end" + ? [`${event.type}:${event.id}`] + : event.type === "media" + ? ["media"] + : [], + ), + ).toEqual([ + "text-start:text-0", + "text-delta:text-0", + "text-end:text-0", + "media", + "text-start:text-1", + "text-delta:text-1", + "text-end:text-1", + ]) + expect(response.message.content).toEqual([ + { type: "text", text: "A" }, + { type: "media", media: Media.base64("AAECAw==", "image/png") }, + { type: "text", text: "B" }, + ]) + }), + ) + it.effect("ignores unknown response parts", () => Effect.gen(function* () { const response = yield* LLMClient.generate(request).pipe(