fix(ai): keep Gemini text after inline images in order

This commit is contained in:
Aiden Cline
2026-11-04 12:23:33 -06:00
Unverified
parent b92e085cf8
commit 73a460e995
2 changed files with 84 additions and 57 deletions
+44 -57
View File
@@ -266,15 +266,21 @@ interface ParserState {
/** Usage of the earlier requests of a response Gemini continued across requests. */
readonly continuedUsage?: ReadonlyArray<GeminiUsage>
readonly lifecycle: Lifecycle.State
readonly reasoningSignature?: string
readonly textSignature?: string
readonly reasoningId?: string
readonly textId?: string
/** At most one text or reasoning block is open; it stays open across continued requests of one response. */
readonly active: ActiveBlock
readonly nextReasoningId: number
readonly nextTextId: number
readonly seenCallIds?: ReadonlySet<string>
}
type OpenBlock = {
readonly _tag: "text" | "reasoning"
readonly id: string
/** Text emits it with the next delta, else at block end; reasoning emits the latest at block end. */
readonly signature?: string
}
type ActiveBlock = { readonly _tag: "none" } | OpenBlock
// =============================================================================
// Request Lowering
// =============================================================================
@@ -593,21 +599,7 @@ const finish = (state: ParserState): ReadonlyArray<LLMEvent> => {
if (finishReason === undefined && state.usage === undefined && state.continuedUsage === undefined) return []
const events: LLMEvent[] = []
let lifecycle = state.lifecycle
if (state.reasoningId !== undefined)
lifecycle = Lifecycle.reasoningEnd(
lifecycle,
events,
state.reasoningId,
signatureMetadata(state.providerMetadataKey, state.reasoningSignature),
)
if (state.textId !== undefined)
lifecycle = Lifecycle.textEnd(
lifecycle,
events,
state.textId,
signatureMetadata(state.providerMetadataKey, state.textSignature),
)
const lifecycle = endBlock(state.lifecycle, events, state.active, state.providerMetadataKey)
Lifecycle.finish(lifecycle, events, {
reason: {
normalized:
@@ -626,6 +618,13 @@ const finish = (state: ParserState): ReadonlyArray<LLMEvent> => {
return events
}
const endBlock = (lifecycle: Lifecycle.State, events: LLMEvent[], active: ActiveBlock, metadataKey: string) => {
if (active._tag === "none") return lifecycle
const metadata = signatureMetadata(metadataKey, active.signature)
if (active._tag === "text") return Lifecycle.textEnd(lifecycle, events, active.id, metadata)
return Lifecycle.reasoningEnd(lifecycle, events, active.id, metadata)
}
const step = (state: ParserState, event: GeminiEvent | NextRequest) => {
// Usage is cumulative within one request, so the next request of the same response counts from zero.
if (event instanceof NextRequest)
@@ -675,26 +674,24 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => {
const events: LLMEvent[] = []
let hasToolCalls = nextState.hasToolCalls
let lifecycle = nextState.lifecycle
let reasoningSignature = nextState.reasoningSignature
let textSignature = nextState.textSignature
let reasoningId = nextState.reasoningId
let textId = nextState.textId
let active = nextState.active
let nextReasoningId = nextState.nextReasoningId
let nextTextId = nextState.nextTextId
// Supplier ids must be tracked across chunks of the same response, not just within one event's parts.
const seenCallIds = new Set(nextState.seenCallIds)
const signed = (signature: string | undefined) => signatureMetadata(state.providerMetadataKey, signature)
const endReasoning = () => {
if (reasoningId === undefined) return
lifecycle = Lifecycle.reasoningEnd(lifecycle, events, reasoningId, signed(reasoningSignature))
reasoningId = undefined
reasoningSignature = undefined
const close = () => {
lifecycle = endBlock(lifecycle, events, active, state.providerMetadataKey)
active = { _tag: "none" }
}
const endText = () => {
if (textId === undefined) return
lifecycle = Lifecycle.textEnd(lifecycle, events, textId, signed(textSignature))
textId = undefined
textSignature = undefined
// Entering a different kind closes the open block first, so blocks never interleave.
const enter = (_tag: OpenBlock["_tag"], signature: string | undefined) => {
if (active._tag !== _tag) {
close()
active = { _tag, id: _tag === "text" ? `text-${nextTextId++}` : `reasoning-${nextReasoningId++}` }
}
active = { ...active, signature: signature ?? active.signature }
return active
}
for (const input of candidate.content.parts ?? []) {
@@ -713,12 +710,9 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => {
)
const part = decoded.value
const signature = "thoughtSignature" in part && part.thoughtSignature ? part.thoughtSignature : undefined
// Gemini attaches replay signatures to thought parts, visible text, or function calls;
// each block kind must retain the signature attached to its own parts.
if (signature !== undefined && "thought" in part && part.thought) reasoningSignature = signature
else if (signature !== undefined && "text" in part) textSignature = signature
// Image-capable Gemini models return generated images as inline data parts; surface them as first-class output.
if ("inlineData" in part) {
close()
lifecycle = Lifecycle.stepStart(lifecycle, events)
events.push(
LLMEvent.media({
@@ -728,23 +722,19 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => {
)
continue
}
if ("text" in part && part.text.length > 0) {
// Gemini attaches replay signatures to thought parts, visible text, or function calls; each block
// keeps the signatures of its own parts, including a trailing empty part that only carries one.
if ("text" in part && (part.text.length > 0 || signature !== undefined)) {
if (part.thought) {
endText()
if (reasoningId === undefined) {
reasoningId = `reasoning-${nextReasoningId}`
nextReasoningId += 1
}
lifecycle = Lifecycle.reasoningDelta(lifecycle, events, reasoningId, part.text, signed(signature))
const block = enter("reasoning", signature)
if (part.text.length > 0)
lifecycle = Lifecycle.reasoningDelta(lifecycle, events, block.id, part.text, signed(signature))
continue
}
endReasoning()
if (textId === undefined) {
textId = `text-${nextTextId}`
nextTextId += 1
}
lifecycle = Lifecycle.textDelta(lifecycle, events, textId, part.text, signed(textSignature))
textSignature = undefined
const block = enter("text", signature)
if (part.text.length === 0) continue
lifecycle = Lifecycle.textDelta(lifecycle, events, block.id, part.text, signed(block.signature))
active = { _tag: "text", id: block.id }
continue
}
@@ -758,8 +748,7 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => {
const duplicate = supplied !== undefined && seenCallIds.has(supplied)
if (supplied !== undefined) seenCallIds.add(supplied)
const id = supplied !== undefined && !duplicate ? supplied : `tool_${crypto.randomUUID().replaceAll("-", "")}`
endReasoning()
endText()
close()
lifecycle = Lifecycle.stepStart(lifecycle, events)
events.push(
LLMEvent.toolCall({
@@ -778,10 +767,7 @@ const step = (state: ParserState, event: GeminiEvent | NextRequest) => {
...nextState,
hasToolCalls,
lifecycle,
reasoningSignature,
textSignature,
reasoningId,
textId,
active,
nextReasoningId,
nextTextId,
seenCallIds,
@@ -904,6 +890,7 @@ export const protocol = Protocol.make({
providerMetadataKey: request.model.route.providerMetadataKey ?? request.model.provider,
hasToolCalls: false,
lifecycle: Lifecycle.initial(),
active: { _tag: "none" },
nextReasoningId: 0,
nextTextId: 0,
}),
+40
View File
@@ -1208,6 +1208,46 @@ describe("Gemini route", () => {
}),
)
it.effect("starts a new text block after an inline image", () =>
Effect.gen(function* () {
const body = sseEvents(
{ candidates: [{ content: { role: "model", parts: [{ text: "A" }] } }] },
{
candidates: [
{
content: { role: "model", parts: [{ inlineData: { mimeType: "image/png", data: "AAECAw==" } }] },
},
],
},
{ candidates: [{ content: { role: "model", parts: [{ text: "B" }] }, finishReason: "STOP" }] },
)
const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body)))
expect(
response.events.flatMap((event) =>
event.type === "text-start" || event.type === "text-delta" || event.type === "text-end"
? [`${event.type}:${event.id}`]
: event.type === "media"
? ["media"]
: [],
),
).toEqual([
"text-start:text-0",
"text-delta:text-0",
"text-end:text-0",
"media",
"text-start:text-1",
"text-delta:text-1",
"text-end:text-1",
])
expect(response.message.content).toEqual([
{ type: "text", text: "A" },
{ type: "media", media: Media.base64("AAECAw==", "image/png") },
{ type: "text", text: "B" },
])
}),
)
it.effect("ignores unknown response parts", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(request).pipe(