From 5d29fc65383a9771386afd0674da1067954f3a54 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 4 Nov 2026 15:25:59 -0600 Subject: [PATCH] refactor(ai): key Anthropic and Bedrock blocks by index as one map --- .../ai/src/protocols/anthropic-messages.ts | 148 +++++++++------- packages/ai/src/protocols/bedrock-converse.ts | 165 +++++++++--------- .../ai/test/provider/bedrock-converse.test.ts | 17 ++ 3 files changed, 177 insertions(+), 153 deletions(-) diff --git a/packages/ai/src/protocols/anthropic-messages.ts b/packages/ai/src/protocols/anthropic-messages.ts index f9de342d05e..765ff625148 100644 --- a/packages/ai/src/protocols/anthropic-messages.ts +++ b/packages/ai/src/protocols/anthropic-messages.ts @@ -508,11 +508,18 @@ const TypedAnthropicEvent = Schema.StructWithRest( const AnthropicEvent = withUntypedFrames(TypedAnthropicEvent) type AnthropicEvent = Schema.Schema.Type +// One entry per open content block index. Reasoning signatures live on the +// lifecycle's open block, so the reasoning entry only records the kind. +type Block = + | { readonly kind: "text" } + | { readonly kind: "reasoning" } + | { readonly kind: "tool"; readonly tool: ToolStream.PendingTool } + | { readonly kind: "compaction"; readonly content: string | null } + interface ParserState { readonly provider: LLMRequest["model"]["provider"] - readonly compactions: Readonly> readonly providerMetadataKey: string - readonly tools: ToolStream.State + readonly blocks: Readonly>> readonly usage?: Usage readonly pendingFinish?: { readonly reason: FinishReasonDetails @@ -1147,15 +1154,22 @@ const onContentBlockStart = ( { ...state, lifecycle, - tools: ToolStream.start(state.tools, event.index, { - id: block.id, - name: block.name ?? "", - input: - block.input !== undefined && (!ProviderShared.isRecord(block.input) || Object.keys(block.input).length > 0) - ? ProviderShared.encodeJson(block.input) - : undefined, - providerExecuted: block.type === "server_tool_use", - }), + blocks: { + ...state.blocks, + [event.index]: { + kind: "tool", + tool: { + id: block.id, + name: block.name ?? "", + input: + block.input !== undefined && + (!ProviderShared.isRecord(block.input) || Object.keys(block.input).length > 0) + ? ProviderShared.encodeJson(block.input) + : "", + providerExecuted: block.type === "server_tool_use", + }, + }, + }, }, [ ...events, @@ -1173,7 +1187,11 @@ const onContentBlockStart = ( const id = `text-${event.index ?? 0}` const lifecycle = Lifecycle.textStart(state.lifecycle, events, id) return [ - { ...state, lifecycle: block.text ? Lifecycle.textDelta(lifecycle, events, id, block.text) : lifecycle }, + { + ...state, + blocks: { ...state.blocks, [event.index ?? 0]: { kind: "text" } }, + lifecycle: block.text ? Lifecycle.textDelta(lifecycle, events, id, block.text) : lifecycle, + }, events, ] } @@ -1194,6 +1212,7 @@ const onContentBlockStart = ( return [ { ...state, + blocks: { ...state.blocks, [event.index ?? 0]: { kind: "reasoning" } }, lifecycle: block.thinking ? Lifecycle.reasoningDelta(lifecycle, events, id, block.thinking, metadata) : lifecycle, @@ -1210,6 +1229,7 @@ const onContentBlockStart = ( return [ { ...state, + blocks: { ...state.blocks, [event.index ?? 0]: { kind: "reasoning" } }, lifecycle: Lifecycle.reasoningStart( state.lifecycle, events, @@ -1227,54 +1247,53 @@ const onContentBlockStart = ( return [{ ...state, lifecycle: Lifecycle.stepStart(state.lifecycle, events) }, [...events, result]] } +// Anthropic always opens a block with content_block_start, so a text, thinking, +// signature, or tool delta for an index that has no matching block is dropped. +// Bedrock has no start event for text or reasoning and opens them on demand. const onContentBlockDelta = Effect.fnUntraced(function* ( state: ParserState, event: AnthropicEvent & { readonly delta: AnthropicStreamDelta }, ) { const delta = event.delta + const index = event.index ?? 0 + const block = state.blocks[index] if (delta.type === "compaction_delta") { - if (event.index === undefined || !(event.index in state.compactions) || delta.content === undefined) + if (event.index === undefined || block?.kind !== "compaction" || delta.content === undefined) return yield* ProviderShared.eventError(ADAPTER, "Compaction delta is missing its block or content") return [ - { ...state, compactions: { ...state.compactions, [event.index]: delta.content } }, + { ...state, blocks: { ...state.blocks, [index]: { kind: "compaction", content: delta.content } } }, NO_EVENTS, ] satisfies StepResult } if (delta.type === "text_delta" && delta.text) { - if (!Lifecycle.isOpen(state.lifecycle, "text", `text-${event.index ?? 0}`)) - return [state, NO_EVENTS] satisfies StepResult + if (block?.kind !== "text") return [state, NO_EVENTS] satisfies StepResult const events: LLMEvent[] = [] return [ - { ...state, lifecycle: Lifecycle.textDelta(state.lifecycle, events, `text-${event.index ?? 0}`, delta.text) }, + { ...state, lifecycle: Lifecycle.textDelta(state.lifecycle, events, `text-${index}`, delta.text) }, events, ] satisfies StepResult } if (delta.type === "thinking_delta" && delta.thinking) { - if (!Lifecycle.isOpen(state.lifecycle, "reasoning", `reasoning-${event.index ?? 0}`)) - return [state, NO_EVENTS] satisfies StepResult + if (block?.kind !== "reasoning") return [state, NO_EVENTS] satisfies StepResult const events: LLMEvent[] = [] return [ - { - ...state, - lifecycle: Lifecycle.reasoningDelta(state.lifecycle, events, `reasoning-${event.index ?? 0}`, delta.thinking), - }, + { ...state, lifecycle: Lifecycle.reasoningDelta(state.lifecycle, events, `reasoning-${index}`, delta.thinking) }, events, ] satisfies StepResult } if (delta.type === "signature_delta" && delta.signature) { - const id = `reasoning-${event.index ?? 0}` - if (!Lifecycle.isOpen(state.lifecycle, "reasoning", id)) return [state, NO_EVENTS] satisfies StepResult + if (block?.kind !== "reasoning") return [state, NO_EVENTS] satisfies StepResult return [ { ...state, lifecycle: Lifecycle.setMetadata( state.lifecycle, "reasoning", - id, + `reasoning-${index}`, providerMetadata(state.providerMetadataKey, { signature: delta.signature }), ), }, @@ -1283,20 +1302,16 @@ const onContentBlockDelta = Effect.fnUntraced(function* ( } if (delta.type === "input_json_delta" && event.index !== undefined) { - if (!delta.partial_json) return [state, NO_EVENTS] satisfies StepResult - if (!state.tools[event.index]) return [state, NO_EVENTS] satisfies StepResult - const result = ToolStream.appendExisting( - ADAPTER, - state.tools, - event.index, - delta.partial_json, - "Anthropic Messages tool argument delta is missing its tool call", - ) - if (ToolStream.isError(result)) return yield* result + if (!delta.partial_json || block?.kind !== "tool") return [state, NO_EVENTS] satisfies StepResult + const result = ToolStream.append({ [index]: block.tool }, index, delta.partial_json) + if (!result) return [state, NO_EVENTS] satisfies StepResult const events: LLMEvent[] = [] const lifecycle = result.events.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle events.push(...result.events) - return [{ ...state, lifecycle, tools: result.tools }, events] satisfies StepResult + return [ + { ...state, lifecycle, blocks: { ...state.blocks, [index]: { kind: "tool", tool: result.tool } } }, + events, + ] satisfies StepResult } return [state, NO_EVENTS] satisfies StepResult @@ -1304,30 +1319,25 @@ const onContentBlockDelta = Effect.fnUntraced(function* ( const onContentBlockStop = Effect.fnUntraced(function* (state: ParserState, event: AnthropicEvent) { if (event.index === undefined) return [state, NO_EVENTS] satisfies StepResult - if (event.index in state.compactions) { - const { [event.index]: content, ...compactions } = state.compactions - const events: LLMEvent[] = [] - const lifecycle = Lifecycle.stepStart(state.lifecycle, events) - events.push( - LLMEvent.compaction({ - provider: state.provider, - text: content, - }), - ) - return [{ ...state, compactions, lifecycle }, events] satisfies StepResult - } - const result = yield* ToolStream.finish(ADAPTER, state.tools, event.index) + const { [event.index]: block, ...blocks } = state.blocks + if (!block) return [state, NO_EVENTS] satisfies StepResult const events: LLMEvent[] = [] - const resultEvents = result.events ?? [] - const lifecycle = resultEvents.length - ? Lifecycle.stepStart(state.lifecycle, events) - : Lifecycle.reasoningEnd( - Lifecycle.textEnd(state.lifecycle, events, `text-${event.index}`), - events, - `reasoning-${event.index}`, - ) - events.push(...resultEvents) - return [{ ...state, lifecycle, tools: result.tools }, events] satisfies StepResult + if (block.kind === "compaction") { + const lifecycle = Lifecycle.stepStart(state.lifecycle, events) + events.push(LLMEvent.compaction({ provider: state.provider, text: block.content })) + return [{ ...state, blocks, lifecycle }, events] satisfies StepResult + } + if (block.kind === "tool") { + const result = yield* ToolStream.finish(ADAPTER, { [event.index]: block.tool }, event.index) + const lifecycle = Lifecycle.stepStart(state.lifecycle, events) + events.push(...(result.events ?? [])) + return [{ ...state, blocks, lifecycle }, events] satisfies StepResult + } + const lifecycle = + block.kind === "text" + ? Lifecycle.textEnd(state.lifecycle, events, `text-${event.index}`) + : Lifecycle.reasoningEnd(state.lifecycle, events, `reasoning-${event.index}`) + return [{ ...state, blocks, lifecycle }, events] satisfies StepResult }) const onMessageDelta = ( @@ -1367,9 +1377,13 @@ const onMessageDelta = ( } const onMessageStop = Effect.fnUntraced(function* (state: ParserState) { - if (Object.keys(state.compactions).length) + const blocks = Object.values(state.blocks) + if (blocks.some((block) => block?.kind === "compaction")) return yield* ProviderShared.eventError(ADAPTER, "Response ended with an incomplete compaction block") - const result = yield* ToolStream.finishAll(ADAPTER, state.tools) + const result = yield* ToolStream.finishAll( + ADAPTER, + blocks.flatMap((block) => (block?.kind === "tool" ? [block.tool] : [])), + ) const events: LLMEvent[] = [] const lifecycle = result.events.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle events.push(...result.events) @@ -1381,7 +1395,7 @@ const onMessageStop = Effect.fnUntraced(function* (state: ParserState) { usage: state.usage, providerMetadata: state.pendingFinish?.providerMetadata, }) - return [{ ...state, lifecycle: finished, tools: result.tools }, events] satisfies StepResult + return [{ ...state, lifecycle: finished, blocks: {} }, events] satisfies StepResult }) // Prefix `error.type` so overloads, rate limits, and quota errors are visible @@ -1450,7 +1464,10 @@ const step = (state: ParserState, event: AnthropicEvent) => { const decoded = Schema.decodeUnknownOption(AnthropicCompactionBlock)(event.content_block) if (event.index === undefined || Option.isNone(decoded)) return invalidStreamEvent(event) return Effect.succeed([ - { ...state, compactions: { ...state.compactions, [event.index]: decoded.value.content } }, + { + ...state, + blocks: { ...state.blocks, [event.index]: { kind: "compaction", content: decoded.value.content } }, + }, NO_EVENTS, ]) } @@ -1504,9 +1521,8 @@ export const protocol = Protocol.make({ event: Protocol.jsonEvent(AnthropicEvent), initial: (request) => ({ provider: request.model.provider, - compactions: {}, providerMetadataKey: request.model.route.providerMetadataKey ?? request.model.provider, - tools: ToolStream.empty(), + blocks: {}, lifecycle: Lifecycle.initial(), }), step, diff --git a/packages/ai/src/protocols/bedrock-converse.ts b/packages/ai/src/protocols/bedrock-converse.ts index 6495524c681..b4899e5276e 100644 --- a/packages/ai/src/protocols/bedrock-converse.ts +++ b/packages/ai/src/protocols/bedrock-converse.ts @@ -690,9 +690,17 @@ const mapUsage = (usage: BedrockUsageSchema | undefined, providerMetadataKey: st }) } +// One entry per open content block index. Bedrock sends no start event for text +// or reasoning, so their first delta opens the block. A delta of a different +// kind for an index that is already open is dropped. +type Block = + | { readonly kind: "text" } + | { readonly kind: "reasoning"; readonly signature?: string; readonly redacted?: ReadonlyArray } + | { readonly kind: "tool"; readonly tool: ToolStream.PendingTool } + interface ParserState { readonly providerMetadataKey: string - readonly tools: ToolStream.State + readonly blocks: Readonly>> // Bedrock splits the finish into `messageStop` (carries `stopReason`) and // `metadata` (carries usage). Hold both in state so `onHalt` can emit exactly // one finish after both chunks have had a chance to arrive. @@ -700,11 +708,12 @@ interface ParserState { readonly usage: Usage | undefined readonly hasToolCalls: boolean readonly lifecycle: Lifecycle.State - readonly reasoningSignatures: Readonly> - readonly reasoningRedactedContent: Readonly> } -const encodeRedactedContent = (chunks: ReadonlyArray) => Base64.encode(concatBytes(chunks)) +const reasoningEndMetadata = (key: string, block: Block & { readonly kind: "reasoning" }) => { + if (block.signature) return providerMetadata(key, { signature: block.signature }) + if (block.redacted) return providerMetadata(key, { redactedData: Base64.encode(concatBytes(block.redacted)) }) +} const step = (state: ParserState, event: BedrockEvent) => Effect.gen(function* () { @@ -716,10 +725,17 @@ const step = (state: ParserState, event: BedrockEvent) => { ...state, lifecycle, - tools: ToolStream.start(state.tools, index, { - id: event.contentBlockStart.start.toolUse.toolUseId, - name: event.contentBlockStart.start.toolUse.name, - }), + blocks: { + ...state.blocks, + [index]: { + kind: "tool", + tool: { + id: event.contentBlockStart.start.toolUse.toolUseId, + name: event.contentBlockStart.start.toolUse.name, + input: "", + }, + }, + }, }, [ ...events, @@ -732,16 +748,15 @@ const step = (state: ParserState, event: BedrockEvent) => } if (event.contentBlockDelta?.delta?.text) { + const index = event.contentBlockDelta.contentBlockIndex + const block = state.blocks[index] + if (block && block.kind !== "text") return [state, []] as const const events: LLMEvent[] = [] return [ { ...state, - lifecycle: Lifecycle.textDelta( - state.lifecycle, - events, - `text-${event.contentBlockDelta.contentBlockIndex}`, - event.contentBlockDelta.delta.text, - ), + blocks: block ? state.blocks : { ...state.blocks, [index]: { kind: "text" } satisfies Block }, + lifecycle: Lifecycle.textDelta(state.lifecycle, events, `text-${index}`, event.contentBlockDelta.delta.text), }, events, ] as const @@ -750,6 +765,8 @@ const step = (state: ParserState, event: BedrockEvent) => if (event.contentBlockDelta?.delta?.reasoningContent) { const index = event.contentBlockDelta.contentBlockIndex const reasoning = event.contentBlockDelta.delta.reasoningContent + const block = state.blocks[index] + if (block && block.kind !== "reasoning") return [state, []] as const const events: LLMEvent[] = [] const redactedChunk = yield* (() => { if (reasoning.redactedContent === undefined) return Effect.succeed(undefined) @@ -764,8 +781,6 @@ const step = (state: ParserState, event: BedrockEvent) => ), ) })() - const redactedChunks = state.reasoningRedactedContent[index] ?? [] - if (redactedChunk !== undefined) redactedChunks.push(redactedChunk) const metadata = (() => { if (reasoning.signature) return providerMetadata(state.providerMetadataKey, { signature: reasoning.signature }) if (redactedChunk === undefined && reasoning.data !== undefined) @@ -777,23 +792,19 @@ const step = (state: ParserState, event: BedrockEvent) => if (redactedChunk !== undefined) return Lifecycle.reasoningStart(state.lifecycle, events, `reasoning-${index}`) return state.lifecycle })() - const reasoningRedactedContent = (() => { - if (redactedChunk !== undefined) return { ...state.reasoningRedactedContent, [index]: redactedChunks } - if (reasoning.data === undefined) return state.reasoningRedactedContent - return Object.fromEntries( - Object.entries(state.reasoningRedactedContent).filter(([key]) => key !== String(index)), - ) - })() - const reasoningSignatures = (() => { - if (!reasoning.signature) return state.reasoningSignatures - return { ...state.reasoningSignatures, [index]: reasoning.signature } + // A plain `data` payload replaces any redacted chunks collected so far. + const redacted = (() => { + if (redactedChunk !== undefined) return [...(block?.redacted ?? []), redactedChunk] + if (reasoning.data === undefined) return block?.redacted })() return [ { ...state, lifecycle, - reasoningSignatures, - reasoningRedactedContent, + blocks: { + ...state.blocks, + [index]: { kind: "reasoning", signature: reasoning.signature || block?.signature, redacted }, + }, }, events, ] as const @@ -802,59 +813,41 @@ const step = (state: ParserState, event: BedrockEvent) => if (event.contentBlockDelta?.delta?.toolUse) { // A delta for a block that is not open, whether it already stopped or never // started, has nothing to attach to and is dropped. - const result = ToolStream.append( - state.tools, - event.contentBlockDelta.contentBlockIndex, - event.contentBlockDelta.delta.toolUse.input, - ) + const index = event.contentBlockDelta.contentBlockIndex + const block = state.blocks[index] + if (block?.kind !== "tool") return [state, []] as const + const result = ToolStream.append({ [index]: block.tool }, index, event.contentBlockDelta.delta.toolUse.input) if (!result) return [state, []] as const const events: LLMEvent[] = [] const lifecycle = result.events.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle events.push(...result.events) - return [{ ...state, lifecycle, tools: result.tools }, events] as const + return [ + { ...state, lifecycle, blocks: { ...state.blocks, [index]: { kind: "tool", tool: result.tool } } }, + events, + ] as const } if (event.contentBlockStop) { const index = event.contentBlockStop.contentBlockIndex - const result = yield* ToolStream.finish(ADAPTER, state.tools, index) + const { [index]: block, ...blocks } = state.blocks + if (!block) return [state, []] as const const events: LLMEvent[] = [] - const resultEvents = result.events ?? [] - const lifecycle = (() => { - if (resultEvents.length) return Lifecycle.stepStart(state.lifecycle, events) - const metadata = (() => { - const signature = state.reasoningSignatures[index] - if (signature) return providerMetadata(state.providerMetadataKey, { signature }) - const redactedContent = state.reasoningRedactedContent[index] - if (redactedContent) - return providerMetadata(state.providerMetadataKey, { - redactedData: encodeRedactedContent(redactedContent), - }) - })() - return Lifecycle.reasoningEnd( - Lifecycle.textEnd(state.lifecycle, events, `text-${index}`), - events, - `reasoning-${index}`, - metadata, - ) - })() - events.push(...resultEvents) - return [ - { - ...state, - hasToolCalls: - resultEvents.some((event) => LLMEvent.is.toolCall(event) || LLMEvent.is.toolInputError(event)) || - state.hasToolCalls, - lifecycle, - tools: result.tools, - reasoningSignatures: Object.fromEntries( - Object.entries(state.reasoningSignatures).filter(([key]) => key !== String(index)), - ), - reasoningRedactedContent: Object.fromEntries( - Object.entries(state.reasoningRedactedContent).filter(([key]) => key !== String(index)), - ), - }, - events, - ] as const + if (block.kind === "tool") { + const result = yield* ToolStream.finish(ADAPTER, { [index]: block.tool }, index) + const lifecycle = Lifecycle.stepStart(state.lifecycle, events) + events.push(...(result.events ?? [])) + return [{ ...state, hasToolCalls: true, lifecycle, blocks }, events] as const + } + const lifecycle = + block.kind === "text" + ? Lifecycle.textEnd(state.lifecycle, events, `text-${index}`) + : Lifecycle.reasoningEnd( + state.lifecycle, + events, + `reasoning-${index}`, + reasoningEndMetadata(state.providerMetadataKey, block), + ) + return [{ ...state, lifecycle, blocks }, events] as const } if (event.messageStop) { @@ -915,18 +908,18 @@ const onHalt = (state: ParserState): ReadonlyArray => { return state.finishReason.normalized })() const events: LLMEvent[] = [] - const lifecycle = Object.entries(state.reasoningRedactedContent).reduce((current, [index, chunks]) => { - const signature = state.reasoningSignatures[Number(index)] - return Lifecycle.reasoningEnd( - current, - events, - `reasoning-${index}`, - providerMetadata( - state.providerMetadataKey, - signature ? { signature } : { redactedData: encodeRedactedContent(chunks) }, - ), - ) - }, state.lifecycle) + const lifecycle = Object.entries(state.blocks).reduce( + (current, [index, block]) => + block?.kind === "reasoning" && block.redacted + ? Lifecycle.reasoningEnd( + current, + events, + `reasoning-${index}`, + reasoningEndMetadata(state.providerMetadataKey, block), + ) + : current, + state.lifecycle, + ) Lifecycle.finish(lifecycle, events, { reason: { ...state.finishReason, @@ -954,13 +947,11 @@ export const protocol = Protocol.make({ event: BedrockEventStream.Frame.pipe(Schema.decodeTo(BedrockEvent)), initial: (request) => ({ providerMetadataKey: request.model.route.providerMetadataKey ?? request.model.provider, - tools: ToolStream.empty(), + blocks: {}, finishReason: undefined, usage: undefined, hasToolCalls: false, lifecycle: Lifecycle.initial(), - reasoningSignatures: {}, - reasoningRedactedContent: {}, }), step, onHalt: (state) => Effect.succeed(onHalt(state)), diff --git a/packages/ai/test/provider/bedrock-converse.test.ts b/packages/ai/test/provider/bedrock-converse.test.ts index 0f4e95b89bd..0dd8d162d76 100644 --- a/packages/ai/test/provider/bedrock-converse.test.ts +++ b/packages/ai/test/provider/bedrock-converse.test.ts @@ -21,6 +21,7 @@ import { import { LLMClient } from "../../src/route.js" import { compileRequest } from "../../src/route/client.js" import { AmazonBedrock } from "../../src/providers.js" +import { BedrockConverse } from "../../src/protocols/bedrock-converse.js" import { it } from "../lib/effect.js" import { withProcessEnv } from "../lib/env.js" import { dynamicResponse, fixedResponse } from "../lib/http.js" @@ -1559,6 +1560,22 @@ describe("Bedrock Converse route", () => { }), ) + it.effect("leaves the previous parser state untouched when a redacted chunk arrives", () => + Effect.gen(function* () { + const stream = BedrockConverse.protocol.stream + const chunk = (redactedContent: string) => ({ + contentBlockDelta: { contentBlockIndex: 0, delta: { reasoningContent: { redactedContent } } }, + }) + const [first] = yield* stream.step(stream.initial(baseRequest), chunk("AQ==")) + yield* stream.step(first, chunk("AgM=")) + const [, events] = yield* stream.step(first, { contentBlockStop: { contentBlockIndex: 0 } }) + + expect(events.find((event) => event.type === "reasoning-end")?.providerMetadata).toEqual({ + bedrock: { redactedData: "AQ==" }, + }) + }), + ) + it.effect("preserves split redacted reasoning when contentBlockStop is missing", () => Effect.gen(function* () { const response = yield* LLMClient.generate(baseRequest).pipe(