refactor(ai): key Anthropic and Bedrock blocks by index as one map

This commit is contained in:
Aiden Cline
2026-11-04 15:25:59 -06:00
Unverified
parent 9af0a6706f
commit 5d29fc6538
3 changed files with 177 additions and 153 deletions
+82 -66
View File
@@ -508,11 +508,18 @@ const TypedAnthropicEvent = Schema.StructWithRest(
const AnthropicEvent = withUntypedFrames(TypedAnthropicEvent)
type AnthropicEvent = Schema.Schema.Type<typeof AnthropicEvent>
// One entry per open content block index. Reasoning signatures live on the
// lifecycle's open block, so the reasoning entry only records the kind.
type Block =
| { readonly kind: "text" }
| { readonly kind: "reasoning" }
| { readonly kind: "tool"; readonly tool: ToolStream.PendingTool }
| { readonly kind: "compaction"; readonly content: string | null }
interface ParserState {
readonly provider: LLMRequest["model"]["provider"]
readonly compactions: Readonly<Record<number, string | null>>
readonly providerMetadataKey: string
readonly tools: ToolStream.State<number>
readonly blocks: Readonly<Partial<Record<number, Block>>>
readonly usage?: Usage
readonly pendingFinish?: {
readonly reason: FinishReasonDetails
@@ -1147,15 +1154,22 @@ const onContentBlockStart = (
{
...state,
lifecycle,
tools: ToolStream.start(state.tools, event.index, {
id: block.id,
name: block.name ?? "",
input:
block.input !== undefined && (!ProviderShared.isRecord(block.input) || Object.keys(block.input).length > 0)
? ProviderShared.encodeJson(block.input)
: undefined,
providerExecuted: block.type === "server_tool_use",
}),
blocks: {
...state.blocks,
[event.index]: {
kind: "tool",
tool: {
id: block.id,
name: block.name ?? "",
input:
block.input !== undefined &&
(!ProviderShared.isRecord(block.input) || Object.keys(block.input).length > 0)
? ProviderShared.encodeJson(block.input)
: "",
providerExecuted: block.type === "server_tool_use",
},
},
},
},
[
...events,
@@ -1173,7 +1187,11 @@ const onContentBlockStart = (
const id = `text-${event.index ?? 0}`
const lifecycle = Lifecycle.textStart(state.lifecycle, events, id)
return [
{ ...state, lifecycle: block.text ? Lifecycle.textDelta(lifecycle, events, id, block.text) : lifecycle },
{
...state,
blocks: { ...state.blocks, [event.index ?? 0]: { kind: "text" } },
lifecycle: block.text ? Lifecycle.textDelta(lifecycle, events, id, block.text) : lifecycle,
},
events,
]
}
@@ -1194,6 +1212,7 @@ const onContentBlockStart = (
return [
{
...state,
blocks: { ...state.blocks, [event.index ?? 0]: { kind: "reasoning" } },
lifecycle: block.thinking
? Lifecycle.reasoningDelta(lifecycle, events, id, block.thinking, metadata)
: lifecycle,
@@ -1210,6 +1229,7 @@ const onContentBlockStart = (
return [
{
...state,
blocks: { ...state.blocks, [event.index ?? 0]: { kind: "reasoning" } },
lifecycle: Lifecycle.reasoningStart(
state.lifecycle,
events,
@@ -1227,54 +1247,53 @@ const onContentBlockStart = (
return [{ ...state, lifecycle: Lifecycle.stepStart(state.lifecycle, events) }, [...events, result]]
}
// Anthropic always opens a block with content_block_start, so a text, thinking,
// signature, or tool delta for an index that has no matching block is dropped.
// Bedrock has no start event for text or reasoning and opens them on demand.
const onContentBlockDelta = Effect.fnUntraced(function* (
state: ParserState,
event: AnthropicEvent & { readonly delta: AnthropicStreamDelta },
) {
const delta = event.delta
const index = event.index ?? 0
const block = state.blocks[index]
if (delta.type === "compaction_delta") {
if (event.index === undefined || !(event.index in state.compactions) || delta.content === undefined)
if (event.index === undefined || block?.kind !== "compaction" || delta.content === undefined)
return yield* ProviderShared.eventError(ADAPTER, "Compaction delta is missing its block or content")
return [
{ ...state, compactions: { ...state.compactions, [event.index]: delta.content } },
{ ...state, blocks: { ...state.blocks, [index]: { kind: "compaction", content: delta.content } } },
NO_EVENTS,
] satisfies StepResult
}
if (delta.type === "text_delta" && delta.text) {
if (!Lifecycle.isOpen(state.lifecycle, "text", `text-${event.index ?? 0}`))
return [state, NO_EVENTS] satisfies StepResult
if (block?.kind !== "text") return [state, NO_EVENTS] satisfies StepResult
const events: LLMEvent[] = []
return [
{ ...state, lifecycle: Lifecycle.textDelta(state.lifecycle, events, `text-${event.index ?? 0}`, delta.text) },
{ ...state, lifecycle: Lifecycle.textDelta(state.lifecycle, events, `text-${index}`, delta.text) },
events,
] satisfies StepResult
}
if (delta.type === "thinking_delta" && delta.thinking) {
if (!Lifecycle.isOpen(state.lifecycle, "reasoning", `reasoning-${event.index ?? 0}`))
return [state, NO_EVENTS] satisfies StepResult
if (block?.kind !== "reasoning") return [state, NO_EVENTS] satisfies StepResult
const events: LLMEvent[] = []
return [
{
...state,
lifecycle: Lifecycle.reasoningDelta(state.lifecycle, events, `reasoning-${event.index ?? 0}`, delta.thinking),
},
{ ...state, lifecycle: Lifecycle.reasoningDelta(state.lifecycle, events, `reasoning-${index}`, delta.thinking) },
events,
] satisfies StepResult
}
if (delta.type === "signature_delta" && delta.signature) {
const id = `reasoning-${event.index ?? 0}`
if (!Lifecycle.isOpen(state.lifecycle, "reasoning", id)) return [state, NO_EVENTS] satisfies StepResult
if (block?.kind !== "reasoning") return [state, NO_EVENTS] satisfies StepResult
return [
{
...state,
lifecycle: Lifecycle.setMetadata(
state.lifecycle,
"reasoning",
id,
`reasoning-${index}`,
providerMetadata(state.providerMetadataKey, { signature: delta.signature }),
),
},
@@ -1283,20 +1302,16 @@ const onContentBlockDelta = Effect.fnUntraced(function* (
}
if (delta.type === "input_json_delta" && event.index !== undefined) {
if (!delta.partial_json) return [state, NO_EVENTS] satisfies StepResult
if (!state.tools[event.index]) return [state, NO_EVENTS] satisfies StepResult
const result = ToolStream.appendExisting(
ADAPTER,
state.tools,
event.index,
delta.partial_json,
"Anthropic Messages tool argument delta is missing its tool call",
)
if (ToolStream.isError(result)) return yield* result
if (!delta.partial_json || block?.kind !== "tool") return [state, NO_EVENTS] satisfies StepResult
const result = ToolStream.append({ [index]: block.tool }, index, delta.partial_json)
if (!result) return [state, NO_EVENTS] satisfies StepResult
const events: LLMEvent[] = []
const lifecycle = result.events.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle
events.push(...result.events)
return [{ ...state, lifecycle, tools: result.tools }, events] satisfies StepResult
return [
{ ...state, lifecycle, blocks: { ...state.blocks, [index]: { kind: "tool", tool: result.tool } } },
events,
] satisfies StepResult
}
return [state, NO_EVENTS] satisfies StepResult
@@ -1304,30 +1319,25 @@ const onContentBlockDelta = Effect.fnUntraced(function* (
const onContentBlockStop = Effect.fnUntraced(function* (state: ParserState, event: AnthropicEvent) {
if (event.index === undefined) return [state, NO_EVENTS] satisfies StepResult
if (event.index in state.compactions) {
const { [event.index]: content, ...compactions } = state.compactions
const events: LLMEvent[] = []
const lifecycle = Lifecycle.stepStart(state.lifecycle, events)
events.push(
LLMEvent.compaction({
provider: state.provider,
text: content,
}),
)
return [{ ...state, compactions, lifecycle }, events] satisfies StepResult
}
const result = yield* ToolStream.finish(ADAPTER, state.tools, event.index)
const { [event.index]: block, ...blocks } = state.blocks
if (!block) return [state, NO_EVENTS] satisfies StepResult
const events: LLMEvent[] = []
const resultEvents = result.events ?? []
const lifecycle = resultEvents.length
? Lifecycle.stepStart(state.lifecycle, events)
: Lifecycle.reasoningEnd(
Lifecycle.textEnd(state.lifecycle, events, `text-${event.index}`),
events,
`reasoning-${event.index}`,
)
events.push(...resultEvents)
return [{ ...state, lifecycle, tools: result.tools }, events] satisfies StepResult
if (block.kind === "compaction") {
const lifecycle = Lifecycle.stepStart(state.lifecycle, events)
events.push(LLMEvent.compaction({ provider: state.provider, text: block.content }))
return [{ ...state, blocks, lifecycle }, events] satisfies StepResult
}
if (block.kind === "tool") {
const result = yield* ToolStream.finish(ADAPTER, { [event.index]: block.tool }, event.index)
const lifecycle = Lifecycle.stepStart(state.lifecycle, events)
events.push(...(result.events ?? []))
return [{ ...state, blocks, lifecycle }, events] satisfies StepResult
}
const lifecycle =
block.kind === "text"
? Lifecycle.textEnd(state.lifecycle, events, `text-${event.index}`)
: Lifecycle.reasoningEnd(state.lifecycle, events, `reasoning-${event.index}`)
return [{ ...state, blocks, lifecycle }, events] satisfies StepResult
})
const onMessageDelta = (
@@ -1367,9 +1377,13 @@ const onMessageDelta = (
}
const onMessageStop = Effect.fnUntraced(function* (state: ParserState) {
if (Object.keys(state.compactions).length)
const blocks = Object.values(state.blocks)
if (blocks.some((block) => block?.kind === "compaction"))
return yield* ProviderShared.eventError(ADAPTER, "Response ended with an incomplete compaction block")
const result = yield* ToolStream.finishAll(ADAPTER, state.tools)
const result = yield* ToolStream.finishAll<number>(
ADAPTER,
blocks.flatMap((block) => (block?.kind === "tool" ? [block.tool] : [])),
)
const events: LLMEvent[] = []
const lifecycle = result.events.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle
events.push(...result.events)
@@ -1381,7 +1395,7 @@ const onMessageStop = Effect.fnUntraced(function* (state: ParserState) {
usage: state.usage,
providerMetadata: state.pendingFinish?.providerMetadata,
})
return [{ ...state, lifecycle: finished, tools: result.tools }, events] satisfies StepResult
return [{ ...state, lifecycle: finished, blocks: {} }, events] satisfies StepResult
})
// Prefix `error.type` so overloads, rate limits, and quota errors are visible
@@ -1450,7 +1464,10 @@ const step = (state: ParserState, event: AnthropicEvent) => {
const decoded = Schema.decodeUnknownOption(AnthropicCompactionBlock)(event.content_block)
if (event.index === undefined || Option.isNone(decoded)) return invalidStreamEvent(event)
return Effect.succeed<StepResult>([
{ ...state, compactions: { ...state.compactions, [event.index]: decoded.value.content } },
{
...state,
blocks: { ...state.blocks, [event.index]: { kind: "compaction", content: decoded.value.content } },
},
NO_EVENTS,
])
}
@@ -1504,9 +1521,8 @@ export const protocol = Protocol.make({
event: Protocol.jsonEvent(AnthropicEvent),
initial: (request) => ({
provider: request.model.provider,
compactions: {},
providerMetadataKey: request.model.route.providerMetadataKey ?? request.model.provider,
tools: ToolStream.empty<number>(),
blocks: {},
lifecycle: Lifecycle.initial(),
}),
step,
+78 -87
View File
@@ -690,9 +690,17 @@ const mapUsage = (usage: BedrockUsageSchema | undefined, providerMetadataKey: st
})
}
// One entry per open content block index. Bedrock sends no start event for text
// or reasoning, so their first delta opens the block. A delta of a different
// kind for an index that is already open is dropped.
type Block =
| { readonly kind: "text" }
| { readonly kind: "reasoning"; readonly signature?: string; readonly redacted?: ReadonlyArray<Uint8Array> }
| { readonly kind: "tool"; readonly tool: ToolStream.PendingTool }
interface ParserState {
readonly providerMetadataKey: string
readonly tools: ToolStream.State<number>
readonly blocks: Readonly<Partial<Record<number, Block>>>
// Bedrock splits the finish into `messageStop` (carries `stopReason`) and
// `metadata` (carries usage). Hold both in state so `onHalt` can emit exactly
// one finish after both chunks have had a chance to arrive.
@@ -700,11 +708,12 @@ interface ParserState {
readonly usage: Usage | undefined
readonly hasToolCalls: boolean
readonly lifecycle: Lifecycle.State
readonly reasoningSignatures: Readonly<Record<number, string>>
readonly reasoningRedactedContent: Readonly<Record<number, Uint8Array[]>>
}
const encodeRedactedContent = (chunks: ReadonlyArray<Uint8Array>) => Base64.encode(concatBytes(chunks))
const reasoningEndMetadata = (key: string, block: Block & { readonly kind: "reasoning" }) => {
if (block.signature) return providerMetadata(key, { signature: block.signature })
if (block.redacted) return providerMetadata(key, { redactedData: Base64.encode(concatBytes(block.redacted)) })
}
const step = (state: ParserState, event: BedrockEvent) =>
Effect.gen(function* () {
@@ -716,10 +725,17 @@ const step = (state: ParserState, event: BedrockEvent) =>
{
...state,
lifecycle,
tools: ToolStream.start(state.tools, index, {
id: event.contentBlockStart.start.toolUse.toolUseId,
name: event.contentBlockStart.start.toolUse.name,
}),
blocks: {
...state.blocks,
[index]: {
kind: "tool",
tool: {
id: event.contentBlockStart.start.toolUse.toolUseId,
name: event.contentBlockStart.start.toolUse.name,
input: "",
},
},
},
},
[
...events,
@@ -732,16 +748,15 @@ const step = (state: ParserState, event: BedrockEvent) =>
}
if (event.contentBlockDelta?.delta?.text) {
const index = event.contentBlockDelta.contentBlockIndex
const block = state.blocks[index]
if (block && block.kind !== "text") return [state, []] as const
const events: LLMEvent[] = []
return [
{
...state,
lifecycle: Lifecycle.textDelta(
state.lifecycle,
events,
`text-${event.contentBlockDelta.contentBlockIndex}`,
event.contentBlockDelta.delta.text,
),
blocks: block ? state.blocks : { ...state.blocks, [index]: { kind: "text" } satisfies Block },
lifecycle: Lifecycle.textDelta(state.lifecycle, events, `text-${index}`, event.contentBlockDelta.delta.text),
},
events,
] as const
@@ -750,6 +765,8 @@ const step = (state: ParserState, event: BedrockEvent) =>
if (event.contentBlockDelta?.delta?.reasoningContent) {
const index = event.contentBlockDelta.contentBlockIndex
const reasoning = event.contentBlockDelta.delta.reasoningContent
const block = state.blocks[index]
if (block && block.kind !== "reasoning") return [state, []] as const
const events: LLMEvent[] = []
const redactedChunk = yield* (() => {
if (reasoning.redactedContent === undefined) return Effect.succeed(undefined)
@@ -764,8 +781,6 @@ const step = (state: ParserState, event: BedrockEvent) =>
),
)
})()
const redactedChunks = state.reasoningRedactedContent[index] ?? []
if (redactedChunk !== undefined) redactedChunks.push(redactedChunk)
const metadata = (() => {
if (reasoning.signature) return providerMetadata(state.providerMetadataKey, { signature: reasoning.signature })
if (redactedChunk === undefined && reasoning.data !== undefined)
@@ -777,23 +792,19 @@ const step = (state: ParserState, event: BedrockEvent) =>
if (redactedChunk !== undefined) return Lifecycle.reasoningStart(state.lifecycle, events, `reasoning-${index}`)
return state.lifecycle
})()
const reasoningRedactedContent = (() => {
if (redactedChunk !== undefined) return { ...state.reasoningRedactedContent, [index]: redactedChunks }
if (reasoning.data === undefined) return state.reasoningRedactedContent
return Object.fromEntries(
Object.entries(state.reasoningRedactedContent).filter(([key]) => key !== String(index)),
)
})()
const reasoningSignatures = (() => {
if (!reasoning.signature) return state.reasoningSignatures
return { ...state.reasoningSignatures, [index]: reasoning.signature }
// A plain `data` payload replaces any redacted chunks collected so far.
const redacted = (() => {
if (redactedChunk !== undefined) return [...(block?.redacted ?? []), redactedChunk]
if (reasoning.data === undefined) return block?.redacted
})()
return [
{
...state,
lifecycle,
reasoningSignatures,
reasoningRedactedContent,
blocks: {
...state.blocks,
[index]: { kind: "reasoning", signature: reasoning.signature || block?.signature, redacted },
},
},
events,
] as const
@@ -802,59 +813,41 @@ const step = (state: ParserState, event: BedrockEvent) =>
if (event.contentBlockDelta?.delta?.toolUse) {
// A delta for a block that is not open, whether it already stopped or never
// started, has nothing to attach to and is dropped.
const result = ToolStream.append(
state.tools,
event.contentBlockDelta.contentBlockIndex,
event.contentBlockDelta.delta.toolUse.input,
)
const index = event.contentBlockDelta.contentBlockIndex
const block = state.blocks[index]
if (block?.kind !== "tool") return [state, []] as const
const result = ToolStream.append({ [index]: block.tool }, index, event.contentBlockDelta.delta.toolUse.input)
if (!result) return [state, []] as const
const events: LLMEvent[] = []
const lifecycle = result.events.length ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle
events.push(...result.events)
return [{ ...state, lifecycle, tools: result.tools }, events] as const
return [
{ ...state, lifecycle, blocks: { ...state.blocks, [index]: { kind: "tool", tool: result.tool } } },
events,
] as const
}
if (event.contentBlockStop) {
const index = event.contentBlockStop.contentBlockIndex
const result = yield* ToolStream.finish(ADAPTER, state.tools, index)
const { [index]: block, ...blocks } = state.blocks
if (!block) return [state, []] as const
const events: LLMEvent[] = []
const resultEvents = result.events ?? []
const lifecycle = (() => {
if (resultEvents.length) return Lifecycle.stepStart(state.lifecycle, events)
const metadata = (() => {
const signature = state.reasoningSignatures[index]
if (signature) return providerMetadata(state.providerMetadataKey, { signature })
const redactedContent = state.reasoningRedactedContent[index]
if (redactedContent)
return providerMetadata(state.providerMetadataKey, {
redactedData: encodeRedactedContent(redactedContent),
})
})()
return Lifecycle.reasoningEnd(
Lifecycle.textEnd(state.lifecycle, events, `text-${index}`),
events,
`reasoning-${index}`,
metadata,
)
})()
events.push(...resultEvents)
return [
{
...state,
hasToolCalls:
resultEvents.some((event) => LLMEvent.is.toolCall(event) || LLMEvent.is.toolInputError(event)) ||
state.hasToolCalls,
lifecycle,
tools: result.tools,
reasoningSignatures: Object.fromEntries(
Object.entries(state.reasoningSignatures).filter(([key]) => key !== String(index)),
),
reasoningRedactedContent: Object.fromEntries(
Object.entries(state.reasoningRedactedContent).filter(([key]) => key !== String(index)),
),
},
events,
] as const
if (block.kind === "tool") {
const result = yield* ToolStream.finish(ADAPTER, { [index]: block.tool }, index)
const lifecycle = Lifecycle.stepStart(state.lifecycle, events)
events.push(...(result.events ?? []))
return [{ ...state, hasToolCalls: true, lifecycle, blocks }, events] as const
}
const lifecycle =
block.kind === "text"
? Lifecycle.textEnd(state.lifecycle, events, `text-${index}`)
: Lifecycle.reasoningEnd(
state.lifecycle,
events,
`reasoning-${index}`,
reasoningEndMetadata(state.providerMetadataKey, block),
)
return [{ ...state, lifecycle, blocks }, events] as const
}
if (event.messageStop) {
@@ -915,18 +908,18 @@ const onHalt = (state: ParserState): ReadonlyArray<LLMEvent> => {
return state.finishReason.normalized
})()
const events: LLMEvent[] = []
const lifecycle = Object.entries(state.reasoningRedactedContent).reduce((current, [index, chunks]) => {
const signature = state.reasoningSignatures[Number(index)]
return Lifecycle.reasoningEnd(
current,
events,
`reasoning-${index}`,
providerMetadata(
state.providerMetadataKey,
signature ? { signature } : { redactedData: encodeRedactedContent(chunks) },
),
)
}, state.lifecycle)
const lifecycle = Object.entries(state.blocks).reduce(
(current, [index, block]) =>
block?.kind === "reasoning" && block.redacted
? Lifecycle.reasoningEnd(
current,
events,
`reasoning-${index}`,
reasoningEndMetadata(state.providerMetadataKey, block),
)
: current,
state.lifecycle,
)
Lifecycle.finish(lifecycle, events, {
reason: {
...state.finishReason,
@@ -954,13 +947,11 @@ export const protocol = Protocol.make({
event: BedrockEventStream.Frame.pipe(Schema.decodeTo(BedrockEvent)),
initial: (request) => ({
providerMetadataKey: request.model.route.providerMetadataKey ?? request.model.provider,
tools: ToolStream.empty<number>(),
blocks: {},
finishReason: undefined,
usage: undefined,
hasToolCalls: false,
lifecycle: Lifecycle.initial(),
reasoningSignatures: {},
reasoningRedactedContent: {},
}),
step,
onHalt: (state) => Effect.succeed(onHalt(state)),
@@ -21,6 +21,7 @@ import {
import { LLMClient } from "../../src/route.js"
import { compileRequest } from "../../src/route/client.js"
import { AmazonBedrock } from "../../src/providers.js"
import { BedrockConverse } from "../../src/protocols/bedrock-converse.js"
import { it } from "../lib/effect.js"
import { withProcessEnv } from "../lib/env.js"
import { dynamicResponse, fixedResponse } from "../lib/http.js"
@@ -1559,6 +1560,22 @@ describe("Bedrock Converse route", () => {
}),
)
it.effect("leaves the previous parser state untouched when a redacted chunk arrives", () =>
Effect.gen(function* () {
const stream = BedrockConverse.protocol.stream
const chunk = (redactedContent: string) => ({
contentBlockDelta: { contentBlockIndex: 0, delta: { reasoningContent: { redactedContent } } },
})
const [first] = yield* stream.step(stream.initial(baseRequest), chunk("AQ=="))
yield* stream.step(first, chunk("AgM="))
const [, events] = yield* stream.step(first, { contentBlockStop: { contentBlockIndex: 0 } })
expect(events.find((event) => event.type === "reasoning-end")?.providerMetadata).toEqual({
bedrock: { redactedData: "AQ==" },
})
}),
)
it.effect("preserves split redacted reasoning when contentBlockStop is missing", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(baseRequest).pipe(