diff --git a/packages/ai/src/protocols/mistral-chat.ts b/packages/ai/src/protocols/mistral-chat.ts index 6e9975c2662..2c4bb14c0f0 100644 --- a/packages/ai/src/protocols/mistral-chat.ts +++ b/packages/ai/src/protocols/mistral-chat.ts @@ -12,7 +12,6 @@ import { ProviderInternalError, UnknownProviderError, Usage, - type FinishReasonDetails, type LLMRequest, type MediaPart, type ToolCallPart, @@ -413,11 +412,6 @@ export const fromRequest = Effect.fn("MistralChat.fromRequest")(function* (reque }) type ToolKey = string | number -interface PendingTool { - readonly id: string - readonly name?: string - readonly input: string -} interface ActiveContent { readonly type: "text" | "reasoning" @@ -427,18 +421,15 @@ interface ActiveContent { } export interface ParserState { - readonly tools: ToolStream.State - readonly pendingTools: Partial> + readonly toolCalls: ToolStream.Slots readonly toolIDs: ReadonlyMap readonly usedToolIDs: ReadonlySet - readonly completedTools: ReadonlyArray readonly latestToolKey?: ToolKey readonly generatedTools: number readonly lifecycle: Lifecycle.State readonly active?: ActiveContent readonly nextContent: number readonly usage?: Usage - readonly finishReason?: FinishReasonDetails } const mapUsage = (usage: MistralEvent["usage"]): Usage | undefined => { @@ -580,13 +571,10 @@ const toolText = (tool: MistralToolDelta) => { return value === null || value === undefined ? "" : ProviderShared.encodeJson(value) } -const appendTools = Effect.fnUntraced(function* ( - initial: ParserState, - events: LLMEvent[], - deltas: ReadonlyArray, -) { - if (deltas.length === 0) return initial +const appendTools = (initial: ParserState, events: LLMEvent[], deltas: ReadonlyArray) => { + if (deltas.length === 0 || initial.toolCalls.status === "finished") return initial let state = closeActive(initial, events) + let slots = initial.toolCalls.slots for (const [position, delta] of deltas.entries()) { const wireID = delta.id?.trim() || undefined const providedID = wireID === "null" ? undefined : wireID @@ -597,54 +585,32 @@ const appendTools = Effect.fnUntraced(function* ( : deltas.length > 1 ? `position:${position}` : (state.latestToolKey ?? `missing:${state.generatedTools}`)) - const existing = state.tools[key] - const pending = state.pendingTools[key] - const source = providedID ?? `generated:${String(key)}` - const normalized = - existing || pending ? { id: existing?.id ?? pending?.id ?? "", state } : normalizeStreamToolID(state, source) - state = normalized.state - const name = existing?.name ?? pending?.name ?? (delta.function?.name?.trim() || undefined) - const text = `${pending?.input ?? ""}${toolText(delta)}` - if (!name) { - state = { - ...state, - pendingTools: { ...state.pendingTools, [key]: { id: normalized.id, input: text } }, - latestToolKey: key, - generatedTools: state.generatedTools + (!providedID && !pending ? 1 : 0), - } - continue - } - const result = ToolStream.appendOrStart( - ADAPTER, - state.tools, - key, - { id: normalized.id, name, text }, - "Mistral Chat tool call delta is missing a name", - ) - if (ToolStream.isError(result)) return yield* result - if (result.events.length > 0) state = { ...state, lifecycle: Lifecycle.stepStart(state.lifecycle, events) } - events.push(...result.events) - const pendingTools = { ...state.pendingTools } - delete pendingTools[key] + const slot = slots.get(key) + const normalized = slot + ? { id: ToolStream.slotID(slot), state } + : normalizeStreamToolID(state, providedID ?? `generated:${String(key)}`) + const next = ToolStream.slotDelta(slots, key, { + id: normalized.id, + name: delta.function?.name?.trim() || undefined, + text: toolText(delta), + freeform: false, + }) + slots = next.slots state = { - ...state, - tools: result.tools, - pendingTools, + ...normalized.state, + lifecycle: next.events.length > 0 ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle, latestToolKey: key, - generatedTools: state.generatedTools + (!providedID && !existing && !pending ? 1 : 0), + generatedTools: state.generatedTools + (!providedID && !slot ? 1 : 0), } + events.push(...next.events) } - return state -}) + return { ...state, toolCalls: { status: "streaming" as const, slots } } +} -const hasLateContent = (event: MistralEvent) => { - const delta = event.choices?.[0]?.delta - if (typeof delta?.content === "string" && delta.content.length > 0) return true - if (Array.isArray(delta?.content) && delta.content.length > 0) return true - return (delta?.tool_calls ?? []).some( +const hasLateToolCall = (event: MistralEvent) => + (event.choices?.[0]?.delta?.tool_calls ?? []).some( (tool) => Boolean(tool.id) || Boolean(tool.function?.name) || tool.function?.arguments !== undefined, ) -} const step = Effect.fnUntraced(function* (state: ParserState, event: MistralEvent) { if (event.error) { @@ -659,19 +625,18 @@ const step = Effect.fnUntraced(function* (state: ParserState, event: MistralEven } const events: LLMEvent[] = [] const usage = mapUsage(event.usage) ?? state.usage - if (state.finishReason) { - if (hasLateContent(event)) - return yield* ProviderShared.eventError( - ADAPTER, - "Mistral Chat received content after the finish reason", - ProviderShared.encodeJson(event), - ) - return [{ ...state, usage }, events] as const - } + // Like OpenAI Chat, text and reasoning after the finish reason still belong to the response; tool calls do not. + if (state.toolCalls.status === "finished" && hasLateToolCall(event)) + return yield* ProviderShared.eventError( + ADAPTER, + "Mistral Chat received content after the finish reason", + ProviderShared.encodeJson(event), + ) const choice = event.choices?.[0] const withContent = choice?.delta?.content == null ? state : appendContent(state, events, choice.delta.content) - const withTools = yield* appendTools(withContent, events, choice?.delta?.tool_calls ?? []) - if (!choice?.finish_reason) return [{ ...withTools, usage }, events] as const + const withTools = appendTools(withContent, events, choice?.delta?.tool_calls ?? []) + if (!choice?.finish_reason || withTools.toolCalls.status === "finished") + return [{ ...withTools, usage }, events] as const const finishReason = { normalized: mapFinishReason(choice.finish_reason), @@ -688,30 +653,21 @@ const step = Effect.fnUntraced(function* (state: ParserState, event: MistralEven }) } const incomplete = finishReason.normalized === "length" || finishReason.normalized === "content-filter" - if (!incomplete && Object.keys(withTools.pendingTools).length > 0) + if (!incomplete && ToolStream.hasBufferedSlot(withTools.toolCalls.slots)) return yield* ProviderShared.eventError( ADAPTER, "Mistral Chat tool call delta is missing a name", ProviderShared.encodeJson(event), ) - const finished = - !incomplete && Object.keys(withTools.tools).length > 0 - ? yield* ToolStream.finishAll(ADAPTER, withTools.tools) - : undefined + const calls = yield* ToolStream.finishSlots(ADAPTER, withTools.toolCalls.slots, finishReason.normalized) return [ - { - ...withTools, - tools: finished?.tools ?? withTools.tools, - completedTools: finished?.events ?? withTools.completedTools, - usage, - finishReason, - }, + { ...withTools, toolCalls: { status: "finished" as const, reason: finishReason, events: calls }, usage }, events, ] as const }) const finishEvents = Effect.fnUntraced(function* (state: ParserState) { - if (!state.finishReason) + if (state.toolCalls.status === "streaming") return yield* new AIError({ reason: new InvalidProviderOutputError({ message: "Mistral Chat stream ended without finish_reason", @@ -721,12 +677,13 @@ const finishEvents = Effect.fnUntraced(function* (state: ParserState) { }) const events: LLMEvent[] = [] const closed = closeActive(state, events) - const lifecycle = closed.completedTools.length > 0 ? Lifecycle.stepStart(closed.lifecycle, events) : closed.lifecycle - events.push(...closed.completedTools) + const calls = state.toolCalls.events + const lifecycle = calls.length > 0 ? Lifecycle.stepStart(closed.lifecycle, events) : closed.lifecycle + events.push(...calls) const reason = - state.finishReason.normalized === "stop" && closed.completedTools.some(LLMEvent.is.toolCall) - ? { ...state.finishReason, normalized: "tool-calls" as const } - : state.finishReason + state.toolCalls.reason.normalized === "stop" && calls.some(LLMEvent.is.toolCall) + ? { ...state.toolCalls.reason, normalized: "tool-calls" as const } + : state.toolCalls.reason Lifecycle.finish(lifecycle, events, { reason, usage: closed.usage }) return events }) @@ -737,11 +694,9 @@ export const protocol = Protocol.make({ stream: { event: MistralStreamEvent, initial: (): ParserState => ({ - tools: ToolStream.empty(), - pendingTools: {}, + toolCalls: { status: "streaming", slots: new Map() }, toolIDs: new Map(), usedToolIDs: new Set(), - completedTools: [], generatedTools: 0, lifecycle: Lifecycle.initial(), nextContent: 0, diff --git a/packages/ai/src/protocols/openai-chat.ts b/packages/ai/src/protocols/openai-chat.ts index cd76637892c..a4c7d5d0793 100644 --- a/packages/ai/src/protocols/openai-chat.ts +++ b/packages/ai/src/protocols/openai-chat.ts @@ -8,13 +8,11 @@ import { HttpTransport } from "../route/transport/index.js" import { Protocol } from "../route/protocol.js" import { AIError, - AIErrorReason, InvalidProviderOutputError, LLMEvent, ProviderInternalError, UnknownProviderError, Usage, - type FinishReasonDetails, type AssistantMessage, type CacheHint, type LLMRequest, @@ -345,23 +343,11 @@ export type OpenAIChatEvent = Schema.Schema.Type const DONE = "[DONE]" as const const OpenAIChatStreamEvent = Schema.Union([Schema.Literal(DONE), Protocol.jsonEvent(OpenAIChatEvent)]) -interface PendingToolDelta { - readonly id?: string - readonly name?: string - readonly input: string - readonly extraContent?: Schema.Schema.Type -} - export interface ParserState { readonly providerMetadataKey: string - readonly tools: ToolStream.State readonly custom: ReadonlyMap - /** Stream indexes of custom tool calls, whose JSON fragment closes when the call finishes. */ - readonly customCalls: ReadonlySet - readonly pendingTools: Partial> - readonly toolCallEvents: ReadonlyArray + readonly toolCalls: ToolStream.Slots readonly usage?: Usage - readonly finishReason?: FinishReasonDetails readonly lifecycle: Lifecycle.State readonly reasoningField?: string /** A scalar reasoning field (`reasoning_content`, ...) has carried text in this stream. */ @@ -370,7 +356,6 @@ export interface ParserState { readonly reasoningDetailsObserved: boolean readonly reasoningEmitted: boolean readonly latestToolIndex?: number - readonly nextToolIndex: number readonly requireFinishReason: boolean } @@ -973,15 +958,48 @@ const mapUsage = (usage: OpenAIChatEvent["usage"], providerMetadataKey: string): }) } -const toolIndexByID = ( - tools: ParserState["tools"], - pendingTools: ParserState["pendingTools"], - id: string | undefined, -) => { - if (!id) return undefined - const entry = Object.entries({ ...pendingTools, ...tools }).find(([, tool]) => tool?.id === id) - return entry ? Number(entry[0]) : undefined -} +// Compatible providers may omit indexes. Prefer durable identity, then use +// batch position for parallel deltas or the latest call for sparse chunks. +const appendTools = ( + state: ParserState, + slots: ReadonlyMap, + deltas: ReadonlyArray, +) => + deltas.reduce( + (result, tool, position) => { + const matched = tool.id + ? [...result.slots].find(([, slot]) => ToolStream.slotID(slot) === tool.id)?.[0] + : undefined + const fallback = deltas.length > 1 ? position : (result.latestToolIndex ?? position) + const fallbackID = ToolStream.slotID(result.slots.get(fallback)) + const index = + tool.index ?? + matched ?? + (tool.id && fallbackID && fallbackID !== tool.id ? Math.max(-1, ...result.slots.keys()) + 1 : fallback) + const slot = result.slots.get(index) + const name = tool.function?.name || tool.custom?.name || undefined + // Custom calls stream raw text; re-encode it as JSON arguments for the call's field. + const raw = tool.custom?.input ?? "" + const text = tool.custom + ? slot?.freeform + ? OpenAIFreeform.streamDelta(raw) + : OpenAIFreeform.streamStart( + state.custom.get((slot?.status === "open" ? slot.tool.name : slot?.name) ?? name ?? ""), + raw, + ) + : (tool.function?.arguments ?? "") + const extraContent = decodeExtraContent(tool.extra_content) + const next = ToolStream.slotDelta(result.slots, index, { + id: tool.id || undefined, + name, + text, + freeform: Boolean(tool.custom), + providerMetadata: extraContent && { [state.providerMetadataKey]: { extraContent } }, + }) + return { slots: next.slots, latestToolIndex: index, events: [...result.events, ...next.events] } + }, + { slots, latestToolIndex: state.latestToolIndex, events: [] as ReadonlyArray }, + ) const reasoningDelta = ( delta: Schema.Schema.Type | null | undefined, @@ -1110,20 +1128,14 @@ const step = (state: ParserState, event: OpenAIChatEvent) => normalized: yield* mapFinishReason(event, rawFinishReason), raw: choice?.native_finish_reason ?? rawFinishReason, } - : state.finishReason + : undefined const delta = choice?.delta const toolDeltas = delta?.tool_calls ?? [] - let tools = state.tools - let pendingTools = state.pendingTools - let customCalls = state.customCalls - let latestToolIndex = state.latestToolIndex - let nextToolIndex = state.nextToolIndex - let lifecycle = state.lifecycle const reasoning = reasoningDelta(delta, state.reasoningField) if ( - state.finishReason !== undefined && + state.toolCalls.status === "finished" && toolDeltas.some( (tool) => Boolean(tool.id) || @@ -1165,74 +1177,17 @@ const step = (state: ParserState, event: OpenAIChatEvent) => if (delta?.refusal) lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.refusal) - // Compatible providers may omit indexes. Prefer durable identity, then use - // batch position for parallel deltas or the latest call for sparse chunks. - for (const [position, tool] of toolDeltas.entries()) { - const matched = toolIndexByID(tools, pendingTools, tool.id || undefined) - const fallback = toolDeltas.length > 1 ? position : (latestToolIndex ?? position) - const fallbackTool = tools[fallback] ?? pendingTools[fallback] - const index = - tool.index ?? matched ?? (tool.id && fallbackTool?.id && fallbackTool.id !== tool.id ? nextToolIndex : fallback) - const current = tools[index] - const pending = pendingTools[index] - const id = current?.id ?? pending?.id ?? (tool.id || undefined) - const name = current?.name ?? pending?.name ?? (tool.function?.name || tool.custom?.name || undefined) - // Custom calls stream raw text; re-encode it as JSON arguments for the call's field. - const opening = Boolean(tool.custom) && !customCalls.has(index) - if (opening) customCalls = new Set([...customCalls, index]) - const raw = tool.custom?.input ?? "" - const fragment = tool.custom - ? opening - ? OpenAIFreeform.streamStart(state.custom.get(name ?? ""), raw) - : OpenAIFreeform.streamDelta(raw) - : (tool.function?.arguments ?? "") - const text = `${pending?.input ?? ""}${fragment}` - const extraContent = pending?.extraContent ?? decodeExtraContent(tool.extra_content) - latestToolIndex = index - nextToolIndex = Math.max(nextToolIndex, index + 1) - if (!current && (!id || !name)) { - pendingTools = { - ...pendingTools, - [index]: { id: id || undefined, name: name || undefined, input: text, extraContent }, - } - continue - } - if (pending) { - pendingTools = { ...pendingTools } - delete pendingTools[index] - } - const result = ToolStream.appendOrStart( - ADAPTER, - tools, - index, - { - id: id || undefined, - name: name || undefined, - text, - providerMetadata: extraContent && { [state.providerMetadataKey]: { extraContent } }, - }, - "OpenAI Chat tool call delta is missing id or name", - ) - if (ToolStream.isError(result)) - return yield* new AIError({ - reason: AIErrorReason.make({ - ...result.reason, - message: result.message, - cause: result.reason.cause, - body: ProviderShared.encodeJson(event), - }), - }) - tools = result.tools - if (result.events.length) lifecycle = Lifecycle.stepStart(lifecycle, events) - events.push(...result.events) - } + const streamed = + state.toolCalls.status === "streaming" ? appendTools(state, state.toolCalls.slots, toolDeltas) : undefined + if (streamed?.events.length) lifecycle = Lifecycle.stepStart(lifecycle, events) + events.push(...(streamed?.events ?? [])) - const incompleteTools = finishReason?.normalized === "content-filter" || finishReason?.normalized === "length" if ( + streamed !== undefined && finishReason !== undefined && - !incompleteTools && - state.finishReason === undefined && - Object.keys(pendingTools).length + finishReason.normalized !== "content-filter" && + finishReason.normalized !== "length" && + ToolStream.hasBufferedSlot(streamed.slots) ) return yield* ProviderShared.eventError( ADAPTER, @@ -1240,48 +1195,40 @@ const step = (state: ParserState, event: OpenAIChatEvent) => ProviderShared.encodeJson(event), ) - // Filtering or truncation terminates the response without confirming pending tool calls. - const finished = - finishReason !== undefined && - !incompleteTools && - state.finishReason === undefined && - Object.keys(tools).length > 0 - ? yield* finishTools(tools, customCalls) - : undefined + const toolCalls: ToolStream.Slots = + streamed === undefined + ? state.toolCalls + : finishReason === undefined + ? { status: "streaming", slots: streamed.slots } + : { + status: "finished", + reason: finishReason, + events: yield* ToolStream.finishSlots( + ADAPTER, + streamed.slots, + finishReason.normalized, + OpenAIFreeform.streamEnd, + ), + } return [ { - providerMetadataKey: state.providerMetadataKey, - tools: finished?.tools ?? tools, - custom: state.custom, - customCalls, - pendingTools, - toolCallEvents: finished?.events ?? state.toolCallEvents, + ...state, + toolCalls, usage, - finishReason: state.finishReason ?? finishReason, lifecycle, reasoningField, reasoningTextObserved, - reasoningDetails: state.reasoningDetails, reasoningDetailsObserved, reasoningEmitted, - latestToolIndex, - nextToolIndex, - requireFinishReason: state.requireFinishReason, + latestToolIndex: streamed?.latestToolIndex ?? state.latestToolIndex, }, events, ] as const }) -// Custom calls close their JSON fragment before every pending call finishes. -const finishTools = Effect.fnUntraced(function* (tools: ParserState["tools"], customCalls: ReadonlySet) { - const closed = OpenAIFreeform.close(tools, customCalls) - const finished = yield* ToolStream.finishAll(ADAPTER, closed.tools) - return { tools: finished.tools, events: [...closed.events, ...finished.events] } -}) - export const finishEvents = Effect.fnUntraced(function* (state: ParserState) { - if (state.finishReason === undefined && state.requireFinishReason) + if (state.toolCalls.status === "streaming" && state.requireFinishReason) return yield* new AIError({ reason: new InvalidProviderOutputError({ message: "OpenAI Chat stream ended without finish_reason", @@ -1291,15 +1238,15 @@ export const finishEvents = Effect.fnUntraced(function* (state: ParserState) { }) const events: LLMEvent[] = [] const toolCallEvents = - state.finishReason === undefined && Object.keys(state.tools).length > 0 - ? (yield* finishTools(state.tools, state.customCalls)).events - : state.toolCallEvents + state.toolCalls.status === "finished" + ? state.toolCalls.events + : yield* ToolStream.finishSlots(ADAPTER, state.toolCalls.slots, undefined, OpenAIFreeform.streamEnd) const hasToolCalls = toolCallEvents.length > 0 - const reason = state.finishReason + const finishReason = state.toolCalls.status === "finished" ? state.toolCalls.reason : undefined + const reason = finishReason ? { - ...state.finishReason, - normalized: - state.finishReason.normalized === "stop" && hasToolCalls ? "tool-calls" : state.finishReason.normalized, + ...finishReason, + normalized: finishReason.normalized === "stop" && hasToolCalls ? "tool-calls" : finishReason.normalized, } : { normalized: hasToolCalls ? ("tool-calls" as const) : ("stop" as const) } // Snapshot details at publish time so the emitted event never observes later @@ -1342,20 +1289,16 @@ export const protocol = Protocol.make({ }, stream: { event: OpenAIChatStreamEvent, - initial: (request) => ({ + initial: (request): ParserState => ({ providerMetadataKey: request.model.route.providerMetadataKey ?? request.model.provider, - tools: ToolStream.empty(), custom: OpenAIFreeform.tools(request), - customCalls: new Set(), - pendingTools: {}, - toolCallEvents: [], + toolCalls: { status: "streaming", slots: new Map() }, lifecycle: Lifecycle.initial(), reasoningField: request.model.compatibility?.reasoningField, reasoningTextObserved: false, reasoningDetails: [], reasoningDetailsObserved: false, reasoningEmitted: false, - nextToolIndex: 0, requireFinishReason: request.model.compatibility?.requireFinishReason ?? true, }), step: (state: ParserState, event) => (event === DONE ? Effect.succeed([state, []] as const) : step(state, event)), diff --git a/packages/ai/src/protocols/openai-freeform.ts b/packages/ai/src/protocols/openai-freeform.ts index 272827f7c93..a1905e648b5 100644 --- a/packages/ai/src/protocols/openai-freeform.ts +++ b/packages/ai/src/protocols/openai-freeform.ts @@ -57,7 +57,7 @@ const field = (custom: Custom | undefined) => custom?.field ?? "input" export const streamStart = (custom: Custom | undefined, raw = "") => `{${encodeJson(field(custom))}:"${streamDelta(raw)}` export const streamDelta = (raw: string) => encodeJson(raw).slice(1, -1) -const STREAM_END = '"}' +export const streamEnd = '"}' const decodeArguments = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown))) @@ -73,9 +73,9 @@ export const close = ( [...keys].reduce<{ readonly tools: ToolStream.State; readonly events: ReadonlyArray }>( (result, key) => { const pending = result.tools[key] - const streamed = pending && Option.getOrUndefined(decodeArguments(`${pending.input}${STREAM_END}`)) + const streamed = pending && Option.getOrUndefined(decodeArguments(`${pending.input}${streamEnd}`)) if (!pending || (final !== undefined && streamed?.[field(final.custom)] !== final.raw)) return result - const appended = ToolStream.append(result.tools, key, STREAM_END) + const appended = ToolStream.append(result.tools, key, streamEnd) return appended ? { tools: appended.tools, events: [...result.events, ...appended.events] } : result }, { tools, events: [] }, diff --git a/packages/ai/src/protocols/utils/tool-stream.ts b/packages/ai/src/protocols/utils/tool-stream.ts index b3615cdae19..ba300d49871 100644 --- a/packages/ai/src/protocols/utils/tool-stream.ts +++ b/packages/ai/src/protocols/utils/tool-stream.ts @@ -1,5 +1,12 @@ import { Effect, Option } from "effect" -import { AIError, LLMEvent, type ProviderMetadata, type ToolCall, type ToolInputError } from "../../schema/index.js" +import { + AIError, + type FinishReasonDetails, + LLMEvent, + type ProviderMetadata, + type ToolCall, + type ToolInputError, +} from "../../schema/index.js" import { eventError, parseToolInput, type ToolAccumulator } from "../shared.js" import { parse } from "./partial-json.js" @@ -98,13 +105,16 @@ const toolCall = (route: string, tool: PendingTool, inputOverride?: string) => { ) } -const finishEvents = (tool: PendingTool, event: ToolCall | ToolInputError): ReadonlyArray => [ +const inputEnd = (tool: PendingTool) => LLMEvent.toolInputEnd({ id: tool.id, name: tool.name, namespace: tool.namespace, providerMetadata: tool.providerMetadata, - }), + }) + +const finishEvents = (tool: PendingTool, event: ToolCall | ToolInputError): ReadonlyArray => [ + inputEnd(tool), event, ] @@ -237,4 +247,101 @@ export const finishAll = (route: string, tools: State) = } }) +/** + * One Chat tool call under its stream key. Chat providers may split the call id + * and name across deltas, so a slot buffers argument text until both arrive and + * only then opens with `tool-input-start`. Freeform calls stream raw text that + * the parser re-encodes as JSON arguments. + */ +export type Slot = + | { + readonly status: "buffering" + readonly freeform: boolean + readonly id?: string + readonly name?: string + readonly input: string + readonly providerMetadata?: ProviderMetadata + } + | { readonly status: "open"; readonly freeform: boolean; readonly tool: PendingTool } + +/** + * Chat tool calls stream until the first finish reason, which settles every + * slot. The settled events wait for the end of the stream so late text and + * reasoning still precede them. + */ +export type Slots = + | { readonly status: "streaming"; readonly slots: ReadonlyMap } + | { readonly status: "finished"; readonly reason: FinishReasonDetails; readonly events: ReadonlyArray } + +export const slotID = (slot: Slot | undefined) => (slot?.status === "open" ? slot.tool.id : slot?.id) + +/** Add one delta to a slot, opening it once both id and name are known. */ +export const slotDelta = ( + slots: ReadonlyMap, + key: K, + delta: { + readonly id?: string + readonly name?: string + readonly text: string + readonly freeform: boolean + readonly providerMetadata?: ProviderMetadata + }, +) => { + const slot = slots.get(key) + if (slot?.status === "open") { + if (delta.text.length === 0) return { slots, events: [] } + const tool = { + ...slot.tool, + input: `${slot.tool.input}${delta.text}`, + providerMetadata: slot.tool.providerMetadata ?? delta.providerMetadata, + } + return { slots: new Map(slots).set(key, { ...slot, tool }), events: [inputDelta(tool, delta.text)] } + } + const id = slot?.id ?? delta.id + const name = slot?.name ?? delta.name + const input = `${slot?.input ?? ""}${delta.text}` + const freeform = slot?.freeform ?? delta.freeform + const providerMetadata = slot?.providerMetadata ?? delta.providerMetadata + if (!id || !name) + return { + slots: new Map(slots).set(key, { status: "buffering", freeform, id, name, input, providerMetadata }), + events: [], + } + const tool = { id, name, input, providerMetadata } + return { + slots: new Map(slots).set(key, { status: "open", freeform, tool }), + events: input.length > 0 ? [inputStart(tool), inputDelta(tool, input)] : [inputStart(tool)], + } +} + +export const hasBufferedSlot = (slots: ReadonlyMap) => + [...slots.values()].some((slot) => slot.status === "buffering") + +/** + * Settle every open slot. A completed response parses each call, appending + * `freeformEnd` to freeform input first. Truncation and filtering abandon open + * calls: their input ends without a call, and the consumer settles them as + * incomplete. Buffered slots never started, so they produce nothing. + */ +export const finishSlots = ( + route: string, + slots: ReadonlyMap, + reason: FinishReasonDetails["normalized"] | undefined, + freeformEnd = "", +) => + Effect.gen(function* () { + const open = [...slots.values()].flatMap((slot) => (slot.status === "open" ? [slot] : [])) + if (reason === "length" || reason === "content-filter") return open.map((slot) => inputEnd(slot.tool)) + const closed = open.map((slot) => + slot.freeform && freeformEnd + ? { tool: { ...slot.tool, input: `${slot.tool.input}${freeformEnd}` }, delta: freeformEnd } + : { tool: slot.tool, delta: "" }, + ) + const deltas = closed.flatMap((item) => (item.delta ? [inputDelta(item.tool, item.delta)] : [])) + const calls = yield* Effect.forEach(closed, (item) => + toolCall(route, item.tool).pipe(Effect.map((event) => finishEvents(item.tool, event))), + ) + return [...deltas, ...calls.flat()] + }) + export * as ToolStream from "./tool-stream.js" diff --git a/packages/ai/src/protocols/venice-chat.ts b/packages/ai/src/protocols/venice-chat.ts index d61f7848a88..6a9de47a9fb 100644 --- a/packages/ai/src/protocols/venice-chat.ts +++ b/packages/ai/src/protocols/venice-chat.ts @@ -6,6 +6,7 @@ import { OpenAIChat } from "./openai-chat.js" import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js" import { cacheControl } from "./utils/cache.js" import { OpenResponsesOptions } from "./utils/open-responses-options.js" +import { ToolStream } from "./utils/tool-stream.js" // --------------------------------------------------------------------------- // Public options and request body @@ -280,8 +281,12 @@ const step = Effect.fn("VeniceChat.step")(function* (state: State, event: typeof for (const call of delta?.tool_calls ?? []) { if (!call.thought_signature) continue const index = call.index ?? result[0].latestToolIndex + const toolCalls = result[0].toolCalls const id = - call.id ?? (index === undefined ? undefined : (result[0].tools[index]?.id ?? result[0].pendingTools[index]?.id)) + call.id ?? + (index === undefined || toolCalls.status === "finished" + ? undefined + : ToolStream.slotID(toolCalls.slots.get(index))) if (id) tools[id] = call.thought_signature } return [ diff --git a/packages/ai/test/provider/mistral-chat.test.ts b/packages/ai/test/provider/mistral-chat.test.ts index 6efdedf04ba..9598a212c2a 100644 --- a/packages/ai/test/provider/mistral-chat.test.ts +++ b/packages/ai/test/provider/mistral-chat.test.ts @@ -674,21 +674,34 @@ describe("Mistral Chat", () => { }) } - const truncated = yield* LLMClient.generate(request).pipe( - Effect.provide( - fixedResponse( - sseEvents( - chunk({ - tool_calls: [{ index: 0, id: "Ab12Cd34E", function: { name: "lookup", arguments: '{"city":' } }], - }), - chunk({}, "length"), + for (const [raw, normalized] of [ + ["length", "length"], + ["content_filter", "content-filter"], + ] as const) { + const truncated = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + chunk({ + tool_calls: [{ index: 0, id: "Ab12Cd34E", function: { name: "lookup", arguments: '{"city":' } }], + }), + chunk({ tool_calls: [{ index: 1, id: "Cd34Ef56G", function: { arguments: "{}" } }] }), + chunk({}, raw), + ), ), ), - ), - ) - expect(truncated.finishReason).toEqual({ normalized: "length", raw: "length" }) - expect(truncated.events.some(LLMEvent.is.toolCall)).toBe(false) - expect(truncated.events.some(LLMEvent.is.toolInputEnd)).toBe(false) + ) + expect(truncated.finishReason).toEqual({ normalized, raw }) + // The open call closes its input without a call; the call that never got a name was never started. + expect(truncated.events.map((event) => event.type)).toEqual([ + "step-start", + "tool-input-start", + "tool-input-delta", + "tool-input-end", + "step-finish", + "finish", + ]) + } }), ) @@ -719,13 +732,34 @@ describe("Mistral Chat", () => { ) expect(missingFinish.message).toContain("without finish_reason") - const lateContent = yield* LLMClient.generate(request).pipe( + const lateText = yield* LLMClient.generate(request).pipe( Effect.provide( - fixedResponse(sseEvents(chunk({}, "stop"), chunk({ content: [{ type: "text", text: "late" }] }))), + fixedResponse( + sseEvents( + chunk({ content: "Hello" }), + chunk({}, "stop"), + chunk({ content: [{ type: "text", text: " late" }] }), + chunk({ content: [{ type: "thinking", thinking: "late thought" }] }), + ), + ), + ), + ) + expect(lateText.text).toBe("Hello late") + expect(lateText.reasoning).toBe("late thought") + expect(lateText.finishReason).toEqual({ normalized: "stop", raw: "stop" }) + + const lateTool = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + chunk({}, "stop"), + chunk({ tool_calls: [{ index: 0, id: "Ab12Cd34E", function: { name: "lookup", arguments: "{}" } }] }), + ), + ), ), Effect.flip, ) - expect(lateContent.message).toContain("content after the finish reason") + expect(lateTool.message).toContain("content after the finish reason") }), ) diff --git a/packages/ai/test/provider/openai-chat.test.ts b/packages/ai/test/provider/openai-chat.test.ts index 6c07ad4c57b..343c14665d8 100644 --- a/packages/ai/test/provider/openai-chat.test.ts +++ b/packages/ai/test/provider/openai-chat.test.ts @@ -2039,7 +2039,7 @@ describe("OpenAI Chat route", () => { }), ) - it.effect("does not finalize streamed tool calls when content is filtered", () => + it.effect("closes streamed tool input without a call when content is filtered", () => Effect.gen(function* () { const body = sseEvents( deltaChunk({ @@ -2068,6 +2068,7 @@ describe("OpenAI Chat route", () => { text: '{"query":"weather"', input: { query: "weather" }, }, + { type: "tool-input-end", id: "call_1", name: "lookup", providerMetadata: undefined }, { type: "step-finish", index: 0, @@ -2094,7 +2095,7 @@ describe("OpenAI Chat route", () => { }), ) - it.effect("does not finalize streamed tool calls when output is truncated", () => + it.effect("closes streamed tool input without a call when output is truncated", () => Effect.gen(function* () { const body = sseEvents( deltaChunk({ @@ -2123,6 +2124,7 @@ describe("OpenAI Chat route", () => { text: '{"query":"weather"}', input: { query: "weather" }, }, + { type: "tool-input-end", id: "call_1", name: "lookup", providerMetadata: undefined }, { type: "step-finish", index: 0, @@ -2149,6 +2151,33 @@ describe("OpenAI Chat route", () => { }), ) + it.effect("opens a buffered tool call once under its stream index", () => + Effect.gen(function* () { + const body = sseEvents( + deltaChunk({ tool_calls: [{ index: 0, id: "call_1", function: { arguments: '{"query"' } }] }), + deltaChunk({ tool_calls: [{ id: "call_1", function: { name: "lookup", arguments: ':"weather"' } }] }), + deltaChunk({ tool_calls: [{ index: 0, function: { name: "lookup", arguments: "}" } }] }), + deltaChunk({}, "tool_calls"), + ) + const response = yield* LLMClient.generate({ + ...request, + tools: [ToolDefinition.make({ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } })], + }).pipe(Effect.provide(fixedResponse(body))) + + expect(response.events.map((event) => event.type)).toEqual([ + "step-start", + "tool-input-start", + "tool-input-delta", + "tool-input-delta", + "tool-input-end", + "tool-call", + "step-finish", + "finish", + ]) + expect(response.toolCalls).toMatchObject([{ id: "call_1", name: "lookup", input: { query: "weather" } }]) + }), + ) + it.effect("maps unknown finish reasons to unknown and keeps the raw value", () => Effect.gen(function* () { const response = yield* LLMClient.generate(request).pipe( diff --git a/packages/ai/test/tool-stream.test.ts b/packages/ai/test/tool-stream.test.ts index 4389ff0b818..4e1bc8a636c 100644 --- a/packages/ai/test/tool-stream.test.ts +++ b/packages/ai/test/tool-stream.test.ts @@ -249,4 +249,40 @@ describe("ToolStream", () => { }) }), ) + + it.effect("buffers a chat slot until id and name arrive, then opens it once", () => + Effect.gen(function* () { + const buffered = ToolStream.slotDelta(new Map(), 0, { id: "call_1", text: '{"query"', freeform: false }) + const opened = ToolStream.slotDelta(buffered.slots, 0, { name: "lookup", text: ':"weather"', freeform: false }) + const repeated = ToolStream.slotDelta(opened.slots, 0, { + id: "call_1", + name: "lookup", + text: "", + freeform: false, + }) + const closed = ToolStream.slotDelta(repeated.slots, 0, { text: "}", freeform: false }) + + expect(buffered.events).toEqual([]) + expect([...buffered.slots.values()].map((slot) => slot.status)).toEqual(["buffering"]) + expect(opened.events.map((event) => event.type)).toEqual(["tool-input-start", "tool-input-delta"]) + expect(repeated.events).toEqual([]) + expect([...closed.slots.values()].map((slot) => slot.status)).toEqual(["open"]) + expect(yield* ToolStream.finishSlots(ADAPTER, closed.slots, "stop")).toEqual([ + { type: "tool-input-end", id: "call_1", name: "lookup" }, + { type: "tool-call", id: "call_1", name: "lookup", input: { query: "weather" } }, + ]) + }), + ) + + it.effect("ends open chat slots without a call on length or content filter", () => + Effect.gen(function* () { + const open = ToolStream.slotDelta(new Map(), 0, { id: "call_1", name: "lookup", text: '{"q', freeform: false }) + const slots = ToolStream.slotDelta(open.slots, 1, { id: "call_2", text: "{}", freeform: false }).slots + + for (const reason of ["length", "content-filter"] as const) + expect(yield* ToolStream.finishSlots(ADAPTER, slots, reason)).toEqual([ + { type: "tool-input-end", id: "call_1", name: "lookup" }, + ]) + }), + ) })