mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
fix(ai): track chat tool calls as explicit slots
This commit is contained in:
@@ -12,7 +12,6 @@ import {
|
||||
ProviderInternalError,
|
||||
UnknownProviderError,
|
||||
Usage,
|
||||
type FinishReasonDetails,
|
||||
type LLMRequest,
|
||||
type MediaPart,
|
||||
type ToolCallPart,
|
||||
@@ -413,11 +412,6 @@ export const fromRequest = Effect.fn("MistralChat.fromRequest")(function* (reque
|
||||
})
|
||||
|
||||
type ToolKey = string | number
|
||||
interface PendingTool {
|
||||
readonly id: string
|
||||
readonly name?: string
|
||||
readonly input: string
|
||||
}
|
||||
|
||||
interface ActiveContent {
|
||||
readonly type: "text" | "reasoning"
|
||||
@@ -427,18 +421,15 @@ interface ActiveContent {
|
||||
}
|
||||
|
||||
export interface ParserState {
|
||||
readonly tools: ToolStream.State<ToolKey>
|
||||
readonly pendingTools: Partial<Record<ToolKey, PendingTool>>
|
||||
readonly toolCalls: ToolStream.Slots<ToolKey>
|
||||
readonly toolIDs: ReadonlyMap<string, string>
|
||||
readonly usedToolIDs: ReadonlySet<string>
|
||||
readonly completedTools: ReadonlyArray<LLMEvent>
|
||||
readonly latestToolKey?: ToolKey
|
||||
readonly generatedTools: number
|
||||
readonly lifecycle: Lifecycle.State
|
||||
readonly active?: ActiveContent
|
||||
readonly nextContent: number
|
||||
readonly usage?: Usage
|
||||
readonly finishReason?: FinishReasonDetails
|
||||
}
|
||||
|
||||
const mapUsage = (usage: MistralEvent["usage"]): Usage | undefined => {
|
||||
@@ -580,13 +571,10 @@ const toolText = (tool: MistralToolDelta) => {
|
||||
return value === null || value === undefined ? "" : ProviderShared.encodeJson(value)
|
||||
}
|
||||
|
||||
const appendTools = Effect.fnUntraced(function* (
|
||||
initial: ParserState,
|
||||
events: LLMEvent[],
|
||||
deltas: ReadonlyArray<MistralToolDelta>,
|
||||
) {
|
||||
if (deltas.length === 0) return initial
|
||||
const appendTools = (initial: ParserState, events: LLMEvent[], deltas: ReadonlyArray<MistralToolDelta>) => {
|
||||
if (deltas.length === 0 || initial.toolCalls.status === "finished") return initial
|
||||
let state = closeActive(initial, events)
|
||||
let slots = initial.toolCalls.slots
|
||||
for (const [position, delta] of deltas.entries()) {
|
||||
const wireID = delta.id?.trim() || undefined
|
||||
const providedID = wireID === "null" ? undefined : wireID
|
||||
@@ -597,54 +585,32 @@ const appendTools = Effect.fnUntraced(function* (
|
||||
: deltas.length > 1
|
||||
? `position:${position}`
|
||||
: (state.latestToolKey ?? `missing:${state.generatedTools}`))
|
||||
const existing = state.tools[key]
|
||||
const pending = state.pendingTools[key]
|
||||
const source = providedID ?? `generated:${String(key)}`
|
||||
const normalized =
|
||||
existing || pending ? { id: existing?.id ?? pending?.id ?? "", state } : normalizeStreamToolID(state, source)
|
||||
state = normalized.state
|
||||
const name = existing?.name ?? pending?.name ?? (delta.function?.name?.trim() || undefined)
|
||||
const text = `${pending?.input ?? ""}${toolText(delta)}`
|
||||
if (!name) {
|
||||
state = {
|
||||
...state,
|
||||
pendingTools: { ...state.pendingTools, [key]: { id: normalized.id, input: text } },
|
||||
latestToolKey: key,
|
||||
generatedTools: state.generatedTools + (!providedID && !pending ? 1 : 0),
|
||||
}
|
||||
continue
|
||||
}
|
||||
const result = ToolStream.appendOrStart(
|
||||
ADAPTER,
|
||||
state.tools,
|
||||
key,
|
||||
{ id: normalized.id, name, text },
|
||||
"Mistral Chat tool call delta is missing a name",
|
||||
)
|
||||
if (ToolStream.isError(result)) return yield* result
|
||||
if (result.events.length > 0) state = { ...state, lifecycle: Lifecycle.stepStart(state.lifecycle, events) }
|
||||
events.push(...result.events)
|
||||
const pendingTools = { ...state.pendingTools }
|
||||
delete pendingTools[key]
|
||||
const slot = slots.get(key)
|
||||
const normalized = slot
|
||||
? { id: ToolStream.slotID(slot), state }
|
||||
: normalizeStreamToolID(state, providedID ?? `generated:${String(key)}`)
|
||||
const next = ToolStream.slotDelta(slots, key, {
|
||||
id: normalized.id,
|
||||
name: delta.function?.name?.trim() || undefined,
|
||||
text: toolText(delta),
|
||||
freeform: false,
|
||||
})
|
||||
slots = next.slots
|
||||
state = {
|
||||
...state,
|
||||
tools: result.tools,
|
||||
pendingTools,
|
||||
...normalized.state,
|
||||
lifecycle: next.events.length > 0 ? Lifecycle.stepStart(state.lifecycle, events) : state.lifecycle,
|
||||
latestToolKey: key,
|
||||
generatedTools: state.generatedTools + (!providedID && !existing && !pending ? 1 : 0),
|
||||
generatedTools: state.generatedTools + (!providedID && !slot ? 1 : 0),
|
||||
}
|
||||
events.push(...next.events)
|
||||
}
|
||||
return state
|
||||
})
|
||||
return { ...state, toolCalls: { status: "streaming" as const, slots } }
|
||||
}
|
||||
|
||||
const hasLateContent = (event: MistralEvent) => {
|
||||
const delta = event.choices?.[0]?.delta
|
||||
if (typeof delta?.content === "string" && delta.content.length > 0) return true
|
||||
if (Array.isArray(delta?.content) && delta.content.length > 0) return true
|
||||
return (delta?.tool_calls ?? []).some(
|
||||
const hasLateToolCall = (event: MistralEvent) =>
|
||||
(event.choices?.[0]?.delta?.tool_calls ?? []).some(
|
||||
(tool) => Boolean(tool.id) || Boolean(tool.function?.name) || tool.function?.arguments !== undefined,
|
||||
)
|
||||
}
|
||||
|
||||
const step = Effect.fnUntraced(function* (state: ParserState, event: MistralEvent) {
|
||||
if (event.error) {
|
||||
@@ -659,19 +625,18 @@ const step = Effect.fnUntraced(function* (state: ParserState, event: MistralEven
|
||||
}
|
||||
const events: LLMEvent[] = []
|
||||
const usage = mapUsage(event.usage) ?? state.usage
|
||||
if (state.finishReason) {
|
||||
if (hasLateContent(event))
|
||||
return yield* ProviderShared.eventError(
|
||||
ADAPTER,
|
||||
"Mistral Chat received content after the finish reason",
|
||||
ProviderShared.encodeJson(event),
|
||||
)
|
||||
return [{ ...state, usage }, events] as const
|
||||
}
|
||||
// Like OpenAI Chat, text and reasoning after the finish reason still belong to the response; tool calls do not.
|
||||
if (state.toolCalls.status === "finished" && hasLateToolCall(event))
|
||||
return yield* ProviderShared.eventError(
|
||||
ADAPTER,
|
||||
"Mistral Chat received content after the finish reason",
|
||||
ProviderShared.encodeJson(event),
|
||||
)
|
||||
const choice = event.choices?.[0]
|
||||
const withContent = choice?.delta?.content == null ? state : appendContent(state, events, choice.delta.content)
|
||||
const withTools = yield* appendTools(withContent, events, choice?.delta?.tool_calls ?? [])
|
||||
if (!choice?.finish_reason) return [{ ...withTools, usage }, events] as const
|
||||
const withTools = appendTools(withContent, events, choice?.delta?.tool_calls ?? [])
|
||||
if (!choice?.finish_reason || withTools.toolCalls.status === "finished")
|
||||
return [{ ...withTools, usage }, events] as const
|
||||
|
||||
const finishReason = {
|
||||
normalized: mapFinishReason(choice.finish_reason),
|
||||
@@ -688,30 +653,21 @@ const step = Effect.fnUntraced(function* (state: ParserState, event: MistralEven
|
||||
})
|
||||
}
|
||||
const incomplete = finishReason.normalized === "length" || finishReason.normalized === "content-filter"
|
||||
if (!incomplete && Object.keys(withTools.pendingTools).length > 0)
|
||||
if (!incomplete && ToolStream.hasBufferedSlot(withTools.toolCalls.slots))
|
||||
return yield* ProviderShared.eventError(
|
||||
ADAPTER,
|
||||
"Mistral Chat tool call delta is missing a name",
|
||||
ProviderShared.encodeJson(event),
|
||||
)
|
||||
const finished =
|
||||
!incomplete && Object.keys(withTools.tools).length > 0
|
||||
? yield* ToolStream.finishAll(ADAPTER, withTools.tools)
|
||||
: undefined
|
||||
const calls = yield* ToolStream.finishSlots(ADAPTER, withTools.toolCalls.slots, finishReason.normalized)
|
||||
return [
|
||||
{
|
||||
...withTools,
|
||||
tools: finished?.tools ?? withTools.tools,
|
||||
completedTools: finished?.events ?? withTools.completedTools,
|
||||
usage,
|
||||
finishReason,
|
||||
},
|
||||
{ ...withTools, toolCalls: { status: "finished" as const, reason: finishReason, events: calls }, usage },
|
||||
events,
|
||||
] as const
|
||||
})
|
||||
|
||||
const finishEvents = Effect.fnUntraced(function* (state: ParserState) {
|
||||
if (!state.finishReason)
|
||||
if (state.toolCalls.status === "streaming")
|
||||
return yield* new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
message: "Mistral Chat stream ended without finish_reason",
|
||||
@@ -721,12 +677,13 @@ const finishEvents = Effect.fnUntraced(function* (state: ParserState) {
|
||||
})
|
||||
const events: LLMEvent[] = []
|
||||
const closed = closeActive(state, events)
|
||||
const lifecycle = closed.completedTools.length > 0 ? Lifecycle.stepStart(closed.lifecycle, events) : closed.lifecycle
|
||||
events.push(...closed.completedTools)
|
||||
const calls = state.toolCalls.events
|
||||
const lifecycle = calls.length > 0 ? Lifecycle.stepStart(closed.lifecycle, events) : closed.lifecycle
|
||||
events.push(...calls)
|
||||
const reason =
|
||||
state.finishReason.normalized === "stop" && closed.completedTools.some(LLMEvent.is.toolCall)
|
||||
? { ...state.finishReason, normalized: "tool-calls" as const }
|
||||
: state.finishReason
|
||||
state.toolCalls.reason.normalized === "stop" && calls.some(LLMEvent.is.toolCall)
|
||||
? { ...state.toolCalls.reason, normalized: "tool-calls" as const }
|
||||
: state.toolCalls.reason
|
||||
Lifecycle.finish(lifecycle, events, { reason, usage: closed.usage })
|
||||
return events
|
||||
})
|
||||
@@ -737,11 +694,9 @@ export const protocol = Protocol.make({
|
||||
stream: {
|
||||
event: MistralStreamEvent,
|
||||
initial: (): ParserState => ({
|
||||
tools: ToolStream.empty<ToolKey>(),
|
||||
pendingTools: {},
|
||||
toolCalls: { status: "streaming", slots: new Map() },
|
||||
toolIDs: new Map(),
|
||||
usedToolIDs: new Set(),
|
||||
completedTools: [],
|
||||
generatedTools: 0,
|
||||
lifecycle: Lifecycle.initial(),
|
||||
nextContent: 0,
|
||||
|
||||
@@ -8,13 +8,11 @@ import { HttpTransport } from "../route/transport/index.js"
|
||||
import { Protocol } from "../route/protocol.js"
|
||||
import {
|
||||
AIError,
|
||||
AIErrorReason,
|
||||
InvalidProviderOutputError,
|
||||
LLMEvent,
|
||||
ProviderInternalError,
|
||||
UnknownProviderError,
|
||||
Usage,
|
||||
type FinishReasonDetails,
|
||||
type AssistantMessage,
|
||||
type CacheHint,
|
||||
type LLMRequest,
|
||||
@@ -345,23 +343,11 @@ export type OpenAIChatEvent = Schema.Schema.Type<typeof OpenAIChatEvent>
|
||||
const DONE = "[DONE]" as const
|
||||
const OpenAIChatStreamEvent = Schema.Union([Schema.Literal(DONE), Protocol.jsonEvent(OpenAIChatEvent)])
|
||||
|
||||
interface PendingToolDelta {
|
||||
readonly id?: string
|
||||
readonly name?: string
|
||||
readonly input: string
|
||||
readonly extraContent?: Schema.Schema.Type<typeof ExtraContent>
|
||||
}
|
||||
|
||||
export interface ParserState {
|
||||
readonly providerMetadataKey: string
|
||||
readonly tools: ToolStream.State<number>
|
||||
readonly custom: ReadonlyMap<string, OpenAIFreeform.Custom>
|
||||
/** Stream indexes of custom tool calls, whose JSON fragment closes when the call finishes. */
|
||||
readonly customCalls: ReadonlySet<number>
|
||||
readonly pendingTools: Partial<Record<number, PendingToolDelta>>
|
||||
readonly toolCallEvents: ReadonlyArray<LLMEvent>
|
||||
readonly toolCalls: ToolStream.Slots<number>
|
||||
readonly usage?: Usage
|
||||
readonly finishReason?: FinishReasonDetails
|
||||
readonly lifecycle: Lifecycle.State
|
||||
readonly reasoningField?: string
|
||||
/** A scalar reasoning field (`reasoning_content`, ...) has carried text in this stream. */
|
||||
@@ -370,7 +356,6 @@ export interface ParserState {
|
||||
readonly reasoningDetailsObserved: boolean
|
||||
readonly reasoningEmitted: boolean
|
||||
readonly latestToolIndex?: number
|
||||
readonly nextToolIndex: number
|
||||
readonly requireFinishReason: boolean
|
||||
}
|
||||
|
||||
@@ -973,15 +958,48 @@ const mapUsage = (usage: OpenAIChatEvent["usage"], providerMetadataKey: string):
|
||||
})
|
||||
}
|
||||
|
||||
const toolIndexByID = (
|
||||
tools: ParserState["tools"],
|
||||
pendingTools: ParserState["pendingTools"],
|
||||
id: string | undefined,
|
||||
) => {
|
||||
if (!id) return undefined
|
||||
const entry = Object.entries({ ...pendingTools, ...tools }).find(([, tool]) => tool?.id === id)
|
||||
return entry ? Number(entry[0]) : undefined
|
||||
}
|
||||
// Compatible providers may omit indexes. Prefer durable identity, then use
|
||||
// batch position for parallel deltas or the latest call for sparse chunks.
|
||||
const appendTools = (
|
||||
state: ParserState,
|
||||
slots: ReadonlyMap<number, ToolStream.Slot>,
|
||||
deltas: ReadonlyArray<OpenAIChatToolCallDelta>,
|
||||
) =>
|
||||
deltas.reduce(
|
||||
(result, tool, position) => {
|
||||
const matched = tool.id
|
||||
? [...result.slots].find(([, slot]) => ToolStream.slotID(slot) === tool.id)?.[0]
|
||||
: undefined
|
||||
const fallback = deltas.length > 1 ? position : (result.latestToolIndex ?? position)
|
||||
const fallbackID = ToolStream.slotID(result.slots.get(fallback))
|
||||
const index =
|
||||
tool.index ??
|
||||
matched ??
|
||||
(tool.id && fallbackID && fallbackID !== tool.id ? Math.max(-1, ...result.slots.keys()) + 1 : fallback)
|
||||
const slot = result.slots.get(index)
|
||||
const name = tool.function?.name || tool.custom?.name || undefined
|
||||
// Custom calls stream raw text; re-encode it as JSON arguments for the call's field.
|
||||
const raw = tool.custom?.input ?? ""
|
||||
const text = tool.custom
|
||||
? slot?.freeform
|
||||
? OpenAIFreeform.streamDelta(raw)
|
||||
: OpenAIFreeform.streamStart(
|
||||
state.custom.get((slot?.status === "open" ? slot.tool.name : slot?.name) ?? name ?? ""),
|
||||
raw,
|
||||
)
|
||||
: (tool.function?.arguments ?? "")
|
||||
const extraContent = decodeExtraContent(tool.extra_content)
|
||||
const next = ToolStream.slotDelta(result.slots, index, {
|
||||
id: tool.id || undefined,
|
||||
name,
|
||||
text,
|
||||
freeform: Boolean(tool.custom),
|
||||
providerMetadata: extraContent && { [state.providerMetadataKey]: { extraContent } },
|
||||
})
|
||||
return { slots: next.slots, latestToolIndex: index, events: [...result.events, ...next.events] }
|
||||
},
|
||||
{ slots, latestToolIndex: state.latestToolIndex, events: [] as ReadonlyArray<LLMEvent> },
|
||||
)
|
||||
|
||||
const reasoningDelta = (
|
||||
delta: Schema.Schema.Type<typeof OpenAIChatDelta> | null | undefined,
|
||||
@@ -1110,20 +1128,14 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
||||
normalized: yield* mapFinishReason(event, rawFinishReason),
|
||||
raw: choice?.native_finish_reason ?? rawFinishReason,
|
||||
}
|
||||
: state.finishReason
|
||||
: undefined
|
||||
const delta = choice?.delta
|
||||
const toolDeltas = delta?.tool_calls ?? []
|
||||
let tools = state.tools
|
||||
let pendingTools = state.pendingTools
|
||||
let customCalls = state.customCalls
|
||||
let latestToolIndex = state.latestToolIndex
|
||||
let nextToolIndex = state.nextToolIndex
|
||||
|
||||
let lifecycle = state.lifecycle
|
||||
|
||||
const reasoning = reasoningDelta(delta, state.reasoningField)
|
||||
if (
|
||||
state.finishReason !== undefined &&
|
||||
state.toolCalls.status === "finished" &&
|
||||
toolDeltas.some(
|
||||
(tool) =>
|
||||
Boolean(tool.id) ||
|
||||
@@ -1165,74 +1177,17 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
||||
|
||||
if (delta?.refusal) lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.refusal)
|
||||
|
||||
// Compatible providers may omit indexes. Prefer durable identity, then use
|
||||
// batch position for parallel deltas or the latest call for sparse chunks.
|
||||
for (const [position, tool] of toolDeltas.entries()) {
|
||||
const matched = toolIndexByID(tools, pendingTools, tool.id || undefined)
|
||||
const fallback = toolDeltas.length > 1 ? position : (latestToolIndex ?? position)
|
||||
const fallbackTool = tools[fallback] ?? pendingTools[fallback]
|
||||
const index =
|
||||
tool.index ?? matched ?? (tool.id && fallbackTool?.id && fallbackTool.id !== tool.id ? nextToolIndex : fallback)
|
||||
const current = tools[index]
|
||||
const pending = pendingTools[index]
|
||||
const id = current?.id ?? pending?.id ?? (tool.id || undefined)
|
||||
const name = current?.name ?? pending?.name ?? (tool.function?.name || tool.custom?.name || undefined)
|
||||
// Custom calls stream raw text; re-encode it as JSON arguments for the call's field.
|
||||
const opening = Boolean(tool.custom) && !customCalls.has(index)
|
||||
if (opening) customCalls = new Set([...customCalls, index])
|
||||
const raw = tool.custom?.input ?? ""
|
||||
const fragment = tool.custom
|
||||
? opening
|
||||
? OpenAIFreeform.streamStart(state.custom.get(name ?? ""), raw)
|
||||
: OpenAIFreeform.streamDelta(raw)
|
||||
: (tool.function?.arguments ?? "")
|
||||
const text = `${pending?.input ?? ""}${fragment}`
|
||||
const extraContent = pending?.extraContent ?? decodeExtraContent(tool.extra_content)
|
||||
latestToolIndex = index
|
||||
nextToolIndex = Math.max(nextToolIndex, index + 1)
|
||||
if (!current && (!id || !name)) {
|
||||
pendingTools = {
|
||||
...pendingTools,
|
||||
[index]: { id: id || undefined, name: name || undefined, input: text, extraContent },
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (pending) {
|
||||
pendingTools = { ...pendingTools }
|
||||
delete pendingTools[index]
|
||||
}
|
||||
const result = ToolStream.appendOrStart(
|
||||
ADAPTER,
|
||||
tools,
|
||||
index,
|
||||
{
|
||||
id: id || undefined,
|
||||
name: name || undefined,
|
||||
text,
|
||||
providerMetadata: extraContent && { [state.providerMetadataKey]: { extraContent } },
|
||||
},
|
||||
"OpenAI Chat tool call delta is missing id or name",
|
||||
)
|
||||
if (ToolStream.isError(result))
|
||||
return yield* new AIError({
|
||||
reason: AIErrorReason.make({
|
||||
...result.reason,
|
||||
message: result.message,
|
||||
cause: result.reason.cause,
|
||||
body: ProviderShared.encodeJson(event),
|
||||
}),
|
||||
})
|
||||
tools = result.tools
|
||||
if (result.events.length) lifecycle = Lifecycle.stepStart(lifecycle, events)
|
||||
events.push(...result.events)
|
||||
}
|
||||
const streamed =
|
||||
state.toolCalls.status === "streaming" ? appendTools(state, state.toolCalls.slots, toolDeltas) : undefined
|
||||
if (streamed?.events.length) lifecycle = Lifecycle.stepStart(lifecycle, events)
|
||||
events.push(...(streamed?.events ?? []))
|
||||
|
||||
const incompleteTools = finishReason?.normalized === "content-filter" || finishReason?.normalized === "length"
|
||||
if (
|
||||
streamed !== undefined &&
|
||||
finishReason !== undefined &&
|
||||
!incompleteTools &&
|
||||
state.finishReason === undefined &&
|
||||
Object.keys(pendingTools).length
|
||||
finishReason.normalized !== "content-filter" &&
|
||||
finishReason.normalized !== "length" &&
|
||||
ToolStream.hasBufferedSlot(streamed.slots)
|
||||
)
|
||||
return yield* ProviderShared.eventError(
|
||||
ADAPTER,
|
||||
@@ -1240,48 +1195,40 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
|
||||
ProviderShared.encodeJson(event),
|
||||
)
|
||||
|
||||
// Filtering or truncation terminates the response without confirming pending tool calls.
|
||||
const finished =
|
||||
finishReason !== undefined &&
|
||||
!incompleteTools &&
|
||||
state.finishReason === undefined &&
|
||||
Object.keys(tools).length > 0
|
||||
? yield* finishTools(tools, customCalls)
|
||||
: undefined
|
||||
const toolCalls: ToolStream.Slots<number> =
|
||||
streamed === undefined
|
||||
? state.toolCalls
|
||||
: finishReason === undefined
|
||||
? { status: "streaming", slots: streamed.slots }
|
||||
: {
|
||||
status: "finished",
|
||||
reason: finishReason,
|
||||
events: yield* ToolStream.finishSlots(
|
||||
ADAPTER,
|
||||
streamed.slots,
|
||||
finishReason.normalized,
|
||||
OpenAIFreeform.streamEnd,
|
||||
),
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
providerMetadataKey: state.providerMetadataKey,
|
||||
tools: finished?.tools ?? tools,
|
||||
custom: state.custom,
|
||||
customCalls,
|
||||
pendingTools,
|
||||
toolCallEvents: finished?.events ?? state.toolCallEvents,
|
||||
...state,
|
||||
toolCalls,
|
||||
usage,
|
||||
finishReason: state.finishReason ?? finishReason,
|
||||
lifecycle,
|
||||
reasoningField,
|
||||
reasoningTextObserved,
|
||||
reasoningDetails: state.reasoningDetails,
|
||||
reasoningDetailsObserved,
|
||||
reasoningEmitted,
|
||||
latestToolIndex,
|
||||
nextToolIndex,
|
||||
requireFinishReason: state.requireFinishReason,
|
||||
latestToolIndex: streamed?.latestToolIndex ?? state.latestToolIndex,
|
||||
},
|
||||
events,
|
||||
] as const
|
||||
})
|
||||
|
||||
// Custom calls close their JSON fragment before every pending call finishes.
|
||||
const finishTools = Effect.fnUntraced(function* (tools: ParserState["tools"], customCalls: ReadonlySet<number>) {
|
||||
const closed = OpenAIFreeform.close(tools, customCalls)
|
||||
const finished = yield* ToolStream.finishAll(ADAPTER, closed.tools)
|
||||
return { tools: finished.tools, events: [...closed.events, ...finished.events] }
|
||||
})
|
||||
|
||||
export const finishEvents = Effect.fnUntraced(function* (state: ParserState) {
|
||||
if (state.finishReason === undefined && state.requireFinishReason)
|
||||
if (state.toolCalls.status === "streaming" && state.requireFinishReason)
|
||||
return yield* new AIError({
|
||||
reason: new InvalidProviderOutputError({
|
||||
message: "OpenAI Chat stream ended without finish_reason",
|
||||
@@ -1291,15 +1238,15 @@ export const finishEvents = Effect.fnUntraced(function* (state: ParserState) {
|
||||
})
|
||||
const events: LLMEvent[] = []
|
||||
const toolCallEvents =
|
||||
state.finishReason === undefined && Object.keys(state.tools).length > 0
|
||||
? (yield* finishTools(state.tools, state.customCalls)).events
|
||||
: state.toolCallEvents
|
||||
state.toolCalls.status === "finished"
|
||||
? state.toolCalls.events
|
||||
: yield* ToolStream.finishSlots(ADAPTER, state.toolCalls.slots, undefined, OpenAIFreeform.streamEnd)
|
||||
const hasToolCalls = toolCallEvents.length > 0
|
||||
const reason = state.finishReason
|
||||
const finishReason = state.toolCalls.status === "finished" ? state.toolCalls.reason : undefined
|
||||
const reason = finishReason
|
||||
? {
|
||||
...state.finishReason,
|
||||
normalized:
|
||||
state.finishReason.normalized === "stop" && hasToolCalls ? "tool-calls" : state.finishReason.normalized,
|
||||
...finishReason,
|
||||
normalized: finishReason.normalized === "stop" && hasToolCalls ? "tool-calls" : finishReason.normalized,
|
||||
}
|
||||
: { normalized: hasToolCalls ? ("tool-calls" as const) : ("stop" as const) }
|
||||
// Snapshot details at publish time so the emitted event never observes later
|
||||
@@ -1342,20 +1289,16 @@ export const protocol = Protocol.make({
|
||||
},
|
||||
stream: {
|
||||
event: OpenAIChatStreamEvent,
|
||||
initial: (request) => ({
|
||||
initial: (request): ParserState => ({
|
||||
providerMetadataKey: request.model.route.providerMetadataKey ?? request.model.provider,
|
||||
tools: ToolStream.empty<number>(),
|
||||
custom: OpenAIFreeform.tools(request),
|
||||
customCalls: new Set<number>(),
|
||||
pendingTools: {},
|
||||
toolCallEvents: [],
|
||||
toolCalls: { status: "streaming", slots: new Map() },
|
||||
lifecycle: Lifecycle.initial(),
|
||||
reasoningField: request.model.compatibility?.reasoningField,
|
||||
reasoningTextObserved: false,
|
||||
reasoningDetails: [],
|
||||
reasoningDetailsObserved: false,
|
||||
reasoningEmitted: false,
|
||||
nextToolIndex: 0,
|
||||
requireFinishReason: request.model.compatibility?.requireFinishReason ?? true,
|
||||
}),
|
||||
step: (state: ParserState, event) => (event === DONE ? Effect.succeed([state, []] as const) : step(state, event)),
|
||||
|
||||
@@ -57,7 +57,7 @@ const field = (custom: Custom | undefined) => custom?.field ?? "input"
|
||||
export const streamStart = (custom: Custom | undefined, raw = "") =>
|
||||
`{${encodeJson(field(custom))}:"${streamDelta(raw)}`
|
||||
export const streamDelta = (raw: string) => encodeJson(raw).slice(1, -1)
|
||||
const STREAM_END = '"}'
|
||||
export const streamEnd = '"}'
|
||||
|
||||
const decodeArguments = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)))
|
||||
|
||||
@@ -73,9 +73,9 @@ export const close = <K extends string | number>(
|
||||
[...keys].reduce<{ readonly tools: ToolStream.State<K>; readonly events: ReadonlyArray<LLMEvent> }>(
|
||||
(result, key) => {
|
||||
const pending = result.tools[key]
|
||||
const streamed = pending && Option.getOrUndefined(decodeArguments(`${pending.input}${STREAM_END}`))
|
||||
const streamed = pending && Option.getOrUndefined(decodeArguments(`${pending.input}${streamEnd}`))
|
||||
if (!pending || (final !== undefined && streamed?.[field(final.custom)] !== final.raw)) return result
|
||||
const appended = ToolStream.append(result.tools, key, STREAM_END)
|
||||
const appended = ToolStream.append(result.tools, key, streamEnd)
|
||||
return appended ? { tools: appended.tools, events: [...result.events, ...appended.events] } : result
|
||||
},
|
||||
{ tools, events: [] },
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
import { Effect, Option } from "effect"
|
||||
import { AIError, LLMEvent, type ProviderMetadata, type ToolCall, type ToolInputError } from "../../schema/index.js"
|
||||
import {
|
||||
AIError,
|
||||
type FinishReasonDetails,
|
||||
LLMEvent,
|
||||
type ProviderMetadata,
|
||||
type ToolCall,
|
||||
type ToolInputError,
|
||||
} from "../../schema/index.js"
|
||||
import { eventError, parseToolInput, type ToolAccumulator } from "../shared.js"
|
||||
import { parse } from "./partial-json.js"
|
||||
|
||||
@@ -98,13 +105,16 @@ const toolCall = (route: string, tool: PendingTool, inputOverride?: string) => {
|
||||
)
|
||||
}
|
||||
|
||||
const finishEvents = (tool: PendingTool, event: ToolCall | ToolInputError): ReadonlyArray<LLMEvent> => [
|
||||
const inputEnd = (tool: PendingTool) =>
|
||||
LLMEvent.toolInputEnd({
|
||||
id: tool.id,
|
||||
name: tool.name,
|
||||
namespace: tool.namespace,
|
||||
providerMetadata: tool.providerMetadata,
|
||||
}),
|
||||
})
|
||||
|
||||
const finishEvents = (tool: PendingTool, event: ToolCall | ToolInputError): ReadonlyArray<LLMEvent> => [
|
||||
inputEnd(tool),
|
||||
event,
|
||||
]
|
||||
|
||||
@@ -237,4 +247,101 @@ export const finishAll = <K extends StreamKey>(route: string, tools: State<K>) =
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* One Chat tool call under its stream key. Chat providers may split the call id
|
||||
* and name across deltas, so a slot buffers argument text until both arrive and
|
||||
* only then opens with `tool-input-start`. Freeform calls stream raw text that
|
||||
* the parser re-encodes as JSON arguments.
|
||||
*/
|
||||
export type Slot =
|
||||
| {
|
||||
readonly status: "buffering"
|
||||
readonly freeform: boolean
|
||||
readonly id?: string
|
||||
readonly name?: string
|
||||
readonly input: string
|
||||
readonly providerMetadata?: ProviderMetadata
|
||||
}
|
||||
| { readonly status: "open"; readonly freeform: boolean; readonly tool: PendingTool }
|
||||
|
||||
/**
|
||||
* Chat tool calls stream until the first finish reason, which settles every
|
||||
* slot. The settled events wait for the end of the stream so late text and
|
||||
* reasoning still precede them.
|
||||
*/
|
||||
export type Slots<K extends StreamKey> =
|
||||
| { readonly status: "streaming"; readonly slots: ReadonlyMap<K, Slot> }
|
||||
| { readonly status: "finished"; readonly reason: FinishReasonDetails; readonly events: ReadonlyArray<LLMEvent> }
|
||||
|
||||
export const slotID = (slot: Slot | undefined) => (slot?.status === "open" ? slot.tool.id : slot?.id)
|
||||
|
||||
/** Add one delta to a slot, opening it once both id and name are known. */
|
||||
export const slotDelta = <K extends StreamKey>(
|
||||
slots: ReadonlyMap<K, Slot>,
|
||||
key: K,
|
||||
delta: {
|
||||
readonly id?: string
|
||||
readonly name?: string
|
||||
readonly text: string
|
||||
readonly freeform: boolean
|
||||
readonly providerMetadata?: ProviderMetadata
|
||||
},
|
||||
) => {
|
||||
const slot = slots.get(key)
|
||||
if (slot?.status === "open") {
|
||||
if (delta.text.length === 0) return { slots, events: [] }
|
||||
const tool = {
|
||||
...slot.tool,
|
||||
input: `${slot.tool.input}${delta.text}`,
|
||||
providerMetadata: slot.tool.providerMetadata ?? delta.providerMetadata,
|
||||
}
|
||||
return { slots: new Map(slots).set(key, { ...slot, tool }), events: [inputDelta(tool, delta.text)] }
|
||||
}
|
||||
const id = slot?.id ?? delta.id
|
||||
const name = slot?.name ?? delta.name
|
||||
const input = `${slot?.input ?? ""}${delta.text}`
|
||||
const freeform = slot?.freeform ?? delta.freeform
|
||||
const providerMetadata = slot?.providerMetadata ?? delta.providerMetadata
|
||||
if (!id || !name)
|
||||
return {
|
||||
slots: new Map(slots).set(key, { status: "buffering", freeform, id, name, input, providerMetadata }),
|
||||
events: [],
|
||||
}
|
||||
const tool = { id, name, input, providerMetadata }
|
||||
return {
|
||||
slots: new Map(slots).set(key, { status: "open", freeform, tool }),
|
||||
events: input.length > 0 ? [inputStart(tool), inputDelta(tool, input)] : [inputStart(tool)],
|
||||
}
|
||||
}
|
||||
|
||||
export const hasBufferedSlot = (slots: ReadonlyMap<StreamKey, Slot>) =>
|
||||
[...slots.values()].some((slot) => slot.status === "buffering")
|
||||
|
||||
/**
|
||||
* Settle every open slot. A completed response parses each call, appending
|
||||
* `freeformEnd` to freeform input first. Truncation and filtering abandon open
|
||||
* calls: their input ends without a call, and the consumer settles them as
|
||||
* incomplete. Buffered slots never started, so they produce nothing.
|
||||
*/
|
||||
export const finishSlots = <K extends StreamKey>(
|
||||
route: string,
|
||||
slots: ReadonlyMap<K, Slot>,
|
||||
reason: FinishReasonDetails["normalized"] | undefined,
|
||||
freeformEnd = "",
|
||||
) =>
|
||||
Effect.gen(function* () {
|
||||
const open = [...slots.values()].flatMap((slot) => (slot.status === "open" ? [slot] : []))
|
||||
if (reason === "length" || reason === "content-filter") return open.map((slot) => inputEnd(slot.tool))
|
||||
const closed = open.map((slot) =>
|
||||
slot.freeform && freeformEnd
|
||||
? { tool: { ...slot.tool, input: `${slot.tool.input}${freeformEnd}` }, delta: freeformEnd }
|
||||
: { tool: slot.tool, delta: "" },
|
||||
)
|
||||
const deltas = closed.flatMap((item) => (item.delta ? [inputDelta(item.tool, item.delta)] : []))
|
||||
const calls = yield* Effect.forEach(closed, (item) =>
|
||||
toolCall(route, item.tool).pipe(Effect.map((event) => finishEvents(item.tool, event))),
|
||||
)
|
||||
return [...deltas, ...calls.flat()]
|
||||
})
|
||||
|
||||
export * as ToolStream from "./tool-stream.js"
|
||||
|
||||
@@ -6,6 +6,7 @@ import { OpenAIChat } from "./openai-chat.js"
|
||||
import { JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared.js"
|
||||
import { cacheControl } from "./utils/cache.js"
|
||||
import { OpenResponsesOptions } from "./utils/open-responses-options.js"
|
||||
import { ToolStream } from "./utils/tool-stream.js"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public options and request body
|
||||
@@ -280,8 +281,12 @@ const step = Effect.fn("VeniceChat.step")(function* (state: State, event: typeof
|
||||
for (const call of delta?.tool_calls ?? []) {
|
||||
if (!call.thought_signature) continue
|
||||
const index = call.index ?? result[0].latestToolIndex
|
||||
const toolCalls = result[0].toolCalls
|
||||
const id =
|
||||
call.id ?? (index === undefined ? undefined : (result[0].tools[index]?.id ?? result[0].pendingTools[index]?.id))
|
||||
call.id ??
|
||||
(index === undefined || toolCalls.status === "finished"
|
||||
? undefined
|
||||
: ToolStream.slotID(toolCalls.slots.get(index)))
|
||||
if (id) tools[id] = call.thought_signature
|
||||
}
|
||||
return [
|
||||
|
||||
@@ -674,21 +674,34 @@ describe("Mistral Chat", () => {
|
||||
})
|
||||
}
|
||||
|
||||
const truncated = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
chunk({
|
||||
tool_calls: [{ index: 0, id: "Ab12Cd34E", function: { name: "lookup", arguments: '{"city":' } }],
|
||||
}),
|
||||
chunk({}, "length"),
|
||||
for (const [raw, normalized] of [
|
||||
["length", "length"],
|
||||
["content_filter", "content-filter"],
|
||||
] as const) {
|
||||
const truncated = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
chunk({
|
||||
tool_calls: [{ index: 0, id: "Ab12Cd34E", function: { name: "lookup", arguments: '{"city":' } }],
|
||||
}),
|
||||
chunk({ tool_calls: [{ index: 1, id: "Cd34Ef56G", function: { arguments: "{}" } }] }),
|
||||
chunk({}, raw),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
expect(truncated.finishReason).toEqual({ normalized: "length", raw: "length" })
|
||||
expect(truncated.events.some(LLMEvent.is.toolCall)).toBe(false)
|
||||
expect(truncated.events.some(LLMEvent.is.toolInputEnd)).toBe(false)
|
||||
)
|
||||
expect(truncated.finishReason).toEqual({ normalized, raw })
|
||||
// The open call closes its input without a call; the call that never got a name was never started.
|
||||
expect(truncated.events.map((event) => event.type)).toEqual([
|
||||
"step-start",
|
||||
"tool-input-start",
|
||||
"tool-input-delta",
|
||||
"tool-input-end",
|
||||
"step-finish",
|
||||
"finish",
|
||||
])
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -719,13 +732,34 @@ describe("Mistral Chat", () => {
|
||||
)
|
||||
expect(missingFinish.message).toContain("without finish_reason")
|
||||
|
||||
const lateContent = yield* LLMClient.generate(request).pipe(
|
||||
const lateText = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(sseEvents(chunk({}, "stop"), chunk({ content: [{ type: "text", text: "late" }] }))),
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
chunk({ content: "Hello" }),
|
||||
chunk({}, "stop"),
|
||||
chunk({ content: [{ type: "text", text: " late" }] }),
|
||||
chunk({ content: [{ type: "thinking", thinking: "late thought" }] }),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
expect(lateText.text).toBe("Hello late")
|
||||
expect(lateText.reasoning).toBe("late thought")
|
||||
expect(lateText.finishReason).toEqual({ normalized: "stop", raw: "stop" })
|
||||
|
||||
const lateTool = yield* LLMClient.generate(request).pipe(
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
chunk({}, "stop"),
|
||||
chunk({ tool_calls: [{ index: 0, id: "Ab12Cd34E", function: { name: "lookup", arguments: "{}" } }] }),
|
||||
),
|
||||
),
|
||||
),
|
||||
Effect.flip,
|
||||
)
|
||||
expect(lateContent.message).toContain("content after the finish reason")
|
||||
expect(lateTool.message).toContain("content after the finish reason")
|
||||
}),
|
||||
)
|
||||
|
||||
|
||||
@@ -2039,7 +2039,7 @@ describe("OpenAI Chat route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("does not finalize streamed tool calls when content is filtered", () =>
|
||||
it.effect("closes streamed tool input without a call when content is filtered", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
deltaChunk({
|
||||
@@ -2068,6 +2068,7 @@ describe("OpenAI Chat route", () => {
|
||||
text: '{"query":"weather"',
|
||||
input: { query: "weather" },
|
||||
},
|
||||
{ type: "tool-input-end", id: "call_1", name: "lookup", providerMetadata: undefined },
|
||||
{
|
||||
type: "step-finish",
|
||||
index: 0,
|
||||
@@ -2094,7 +2095,7 @@ describe("OpenAI Chat route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("does not finalize streamed tool calls when output is truncated", () =>
|
||||
it.effect("closes streamed tool input without a call when output is truncated", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
deltaChunk({
|
||||
@@ -2123,6 +2124,7 @@ describe("OpenAI Chat route", () => {
|
||||
text: '{"query":"weather"}',
|
||||
input: { query: "weather" },
|
||||
},
|
||||
{ type: "tool-input-end", id: "call_1", name: "lookup", providerMetadata: undefined },
|
||||
{
|
||||
type: "step-finish",
|
||||
index: 0,
|
||||
@@ -2149,6 +2151,33 @@ describe("OpenAI Chat route", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("opens a buffered tool call once under its stream index", () =>
|
||||
Effect.gen(function* () {
|
||||
const body = sseEvents(
|
||||
deltaChunk({ tool_calls: [{ index: 0, id: "call_1", function: { arguments: '{"query"' } }] }),
|
||||
deltaChunk({ tool_calls: [{ id: "call_1", function: { name: "lookup", arguments: ':"weather"' } }] }),
|
||||
deltaChunk({ tool_calls: [{ index: 0, function: { name: "lookup", arguments: "}" } }] }),
|
||||
deltaChunk({}, "tool_calls"),
|
||||
)
|
||||
const response = yield* LLMClient.generate({
|
||||
...request,
|
||||
tools: [ToolDefinition.make({ name: "lookup", description: "Lookup data", inputSchema: { type: "object" } })],
|
||||
}).pipe(Effect.provide(fixedResponse(body)))
|
||||
|
||||
expect(response.events.map((event) => event.type)).toEqual([
|
||||
"step-start",
|
||||
"tool-input-start",
|
||||
"tool-input-delta",
|
||||
"tool-input-delta",
|
||||
"tool-input-end",
|
||||
"tool-call",
|
||||
"step-finish",
|
||||
"finish",
|
||||
])
|
||||
expect(response.toolCalls).toMatchObject([{ id: "call_1", name: "lookup", input: { query: "weather" } }])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("maps unknown finish reasons to unknown and keeps the raw value", () =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* LLMClient.generate(request).pipe(
|
||||
|
||||
@@ -249,4 +249,40 @@ describe("ToolStream", () => {
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("buffers a chat slot until id and name arrive, then opens it once", () =>
|
||||
Effect.gen(function* () {
|
||||
const buffered = ToolStream.slotDelta(new Map(), 0, { id: "call_1", text: '{"query"', freeform: false })
|
||||
const opened = ToolStream.slotDelta(buffered.slots, 0, { name: "lookup", text: ':"weather"', freeform: false })
|
||||
const repeated = ToolStream.slotDelta(opened.slots, 0, {
|
||||
id: "call_1",
|
||||
name: "lookup",
|
||||
text: "",
|
||||
freeform: false,
|
||||
})
|
||||
const closed = ToolStream.slotDelta(repeated.slots, 0, { text: "}", freeform: false })
|
||||
|
||||
expect(buffered.events).toEqual([])
|
||||
expect([...buffered.slots.values()].map((slot) => slot.status)).toEqual(["buffering"])
|
||||
expect(opened.events.map((event) => event.type)).toEqual(["tool-input-start", "tool-input-delta"])
|
||||
expect(repeated.events).toEqual([])
|
||||
expect([...closed.slots.values()].map((slot) => slot.status)).toEqual(["open"])
|
||||
expect(yield* ToolStream.finishSlots(ADAPTER, closed.slots, "stop")).toEqual([
|
||||
{ type: "tool-input-end", id: "call_1", name: "lookup" },
|
||||
{ type: "tool-call", id: "call_1", name: "lookup", input: { query: "weather" } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("ends open chat slots without a call on length or content filter", () =>
|
||||
Effect.gen(function* () {
|
||||
const open = ToolStream.slotDelta(new Map(), 0, { id: "call_1", name: "lookup", text: '{"q', freeform: false })
|
||||
const slots = ToolStream.slotDelta(open.slots, 1, { id: "call_2", text: "{}", freeform: false }).slots
|
||||
|
||||
for (const reason of ["length", "content-filter"] as const)
|
||||
expect(yield* ToolStream.finishSlots(ADAPTER, slots, reason)).toEqual([
|
||||
{ type: "tool-input-end", id: "call_1", name: "lookup" },
|
||||
])
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user