diff --git a/packages/ai/src/llm.ts b/packages/ai/src/llm.ts index 862006fcc05..2f1d660d84a 100644 --- a/packages/ai/src/llm.ts +++ b/packages/ai/src/llm.ts @@ -19,6 +19,7 @@ import { AIError, LLMEvent, LLMResponse, + ResponseFold, Message, LanguageModel, SystemPart, @@ -173,7 +174,7 @@ const stepStream = ( after: (step: Step) => Stream.Stream, ): Stream.Stream => Stream.suspend(() => { - let state = LLMResponse.empty() + let state = ResponseFold.empty() let finish: Finish | undefined return LLMClient.stream(request, options).pipe( Stream.map((event) => @@ -183,15 +184,15 @@ const stepStream = ( ), Stream.tap((event) => Effect.sync(() => { - state = LLMResponse.reduce(state, event) + state = ResponseFold.reduce(state, event) if (event.type === "finish") finish = event }), ), Stream.filter((event) => event.type !== "finish"), - Stream.mapError((error) => withPartial(error, LLMResponse.partial(state))), + Stream.mapError((error) => withPartial(error, ResponseFold.partial(state))), Stream.concat( Stream.suspend(() => { - const response = LLMResponse.complete(state) + const response = ResponseFold.complete(state) return response && finish ? after({ response, finish }) : Stream.empty }), ), diff --git a/packages/ai/src/loop.ts b/packages/ai/src/loop.ts index e82116b7eed..cda7cacd62c 100644 --- a/packages/ai/src/loop.ts +++ b/packages/ai/src/loop.ts @@ -3,6 +3,7 @@ import { AIError, InvalidProviderOutputError, LLMResponse, + ResponseFold, Message, Usage, LLMEvent, @@ -138,7 +139,7 @@ export function collect( return Effect.suspend(() => { const steps: LLMResponse[] = [] const messages: Message[] = [] - let step = LLMResponse.empty() + let step = ResponseFold.empty() // Between a `step-finish` and the next `step-start`, tool results come from the loop running tools. let betweenSteps = false // The last step's assistant message, and its calls with invalid input that the loop may answer. @@ -156,7 +157,7 @@ export function collect( steps.push(response) assistant = messages.push(response.message) - 1 invalid = step.events.filter(LLMEvent.is.toolInputError).map(invalidCall) - step = LLMResponse.empty() + step = ResponseFold.empty() } return Stream.runForEach(events, (event) => Effect.sync(() => { @@ -180,7 +181,7 @@ export function collect( return } if (event.type === "step-start") betweenSteps = false - step = LLMResponse.reduce(step, event) + step = ResponseFold.reduce(step, event) if (event.type !== "step-finish") return close(event.reason, event.usage) betweenSteps = true diff --git a/packages/ai/src/route/client.ts b/packages/ai/src/route/client.ts index de9a8d48b86..ca3b97f55fd 100644 --- a/packages/ai/src/route/client.ts +++ b/packages/ai/src/route/client.ts @@ -22,6 +22,7 @@ import { GenerationOptions, HttpOptions, LLMResponse, + ResponseFold, LanguageModel, Message, LLMEvent, @@ -662,16 +663,16 @@ const streamRequestWith = (runtime: TransportRuntime) => (request: LLMRequest, o export const generateWith = (stream: Interface["stream"]) => Effect.fn("LLM.generate")(function* (request: LLMRequest, options?: StreamOptions) { - let state = LLMResponse.empty() + let state = ResponseFold.empty() yield* stream(request, options).pipe( - Stream.runForEach((event) => Effect.sync(() => (state = LLMResponse.reduce(state, event)))), - Effect.mapError((error) => withPartial(error, LLMResponse.partial(state))), + Stream.runForEach((event) => Effect.sync(() => (state = ResponseFold.reduce(state, event)))), + Effect.mapError((error) => withPartial(error, ResponseFold.partial(state))), ) - const response = LLMResponse.complete(state) + const response = ResponseFold.complete(state) if (response) return response return yield* withPartial( incompleteStreamError(`${request.model.provider}/${request.model.route.id}`), - LLMResponse.partial(state), + ResponseFold.partial(state), ) }) diff --git a/packages/ai/src/schema/events.ts b/packages/ai/src/schema/events.ts index 8243b612cc9..fe41debfcbe 100644 --- a/packages/ai/src/schema/events.ts +++ b/packages/ai/src/schema/events.ts @@ -452,7 +452,7 @@ interface ToolInputAssembly { } interface ResponseState { - readonly events: ReadonlyArray + readonly events: LLMEvent[] readonly message: Message readonly usage?: Usage readonly finishReason?: FinishReasonDetails @@ -470,18 +470,16 @@ const emptyResponseState = (): ResponseState => ({ }) const appendEvent = (state: ResponseState, event: LLMEvent): ResponseState => { - const events = [...state.events, event] + state.events.push(event) if (LLMEvent.is.finish(event)) { return { ...state, - events, usage: event.usage ?? state.usage, finishReason: event.reason, } } return { ...state, - events, usage: "usage" in event && event.usage !== undefined ? event.usage : state.usage, } } @@ -692,17 +690,20 @@ export class LLMResponse extends Schema.Class("LLM.Response")({ } } -export namespace LLMResponse { +/** + * Internal fold that assembles one provider attempt. Not exported from the package. + * A state has a single owner: `reduce` appends to the events buffer in place, so earlier states must not be reused. + */ +export namespace ResponseFold { export type State = ResponseState - export type Output = LLMResponse | { readonly events: ReadonlyArray; readonly usage?: Usage } - /** Initial reducer state for assembling one provider attempt. */ + /** Initial state with a fresh events buffer. */ export const empty = emptyResponseState - /** Purely fold one provider-neutral event into the attempt assembly state. */ + /** Fold one provider-neutral event into the state. */ export const reduce = reduceResponseState - /** Return a completed response only after the terminal finish. */ + /** Return a completed response only after the terminal finish. The response gets its own copy of the events. */ export const complete = (state: State): LLMResponse | undefined => state.finishReason === undefined ? undefined @@ -713,14 +714,19 @@ export namespace LLMResponse { finishReason: state.finishReason, }) - /** Convenience reducer for callers that already have a collected event list. */ - export const fromEvents = (events: ReadonlyArray) => complete(events.reduce(reduce, empty())) - /** Output that arrived before a failure, or `undefined` if only `step-start` did; an unfinished response reports `error`. */ export const partial = (state: State) => state.events.every(LLMEvent.is.stepStart) ? undefined : complete({ ...state, finishReason: state.finishReason ?? { normalized: "error" } }) +} + +export namespace LLMResponse { + export type Output = LLMResponse | { readonly events: ReadonlyArray; readonly usage?: Usage } + + /** Assemble a response from a collected event list, or `undefined` if it has no terminal finish. */ + export const fromEvents = (events: ReadonlyArray) => + ResponseFold.complete(events.reduce(ResponseFold.reduce, ResponseFold.empty())) /** Concatenate assistant text from a response or collected event list. */ export const text = (response: Output) => responseText(response.events) diff --git a/packages/ai/test/provider/openai-chat.test.ts b/packages/ai/test/provider/openai-chat.test.ts index fc99edd405e..07b0bf8aebf 100644 --- a/packages/ai/test/provider/openai-chat.test.ts +++ b/packages/ai/test/provider/openai-chat.test.ts @@ -1787,8 +1787,8 @@ describe("OpenAI Chat route", () => { publishedEndMetadata.push(decodeJson(encodeJson(event.providerMetadata))) }), ), - Stream.runFold(LLMResponse.empty, LLMResponse.reduce), - Effect.map((state) => LLMResponse.complete(state)!), + Stream.runCollect, + Effect.map((events) => LLMResponse.fromEvents(events)!), Effect.provide( fixedResponse( sseEvents( diff --git a/packages/ai/test/response.test.ts b/packages/ai/test/response.test.ts index d85789fa525..d7a8c620f2f 100644 --- a/packages/ai/test/response.test.ts +++ b/packages/ai/test/response.test.ts @@ -1,7 +1,8 @@ import { describe, expect, test } from "bun:test" import { LLMEvent, LLMResponse } from "../src/effect.js" +import { ResponseFold } from "../src/schema/index.js" -const reduce = (events: ReadonlyArray) => events.reduce(LLMResponse.reduce, LLMResponse.empty()) +const reduce = (events: ReadonlyArray) => events.reduce(ResponseFold.reduce, ResponseFold.empty()) const finishEvents = (events: ReadonlyArray) => events.filter(LLMEvent.is.finish) describe("LLMResponse reducer", () => { @@ -45,7 +46,7 @@ describe("LLMResponse reducer", () => { test("preserves partial content without completing a failed stream", () => { const state = reduce([LLMEvent.textStart({ id: "t1" }), LLMEvent.textDelta({ id: "t1", text: "partial" })]) - expect(LLMResponse.complete(state)).toBeUndefined() + expect(ResponseFold.complete(state)).toBeUndefined() expect(state.message.content).toEqual([{ type: "text", text: "partial" }]) }) @@ -56,10 +57,37 @@ describe("LLMResponse reducer", () => { LLMEvent.textEnd({ id: "t1" }), ]) - expect(LLMResponse.complete(state)).toBeUndefined() + expect(ResponseFold.complete(state)).toBeUndefined() expect(state.message.content).toEqual([{ type: "text", text: "partial" }]) }) + test("assembles a long text stream", () => { + const deltas = Array.from({ length: 50_000 }, (_, index) => LLMEvent.textDelta({ id: "t1", text: `${index % 10}` })) + const response = LLMResponse.fromEvents([ + LLMEvent.textStart({ id: "t1" }), + ...deltas, + LLMEvent.finish({ reason: { normalized: "stop" } }), + ]) + + expect(response?.events).toHaveLength(50_002) + expect(response?.text).toBe("0123456789".repeat(5_000)) + }) + + test("keeps completed responses independent of later folding", () => { + const state = reduce([ + LLMEvent.textStart({ id: "t1" }), + LLMEvent.textDelta({ id: "t1", text: "a" }), + LLMEvent.finish({ reason: { normalized: "stop" } }), + ]) + const response = ResponseFold.complete(state) + const partial = ResponseFold.partial(state) + ResponseFold.reduce(state, LLMEvent.textDelta({ id: "t1", text: "b" })) + + expect(response?.events).toHaveLength(3) + expect(partial?.events).toHaveLength(3) + expect(ResponseFold.empty().events).not.toBe(ResponseFold.empty().events) + }) + test("uses terminal usage when present and keeps prior usage when finish omits it", () => { const withFinishUsage = LLMResponse.fromEvents([ LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" }, usage: { inputTokens: 3 } }),