perf(ai): make the response fold internal and stop copying events per chunk (#61194)

This commit is contained in:
Aiden Cline
2026-11-01 23:33:51 -06:00
committed by GitHub
Unverified
parent 0b211e7d99
commit 056d5f0343
6 changed files with 66 additions and 29 deletions
+5 -4
View File
@@ -19,6 +19,7 @@ import {
AIError,
LLMEvent,
LLMResponse,
ResponseFold,
Message,
LanguageModel,
SystemPart,
@@ -173,7 +174,7 @@ const stepStream = (
after: (step: Step) => Stream.Stream<LLMEvent, AIError, Service>,
): Stream.Stream<LLMEvent, AIError, Service> =>
Stream.suspend(() => {
let state = LLMResponse.empty()
let state = ResponseFold.empty()
let finish: Finish | undefined
return LLMClient.stream(request, options).pipe(
Stream.map((event) =>
@@ -183,15 +184,15 @@ const stepStream = (
),
Stream.tap((event) =>
Effect.sync(() => {
state = LLMResponse.reduce(state, event)
state = ResponseFold.reduce(state, event)
if (event.type === "finish") finish = event
}),
),
Stream.filter((event) => event.type !== "finish"),
Stream.mapError((error) => withPartial(error, LLMResponse.partial(state))),
Stream.mapError((error) => withPartial(error, ResponseFold.partial(state))),
Stream.concat(
Stream.suspend(() => {
const response = LLMResponse.complete(state)
const response = ResponseFold.complete(state)
return response && finish ? after({ response, finish }) : Stream.empty
}),
),
+4 -3
View File
@@ -3,6 +3,7 @@ import {
AIError,
InvalidProviderOutputError,
LLMResponse,
ResponseFold,
Message,
Usage,
LLMEvent,
@@ -138,7 +139,7 @@ export function collect<E, R>(
return Effect.suspend(() => {
const steps: LLMResponse[] = []
const messages: Message[] = []
let step = LLMResponse.empty()
let step = ResponseFold.empty()
// Between a `step-finish` and the next `step-start`, tool results come from the loop running tools.
let betweenSteps = false
// The last step's assistant message, and its calls with invalid input that the loop may answer.
@@ -156,7 +157,7 @@ export function collect<E, R>(
steps.push(response)
assistant = messages.push(response.message) - 1
invalid = step.events.filter(LLMEvent.is.toolInputError).map(invalidCall)
step = LLMResponse.empty()
step = ResponseFold.empty()
}
return Stream.runForEach(events, (event) =>
Effect.sync(() => {
@@ -180,7 +181,7 @@ export function collect<E, R>(
return
}
if (event.type === "step-start") betweenSteps = false
step = LLMResponse.reduce(step, event)
step = ResponseFold.reduce(step, event)
if (event.type !== "step-finish") return
close(event.reason, event.usage)
betweenSteps = true
+6 -5
View File
@@ -22,6 +22,7 @@ import {
GenerationOptions,
HttpOptions,
LLMResponse,
ResponseFold,
LanguageModel,
Message,
LLMEvent,
@@ -662,16 +663,16 @@ const streamRequestWith = (runtime: TransportRuntime) => (request: LLMRequest, o
export const generateWith = (stream: Interface["stream"]) =>
Effect.fn("LLM.generate")(function* (request: LLMRequest, options?: StreamOptions) {
let state = LLMResponse.empty()
let state = ResponseFold.empty()
yield* stream(request, options).pipe(
Stream.runForEach((event) => Effect.sync(() => (state = LLMResponse.reduce(state, event)))),
Effect.mapError((error) => withPartial(error, LLMResponse.partial(state))),
Stream.runForEach((event) => Effect.sync(() => (state = ResponseFold.reduce(state, event)))),
Effect.mapError((error) => withPartial(error, ResponseFold.partial(state))),
)
const response = LLMResponse.complete(state)
const response = ResponseFold.complete(state)
if (response) return response
return yield* withPartial(
incompleteStreamError(`${request.model.provider}/${request.model.route.id}`),
LLMResponse.partial(state),
ResponseFold.partial(state),
)
})
+18 -12
View File
@@ -452,7 +452,7 @@ interface ToolInputAssembly {
}
interface ResponseState {
readonly events: ReadonlyArray<LLMEvent>
readonly events: LLMEvent[]
readonly message: Message
readonly usage?: Usage
readonly finishReason?: FinishReasonDetails
@@ -470,18 +470,16 @@ const emptyResponseState = (): ResponseState => ({
})
const appendEvent = (state: ResponseState, event: LLMEvent): ResponseState => {
const events = [...state.events, event]
state.events.push(event)
if (LLMEvent.is.finish(event)) {
return {
...state,
events,
usage: event.usage ?? state.usage,
finishReason: event.reason,
}
}
return {
...state,
events,
usage: "usage" in event && event.usage !== undefined ? event.usage : state.usage,
}
}
@@ -692,17 +690,20 @@ export class LLMResponse extends Schema.Class<LLMResponse>("LLM.Response")({
}
}
export namespace LLMResponse {
/**
* Internal fold that assembles one provider attempt. Not exported from the package.
* A state has a single owner: `reduce` appends to the events buffer in place, so earlier states must not be reused.
*/
export namespace ResponseFold {
export type State = ResponseState
export type Output = LLMResponse | { readonly events: ReadonlyArray<LLMEvent>; readonly usage?: Usage }
/** Initial reducer state for assembling one provider attempt. */
/** Initial state with a fresh events buffer. */
export const empty = emptyResponseState
/** Purely fold one provider-neutral event into the attempt assembly state. */
/** Fold one provider-neutral event into the state. */
export const reduce = reduceResponseState
/** Return a completed response only after the terminal finish. */
/** Return a completed response only after the terminal finish. The response gets its own copy of the events. */
export const complete = (state: State): LLMResponse | undefined =>
state.finishReason === undefined
? undefined
@@ -713,14 +714,19 @@ export namespace LLMResponse {
finishReason: state.finishReason,
})
/** Convenience reducer for callers that already have a collected event list. */
export const fromEvents = (events: ReadonlyArray<LLMEvent>) => complete(events.reduce(reduce, empty()))
/** Output that arrived before a failure, or `undefined` if only `step-start` did; an unfinished response reports `error`. */
export const partial = (state: State) =>
state.events.every(LLMEvent.is.stepStart)
? undefined
: complete({ ...state, finishReason: state.finishReason ?? { normalized: "error" } })
}
export namespace LLMResponse {
export type Output = LLMResponse | { readonly events: ReadonlyArray<LLMEvent>; readonly usage?: Usage }
/** Assemble a response from a collected event list, or `undefined` if it has no terminal finish. */
export const fromEvents = (events: ReadonlyArray<LLMEvent>) =>
ResponseFold.complete(events.reduce(ResponseFold.reduce, ResponseFold.empty()))
/** Concatenate assistant text from a response or collected event list. */
export const text = (response: Output) => responseText(response.events)
@@ -1787,8 +1787,8 @@ describe("OpenAI Chat route", () => {
publishedEndMetadata.push(decodeJson(encodeJson(event.providerMetadata)))
}),
),
Stream.runFold(LLMResponse.empty, LLMResponse.reduce),
Effect.map((state) => LLMResponse.complete(state)!),
Stream.runCollect,
Effect.map((events) => LLMResponse.fromEvents(events)!),
Effect.provide(
fixedResponse(
sseEvents(
+31 -3
View File
@@ -1,7 +1,8 @@
import { describe, expect, test } from "bun:test"
import { LLMEvent, LLMResponse } from "../src/effect.js"
import { ResponseFold } from "../src/schema/index.js"
const reduce = (events: ReadonlyArray<LLMEvent>) => events.reduce(LLMResponse.reduce, LLMResponse.empty())
const reduce = (events: ReadonlyArray<LLMEvent>) => events.reduce(ResponseFold.reduce, ResponseFold.empty())
const finishEvents = (events: ReadonlyArray<LLMEvent>) => events.filter(LLMEvent.is.finish)
describe("LLMResponse reducer", () => {
@@ -45,7 +46,7 @@ describe("LLMResponse reducer", () => {
test("preserves partial content without completing a failed stream", () => {
const state = reduce([LLMEvent.textStart({ id: "t1" }), LLMEvent.textDelta({ id: "t1", text: "partial" })])
expect(LLMResponse.complete(state)).toBeUndefined()
expect(ResponseFold.complete(state)).toBeUndefined()
expect(state.message.content).toEqual([{ type: "text", text: "partial" }])
})
@@ -56,10 +57,37 @@ describe("LLMResponse reducer", () => {
LLMEvent.textEnd({ id: "t1" }),
])
expect(LLMResponse.complete(state)).toBeUndefined()
expect(ResponseFold.complete(state)).toBeUndefined()
expect(state.message.content).toEqual([{ type: "text", text: "partial" }])
})
test("assembles a long text stream", () => {
const deltas = Array.from({ length: 50_000 }, (_, index) => LLMEvent.textDelta({ id: "t1", text: `${index % 10}` }))
const response = LLMResponse.fromEvents([
LLMEvent.textStart({ id: "t1" }),
...deltas,
LLMEvent.finish({ reason: { normalized: "stop" } }),
])
expect(response?.events).toHaveLength(50_002)
expect(response?.text).toBe("0123456789".repeat(5_000))
})
test("keeps completed responses independent of later folding", () => {
const state = reduce([
LLMEvent.textStart({ id: "t1" }),
LLMEvent.textDelta({ id: "t1", text: "a" }),
LLMEvent.finish({ reason: { normalized: "stop" } }),
])
const response = ResponseFold.complete(state)
const partial = ResponseFold.partial(state)
ResponseFold.reduce(state, LLMEvent.textDelta({ id: "t1", text: "b" }))
expect(response?.events).toHaveLength(3)
expect(partial?.events).toHaveLength(3)
expect(ResponseFold.empty().events).not.toBe(ResponseFold.empty().events)
})
test("uses terminal usage when present and keeps prior usage when finish omits it", () => {
const withFinishUsage = LLMResponse.fromEvents([
LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" }, usage: { inputTokens: 3 } }),