mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 13:05:14 +03:00
perf(ai): make the response fold internal and stop copying events per chunk (#61194)
This commit is contained in:
committed by
GitHub
Unverified
parent
0b211e7d99
commit
056d5f0343
@@ -19,6 +19,7 @@ import {
|
||||
AIError,
|
||||
LLMEvent,
|
||||
LLMResponse,
|
||||
ResponseFold,
|
||||
Message,
|
||||
LanguageModel,
|
||||
SystemPart,
|
||||
@@ -173,7 +174,7 @@ const stepStream = (
|
||||
after: (step: Step) => Stream.Stream<LLMEvent, AIError, Service>,
|
||||
): Stream.Stream<LLMEvent, AIError, Service> =>
|
||||
Stream.suspend(() => {
|
||||
let state = LLMResponse.empty()
|
||||
let state = ResponseFold.empty()
|
||||
let finish: Finish | undefined
|
||||
return LLMClient.stream(request, options).pipe(
|
||||
Stream.map((event) =>
|
||||
@@ -183,15 +184,15 @@ const stepStream = (
|
||||
),
|
||||
Stream.tap((event) =>
|
||||
Effect.sync(() => {
|
||||
state = LLMResponse.reduce(state, event)
|
||||
state = ResponseFold.reduce(state, event)
|
||||
if (event.type === "finish") finish = event
|
||||
}),
|
||||
),
|
||||
Stream.filter((event) => event.type !== "finish"),
|
||||
Stream.mapError((error) => withPartial(error, LLMResponse.partial(state))),
|
||||
Stream.mapError((error) => withPartial(error, ResponseFold.partial(state))),
|
||||
Stream.concat(
|
||||
Stream.suspend(() => {
|
||||
const response = LLMResponse.complete(state)
|
||||
const response = ResponseFold.complete(state)
|
||||
return response && finish ? after({ response, finish }) : Stream.empty
|
||||
}),
|
||||
),
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
AIError,
|
||||
InvalidProviderOutputError,
|
||||
LLMResponse,
|
||||
ResponseFold,
|
||||
Message,
|
||||
Usage,
|
||||
LLMEvent,
|
||||
@@ -138,7 +139,7 @@ export function collect<E, R>(
|
||||
return Effect.suspend(() => {
|
||||
const steps: LLMResponse[] = []
|
||||
const messages: Message[] = []
|
||||
let step = LLMResponse.empty()
|
||||
let step = ResponseFold.empty()
|
||||
// Between a `step-finish` and the next `step-start`, tool results come from the loop running tools.
|
||||
let betweenSteps = false
|
||||
// The last step's assistant message, and its calls with invalid input that the loop may answer.
|
||||
@@ -156,7 +157,7 @@ export function collect<E, R>(
|
||||
steps.push(response)
|
||||
assistant = messages.push(response.message) - 1
|
||||
invalid = step.events.filter(LLMEvent.is.toolInputError).map(invalidCall)
|
||||
step = LLMResponse.empty()
|
||||
step = ResponseFold.empty()
|
||||
}
|
||||
return Stream.runForEach(events, (event) =>
|
||||
Effect.sync(() => {
|
||||
@@ -180,7 +181,7 @@ export function collect<E, R>(
|
||||
return
|
||||
}
|
||||
if (event.type === "step-start") betweenSteps = false
|
||||
step = LLMResponse.reduce(step, event)
|
||||
step = ResponseFold.reduce(step, event)
|
||||
if (event.type !== "step-finish") return
|
||||
close(event.reason, event.usage)
|
||||
betweenSteps = true
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
GenerationOptions,
|
||||
HttpOptions,
|
||||
LLMResponse,
|
||||
ResponseFold,
|
||||
LanguageModel,
|
||||
Message,
|
||||
LLMEvent,
|
||||
@@ -662,16 +663,16 @@ const streamRequestWith = (runtime: TransportRuntime) => (request: LLMRequest, o
|
||||
|
||||
export const generateWith = (stream: Interface["stream"]) =>
|
||||
Effect.fn("LLM.generate")(function* (request: LLMRequest, options?: StreamOptions) {
|
||||
let state = LLMResponse.empty()
|
||||
let state = ResponseFold.empty()
|
||||
yield* stream(request, options).pipe(
|
||||
Stream.runForEach((event) => Effect.sync(() => (state = LLMResponse.reduce(state, event)))),
|
||||
Effect.mapError((error) => withPartial(error, LLMResponse.partial(state))),
|
||||
Stream.runForEach((event) => Effect.sync(() => (state = ResponseFold.reduce(state, event)))),
|
||||
Effect.mapError((error) => withPartial(error, ResponseFold.partial(state))),
|
||||
)
|
||||
const response = LLMResponse.complete(state)
|
||||
const response = ResponseFold.complete(state)
|
||||
if (response) return response
|
||||
return yield* withPartial(
|
||||
incompleteStreamError(`${request.model.provider}/${request.model.route.id}`),
|
||||
LLMResponse.partial(state),
|
||||
ResponseFold.partial(state),
|
||||
)
|
||||
})
|
||||
|
||||
|
||||
@@ -452,7 +452,7 @@ interface ToolInputAssembly {
|
||||
}
|
||||
|
||||
interface ResponseState {
|
||||
readonly events: ReadonlyArray<LLMEvent>
|
||||
readonly events: LLMEvent[]
|
||||
readonly message: Message
|
||||
readonly usage?: Usage
|
||||
readonly finishReason?: FinishReasonDetails
|
||||
@@ -470,18 +470,16 @@ const emptyResponseState = (): ResponseState => ({
|
||||
})
|
||||
|
||||
const appendEvent = (state: ResponseState, event: LLMEvent): ResponseState => {
|
||||
const events = [...state.events, event]
|
||||
state.events.push(event)
|
||||
if (LLMEvent.is.finish(event)) {
|
||||
return {
|
||||
...state,
|
||||
events,
|
||||
usage: event.usage ?? state.usage,
|
||||
finishReason: event.reason,
|
||||
}
|
||||
}
|
||||
return {
|
||||
...state,
|
||||
events,
|
||||
usage: "usage" in event && event.usage !== undefined ? event.usage : state.usage,
|
||||
}
|
||||
}
|
||||
@@ -692,17 +690,20 @@ export class LLMResponse extends Schema.Class<LLMResponse>("LLM.Response")({
|
||||
}
|
||||
}
|
||||
|
||||
export namespace LLMResponse {
|
||||
/**
|
||||
* Internal fold that assembles one provider attempt. Not exported from the package.
|
||||
* A state has a single owner: `reduce` appends to the events buffer in place, so earlier states must not be reused.
|
||||
*/
|
||||
export namespace ResponseFold {
|
||||
export type State = ResponseState
|
||||
export type Output = LLMResponse | { readonly events: ReadonlyArray<LLMEvent>; readonly usage?: Usage }
|
||||
|
||||
/** Initial reducer state for assembling one provider attempt. */
|
||||
/** Initial state with a fresh events buffer. */
|
||||
export const empty = emptyResponseState
|
||||
|
||||
/** Purely fold one provider-neutral event into the attempt assembly state. */
|
||||
/** Fold one provider-neutral event into the state. */
|
||||
export const reduce = reduceResponseState
|
||||
|
||||
/** Return a completed response only after the terminal finish. */
|
||||
/** Return a completed response only after the terminal finish. The response gets its own copy of the events. */
|
||||
export const complete = (state: State): LLMResponse | undefined =>
|
||||
state.finishReason === undefined
|
||||
? undefined
|
||||
@@ -713,14 +714,19 @@ export namespace LLMResponse {
|
||||
finishReason: state.finishReason,
|
||||
})
|
||||
|
||||
/** Convenience reducer for callers that already have a collected event list. */
|
||||
export const fromEvents = (events: ReadonlyArray<LLMEvent>) => complete(events.reduce(reduce, empty()))
|
||||
|
||||
/** Output that arrived before a failure, or `undefined` if only `step-start` did; an unfinished response reports `error`. */
|
||||
export const partial = (state: State) =>
|
||||
state.events.every(LLMEvent.is.stepStart)
|
||||
? undefined
|
||||
: complete({ ...state, finishReason: state.finishReason ?? { normalized: "error" } })
|
||||
}
|
||||
|
||||
export namespace LLMResponse {
|
||||
export type Output = LLMResponse | { readonly events: ReadonlyArray<LLMEvent>; readonly usage?: Usage }
|
||||
|
||||
/** Assemble a response from a collected event list, or `undefined` if it has no terminal finish. */
|
||||
export const fromEvents = (events: ReadonlyArray<LLMEvent>) =>
|
||||
ResponseFold.complete(events.reduce(ResponseFold.reduce, ResponseFold.empty()))
|
||||
|
||||
/** Concatenate assistant text from a response or collected event list. */
|
||||
export const text = (response: Output) => responseText(response.events)
|
||||
|
||||
@@ -1787,8 +1787,8 @@ describe("OpenAI Chat route", () => {
|
||||
publishedEndMetadata.push(decodeJson(encodeJson(event.providerMetadata)))
|
||||
}),
|
||||
),
|
||||
Stream.runFold(LLMResponse.empty, LLMResponse.reduce),
|
||||
Effect.map((state) => LLMResponse.complete(state)!),
|
||||
Stream.runCollect,
|
||||
Effect.map((events) => LLMResponse.fromEvents(events)!),
|
||||
Effect.provide(
|
||||
fixedResponse(
|
||||
sseEvents(
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { LLMEvent, LLMResponse } from "../src/effect.js"
|
||||
import { ResponseFold } from "../src/schema/index.js"
|
||||
|
||||
const reduce = (events: ReadonlyArray<LLMEvent>) => events.reduce(LLMResponse.reduce, LLMResponse.empty())
|
||||
const reduce = (events: ReadonlyArray<LLMEvent>) => events.reduce(ResponseFold.reduce, ResponseFold.empty())
|
||||
const finishEvents = (events: ReadonlyArray<LLMEvent>) => events.filter(LLMEvent.is.finish)
|
||||
|
||||
describe("LLMResponse reducer", () => {
|
||||
@@ -45,7 +46,7 @@ describe("LLMResponse reducer", () => {
|
||||
test("preserves partial content without completing a failed stream", () => {
|
||||
const state = reduce([LLMEvent.textStart({ id: "t1" }), LLMEvent.textDelta({ id: "t1", text: "partial" })])
|
||||
|
||||
expect(LLMResponse.complete(state)).toBeUndefined()
|
||||
expect(ResponseFold.complete(state)).toBeUndefined()
|
||||
expect(state.message.content).toEqual([{ type: "text", text: "partial" }])
|
||||
})
|
||||
|
||||
@@ -56,10 +57,37 @@ describe("LLMResponse reducer", () => {
|
||||
LLMEvent.textEnd({ id: "t1" }),
|
||||
])
|
||||
|
||||
expect(LLMResponse.complete(state)).toBeUndefined()
|
||||
expect(ResponseFold.complete(state)).toBeUndefined()
|
||||
expect(state.message.content).toEqual([{ type: "text", text: "partial" }])
|
||||
})
|
||||
|
||||
test("assembles a long text stream", () => {
|
||||
const deltas = Array.from({ length: 50_000 }, (_, index) => LLMEvent.textDelta({ id: "t1", text: `${index % 10}` }))
|
||||
const response = LLMResponse.fromEvents([
|
||||
LLMEvent.textStart({ id: "t1" }),
|
||||
...deltas,
|
||||
LLMEvent.finish({ reason: { normalized: "stop" } }),
|
||||
])
|
||||
|
||||
expect(response?.events).toHaveLength(50_002)
|
||||
expect(response?.text).toBe("0123456789".repeat(5_000))
|
||||
})
|
||||
|
||||
test("keeps completed responses independent of later folding", () => {
|
||||
const state = reduce([
|
||||
LLMEvent.textStart({ id: "t1" }),
|
||||
LLMEvent.textDelta({ id: "t1", text: "a" }),
|
||||
LLMEvent.finish({ reason: { normalized: "stop" } }),
|
||||
])
|
||||
const response = ResponseFold.complete(state)
|
||||
const partial = ResponseFold.partial(state)
|
||||
ResponseFold.reduce(state, LLMEvent.textDelta({ id: "t1", text: "b" }))
|
||||
|
||||
expect(response?.events).toHaveLength(3)
|
||||
expect(partial?.events).toHaveLength(3)
|
||||
expect(ResponseFold.empty().events).not.toBe(ResponseFold.empty().events)
|
||||
})
|
||||
|
||||
test("uses terminal usage when present and keeps prior usage when finish omits it", () => {
|
||||
const withFinishUsage = LLMResponse.fromEvents([
|
||||
LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" }, usage: { inputTokens: 3 } }),
|
||||
|
||||
Reference in New Issue
Block a user