fix(core): shorten compaction transcripts oldest first before dropping history (#58015)

This commit is contained in:
Aiden Cline
2026-10-22 14:14:32 -05:00
committed by GitHub
Unverified
parent 0030a705a3
commit ac68d6a929
4 changed files with 176 additions and 67 deletions
+98 -35
View File
@@ -117,7 +117,7 @@ type Precomputed = {
const NOTHING_TO_COMPACT: Failure = { error: { type: "compaction.unavailable", message: "Nothing to compact yet" } }
/** After each "too long" rejection, the next attempt aims at this share of the first rejected request's size. */
const SHRINK_STEPS = [0.7, 0.5, 0.35]
const SHRINK_STEPS = [0.85, 0.7, 0.5, 0.35]
// The least of the window kept free for the last reply before compaction and for the summary itself.
const RESERVE_MIN = 16_000
/**
@@ -130,6 +130,17 @@ const RESERVE_SHARE = 0.15
const UNKNOWN_WINDOW = 200_000
/** A summary gets at least this much output, or the room left if less, even when the estimate margin leaves less. */
const SUMMARY_OUTPUT_FLOOR = 32_000
/**
* How far one exchange of a summary transcript is shortened, in the order the levels apply: whole; tool outputs cut;
* tool outputs left out and tool inputs cut; only the user's text, cut; omitted.
*/
const OUTPUTS_CLIPPED = 1
const OUTPUTS_REMOVED = 2
const USER_ONLY = 3
const OMITTED = 4
const TRANSCRIPT_LEVELS = [0, OUTPUTS_CLIPPED, OUTPUTS_REMOVED, USER_ONLY, OMITTED]
/** Characters kept, from the start and end, of a cut tool input, tool output, or user message. */
const TRANSCRIPT_CLIP = 2_000
const IMAGE_TOKEN_ESTIMATE = 1_500
const PDF_TOKEN_ESTIMATE = 2_000
/** Share of the compaction ceiling at which a background summary starts. */
@@ -549,11 +560,12 @@ export const layer = Layer.effect(
* and otherwise `shrink.history` as text aimed at `target` (see `flattenAndDropOldest`).
* Limits and estimates can be wrong, so until the provider rejects something as too long, a request that cannot
* be made to fit is sent whole: unchanged, or as text once its media has to go. A "too long" rejection means the
* estimate ran low, so the next attempts aim at 70%, 50%, then 35% of the first rejected request's estimate, and a
* rejection after those gives up. A "payload too large" rejection is about bytes, which inline media almost always
* accounts for, so one on the unchanged request resends it as text, which carries no media, without counting; one
* on text counts as "too long". Other provider errors resend the same request under the session's retry policy. A
* `Failure` from `send` is a reply that cannot be used, and is never retried.
* estimate ran low, so the next attempts aim at 85%, 70%, 50%, then 35% of the first rejected request's estimate,
* each shortening the transcript again from scratch, and a rejection after those gives up. A "payload too large"
* rejection is about bytes, which inline media almost always accounts for, so one on the unchanged request resends
* it as text, which carries no media, without counting; one on text counts as "too long". Other provider errors
* resend the same request under the session's retry policy. A `Failure` from `send` is a reply that cannot be
* used, and is never retried.
*/
const deliver = Effect.fnUntraced(function* (
trigger: Trigger,
@@ -637,9 +649,13 @@ export const layer = Layer.effect(
* History that has to change is sent as text rather than as edited messages: `[previous compaction][transcript]`,
* the conversation since that compaction as one text message. A previous summary always stays in front, while
* its verbatim recent part becomes the oldest transcript entry; a previous native window stays exactly as the
* provider made it. The transcript leaves out reasoning, media, and instruction updates, and cuts long tool
* outputs. Only if it still does not fit are whole exchanges dropped, oldest first. Undefined when even the
* newest exchange does not fit.
* provider made it. The transcript has no tool definitions, which the summary does not need and which no longer
* share a cached prefix, and leaves out reasoning, media, and instruction updates.
*
* When the whole transcript does not fit, exchanges are shortened oldest first, one level at a time, until it
* does (see `flattenMessage` for the levels): every exchange reaches a level before any reaches the next, so
* detail is lost from the oldest history first and whole exchanges are omitted only as a last resort. The newest
* exchange is never omitted; undefined when the transcript cannot fit even so.
*/
const flattenAndDropOldest = (request: LLMRequest, context: SessionContext.Loaded, target: number) => {
const previous = previousCompaction(context.messages)
@@ -657,30 +673,58 @@ export const layer = Layer.effect(
: request.messages.slice(0, end)
const recent = summary ? SessionRecentText.kept(summary) : ""
// A user message and everything after it, up to the next one, is one exchange: kept or dropped whole.
const exchanges = request.messages.slice(end).reduce<string[]>(
(groups, message) => {
const text = flattenMessage(message)
if (!text) return groups
if (message.role === "user" || groups.length === 0) groups.push(text)
else groups[groups.length - 1] += `\n${text}`
return groups
},
recent ? [recent] : [],
)
// A user message and everything after it, up to the next one, is one exchange. Each is rendered at every level
// once, so shortening is running arithmetic over these sizes. The previous summary's recent part is already
// shortened text: whole until it is omitted.
const exchanges = [
...(recent ? [TRANSCRIPT_LEVELS.map((level) => (level === OMITTED ? "" : recent))] : []),
...request.messages
.slice(end)
.reduce<Message[][]>((groups, message) => {
if (message.role === "user" || groups.length === 0) groups.push([message])
else groups[groups.length - 1].push(message)
return groups
}, [])
.map((group) =>
TRANSCRIPT_LEVELS.map((level) =>
group
.map((message) => flattenMessage(message, level))
.filter(Boolean)
.join("\n"),
),
)
.filter((renders) => renders[0]),
]
if (exchanges.length === 0) return undefined
const note = (omitted: number) =>
omitted ? `[${omitted} older ${omitted === 1 ? "exchange" : "exchanges"} omitted]\n\n` : ""
const size = (text: string) => (text ? Token.estimate(text) + 1 : 0)
// Room for the longest possible note is set aside before choosing what to keep.
const room =
target -
estimateRequest({ system: request.system, tools: request.tools, messages: lead }) -
estimateRequest({ system: request.system, tools: [], messages: lead }) -
Token.estimate(note(exchanges.length))
const kept = exchanges.slice(oldestToDrop(exchanges, (text) => Token.estimate(text) + 1, room))
if (kept.length === 0) return undefined
return LLMRequest.update(request, {
messages: [...lead, Message.user(note(exchanges.length - kept.length) + kept.join("\n\n"))],
})
const levels = exchanges.map(() => 0)
let total = exchanges.reduce((sum, renders) => sum + size(renders[0]), 0)
const newest = exchanges.length - 1
for (const level of TRANSCRIPT_LEVELS.slice(1)) {
for (let index = 0; index < exchanges.length && total > room; index++) {
// The newest exchange is what the summary most needs; it is shortened but never left out.
if (index === newest && !exchanges[index][level]) continue
total += size(exchanges[index][level]) - size(exchanges[index][levels[index]])
levels[index] = level
}
}
if (total > room) return undefined
// Includes exchanges with no user text, which are already gone at `USER_ONLY`.
const omitted = levels.filter((level, index) => !exchanges[index][level]).length
const text = exchanges
.map((renders, index) => renders[levels[index]])
.filter(Boolean)
.join("\n\n")
return LLMRequest.update(request, { tools: [], messages: [...lead, Message.user(note(omitted) + text)] })
}
const stream = (
@@ -963,29 +1007,48 @@ const oldestToDrop = <T>(items: ReadonlyArray<T>, size: (item: T) => number, bud
return start
}
/** One request message as a flattened transcript line. Empty for what the transcript leaves out. */
const flattenMessage = (message: Message) => {
if (message.role === "system") return ""
/** One request message as transcript lines at a shortening level. Empty for what that level leaves out. */
const flattenMessage = (message: Message, level: number) => {
if (message.role === "system" || level === OMITTED) return ""
if (level >= USER_ONLY && message.role !== "user") return ""
return message.content
.flatMap((part) => {
if (part.type === "text")
return part.text ? [`[${message.role === "user" ? "User" : "Assistant"}]: ${part.text}`] : []
if (part.type === "text") {
if (!part.text) return []
const text = level >= USER_ONLY ? clip(part.text) : part.text
return [`[${message.role === "user" ? "User" : "Assistant"}]: ${text}`]
}
if (level >= USER_ONLY) return []
if (part.type === "media") return [`[${part.media.mediaType} omitted]`]
if (part.type === "tool-call") return [`[Assistant tool call]: ${part.name}(${JSON.stringify(part.input) ?? ""})`]
if (part.type === "tool-call") {
const input = JSON.stringify(part.input) ?? ""
return [`[Assistant tool call]: ${part.name}(${level >= OUTPUTS_REMOVED ? clip(input) : input})`]
}
if (part.type !== "tool-result") return []
const result = part.result
const label = result.type === "error" ? "Tool error" : "Tool result"
if (level >= OUTPUTS_REMOVED) return [`[${label} omitted]`]
const output =
result.type === "content"
? SessionRecentText.serializeToolContent(result.value)
: typeof result.value === "string"
? result.value
: (JSON.stringify(result.value) ?? "")
const label = result.type === "error" ? "Tool error" : "Tool result"
return [`[${label}]: ${SessionRecentText.truncateToolOutput(output)}`]
return [`[${label}]: ${level >= OUTPUTS_CLIPPED ? clip(output) : output}`]
})
.join("\n")
}
/** `value` cut to `TRANSCRIPT_CLIP` characters from its start and end, never splitting a surrogate pair. */
const clip = (value: string) => {
if (value.length <= TRANSCRIPT_CLIP) return value
const chars = Array.from(value)
if (chars.length <= TRANSCRIPT_CLIP) return value
const half = TRANSCRIPT_CLIP / 2
const omitted = chars.length - TRANSCRIPT_CLIP
return `${chars.slice(0, half).join("")}\n[… ${omitted} characters omitted …]\n${chars.slice(-half).join("")}`
}
/** The newest whole, real user messages within `keep` tokens: no synthetic guidance, no half of an attachment. */
export const recentUserMessages = (
messages: ReadonlyArray<SessionMessage.Info>,
+3 -1
View File
@@ -39,7 +39,9 @@ export const messageToText = (message: SessionMessage.Info): string => {
return message.content
.flatMap((part) => {
if (part.type === "text") return [`[Assistant]: ${part.text}`]
if (part.type === "reasoning") return part.text ? [`[Assistant reasoning]: ${part.text}`] : []
// Providers return summaries of the real reasoning, and written out as text it can read like an attempt to
// extract the model's thinking.
if (part.type === "reasoning") return []
const input = typeof part.state.input === "string" ? part.state.input : JSON.stringify(part.state.input)
const call = `[Assistant tool call]: ${part.name}(${input})`
@@ -722,14 +722,17 @@ it.effect("compaction lowers the output limit when it sends a request it could n
Effect.gen(function* () {
requests = []
const session = yield* insertSession(Session.ID.make("ses_unshrinkable_compaction"))
// One exchange estimated at 250k tokens, past the 200k window, so no older exchange can be dropped.
// A previous summary estimated at 250k tokens, past the 200k window; it always stays whole.
const messages = [
SessionMessage.User.make({
SessionMessage.CompactionCompleted.make({
id: SessionMessage.ID.create(),
type: "user",
text: "x".repeat(1_000_000),
type: "compaction",
status: "completed",
reason: "manual",
summary: "x".repeat(1_000_000),
time: { created: DateTime.makeUnsafe(0) },
}),
textUser("Next question"),
]
expect(yield* compactManually(session, messages)).toEqual({ status: "completed" })
+68 -27
View File
@@ -2829,7 +2829,7 @@ describe("SessionRunnerLLM", () => {
})
}
scenario("stops after three smaller compaction inputs overflow", function* (s) {
scenario("stops after four smaller compaction inputs overflow", function* (s) {
// Large enough that the conversation, not the system prompt, is most of the request.
const filler = "context ".repeat(1_000)
yield* s.llm.push(...Array.from({ length: 8 }, (_, index) => TestLLM.text(`Answer ${index}`, `answer-${index}`)))
@@ -2843,17 +2843,18 @@ describe("SessionRunnerLLM", () => {
Stream.fail(
new AIError({ reason: new InvalidRequestError({ message: "Too long", classification: "context-overflow" }) }),
)
yield* s.llm.push(overflow(), overflow(), overflow(), overflow())
yield* s.llm.push(overflow(), overflow(), overflow(), overflow(), overflow())
const compaction = yield* s.session.compact({ sessionID })
yield* s.resume
expect(s.requests).toHaveLength(4)
expect(s.requests).toHaveLength(5)
expect(userTexts(s.requests[2])[0].length).toBeLessThan(userTexts(s.requests[1])[0].length)
expect(userTexts(s.requests[3])[0].length).toBeLessThan(userTexts(s.requests[2])[0].length)
expect(userTexts(s.requests[4])[0].length).toBeLessThan(userTexts(s.requests[3])[0].length)
expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({ status: "failed" })
yield* s.llm.push(TestLLM.text("Continued", "continued"))
yield* s.runPrompt("Continue")
expect(userTexts(s.requests[4])).toContain(`Request 0: ${filler}`)
expect(userTexts(s.requests[5])).toContain(`Request 0: ${filler}`)
})
scenario("resends compaction as text after payload too large, then shrinks later rejections", function* (s) {
@@ -2873,12 +2874,12 @@ describe("SessionRunnerLLM", () => {
)
s.currentModel = unknownContextModel
s.requests.length = 0
yield* s.llm.push(...Array.from({ length: 5 }, () => Stream.fail(payloadTooLarge())))
yield* s.llm.push(...Array.from({ length: 6 }, () => Stream.fail(payloadTooLarge())))
const compaction = yield* s.session.compact({ sessionID })
yield* s.resume
// The first rejection resends everything as text, which carries no media; later ones shrink like "too long".
expect(s.requests).toHaveLength(5)
expect(s.requests).toHaveLength(6)
expect(s.requests[0]?.messages.some((message) => message.content.some((part) => part.type === "media"))).toBeTrue()
expect(s.requests[1]?.messages.every((message) => message.role === "user")).toBeTrue()
expect(userTexts(s.requests[1])[0]).toContain("[image/png omitted]")
@@ -2886,10 +2887,11 @@ describe("SessionRunnerLLM", () => {
expect(userTexts(s.requests[2])[0].length).toBeLessThan(userTexts(s.requests[1])[0].length)
expect(userTexts(s.requests[3])[0].length).toBeLessThan(userTexts(s.requests[2])[0].length)
expect(userTexts(s.requests[4])[0].length).toBeLessThan(userTexts(s.requests[3])[0].length)
expect(userTexts(s.requests[5])[0].length).toBeLessThan(userTexts(s.requests[4])[0].length)
expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({ status: "failed" })
})
scenario("resends whole history as text after payload too large when it cannot be shortened", function* (s) {
scenario("shortens the only exchange's user text when it does not fit", function* (s) {
const image = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
yield* s.session.prompt({
sessionID,
@@ -2901,16 +2903,15 @@ describe("SessionRunnerLLM", () => {
yield* s.resume
s.currentModel = testModel("smaller-history", { context: 7_000, output: 1_000 })
s.requests.length = 0
yield* s.llm.push(Stream.fail(payloadTooLarge()), TestLLM.text("## Objective\n- Recovered", "summary"))
yield* s.llm.push(TestLLM.text("## Objective\n- Recovered", "summary"))
const compaction = yield* s.session.compact({ sessionID })
yield* s.resume
// Even the latest exchange is over the estimated limit, so the first send is unchanged and the second keeps it all.
expect(s.requests).toHaveLength(2)
expect(s.requests[0]?.messages.some((message) => message.content.some((part) => part.type === "media"))).toBeTrue()
expect(s.requests[1]?.messages.every((message) => message.role === "user")).toBeTrue()
expect(userTexts(s.requests[1])[0]).toContain("[image/png omitted]")
expect(userTexts(s.requests[1])[0]).toContain(`Request 0: ${"x".repeat(40_000)}`)
// The only exchange is over the estimated limit, so its text is cut to its start and end rather than sent whole.
expect(s.requests).toHaveLength(1)
expect(s.requests[0]?.messages.every((message) => message.role === "user")).toBeTrue()
expect(userTexts(s.requests[0])[0]).toContain("characters omitted")
expect(userTexts(s.requests[0])[0]).not.toContain("x".repeat(2_001))
expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({ status: "completed" })
})
@@ -2952,14 +2953,14 @@ describe("SessionRunnerLLM", () => {
)
yield* s.runPrompt("Continue")
// The history fits the model's window by estimate, but the provider just rejected it, so older exchanges go.
// The history fits the model's window by estimate, but the provider just rejected it, so older exchanges are cut.
expect(s.requests).toHaveLength(3)
expect(userTexts(s.requests[1])[0]).toContain("older exchanges omitted")
expect(userTexts(s.requests[1])[0]).not.toContain("Request 0:")
expect(userTexts(s.requests[1])[0]).toContain("characters omitted")
expect(userTexts(s.requests[1])[0]).not.toContain(`Request 0: ${filler}`)
expect(userTexts(s.requests[1])[0]).toContain("Request 3:")
})
scenario("serializes history and omits media after a summary input overflow", function* (s) {
scenario("serializes history without tools or media after a summary input overflow", function* (s) {
const image = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
yield* s.session.prompt({
sessionID,
@@ -2995,20 +2996,61 @@ describe("SessionRunnerLLM", () => {
expect(s.requests[1]?.messages.every((message) => message.role === "user")).toBeTrue()
expect(userTexts(s.requests[1])[0]).toContain("[image/png omitted]")
expect(userTexts(s.requests[1])[0]).toContain("[Tool result]:")
expect(userTexts(s.requests[1])[0]).toContain(`${"x".repeat(1_250)}\n[truncated]`)
expect(userTexts(s.requests[1])[0]).not.toContain("x".repeat(1_251))
// Tool output is only cut when the transcript would not fit whole.
expect(userTexts(s.requests[1])[0]).toContain("x".repeat(5_000))
expect(s.requests[1]?.system).toEqual(s.requests[0]?.system)
expect(s.requests[1]?.tools).toEqual(s.requests[0]?.tools)
expect(s.requests[1]?.tools).toEqual([])
expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({ status: "completed" })
})
scenario("drops tool output in serialized compaction history before omitting exchanges", function* (s) {
const service = yield* SessionCompaction.Service
yield* service.transform((editor) => editor.configure({ buffer: 3_000 }))
s.currentModel = testModel("large-history", { context: 1_000_000, output: 32_000 })
yield* s.llm.push(
...Array.from({ length: 2 }, (_, index) =>
TestLLM.stop(
...Array.from({ length: 12 }, (_, call) => [
LLMEvent.toolCall({
id: `hosted-${index}-${call}`,
name: "web_search",
input: { query: "q" },
providerExecuted: true,
}),
LLMEvent.toolResult({
id: `hosted-${index}-${call}`,
name: "web_search",
result: { type: "text", value: "x".repeat(2_000) },
providerExecuted: true,
}),
]).flat(),
LLMEvent.textStart({ id: `answer-${index}` }),
LLMEvent.textDelta({ id: `answer-${index}`, text: `Answer ${index}` }),
LLMEvent.textEnd({ id: `answer-${index}` }),
),
),
)
yield* Effect.forEach([0, 1], (index) => s.runPrompt(`Request ${index}`))
s.currentModel = testModel("smaller-history", { context: 7_000, output: 1_000 })
s.requests.length = 0
yield* s.llm.push(TestLLM.text("## Objective\n- Recovered", "summary"))
yield* s.session.compact({ sessionID })
yield* s.resume
expect(s.requests).toHaveLength(1)
expect(userTexts(s.requests[0])[0]).toContain("Request 0")
// The oldest exchange loses its tool outputs rather than being dropped.
expect(userTexts(s.requests[0])[0]).toContain("[Tool result omitted]")
expect(userTexts(s.requests[0])[0]).not.toMatch(/older exchanges? omitted/)
})
scenario("fits serialized compaction history by omitting oldest exchanges", function* (s) {
const service = yield* SessionCompaction.Service
yield* service.transform((editor) => editor.configure({ buffer: 3_000 }))
s.currentModel = testModel("large-history", { context: 1_000_000, output: 32_000 })
yield* s.llm.push(...Array.from({ length: 4 }, (_, index) => TestLLM.text(`Answer ${index}`, `answer-${index}`)))
yield* s.llm.push(...Array.from({ length: 16 }, (_, index) => TestLLM.text(`Answer ${index}`, `answer-${index}`)))
yield* Effect.forEach(
Array.from({ length: 4 }, (_, index) => index),
Array.from({ length: 16 }, (_, index) => index),
(index) => s.runPrompt(`Request ${index}: ${"x".repeat(8_000)}`),
)
s.currentModel = testModel("smaller-history", { context: 7_000, output: 1_000 })
@@ -3017,13 +3059,12 @@ describe("SessionRunnerLLM", () => {
const compaction = yield* s.session.compact({ sessionID })
yield* s.resume
// Even cut to the user's text, the older exchanges do not all fit, so the oldest are omitted.
expect(s.requests).toHaveLength(1)
expect(userTexts(s.requests[0])[0]).toContain("older exchanges omitted")
expect(userTexts(s.requests[0])[0]).not.toContain("Request 0:")
expect(userTexts(s.requests[0])[0]).not.toContain("Request 1:")
// The latest exchange is the retained tail, so shrinking drops only exchanges before it.
expect(userTexts(s.requests[0])[0]).toContain("Request 2:")
expect(userTexts(s.requests[0])[0]).not.toContain("Request 3:")
// The newest exchanges are the retained tail, so shrinking drops only exchanges before them.
expect(userTexts(s.requests[0])[0]).not.toContain("Request 15:")
expect((yield* s.messages).find((message) => message.id === compaction.id)).toMatchObject({
status: "completed",
summary: "## Objective\n- Recovered",