fix(core): retry failed streams cleanly and keep finished work on interrupt

This commit is contained in:
Aiden Cline
2026-11-03 15:24:25 -06:00
Unverified
parent c7e008d29e
commit 56c33e146f
6 changed files with 529 additions and 151 deletions
+5 -13
View File
@@ -34,9 +34,6 @@ import { Plugin } from "../../plugin.js"
import { PluginHooks } from "../../plugin/hooks.js"
import { MAX_STEPS_PROMPT } from "./max-steps.js"
const CONTINUE_AFTER_INCOMPLETE_STREAM =
"The previous response was interrupted. Continue from where you left off without repeating completed content."
const CONTINUE_AFTER_OUTPUT_LIMIT =
"Your last response hit the output token limit (stop reason: length). Do not apologize, recap, or repeat yourself. Break the remaining work into smaller pieces."
@@ -178,7 +175,9 @@ const layer = Layer.effect(
// The Session would go idle; a plugin may admit input to keep this execution working.
yield* restore(
SessionBackground.outstanding(jobs, store, sessionID).pipe(
Effect.flatMap((background) => hooks.trigger("session", "experimental.idle", { sessionID, background })),
Effect.flatMap((background) =>
hooks.trigger("session", "experimental.idle", { sessionID, background }),
),
),
)
if (!(yield* SessionInbox.has(db, sessionID, "input"))) return DrainResult.Complete()
@@ -321,20 +320,13 @@ const layer = Layer.effect(
)
return outcome.needsContinuation
}),
Retry: (outcome) =>
retry.wait({
decision: outcome.decision,
error: outcome.error,
assistantMessageID,
}),
Continue: Effect.fnUntraced(function* (outcome) {
Retry: Effect.fnUntraced(function* (outcome) {
yield* retry.wait({
decision: outcome.decision,
error: outcome.error,
assistantMessageID,
})
yield* bus.publish(SessionEvent.Synthetic, { sessionID, text: CONTINUE_AFTER_INCOMPLETE_STREAM })
assistantMessageID = SessionMessage.ID.create()
if (outcome.freshAssistant) assistantMessageID = SessionMessage.ID.create()
}),
OutputLimit: Effect.fn("SessionRunner.continueOutputLimit")(function* (outcome) {
// The current response is included. User input or any other finish breaks the streak.
+5 -12
View File
@@ -29,10 +29,10 @@ import { SessionRunnerRetry } from "./retry.js"
export type Outcome = Data.TaggedEnum<{
Completed: { readonly needsContinuation: boolean }
Retry: { readonly error: SessionError.Error; readonly decision: SessionRunnerRetry.Decision }
Continue: {
Retry: {
readonly error: SessionError.Error
readonly decision: SessionRunnerRetry.Decision
readonly freshAssistant?: boolean
}
RecoverFull: {}
Compacted: {}
@@ -208,7 +208,7 @@ export const make = Effect.gen(function* () {
),
)
: undefined
if (llmFailure && llmError && retry?.retry && !recorded.outputStarted) {
if (llmError && retry?.retry && !recorded.outputStarted) {
// Retry state projects onto the existing assistant, even before it has produced output.
yield* publisher.startAssistant()
return Outcome.Retry({ error: llmError, decision: retry })
@@ -269,15 +269,8 @@ export const make = Effect.gen(function* () {
})
}
if (
llmFailure &&
llmError &&
retry?.retry &&
record.outputStarted &&
tools.declines.length === 0 &&
!tools.interrupted
)
return Outcome.Continue({ error: llmError, decision: retry })
if (llmError && retry?.retry && tools.declines.length === 0 && !tools.interrupted)
return Outcome.Retry({ error: llmError, decision: retry, freshAssistant: true })
if (Exit.isFailure(stream)) return yield* Effect.failCause(stream.cause)
if (tools.declines.length > 0) {
@@ -139,28 +139,47 @@ const toolResult = (tool: SessionMessage.AssistantTool, providerMetadata: Provid
}
}
// A call cut off before it ran (by Esc, a crash or a failed stream) is not replayed. Input
// errors stay so the model can reissue the call.
const isAbandonedTool = (item: SessionMessage.AssistantContent) =>
item.type === "tool" &&
item.time.ran === undefined &&
!(item.state.status === "error" && item.state.error.type.startsWith("tool.input"))
const assistant = (message: SessionMessage.Assistant, model: Model.Ref, providerMetadataKey: string) => {
const sameProvider = message.model.providerID === model.providerID
const sameModel = sameProvider && message.model.id === model.id
const reuseProviderMetadata = sameModel && message.error === undefined
const reasoningInterrupted = message.content.some(
(item) => item.type === "reasoning" && item.time !== undefined && item.time.completed === undefined,
)
const content = message.content.flatMap((item): Array<AssistantMessage["content"][number]> => {
// A failed stream is retried from what had already happened: the calls that ran and everything before
// them. The rest was cut off, and dropping it keeps the request from ending in a partial reply.
const ran = message.content.findLastIndex((item) => item.type === "tool" && item.time.ran !== undefined)
const kept =
message.error === undefined || message.error.type === "aborted"
? message.content
: message.content.slice(0, ran + 1)
const items = kept.filter((item) => !isAbandonedTool(item))
// A stream cut by Esc, a crash or a failure may end mid-block, so its last block's provider state is not
// replayed, and reasoning is only replayed once the model moved on to output.
const cut = message.error !== undefined || message.finish === undefined
const tail = cut ? message.content.at(-1) : undefined
const content = items.flatMap((item, index): Array<AssistantMessage["content"][number]> => {
if (item.type === "text")
return [
{
type: "text",
text: item.text,
// Text can carry provider-bound state (e.g. Gemini thought signatures),
// which is only replayable against the model that produced it.
providerMetadata: reuseProviderMetadata ? providerMetadata(providerMetadataKey, item.native) : undefined,
providerMetadata: sameModel && item !== tail ? providerMetadata(providerMetadataKey, item.native) : undefined,
},
]
// Let the destination adapter handle readable reasoning after a model/provider switch.
if (item.type === "reasoning") {
if (message.error !== undefined || reasoningInterrupted) return []
return reuseProviderMetadata
if (
cut &&
!items
.slice(index + 1)
.some((after) => after.type === "tool" || (after.type === "text" && after.text.trim() !== ""))
)
return []
return sameModel
? [
{
type: "reasoning",
@@ -175,13 +194,7 @@ const assistant = (message: SessionMessage.Assistant, model: Model.Ref, provider
// Call-side metadata is model-scoped proof of generation (Gemini thought
// signatures, OpenAI encrypted reasoning): only the producing model may
// replay it.
const reuseToolProviderMetadata =
reuseProviderMetadata ||
(sameModel && item.executed === true && (item.state.status === "completed" || item.state.status === "error"))
const call = toolCall(
item,
reuseToolProviderMetadata ? providerMetadata(providerMetadataKey, item.native) : undefined,
)
const call = toolCall(item, sameModel ? providerMetadata(providerMetadataKey, item.native) : undefined)
if (item.executed !== true) return [call]
// Hosted tools (e.g. google_search) run inside the provider, so their
// native data is provider-format result data rather than model-scoped
@@ -190,15 +203,13 @@ const assistant = (message: SessionMessage.Assistant, model: Model.Ref, provider
return result ? [call, result] : [call]
})
const meaningful = content.filter((part) => {
if (part.type === "text") return part.text !== ""
if (part.type === "text") return cut ? part.text.trim() !== "" : part.text !== ""
if (part.type !== "reasoning") return true
return part.text !== "" || (part.providerMetadata !== undefined && Object.keys(part.providerMetadata).length > 0)
})
const results = message.content
const results = items
.filter((item): item is SessionMessage.AssistantTool => item.type === "tool" && item.executed !== true)
.map((item) =>
toolResult(item, reuseProviderMetadata ? providerMetadata(providerMetadataKey, item.native) : undefined),
)
.map((item) => toolResult(item, sameModel ? providerMetadata(providerMetadataKey, item.native) : undefined))
.filter((message) => message !== undefined)
.map(Message.tool)
if (meaningful.length === 0) return results
@@ -476,7 +476,7 @@ it.effect("auto compaction estimates current content against the buffered prompt
id: "call_read",
name: "read",
state: { status: "completed", input: {}, content: [{ type: "text", text: "x".repeat(4_000) }] },
time: { created: DateTime.makeUnsafe(0) },
time: { created: DateTime.makeUnsafe(0), ran: DateTime.makeUnsafe(0) },
})
const grown = { ...input(89_000, contextLimited), messages: [{ ...assistant, content: [tool] }] }
expect(SessionCompaction.estimateContext(grown)).toBe(90_000)
+328 -17
View File
@@ -60,6 +60,7 @@ describe("toLLMMessages", () => {
agent: build,
model: { id: Model.ID.make("model"), providerID: Provider.ID.make("provider") },
content,
finish: "stop",
time: { created, completed: created },
})
const messages = toLLMMessages(
@@ -746,7 +747,7 @@ Earlier work
input: { path: "README.md" },
metadata: { type: "media", mime: "image/png" },
}),
time: { created },
time: { created, ran: created },
}),
SessionMessage.AssistantTool.make({
type: "tool",
@@ -765,7 +766,7 @@ Earlier work
},
],
}),
time: { created, completed: created },
time: { created, completed: created, ran: created },
}),
SessionMessage.AssistantTool.make({
type: "tool",
@@ -778,7 +779,7 @@ Earlier work
input: { query: "Effect" },
content: [{ type: "text", text: "Found it" }],
}),
time: { created, completed: created },
time: { created, completed: created, ran: created },
}),
SessionMessage.AssistantTool.make({
type: "tool",
@@ -791,9 +792,10 @@ Earlier work
input: { path: "README.md" },
error: { type: "unknown", message: "Denied" },
}),
time: { created, completed: created },
time: { created, completed: created, ran: created },
}),
],
finish: "stop",
time: { created, completed: created },
}),
],
@@ -804,7 +806,6 @@ Earlier work
expect(messages[0]?.content).toEqual([
{ type: "text", text: "Checking" },
{ type: "reasoning", text: "Think", providerMetadata: { provider: { signature: "sig_1" } } },
{ type: "tool-call", id: "pending", name: "read", input: { path: "README.md" } },
{ type: "tool-call", id: "running", name: "read", input: { path: "README.md" } },
{
type: "tool-call",
@@ -879,6 +880,7 @@ Earlier work
native: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" },
}),
],
finish: "stop",
time: { created, completed: created },
}),
],
@@ -910,6 +912,7 @@ Earlier work
native: { signature: "signed" },
}),
],
finish: "stop",
time: { created, completed: created },
}),
],
@@ -922,7 +925,7 @@ Earlier work
])
})
test("drops failed assistant reasoning while preserving settled hosted tools", () => {
test("keeps finished reasoning and settled hosted tools from a failed stream", () => {
const messages = toLLMMessages(
[
SessionMessage.Assistant.make({
@@ -933,7 +936,7 @@ Earlier work
content: [
SessionMessage.AssistantReasoning.make({
type: "reasoning",
text: "Partial thought",
text: "Finished thought",
native: { itemId: "rs_failed", reasoningEncryptedContent: null },
}),
SessionMessage.AssistantTool.make({
@@ -947,7 +950,7 @@ Earlier work
input: { query: "Effect" },
content: [{ type: "text", text: '{"found":true}' }],
}),
time: { created, completed: created },
time: { created, completed: created, ran: created },
}),
SessionMessage.AssistantTool.make({
type: "tool",
@@ -960,7 +963,7 @@ Earlier work
input: { query: "Effect" },
error: { type: "unknown", message: "Step interrupted" },
}),
time: { created, completed: created },
time: { created, completed: created, ran: created },
}),
],
finish: "error",
@@ -972,6 +975,11 @@ Earlier work
)
expect(messages[0]?.content).toEqual([
{
type: "reasoning",
text: "Finished thought",
providerMetadata: { provider: { itemId: "rs_failed", reasoningEncryptedContent: null } },
},
{
type: "tool-call",
id: "hosted-completed",
@@ -1071,7 +1079,7 @@ Earlier work
input: { query: "Effect" },
content: [{ type: "text", text: '{"status":"completed"}' }],
}),
time: { created, completed: created },
time: { created, completed: created, ran: created },
}),
SessionMessage.AssistantTool.make({
type: "tool",
@@ -1084,9 +1092,10 @@ Earlier work
input: { path: "README.md" },
content: [{ type: "text", text: "Hello" }],
}),
time: { created, completed: created },
time: { created, completed: created, ran: created },
}),
],
finish: "stop",
time: { created, completed: created },
}),
],
@@ -1151,6 +1160,7 @@ Earlier work
native: { reasoningEncryptedContent: "encrypted" },
}),
],
finish: "stop",
time: { created, completed: created },
}),
],
@@ -1166,11 +1176,11 @@ Earlier work
])
})
test("drops assistant text provider state across model changes and failures", () => {
const messages = toLLMMessages(
test("drops assistant text provider state across model changes, aborts, and provider failures", () => {
const switched = toLLMMessages(
[
SessionMessage.Assistant.make({
id: id("assistant-phase"),
id: id("assistant-switched"),
type: "assistant",
agent: build,
model: { id: Model.ID.make("old"), providerID: Provider.ID.make("provider") },
@@ -1181,20 +1191,320 @@ Earlier work
native: { phase: "commentary" },
}),
],
error: { type: "provider.unknown", message: "Interrupted after commentary" },
finish: "stop",
time: { created, completed: created },
}),
],
Model.Ref.make({ id: Model.ID.make("new"), providerID: Provider.ID.make("provider") }),
)
expect(messages[0]?.content).toEqual([
expect(switched[0]?.content).toEqual([
{
type: "text",
text: "Checking.",
providerMetadata: undefined,
},
])
const aborted = toLLMMessages(
[
SessionMessage.Assistant.make({
id: id("assistant-aborted"),
type: "assistant",
agent: build,
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
content: [
SessionMessage.AssistantReasoning.make({
type: "reasoning",
text: "Finished thought",
native: { signature: "sig_kept" },
time: { created, completed: created },
}),
SessionMessage.AssistantText.make({
type: "text",
text: "Closed commentary.",
native: { phase: "commentary" },
}),
SessionMessage.AssistantTool.make({
type: "tool",
id: "uncalled-stream",
name: "write",
state: SessionMessage.ToolStateError.make({
status: "error",
input: {},
error: { type: "aborted", message: "Step interrupted" },
}),
time: { created, completed: created },
}),
],
error: { type: "aborted", message: "Step interrupted" },
time: { created, completed: created },
}),
SessionMessage.Assistant.make({
id: id("assistant-aborted-mid-text"),
type: "assistant",
agent: build,
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
content: [
SessionMessage.AssistantText.make({
type: "text",
text: "Cut off sentence",
native: { phase: "final_answer" },
}),
],
error: { type: "aborted", message: "Step interrupted" },
time: { created, completed: created },
}),
],
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
)
expect(aborted.map((item) => item.content)).toEqual([
[
{
type: "reasoning",
text: "Finished thought",
providerMetadata: { provider: { signature: "sig_kept" } },
},
{
type: "text",
text: "Closed commentary.",
providerMetadata: { provider: { phase: "commentary" } },
},
],
[
{
type: "text",
text: "Cut off sentence",
providerMetadata: undefined,
},
],
])
// A failed stream keeps only what had happened by its last call that ran, whether it was retried,
// retries ran out, or Esc landed during the retry wait. With no call that ran, nothing is kept.
const failedTurn = (value: string, content: SessionMessage.Assistant["content"]) =>
SessionMessage.Assistant.make({
id: id(value),
type: "assistant",
agent: build,
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
content,
error: { type: "provider.unknown", message: "Stream dropped" },
finish: "error",
time: { created, completed: created },
})
const failedTurns = toLLMMessages(
[
failedTurn("retries-exhausted", [
SessionMessage.AssistantText.make({
type: "text",
text: "Partial visible answer",
native: { phase: "commentary" },
}),
]),
SessionMessage.Idle.make({ id: id("idle-failed"), type: "idle", outcome: "failed", time: { created } }),
failedTurn("tool-ran-before-failure", [
SessionMessage.AssistantText.make({ type: "text", text: "Reading first", native: { phase: "commentary" } }),
SessionMessage.AssistantTool.make({
type: "tool",
id: "ran",
name: "read",
state: SessionMessage.ToolStateCompleted.make({
status: "completed",
input: { path: "README.md" },
content: [{ type: "text", text: "Hello" }],
}),
time: { created, completed: created, ran: created },
}),
SessionMessage.AssistantText.make({ type: "text", text: "Now I'll run the te" }),
]),
],
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
)
expect(failedTurns.map((item) => ({ role: item.role, types: item.content.map((part) => part.type) }))).toEqual([
{ role: "assistant", types: ["text", "tool-call"] },
{ role: "tool", types: ["tool-result"] },
])
expect(failedTurns[0]?.content[0]).toEqual({
type: "text",
text: "Reading first",
providerMetadata: { provider: { phase: "commentary" } },
})
const supersededFailed = toLLMMessages(
[
SessionMessage.Assistant.make({
id: id("assistant-superseded"),
type: "assistant",
agent: build,
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
content: [
SessionMessage.AssistantText.make({
type: "text",
text: "Partial broken stream",
native: { phase: "commentary" },
}),
],
retry: {
attempt: 1,
at: created,
error: { type: "provider.unknown", message: "Stream dropped" },
},
error: { type: "provider.unknown", message: "Stream dropped" },
time: { created, completed: created },
}),
SessionMessage.ModelSelected.make({
id: id("model-switched-during-retry"),
type: "model-switched",
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
time: { created },
}),
SessionMessage.Assistant.make({
id: id("assistant-replacement"),
type: "assistant",
agent: build,
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
content: [
SessionMessage.AssistantText.make({
type: "text",
text: "Clean retry answer",
native: { phase: "final_answer" },
}),
],
finish: "stop",
time: { created, completed: created },
}),
],
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
)
expect(supersededFailed).toHaveLength(1)
expect(supersededFailed[0]?.content).toEqual([
{
type: "text",
text: "Clean retry answer",
providerMetadata: { provider: { phase: "final_answer" } },
},
])
// A crashed step is stamped completed when the next step starts but never gets a finish. Cut turns drop
// calls that never ran and only replay reasoning the model moved past, so a reasoning-only cut is omitted.
const cutAssistant = (
value: string,
content: SessionMessage.Assistant["content"],
error?: SessionMessage.Assistant["error"],
) =>
SessionMessage.Assistant.make({
id: id(value),
type: "assistant",
agent: build,
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
content,
error,
finish: error === undefined ? undefined : "error",
time: { created, completed: created },
})
const signed = (text: string, native: SessionMessage.Native, completed = true) =>
SessionMessage.AssistantReasoning.make({
type: "reasoning",
text,
native,
time: completed ? { created, completed: created } : { created },
})
const uncalled = (callID: string, state: SessionMessage.AssistantTool["state"], executed?: boolean) =>
SessionMessage.AssistantTool.make({ type: "tool", id: callID, name: "echo", executed, state, time: { created } })
const stepAborted = { type: "aborted" as const, message: "Step interrupted" }
const edgeAborts = toLLMMessages(
[
cutAssistant("crash-cut-reasoning-and-streaming-tool", [
signed("Signed thought before crash", { signature: "sig_crash" }),
uncalled("stale-streaming", SessionMessage.ToolStateStreaming.make({ status: "streaming", input: '{"te' })),
]),
cutAssistant(
"whitespace-between-thinking-and-aborted-tool",
[
signed("Signed thought", { signature: "sig_ws" }),
SessionMessage.AssistantText.make({ type: "text", text: "\n\n" }),
uncalled(
"aborted-uncalled",
SessionMessage.ToolStateError.make({ status: "error", input: {}, error: stepAborted }),
),
],
stepAborted,
),
cutAssistant(
"openai-multipart-summary-cut",
[
signed("Part 0", { itemId: "rs_cut", reasoningEncryptedContent: null }),
signed("Part 1", { itemId: "rs_cut", reasoningEncryptedContent: null }, false),
],
stepAborted,
),
cutAssistant(
"openai-multipart-summary-then-text",
[
signed("Part 0", { itemId: "rs_done", reasoningEncryptedContent: null }),
signed("Part 1", { itemId: "rs_done", reasoningEncryptedContent: "encrypted" }),
SessionMessage.AssistantText.make({
type: "text",
text: "Visible answer before Esc",
native: { itemId: "msg" },
}),
],
stepAborted,
),
],
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
)
expect(edgeAborts).toHaveLength(1)
expect(edgeAborts[0]?.content).toEqual([
{
type: "reasoning",
text: "Part 0",
providerMetadata: { provider: { itemId: "rs_done", reasoningEncryptedContent: null } },
},
{
type: "reasoning",
text: "Part 1",
providerMetadata: { provider: { itemId: "rs_done", reasoningEncryptedContent: "encrypted" } },
},
{ type: "text", text: "Visible answer before Esc", providerMetadata: undefined },
])
// A cut stream drops a hosted call that never ran (marked executed at input start) but keeps input
// feedback so the model can reissue a malformed call.
const failedWithCalls = toLLMMessages(
[
cutAssistant(
"terminal-failure-with-uncalled-calls",
[
SessionMessage.AssistantText.make({ type: "text", text: "Searching" }),
uncalled(
"hosted-uncalled",
SessionMessage.ToolStateError.make({
status: "error",
input: {},
error: stepAborted,
}),
true,
),
uncalled(
"malformed",
SessionMessage.ToolStateError.make({
status: "error",
input: { raw: "{" },
error: { type: "tool.input-json", message: "Reissue the call with valid JSON arguments." },
}),
),
],
stepAborted,
),
],
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
)
expect(failedWithCalls.map((message) => message.content.map((part) => part.type))).toEqual([
["text", "tool-call"],
["tool-result"],
])
expect(failedWithCalls[0]?.content[1]).toMatchObject({ type: "tool-call", id: "malformed" })
})
test("preserves assistant text provider state for the same model", () => {
@@ -1212,6 +1522,7 @@ Earlier work
native: { phase: "commentary" },
}),
],
finish: "stop",
time: { created, completed: created },
}),
],
+157 -86
View File
@@ -681,9 +681,6 @@ const incompleteStream = () =>
}),
})
const INCOMPLETE_STREAM_CONTINUATION =
"The previous response was interrupted. Continue from where you left off without repeating completed content."
const providerRejected = (input: { readonly message: string; readonly classification?: "context-overflow" }) =>
new AIError({ reason: new InvalidRequestError(input) })
@@ -1619,7 +1616,10 @@ describe("SessionRunnerLLM", () => {
}),
)
yield* s.admit("Echo before moving")
yield* s.llm.push(TestLLM.toolCall("call-move", "echo", { text: "moving" }), TestLLM.text("Done", "text-after-move"))
yield* s.llm.push(
TestLLM.toolCall("call-move", "echo", { text: "moving" }),
TestLLM.text("Done", "text-after-move"),
)
const tools = yield* s.blockTools()
const run = yield* s.resume.pipe(Effect.forkChild)
yield* tools.started
@@ -5120,7 +5120,7 @@ describe("SessionRunnerLLM", () => {
yield* s.resume
expect(s.requests).toHaveLength(1)
expect(messageRoles(s.requests[0])).toEqual(["user", "assistant", "tool"])
expect(messageRoles(s.requests[0])).toEqual(["user"])
expect(yield* s.context).toMatchObject([
Expected.user("Recover interrupted tool input"),
Expected.assistant({}, [Expected.failedTool({ id: "call-pending-interrupted" }, {})]),
@@ -5665,7 +5665,20 @@ describe("SessionRunnerLLM", () => {
scenario("durably fails blocked local tools when interrupted while awaiting settlement", function* (s) {
yield* s.admit("Interrupt tool settlement")
const tools = yield* s.blockTools()
yield* s.llm.push(TestLLM.toolCall("call-await-interrupt", "echo", { text: "blocked" }))
yield* s.llm.push([
LLMEvent.stepStart({ index: 0 }),
LLMEvent.reasoningStart({ id: "r1" }),
LLMEvent.reasoningDelta({ id: "r1", text: "Plan before tool" }),
LLMEvent.reasoningEnd({ id: "r1", providerMetadata: { openai: { signature: "sig_tool_interrupt" } } }),
LLMEvent.toolCall({
id: "call-await-interrupt",
name: "echo",
input: { text: "blocked" },
providerMetadata: { openai: { thoughtSignature: "tool_sig_1" } },
}),
LLMEvent.stepFinish({ index: 0, reason: { normalized: "tool-calls" } }),
LLMEvent.finish({ reason: { normalized: "tool-calls" } }),
])
const streamed = yield* s.bus.subscribe(SessionEvent.Step.Streamed).pipe(
Stream.filter((event) => event.data.sessionID === sessionID),
Stream.runHead,
@@ -5683,6 +5696,7 @@ describe("SessionRunnerLLM", () => {
expect(yield* s.context).toMatchObject([
Expected.user("Interrupt tool settlement"),
Expected.assistant({ finish: "error", error: { type: "aborted", message: "Step interrupted" } }, [
Expected.reasoning("Plan before tool"),
Expected.failedTool(
{ id: "call-await-interrupt" },
{ error: { type: "aborted", message: "Tool execution interrupted" } },
@@ -5695,6 +5709,83 @@ describe("SessionRunnerLLM", () => {
expect(eventTypes).not.toContain("session.step.ended.1")
expect(eventTypes).not.toContain("session.retry.scheduled.1")
expect(s.requests).toHaveLength(1)
yield* s.llm.push(TestLLM.text("Resumed with signatures intact", "resumed"))
yield* s.runPrompt("Continue after cancelling tool")
expect(s.requests[1]?.messages).toMatchObject([
{ role: "user", content: [{ type: "text", text: "Interrupt tool settlement" }] },
{
role: "assistant",
content: [
{
type: "reasoning",
text: "Plan before tool",
providerMetadata: { openai: { signature: "sig_tool_interrupt" } },
},
{
type: "tool-call",
id: "call-await-interrupt",
name: "echo",
providerMetadata: { openai: { thoughtSignature: "tool_sig_1" } },
},
],
},
{ role: "tool", content: [{ type: "tool-result", id: "call-await-interrupt" }] },
{ role: "user", content: [{ type: "text", text: "Continue after cancelling tool" }] },
])
})
scenario("keeps closed blocks and drops uncalled tool inputs when interrupted mid-stream", function* (s) {
yield* s.admit("Interrupt mid-second-tool")
const tools = yield* s.blockTools()
const streamingSecond = yield* Deferred.make<void>()
yield* s.llm.push(
Stream.concat(
Stream.fromIterable([
LLMEvent.stepStart({ index: 0 }),
LLMEvent.reasoningStart({ id: "r1" }),
LLMEvent.reasoningDelta({ id: "r1", text: "Closed thought" }),
LLMEvent.reasoningEnd({ id: "r1", providerMetadata: { openai: { signature: "sig_closed" } } }),
LLMEvent.toolCall({
id: "call-first",
name: "echo",
input: { text: "first" },
providerMetadata: { openai: { thoughtSignature: "tool_sig_first" } },
}),
LLMEvent.toolInputStart({ id: "call-second-cut", name: "echo" }),
LLMEvent.toolInputDelta({ id: "call-second-cut", name: "echo", text: '{"text":"half' }),
]),
Stream.fromEffect(Deferred.succeed(streamingSecond, undefined)).pipe(Stream.flatMap(() => Stream.never)),
),
)
const runner = yield* SessionRunner.Service
const run = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild)
yield* tools.started
yield* Deferred.await(streamingSecond)
yield* Fiber.interrupt(run)
expect(Exit.isFailure(yield* Fiber.await(run))).toBe(true)
yield* s.llm.push(TestLLM.text("Continued cleanly", "continued"))
yield* s.runPrompt("Proceed without second tool")
expect(s.requests[1]?.messages).toMatchObject([
{ role: "user", content: [{ type: "text", text: "Interrupt mid-second-tool" }] },
{
role: "assistant",
content: [
{ type: "reasoning", text: "Closed thought", providerMetadata: { openai: { signature: "sig_closed" } } },
{
type: "tool-call",
id: "call-first",
name: "echo",
input: { text: "first" },
providerMetadata: { openai: { thoughtSignature: "tool_sig_first" } },
},
],
},
{ role: "tool", content: [{ type: "tool-result", id: "call-first" }] },
{ role: "user", content: [{ type: "text", text: "Proceed without second tool" }] },
])
})
scenario("forces a text response on an agent's configured final step", function* (s) {
@@ -5766,7 +5857,11 @@ describe("SessionRunnerLLM", () => {
expect(s.requests).toHaveLength(1)
expect(yield* s.context).toMatchObject([
Expected.user("Fail durably"),
{ type: "assistant", finish: "error", error: { type: "provider.invalid-request", message: "Provider unavailable" } },
{
type: "assistant",
finish: "error",
error: { type: "provider.invalid-request", message: "Provider unavailable" },
},
])
})
@@ -5778,7 +5873,11 @@ describe("SessionRunnerLLM", () => {
expect(s.requests).toHaveLength(1)
expect(yield* s.context).toMatchObject([
Expected.user("Fail before step"),
{ type: "assistant", finish: "error", error: { type: "provider.invalid-request", message: "Provider unavailable" } },
{
type: "assistant",
finish: "error",
error: { type: "provider.invalid-request", message: "Provider unavailable" },
},
])
})
@@ -6106,7 +6205,10 @@ describe("SessionRunnerLLM", () => {
// the window instead of rejecting it.
s.currentModel = recoveryModel
yield* s.llm.push(
TestLLM.complete({ reason: { normalized: "length" }, usage: { inputTokens: 199_994, nonCachedInputTokens: 199_994, outputTokens: 0 } }),
TestLLM.complete({
reason: { normalized: "length" },
usage: { inputTokens: 199_994, nonCachedInputTokens: 199_994, outputTokens: 0 },
}),
TestLLM.text("## Objective\n- summary", "summary"),
TestLLM.text("Done", "done"),
)
@@ -6170,7 +6272,7 @@ describe("SessionRunnerLLM", () => {
})
}
scenario("continues an incomplete stream after observable text", function* (s) {
scenario("retries an incomplete stream cleanly from the prompt when no tool ran", function* (s) {
const failure = incompleteStream()
yield* s.admit("Continue partial output")
yield* s.llm.push(
@@ -6181,7 +6283,7 @@ describe("SessionRunnerLLM", () => {
LLMEvent.textDelta({ id: "partial-rate-limit", text: "Partial" }),
),
)
yield* s.llm.push(TestLLM.text(" continuation", "continued-text"))
yield* s.llm.push(TestLLM.text("Recovered", "continued-text"))
const scheduled = yield* subscribeRetries(s)
const run = yield* s.resume.pipe(Effect.forkChild)
@@ -6190,38 +6292,24 @@ describe("SessionRunnerLLM", () => {
yield* Fiber.join(run)
expect(s.requests).toHaveLength(2)
expect(s.requests[1]?.messages.at(-2)).toMatchObject({
role: "assistant",
content: [{ type: "text", text: "Partial" }],
})
expect(s.requests[1]?.messages.at(-1)).toMatchObject({
role: "user",
content: [
{
type: "text",
text: INCOMPLETE_STREAM_CONTINUATION,
},
],
})
expect(s.requests[1]?.messages).toMatchObject([
{ role: "user", content: [{ type: "text", text: "Continue partial output" }] },
])
const context = yield* s.context
expect(context).toMatchObject([
Expected.user("Continue partial output"),
Expected.assistant({ finish: "error", error: { type: "provider.invalid-output" } }, [Expected.text("Partial")]),
{
type: "synthetic",
text: INCOMPLETE_STREAM_CONTINUATION,
},
Expected.assistant({ finish: "stop" }, [Expected.text(" continuation")]),
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
])
const assistants = context.filter((message) => message.type === "assistant")
expect(new Set(assistants.map((message) => message.id)).size).toBe(2)
expect(context.find((message) => message.type === "synthetic")?.description).toBeUndefined()
expect(context.some((message) => message.type === "synthetic")).toBe(false)
expect(yield* recordedEventTypes(sessionID)).toContain("session.retry.scheduled.1")
yield* replaySessionProjection(sessionID)
expect(yield* s.context).toMatchObject(context)
})
scenario("continues an unknown finish after observable text", function* (s) {
scenario("retries an unknown finish cleanly after observable text when no tool ran", function* (s) {
yield* s.admit("Continue unknown finish")
yield* s.llm.push([
LLMEvent.stepStart({ index: 0 }),
@@ -6231,7 +6319,7 @@ describe("SessionRunnerLLM", () => {
LLMEvent.stepFinish({ index: 0, reason: { normalized: "unknown" } }),
LLMEvent.finish({ reason: { normalized: "unknown" } }),
])
yield* s.llm.push(TestLLM.text(" continuation", "unknown-continuation"))
yield* s.llm.push(TestLLM.text("Recovered", "unknown-continuation"))
const scheduled = yield* subscribeRetries(s)
const run = yield* s.resume.pipe(Effect.forkChild)
@@ -6240,19 +6328,17 @@ describe("SessionRunnerLLM", () => {
yield* Fiber.join(run)
expect(s.requests).toHaveLength(2)
expect(s.requests[1]?.messages.at(-1)).toMatchObject({
role: "user",
content: [{ type: "text", text: INCOMPLETE_STREAM_CONTINUATION }],
})
expect(s.requests[1]?.messages).toMatchObject([
{ role: "user", content: [{ type: "text", text: "Continue unknown finish" }] },
])
expect(yield* s.context).toMatchObject([
{ type: "user" },
Expected.assistant({ finish: "error" }, [Expected.text("Partial")]),
{ type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
Expected.assistant({ finish: "stop" }, [Expected.text(" continuation")]),
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
])
})
scenario("continues after a mid-stream rate limit honoring retry-after", function* (s) {
scenario("retries after a mid-stream rate limit honoring retry-after without partial text", function* (s) {
yield* s.admit("Continue after rate limit")
yield* s.llm.push(
TestLLM.failAfter(
@@ -6262,7 +6348,7 @@ describe("SessionRunnerLLM", () => {
LLMEvent.textDelta({ id: "rate-limited-partial", text: "Partial" }),
),
)
yield* s.llm.push(TestLLM.text(" continuation", "rate-limit-continuation"))
yield* s.llm.push(TestLLM.text("Recovered", "rate-limit-continuation"))
const scheduled = yield* subscribeRetries(s)
const run = yield* s.resume.pipe(Effect.forkChild)
@@ -6273,24 +6359,18 @@ describe("SessionRunnerLLM", () => {
yield* Fiber.join(run)
expect(s.requests).toHaveLength(2)
expect(s.requests[1]?.messages.at(-2)).toMatchObject({
role: "assistant",
content: [{ type: "text", text: "Partial" }],
})
expect(s.requests[1]?.messages.at(-1)).toMatchObject({
role: "user",
content: [{ type: "text", text: INCOMPLETE_STREAM_CONTINUATION }],
})
expect(s.requests[1]?.messages).toMatchObject([
{ role: "user", content: [{ type: "text", text: "Continue after rate limit" }] },
])
expect(yield* recordedEventTypes(sessionID)).toContain("session.retry.scheduled.1")
expect(yield* s.context).toMatchObject([
Expected.user("Continue after rate limit"),
Expected.assistant({ finish: "error", error: { type: "provider.rate-limit" } }, [Expected.text("Partial")]),
{ type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
Expected.assistant({ finish: "stop" }, [Expected.text(" continuation")]),
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
])
})
scenario("continues after an unrecognized mid-stream provider failure", function* (s) {
scenario("retries after an unrecognized mid-stream provider failure when no tool ran", function* (s) {
const failure = new AIError({ reason: new UnknownProviderError({ message: "Provider returned error" }) })
yield* s.admit("Continue after unknown failure")
yield* s.llm.push(
@@ -6301,7 +6381,7 @@ describe("SessionRunnerLLM", () => {
LLMEvent.textDelta({ id: "unknown-failure-partial", text: "Partial" }),
),
)
yield* s.llm.push(TestLLM.text(" continuation", "unknown-failure-continuation"))
yield* s.llm.push(TestLLM.text("Recovered", "unknown-failure-continuation"))
const scheduled = yield* subscribeRetries(s)
const run = yield* s.resume.pipe(Effect.forkChild)
@@ -6310,19 +6390,17 @@ describe("SessionRunnerLLM", () => {
yield* Fiber.join(run)
expect(s.requests).toHaveLength(2)
expect(s.requests[1]?.messages.at(-1)).toMatchObject({
role: "user",
content: [{ type: "text", text: INCOMPLETE_STREAM_CONTINUATION }],
})
expect(s.requests[1]?.messages).toMatchObject([
{ role: "user", content: [{ type: "text", text: "Continue after unknown failure" }] },
])
expect(yield* s.context).toMatchObject([
Expected.user("Continue after unknown failure"),
Expected.assistant({ finish: "error", error: { type: "provider.unknown" } }, [Expected.text("Partial")]),
{ type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
Expected.assistant({ finish: "stop" }, [Expected.text(" continuation")]),
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
])
})
scenario("drops interrupted reasoning before continuing an incomplete stream", function* (s) {
scenario("drops interrupted reasoning and retries cleanly without continuation prompt", function* (s) {
yield* s.admit("Continue interrupted reasoning")
yield* s.llm.push(
TestLLM.failAfter(
@@ -6340,20 +6418,18 @@ describe("SessionRunnerLLM", () => {
yield* TestClock.adjust("2400 millis")
yield* Fiber.join(run)
expect(s.requests[1]?.messages.slice(-2)).toMatchObject([
expect(s.requests[1]?.messages).toMatchObject([
{ role: "user", content: [{ type: "text", text: "Continue interrupted reasoning" }] },
{ role: "user", content: [{ type: "text", text: INCOMPLETE_STREAM_CONTINUATION }] },
])
expect(yield* s.context).toMatchObject([
{ type: "user" },
Expected.assistant({ finish: "error" }, [Expected.reasoning("Partial thought")]),
{ type: "synthetic" },
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
])
})
for (const failure of [streamDisconnected, websocketDisconnected]) {
scenario(`continues after ${failure.name} with durable reasoning state`, function* (s) {
scenario(`retries cleanly after ${failure.name} during reasoning`, function* (s) {
yield* s.admit("Recover disconnected reasoning")
yield* s.llm.push(
TestLLM.failAfter(
@@ -6377,9 +6453,8 @@ describe("SessionRunnerLLM", () => {
expect(s.requests).toHaveLength(2)
expect(yield* recordedEventTypes(sessionID)).toContain("session.retry.scheduled.1")
expect(s.requests[1]?.messages.slice(-2)).toMatchObject([
expect(s.requests[1]?.messages).toMatchObject([
{ role: "user", content: [{ type: "text", text: "Recover disconnected reasoning" }] },
{ role: "user", content: [{ type: "text", text: INCOMPLETE_STREAM_CONTINUATION }] },
])
expect(yield* s.context).toMatchObject([
{ type: "user" },
@@ -6390,13 +6465,12 @@ describe("SessionRunnerLLM", () => {
native: { itemId: "rs_disconnected", reasoningEncryptedContent: "encrypted-state" },
},
]),
{ type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
])
})
}
scenario("continues an incomplete stream after settling a local tool", function* (s) {
scenario("retries an incomplete stream from a settled local tool", function* (s) {
yield* s.admit("Continue after tool")
yield* s.llm.push(
TestLLM.failAfter(
@@ -6414,25 +6488,16 @@ describe("SessionRunnerLLM", () => {
yield* Fiber.join(run)
expect(s.executions).toEqual(["settled"])
expect(s.requests[1]?.messages.slice(-3)).toMatchObject([
expect(s.requests[1]?.messages.slice(-2)).toMatchObject([
{
role: "assistant",
content: [{ type: "tool-call", id: "call-before-close", name: "echo", input: { text: "settled" } }],
},
{ role: "tool", content: [{ type: "tool-result", id: "call-before-close" }] },
{
role: "user",
content: [
{
type: "text",
text: INCOMPLETE_STREAM_CONTINUATION,
},
],
},
])
})
scenario("continues an incomplete stream after settling a local tool defect", function* (s) {
scenario("retries an incomplete stream from a settled local tool defect", function* (s) {
yield* s.admit("Continue after tool defect")
yield* s.llm.push(
TestLLM.failAfter(
@@ -6449,7 +6514,7 @@ describe("SessionRunnerLLM", () => {
yield* TestClock.adjust("2400 millis")
yield* Fiber.join(run)
expect(messageRoles(s.requests[1])).toEqual(["user", "assistant", "tool", "user"])
expect(messageRoles(s.requests[1])).toEqual(["user", "assistant", "tool"])
expect(yield* s.context).toMatchObject([
{ type: "user" },
Expected.assistant({}, [
@@ -6458,13 +6523,12 @@ describe("SessionRunnerLLM", () => {
{ error: { type: "unknown", message: "unexpected tool defect" } },
),
]),
{ type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
])
})
scenario(
"shares retry accounting and assistant identity across transparent retries and partial continuations",
"shares retry accounting and assistant identity across transparent retries and retries after output",
function* (s) {
const scheduled = yield* subscribeRetries(s)
yield* s.admit("Mix retry paths")
@@ -6475,7 +6539,7 @@ describe("SessionRunnerLLM", () => {
LLMEvent.textStart({ id: "mixed-partial" }),
LLMEvent.textDelta({ id: "mixed-partial", text: "Partial" }),
)
// Alternate transparent failures and partial continuations until the retry allowance is spent.
// Alternate transparent failures and failures after output until the retry allowance is spent.
const outcomes = RETRY_GAPS.map((_, index) => (index % 2 === 0 ? Stream.fail(failure) : partial))
yield* s.llm.push(...outcomes, partial)
const run = yield* s.resume.pipe(Effect.forkChild)
@@ -6486,7 +6550,7 @@ describe("SessionRunnerLLM", () => {
}
expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(failure)
expect(s.requests).toHaveLength(RETRY_GAPS.length + 1)
// A transparent retry keeps the assistant identity; a partial continuation starts a new one.
// A transparent retry keeps the assistant identity; a retry after output starts a new one.
for (const [index, identity] of identities.entries()) {
if (index === 0) continue
if (index % 2 === 1) expect(identity).toBe(identities[index - 1])
@@ -6495,14 +6559,14 @@ describe("SessionRunnerLLM", () => {
const partials = outcomes.filter((outcome) => outcome === partial).length
const messages = yield* s.context
expect(messages.filter((message) => message.type === "assistant")).toHaveLength(partials + 1)
expect(messages.filter((message) => message.type === "synthetic")).toHaveLength(partials)
expect(messages.filter((message) => message.type === "synthetic")).toHaveLength(0)
const events = yield* recordedEventTypes(sessionID)
expect(events.filter((type) => type === "session.retry.scheduled.1")).toHaveLength(RETRY_GAPS.length)
expect(events.filter((type) => type === "session.step.failed.1")).toHaveLength(partials + 1)
},
)
scenario("stops incomplete stream continuations once the retry allowance is spent", function* (s) {
scenario("stops incomplete stream retries once the retry allowance is spent", function* (s) {
yield* s.admit("Exhaust partial continuations")
const failure = incompleteStream()
yield* s.llm.always(
@@ -6524,7 +6588,14 @@ describe("SessionRunnerLLM", () => {
expect(s.requests).toHaveLength(RETRY_GAPS.length + 1)
const context = yield* s.context
expect(context.filter((message) => message.type === "assistant")).toHaveLength(RETRY_GAPS.length + 1)
expect(context.filter((message) => message.type === "synthetic")).toHaveLength(RETRY_GAPS.length)
expect(context.filter((message) => message.type === "synthetic")).toHaveLength(0)
yield* s.llm.always(TestLLM.text("Continued after exhaustion", "after-exhaustion"))
yield* s.runPrompt("Continue please")
expect(s.requests.at(-1)?.messages).toMatchObject([
{ role: "user", content: [{ type: "text", text: "Exhaust partial continuations" }] },
{ role: "user", content: [{ type: "text", text: "Continue please" }] },
])
})
scenario("stops once the retry allowance is spent", function* (s) {