mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
fix(core): retry failed streams cleanly and keep finished work on interrupt (#60674)
This commit is contained in:
committed by
GitHub
Unverified
parent
f454acc6b9
commit
eccfb829bd
@@ -34,9 +34,6 @@ import { Plugin } from "../../plugin.js"
|
||||
import { PluginHooks } from "../../plugin/hooks.js"
|
||||
import { MAX_STEPS_PROMPT } from "./max-steps.js"
|
||||
|
||||
const CONTINUE_AFTER_INCOMPLETE_STREAM =
|
||||
"The previous response was interrupted. Continue from where you left off without repeating completed content."
|
||||
|
||||
const CONTINUE_AFTER_OUTPUT_LIMIT =
|
||||
"Your last response hit the output token limit (stop reason: length). Do not apologize, recap, or repeat yourself. Break the remaining work into smaller pieces."
|
||||
|
||||
@@ -181,7 +178,9 @@ const layer = Layer.effect(
|
||||
// The Session would go idle; a plugin may admit input to keep this execution working.
|
||||
yield* restore(
|
||||
SessionBackground.outstanding(jobs, store, sessionID).pipe(
|
||||
Effect.flatMap((background) => hooks.trigger("session", "experimental.idle", { sessionID, background })),
|
||||
Effect.flatMap((background) =>
|
||||
hooks.trigger("session", "experimental.idle", { sessionID, background }),
|
||||
),
|
||||
),
|
||||
)
|
||||
if (!(yield* SessionInbox.has(db, sessionID, "input"))) return DrainResult.Complete()
|
||||
@@ -334,16 +333,7 @@ const layer = Layer.effect(
|
||||
error: outcome.error,
|
||||
assistantMessageID,
|
||||
})
|
||||
}),
|
||||
Continue: Effect.fnUntraced(function* (outcome) {
|
||||
retryAttempt = outcome.decision.attempt
|
||||
yield* retry.wait({
|
||||
decision: outcome.decision,
|
||||
error: outcome.error,
|
||||
assistantMessageID,
|
||||
})
|
||||
yield* bus.publish(SessionEvent.Synthetic, { sessionID, text: CONTINUE_AFTER_INCOMPLETE_STREAM })
|
||||
assistantMessageID = SessionMessage.ID.create()
|
||||
if (outcome.freshAssistant) assistantMessageID = SessionMessage.ID.create()
|
||||
}),
|
||||
OutputLimit: Effect.fn("SessionRunner.continueOutputLimit")(function* (outcome) {
|
||||
// The current response is included. User input or any other finish breaks the streak.
|
||||
|
||||
@@ -29,10 +29,10 @@ import { SessionRunnerRetry } from "./retry.js"
|
||||
|
||||
export type Outcome = Data.TaggedEnum<{
|
||||
Completed: { readonly needsContinuation: boolean }
|
||||
Retry: { readonly error: SessionError.Error; readonly decision: SessionRunnerRetry.Decision }
|
||||
Continue: {
|
||||
Retry: {
|
||||
readonly error: SessionError.Error
|
||||
readonly decision: SessionRunnerRetry.Decision
|
||||
readonly freshAssistant?: boolean
|
||||
}
|
||||
RecoverFull: {}
|
||||
Compacted: {}
|
||||
@@ -220,7 +220,7 @@ export const make = Effect.gen(function* () {
|
||||
),
|
||||
)
|
||||
: undefined
|
||||
if (llmFailure && llmError && retry?.retry && !recorded.outputStarted) {
|
||||
if (llmError && retry?.retry && !recorded.outputStarted) {
|
||||
// Retry state projects onto the existing assistant, even before it has produced output.
|
||||
yield* publisher.startAssistant()
|
||||
return Outcome.Retry({ error: llmError, decision: retry })
|
||||
@@ -282,15 +282,8 @@ export const make = Effect.gen(function* () {
|
||||
})
|
||||
}
|
||||
|
||||
if (
|
||||
llmFailure &&
|
||||
llmError &&
|
||||
retry?.retry &&
|
||||
record.outputStarted &&
|
||||
tools.declines.length === 0 &&
|
||||
!tools.interrupted
|
||||
)
|
||||
return Outcome.Continue({ error: llmError, decision: retry })
|
||||
if (llmError && retry?.retry && tools.declines.length === 0 && !tools.interrupted)
|
||||
return Outcome.Retry({ error: llmError, decision: retry, freshAssistant: true })
|
||||
|
||||
if (Exit.isFailure(stream)) return yield* Effect.failCause(stream.cause)
|
||||
if (tools.declines.length > 0) {
|
||||
|
||||
@@ -139,28 +139,55 @@ const toolResult = (tool: SessionMessage.AssistantTool, providerMetadata: Provid
|
||||
}
|
||||
}
|
||||
|
||||
// Sessions migrated from V1 never recorded `time.ran`, so a completed call also counts as run.
|
||||
const ranTool = (item: SessionMessage.AssistantContent) =>
|
||||
item.type === "tool" && (item.time.ran !== undefined || item.state.status === "completed")
|
||||
|
||||
const assistant = (message: SessionMessage.Assistant, model: Model.Ref, providerMetadataKey: string) => {
|
||||
const sameProvider = message.model.providerID === model.providerID
|
||||
const sameModel = sameProvider && message.model.id === model.id
|
||||
const reuseProviderMetadata = sameModel && message.error === undefined
|
||||
const reasoningInterrupted = message.content.some(
|
||||
(item) => item.type === "reasoning" && item.time !== undefined && item.time.completed === undefined,
|
||||
)
|
||||
const content = message.content.flatMap((item): Array<AssistantMessage["content"][number]> => {
|
||||
// A failed stream is retried from what had already happened: the local calls that ran and everything
|
||||
// before them. Cutting after a local call's result keeps the retried request from ending on an assistant
|
||||
// reply; hosted calls after it are provider-side reads, so the retry can redo them.
|
||||
const kept =
|
||||
message.error === undefined || message.error.type === "aborted"
|
||||
? message.content
|
||||
: message.content.slice(
|
||||
0,
|
||||
message.content.findLastIndex((item) => item.type === "tool" && item.executed !== true && ranTool(item)) + 1,
|
||||
)
|
||||
// A stream cut by Esc, a crash or a failure may end mid-block, so its last block's provider state is not
|
||||
// replayed, and reasoning is only replayed once the model moved on to output. Calls it cut off before
|
||||
// they ran are dropped; input errors stay so the model can reissue the call.
|
||||
const cut = message.error !== undefined || message.finish === undefined
|
||||
const tail = cut ? message.content.at(-1) : undefined
|
||||
const items = cut
|
||||
? kept.filter(
|
||||
(item) =>
|
||||
item.type !== "tool" ||
|
||||
ranTool(item) ||
|
||||
(item.state.status === "error" && item.state.error.type.startsWith("tool.input")),
|
||||
)
|
||||
: kept
|
||||
const content = items.flatMap((item, index): Array<AssistantMessage["content"][number]> => {
|
||||
if (item.type === "text")
|
||||
return [
|
||||
{
|
||||
type: "text",
|
||||
text: item.text,
|
||||
// Text can carry provider-bound state (e.g. Gemini thought signatures),
|
||||
// which is only replayable against the model that produced it.
|
||||
providerMetadata: reuseProviderMetadata ? providerMetadata(providerMetadataKey, item.native) : undefined,
|
||||
providerMetadata: sameModel && item !== tail ? providerMetadata(providerMetadataKey, item.native) : undefined,
|
||||
},
|
||||
]
|
||||
// Let the destination adapter handle readable reasoning after a model/provider switch.
|
||||
if (item.type === "reasoning") {
|
||||
if (message.error !== undefined || reasoningInterrupted) return []
|
||||
return reuseProviderMetadata
|
||||
if (
|
||||
cut &&
|
||||
!items
|
||||
.slice(index + 1)
|
||||
.some((after) => after.type === "tool" || (after.type === "text" && after.text.trim() !== ""))
|
||||
)
|
||||
return []
|
||||
return sameModel
|
||||
? [
|
||||
{
|
||||
type: "reasoning",
|
||||
@@ -175,13 +202,7 @@ const assistant = (message: SessionMessage.Assistant, model: Model.Ref, provider
|
||||
// Call-side metadata is model-scoped proof of generation (Gemini thought
|
||||
// signatures, OpenAI encrypted reasoning): only the producing model may
|
||||
// replay it.
|
||||
const reuseToolProviderMetadata =
|
||||
reuseProviderMetadata ||
|
||||
(sameModel && item.executed === true && (item.state.status === "completed" || item.state.status === "error"))
|
||||
const call = toolCall(
|
||||
item,
|
||||
reuseToolProviderMetadata ? providerMetadata(providerMetadataKey, item.native) : undefined,
|
||||
)
|
||||
const call = toolCall(item, sameModel ? providerMetadata(providerMetadataKey, item.native) : undefined)
|
||||
if (item.executed !== true) return [call]
|
||||
// Hosted tools (e.g. google_search) run inside the provider, so their
|
||||
// native data is provider-format result data rather than model-scoped
|
||||
@@ -190,15 +211,13 @@ const assistant = (message: SessionMessage.Assistant, model: Model.Ref, provider
|
||||
return result ? [call, result] : [call]
|
||||
})
|
||||
const meaningful = content.filter((part) => {
|
||||
if (part.type === "text") return part.text !== ""
|
||||
if (part.type === "text") return cut ? part.text.trim() !== "" : part.text !== ""
|
||||
if (part.type !== "reasoning") return true
|
||||
return part.text !== "" || (part.providerMetadata !== undefined && Object.keys(part.providerMetadata).length > 0)
|
||||
})
|
||||
const results = message.content
|
||||
const results = items
|
||||
.filter((item): item is SessionMessage.AssistantTool => item.type === "tool" && item.executed !== true)
|
||||
.map((item) =>
|
||||
toolResult(item, reuseProviderMetadata ? providerMetadata(providerMetadataKey, item.native) : undefined),
|
||||
)
|
||||
.map((item) => toolResult(item, sameModel ? providerMetadata(providerMetadataKey, item.native) : undefined))
|
||||
.filter((message) => message !== undefined)
|
||||
.map(Message.tool)
|
||||
if (meaningful.length === 0) return results
|
||||
|
||||
@@ -476,7 +476,7 @@ it.effect("auto compaction estimates current content against the buffered prompt
|
||||
id: "call_read",
|
||||
name: "read",
|
||||
state: { status: "completed", input: {}, content: [{ type: "text", text: "x".repeat(4_000) }] },
|
||||
time: { created: DateTime.makeUnsafe(0) },
|
||||
time: { created: DateTime.makeUnsafe(0), ran: DateTime.makeUnsafe(0) },
|
||||
})
|
||||
const grown = { ...input(89_000, contextLimited), messages: [{ ...assistant, content: [tool] }] }
|
||||
expect(SessionCompaction.estimateContext(grown)).toBe(90_000)
|
||||
|
||||
@@ -60,6 +60,7 @@ describe("toLLMMessages", () => {
|
||||
agent: build,
|
||||
model: { id: Model.ID.make("model"), providerID: Provider.ID.make("provider") },
|
||||
content,
|
||||
finish: "stop",
|
||||
time: { created, completed: created },
|
||||
})
|
||||
const messages = toLLMMessages(
|
||||
@@ -746,7 +747,7 @@ Earlier work
|
||||
input: { path: "README.md" },
|
||||
metadata: { type: "media", mime: "image/png" },
|
||||
}),
|
||||
time: { created },
|
||||
time: { created, ran: created },
|
||||
}),
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
@@ -765,7 +766,7 @@ Earlier work
|
||||
},
|
||||
],
|
||||
}),
|
||||
time: { created, completed: created },
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
@@ -778,7 +779,7 @@ Earlier work
|
||||
input: { query: "Effect" },
|
||||
content: [{ type: "text", text: "Found it" }],
|
||||
}),
|
||||
time: { created, completed: created },
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
@@ -791,9 +792,10 @@ Earlier work
|
||||
input: { path: "README.md" },
|
||||
error: { type: "unknown", message: "Denied" },
|
||||
}),
|
||||
time: { created, completed: created },
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
],
|
||||
finish: "stop",
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
@@ -879,6 +881,7 @@ Earlier work
|
||||
native: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" },
|
||||
}),
|
||||
],
|
||||
finish: "stop",
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
@@ -910,6 +913,7 @@ Earlier work
|
||||
native: { signature: "signed" },
|
||||
}),
|
||||
],
|
||||
finish: "stop",
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
@@ -922,7 +926,7 @@ Earlier work
|
||||
])
|
||||
})
|
||||
|
||||
test("drops failed assistant reasoning while preserving settled hosted tools", () => {
|
||||
test("retries a failed stream from its last local call that ran", () => {
|
||||
const messages = toLLMMessages(
|
||||
[
|
||||
SessionMessage.Assistant.make({
|
||||
@@ -933,7 +937,7 @@ Earlier work
|
||||
content: [
|
||||
SessionMessage.AssistantReasoning.make({
|
||||
type: "reasoning",
|
||||
text: "Partial thought",
|
||||
text: "Finished thought",
|
||||
native: { itemId: "rs_failed", reasoningEncryptedContent: null },
|
||||
}),
|
||||
SessionMessage.AssistantTool.make({
|
||||
@@ -947,7 +951,7 @@ Earlier work
|
||||
input: { query: "Effect" },
|
||||
content: [{ type: "text", text: '{"found":true}' }],
|
||||
}),
|
||||
time: { created, completed: created },
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
@@ -960,18 +964,73 @@ Earlier work
|
||||
input: { query: "Effect" },
|
||||
error: { type: "unknown", message: "Step interrupted" },
|
||||
}),
|
||||
time: { created, completed: created },
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
id: "local-ran",
|
||||
name: "read",
|
||||
state: SessionMessage.ToolStateCompleted.make({
|
||||
status: "completed",
|
||||
input: { path: "README.md" },
|
||||
content: [{ type: "text", text: "Hello" }],
|
||||
}),
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
id: "hosted-after-local",
|
||||
name: "web_search",
|
||||
executed: true,
|
||||
state: SessionMessage.ToolStateError.make({
|
||||
status: "error",
|
||||
input: { query: "Later" },
|
||||
error: { type: "tool.result-missing", message: "Provider did not return a tool result" },
|
||||
}),
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
],
|
||||
finish: "error",
|
||||
error: { type: "unknown", message: "Step interrupted" },
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
// Hosted calls alone are provider-side reads the retry can redo, so the request never ends on an
|
||||
// assistant reply.
|
||||
SessionMessage.Assistant.make({
|
||||
id: id("assistant-failed-hosted-only"),
|
||||
type: "assistant",
|
||||
agent: build,
|
||||
model: { id: Model.ID.make("model"), providerID: Provider.ID.make("provider") },
|
||||
content: [
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
id: "hosted-only",
|
||||
name: "web_search",
|
||||
executed: true,
|
||||
state: SessionMessage.ToolStateCompleted.make({
|
||||
status: "completed",
|
||||
input: { query: "Effect" },
|
||||
content: [{ type: "text", text: '{"found":true}' }],
|
||||
}),
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
],
|
||||
finish: "error",
|
||||
error: { type: "provider.unknown", message: "Stream dropped" },
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
model,
|
||||
)
|
||||
|
||||
expect(messages.map((message) => message.role)).toEqual(["assistant", "tool"])
|
||||
expect(messages[1]?.content).toMatchObject([{ type: "tool-result", id: "local-ran" }])
|
||||
expect(messages[0]?.content).toEqual([
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Finished thought",
|
||||
providerMetadata: { provider: { itemId: "rs_failed", reasoningEncryptedContent: null } },
|
||||
},
|
||||
{
|
||||
type: "tool-call",
|
||||
id: "hosted-completed",
|
||||
@@ -1012,6 +1071,7 @@ Earlier work
|
||||
cache: undefined,
|
||||
providerMetadata: { provider: { itemId: "hosted_failed" } },
|
||||
},
|
||||
{ type: "tool-call", id: "local-ran", name: "read", input: { path: "README.md" } },
|
||||
])
|
||||
})
|
||||
|
||||
@@ -1071,7 +1131,7 @@ Earlier work
|
||||
input: { query: "Effect" },
|
||||
content: [{ type: "text", text: '{"status":"completed"}' }],
|
||||
}),
|
||||
time: { created, completed: created },
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
@@ -1084,9 +1144,10 @@ Earlier work
|
||||
input: { path: "README.md" },
|
||||
content: [{ type: "text", text: "Hello" }],
|
||||
}),
|
||||
time: { created, completed: created },
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
],
|
||||
finish: "stop",
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
@@ -1151,6 +1212,7 @@ Earlier work
|
||||
native: { reasoningEncryptedContent: "encrypted" },
|
||||
}),
|
||||
],
|
||||
finish: "stop",
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
@@ -1166,11 +1228,11 @@ Earlier work
|
||||
])
|
||||
})
|
||||
|
||||
test("drops assistant text provider state across model changes and failures", () => {
|
||||
const messages = toLLMMessages(
|
||||
test("drops assistant text provider state across model changes, aborts, and provider failures", () => {
|
||||
const switched = toLLMMessages(
|
||||
[
|
||||
SessionMessage.Assistant.make({
|
||||
id: id("assistant-phase"),
|
||||
id: id("assistant-switched"),
|
||||
type: "assistant",
|
||||
agent: build,
|
||||
model: { id: Model.ID.make("old"), providerID: Provider.ID.make("provider") },
|
||||
@@ -1181,20 +1243,357 @@ Earlier work
|
||||
native: { phase: "commentary" },
|
||||
}),
|
||||
],
|
||||
error: { type: "provider.unknown", message: "Interrupted after commentary" },
|
||||
finish: "stop",
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
Model.Ref.make({ id: Model.ID.make("new"), providerID: Provider.ID.make("provider") }),
|
||||
)
|
||||
|
||||
expect(messages[0]?.content).toEqual([
|
||||
expect(switched[0]?.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: "Checking.",
|
||||
providerMetadata: undefined,
|
||||
},
|
||||
])
|
||||
|
||||
const aborted = toLLMMessages(
|
||||
[
|
||||
SessionMessage.Assistant.make({
|
||||
id: id("assistant-aborted"),
|
||||
type: "assistant",
|
||||
agent: build,
|
||||
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
|
||||
content: [
|
||||
SessionMessage.AssistantReasoning.make({
|
||||
type: "reasoning",
|
||||
text: "Finished thought",
|
||||
native: { signature: "sig_kept" },
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
SessionMessage.AssistantText.make({
|
||||
type: "text",
|
||||
text: "Closed commentary.",
|
||||
native: { phase: "commentary" },
|
||||
}),
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
id: "uncalled-stream",
|
||||
name: "write",
|
||||
state: SessionMessage.ToolStateError.make({
|
||||
status: "error",
|
||||
input: {},
|
||||
error: { type: "aborted", message: "Step interrupted" },
|
||||
}),
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
error: { type: "aborted", message: "Step interrupted" },
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
SessionMessage.Assistant.make({
|
||||
id: id("assistant-aborted-mid-text"),
|
||||
type: "assistant",
|
||||
agent: build,
|
||||
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
|
||||
content: [
|
||||
SessionMessage.AssistantText.make({
|
||||
type: "text",
|
||||
text: "Cut off sentence",
|
||||
native: { phase: "final_answer" },
|
||||
}),
|
||||
],
|
||||
error: { type: "aborted", message: "Step interrupted" },
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
|
||||
)
|
||||
expect(aborted.map((item) => item.content)).toEqual([
|
||||
[
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Finished thought",
|
||||
providerMetadata: { provider: { signature: "sig_kept" } },
|
||||
},
|
||||
{
|
||||
type: "text",
|
||||
text: "Closed commentary.",
|
||||
providerMetadata: { provider: { phase: "commentary" } },
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
type: "text",
|
||||
text: "Cut off sentence",
|
||||
providerMetadata: undefined,
|
||||
},
|
||||
],
|
||||
])
|
||||
|
||||
// A failed stream keeps only what had happened by its last call that ran, whether it was retried,
|
||||
// retries ran out, or Esc landed during the retry wait. With no call that ran, nothing is kept.
|
||||
const failedTurn = (value: string, content: SessionMessage.Assistant["content"]) =>
|
||||
SessionMessage.Assistant.make({
|
||||
id: id(value),
|
||||
type: "assistant",
|
||||
agent: build,
|
||||
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
|
||||
content,
|
||||
error: { type: "provider.unknown", message: "Stream dropped" },
|
||||
finish: "error",
|
||||
time: { created, completed: created },
|
||||
})
|
||||
const failedTurns = toLLMMessages(
|
||||
[
|
||||
failedTurn("retries-exhausted", [
|
||||
SessionMessage.AssistantText.make({
|
||||
type: "text",
|
||||
text: "Partial visible answer",
|
||||
native: { phase: "commentary" },
|
||||
}),
|
||||
]),
|
||||
SessionMessage.Idle.make({ id: id("idle-failed"), type: "idle", outcome: "failed", time: { created } }),
|
||||
failedTurn("tool-ran-before-failure", [
|
||||
SessionMessage.AssistantText.make({ type: "text", text: "Reading first", native: { phase: "commentary" } }),
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
id: "ran",
|
||||
name: "read",
|
||||
state: SessionMessage.ToolStateCompleted.make({
|
||||
status: "completed",
|
||||
input: { path: "README.md" },
|
||||
content: [{ type: "text", text: "Hello" }],
|
||||
}),
|
||||
time: { created, completed: created, ran: created },
|
||||
}),
|
||||
SessionMessage.AssistantText.make({ type: "text", text: "Now I'll run the te" }),
|
||||
]),
|
||||
],
|
||||
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
|
||||
)
|
||||
expect(failedTurns.map((item) => ({ role: item.role, types: item.content.map((part) => part.type) }))).toEqual([
|
||||
{ role: "assistant", types: ["text", "tool-call"] },
|
||||
{ role: "tool", types: ["tool-result"] },
|
||||
])
|
||||
expect(failedTurns[0]?.content[0]).toEqual({
|
||||
type: "text",
|
||||
text: "Reading first",
|
||||
providerMetadata: { provider: { phase: "commentary" } },
|
||||
})
|
||||
|
||||
const supersededFailed = toLLMMessages(
|
||||
[
|
||||
SessionMessage.Assistant.make({
|
||||
id: id("assistant-superseded"),
|
||||
type: "assistant",
|
||||
agent: build,
|
||||
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
|
||||
content: [
|
||||
SessionMessage.AssistantText.make({
|
||||
type: "text",
|
||||
text: "Partial broken stream",
|
||||
native: { phase: "commentary" },
|
||||
}),
|
||||
],
|
||||
retry: {
|
||||
attempt: 1,
|
||||
at: created,
|
||||
error: { type: "provider.unknown", message: "Stream dropped" },
|
||||
},
|
||||
error: { type: "provider.unknown", message: "Stream dropped" },
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
SessionMessage.ModelSelected.make({
|
||||
id: id("model-switched-during-retry"),
|
||||
type: "model-switched",
|
||||
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
|
||||
time: { created },
|
||||
}),
|
||||
SessionMessage.Assistant.make({
|
||||
id: id("assistant-replacement"),
|
||||
type: "assistant",
|
||||
agent: build,
|
||||
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
|
||||
content: [
|
||||
SessionMessage.AssistantText.make({
|
||||
type: "text",
|
||||
text: "Clean retry answer",
|
||||
native: { phase: "final_answer" },
|
||||
}),
|
||||
],
|
||||
finish: "stop",
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
|
||||
)
|
||||
expect(supersededFailed).toHaveLength(1)
|
||||
expect(supersededFailed[0]?.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: "Clean retry answer",
|
||||
providerMetadata: { provider: { phase: "final_answer" } },
|
||||
},
|
||||
])
|
||||
|
||||
// A crashed step is stamped completed when the next step starts but never gets a finish. Cut turns drop
|
||||
// calls that never ran and only replay reasoning the model moved past, so a reasoning-only cut is omitted.
|
||||
const cutAssistant = (
|
||||
value: string,
|
||||
content: SessionMessage.Assistant["content"],
|
||||
error?: SessionMessage.Assistant["error"],
|
||||
) =>
|
||||
SessionMessage.Assistant.make({
|
||||
id: id(value),
|
||||
type: "assistant",
|
||||
agent: build,
|
||||
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
|
||||
content,
|
||||
error,
|
||||
finish: error === undefined ? undefined : "error",
|
||||
time: { created, completed: created },
|
||||
})
|
||||
const signed = (text: string, native: SessionMessage.Native, completed = true) =>
|
||||
SessionMessage.AssistantReasoning.make({
|
||||
type: "reasoning",
|
||||
text,
|
||||
native,
|
||||
time: completed ? { created, completed: created } : { created },
|
||||
})
|
||||
const uncalled = (callID: string, state: SessionMessage.AssistantTool["state"], executed?: boolean) =>
|
||||
SessionMessage.AssistantTool.make({ type: "tool", id: callID, name: "echo", executed, state, time: { created } })
|
||||
const stepAborted = { type: "aborted" as const, message: "Step interrupted" }
|
||||
const edgeAborts = toLLMMessages(
|
||||
[
|
||||
cutAssistant("crash-cut-reasoning-and-streaming-tool", [
|
||||
signed("Signed thought before crash", { signature: "sig_crash" }),
|
||||
uncalled("stale-streaming", SessionMessage.ToolStateStreaming.make({ status: "streaming", input: '{"te' })),
|
||||
]),
|
||||
cutAssistant(
|
||||
"whitespace-between-thinking-and-aborted-tool",
|
||||
[
|
||||
signed("Signed thought", { signature: "sig_ws" }),
|
||||
SessionMessage.AssistantText.make({ type: "text", text: "\n\n" }),
|
||||
uncalled(
|
||||
"aborted-uncalled",
|
||||
SessionMessage.ToolStateError.make({ status: "error", input: {}, error: stepAborted }),
|
||||
),
|
||||
],
|
||||
stepAborted,
|
||||
),
|
||||
cutAssistant(
|
||||
"openai-multipart-summary-cut",
|
||||
[
|
||||
signed("Part 0", { itemId: "rs_cut", reasoningEncryptedContent: null }),
|
||||
signed("Part 1", { itemId: "rs_cut", reasoningEncryptedContent: null }, false),
|
||||
],
|
||||
stepAborted,
|
||||
),
|
||||
cutAssistant(
|
||||
"openai-multipart-summary-then-text",
|
||||
[
|
||||
signed("Part 0", { itemId: "rs_done", reasoningEncryptedContent: null }),
|
||||
signed("Part 1", { itemId: "rs_done", reasoningEncryptedContent: "encrypted" }),
|
||||
SessionMessage.AssistantText.make({
|
||||
type: "text",
|
||||
text: "Visible answer before Esc",
|
||||
native: { itemId: "msg" },
|
||||
}),
|
||||
],
|
||||
stepAborted,
|
||||
),
|
||||
],
|
||||
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
|
||||
)
|
||||
expect(edgeAborts).toHaveLength(1)
|
||||
expect(edgeAborts[0]?.content).toEqual([
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Part 0",
|
||||
providerMetadata: { provider: { itemId: "rs_done", reasoningEncryptedContent: null } },
|
||||
},
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Part 1",
|
||||
providerMetadata: { provider: { itemId: "rs_done", reasoningEncryptedContent: "encrypted" } },
|
||||
},
|
||||
{ type: "text", text: "Visible answer before Esc", providerMetadata: undefined },
|
||||
])
|
||||
|
||||
// A cut stream drops a hosted call that never ran (marked executed at input start) but keeps input
|
||||
// feedback so the model can reissue a malformed call.
|
||||
const failedWithCalls = toLLMMessages(
|
||||
[
|
||||
cutAssistant(
|
||||
"terminal-failure-with-uncalled-calls",
|
||||
[
|
||||
SessionMessage.AssistantText.make({ type: "text", text: "Searching" }),
|
||||
uncalled(
|
||||
"hosted-uncalled",
|
||||
SessionMessage.ToolStateError.make({
|
||||
status: "error",
|
||||
input: {},
|
||||
error: stepAborted,
|
||||
}),
|
||||
true,
|
||||
),
|
||||
uncalled(
|
||||
"malformed",
|
||||
SessionMessage.ToolStateError.make({
|
||||
status: "error",
|
||||
input: { raw: "{" },
|
||||
error: { type: "tool.input-json", message: "Reissue the call with valid JSON arguments." },
|
||||
}),
|
||||
),
|
||||
],
|
||||
stepAborted,
|
||||
),
|
||||
],
|
||||
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
|
||||
)
|
||||
expect(failedWithCalls.map((message) => message.content.map((part) => part.type))).toEqual([
|
||||
["text", "tool-call"],
|
||||
["tool-result"],
|
||||
])
|
||||
expect(failedWithCalls[0]?.content[1]).toMatchObject({ type: "tool-call", id: "malformed" })
|
||||
})
|
||||
|
||||
test("replays tools migrated from V1, which never recorded when they ran", () => {
|
||||
const migrated = (value: string, error?: SessionMessage.Assistant["error"]) =>
|
||||
SessionMessage.Assistant.make({
|
||||
id: id(value),
|
||||
type: "assistant",
|
||||
agent: build,
|
||||
model: { id: Model.ID.make("same"), providerID: Provider.ID.make("provider") },
|
||||
content: [
|
||||
SessionMessage.AssistantTool.make({
|
||||
type: "tool",
|
||||
id: `${value}-read`,
|
||||
name: "read",
|
||||
state: SessionMessage.ToolStateCompleted.make({
|
||||
status: "completed",
|
||||
input: { path: "README.md" },
|
||||
content: [{ type: "text", text: "Hello" }],
|
||||
}),
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
error,
|
||||
finish: error === undefined ? "tool-calls" : "error",
|
||||
time: { created, completed: created },
|
||||
})
|
||||
const messages = toLLMMessages(
|
||||
[
|
||||
migrated("finished"),
|
||||
migrated("aborted", { type: "aborted", message: "Step interrupted" }),
|
||||
migrated("failed", { type: "provider.unknown", message: "Stream dropped" }),
|
||||
],
|
||||
Model.Ref.make({ id: Model.ID.make("same"), providerID: Provider.ID.make("provider") }),
|
||||
)
|
||||
expect(
|
||||
messages.map((message) => message.content.map((part) => (part.type === "tool-call" ? part.id : part.type))),
|
||||
).toEqual([["finished-read"], ["tool-result"], ["aborted-read"], ["tool-result"], ["failed-read"], ["tool-result"]])
|
||||
})
|
||||
|
||||
test("preserves assistant text provider state for the same model", () => {
|
||||
@@ -1212,6 +1611,7 @@ Earlier work
|
||||
native: { phase: "commentary" },
|
||||
}),
|
||||
],
|
||||
finish: "stop",
|
||||
time: { created, completed: created },
|
||||
}),
|
||||
],
|
||||
|
||||
@@ -697,9 +697,6 @@ const incompleteStream = () =>
|
||||
}),
|
||||
})
|
||||
|
||||
const INCOMPLETE_STREAM_CONTINUATION =
|
||||
"The previous response was interrupted. Continue from where you left off without repeating completed content."
|
||||
|
||||
const providerRejected = (input: { readonly message: string; readonly classification?: "context-overflow" }) =>
|
||||
new AIError({ reason: new InvalidRequestError(input) })
|
||||
|
||||
@@ -5222,7 +5219,7 @@ describe("SessionRunnerLLM", () => {
|
||||
yield* s.resume
|
||||
|
||||
expect(s.requests).toHaveLength(1)
|
||||
expect(messageRoles(s.requests[0])).toEqual(["user", "assistant", "tool"])
|
||||
expect(messageRoles(s.requests[0])).toEqual(["user"])
|
||||
expect(yield* s.context).toMatchObject([
|
||||
Expected.user("Recover interrupted tool input"),
|
||||
Expected.assistant({}, [Expected.failedTool({ id: "call-pending-interrupted" }, {})]),
|
||||
@@ -5767,7 +5764,20 @@ describe("SessionRunnerLLM", () => {
|
||||
scenario("durably fails blocked local tools when interrupted while awaiting settlement", function* (s) {
|
||||
yield* s.admit("Interrupt tool settlement")
|
||||
const tools = yield* s.blockTools()
|
||||
yield* s.llm.push(TestLLM.toolCall("call-await-interrupt", "echo", { text: "blocked" }))
|
||||
yield* s.llm.push([
|
||||
LLMEvent.stepStart({ index: 0 }),
|
||||
LLMEvent.reasoningStart({ id: "r1" }),
|
||||
LLMEvent.reasoningDelta({ id: "r1", text: "Plan before tool" }),
|
||||
LLMEvent.reasoningEnd({ id: "r1", providerMetadata: { openai: { signature: "sig_tool_interrupt" } } }),
|
||||
LLMEvent.toolCall({
|
||||
id: "call-await-interrupt",
|
||||
name: "echo",
|
||||
input: { text: "blocked" },
|
||||
providerMetadata: { openai: { thoughtSignature: "tool_sig_1" } },
|
||||
}),
|
||||
LLMEvent.stepFinish({ index: 0, reason: { normalized: "tool-calls" } }),
|
||||
LLMEvent.finish({ reason: { normalized: "tool-calls" } }),
|
||||
])
|
||||
const streamed = yield* s.bus.subscribe(SessionEvent.Step.Streamed).pipe(
|
||||
Stream.filter((event) => event.data.sessionID === sessionID),
|
||||
Stream.runHead,
|
||||
@@ -5785,6 +5795,7 @@ describe("SessionRunnerLLM", () => {
|
||||
expect(yield* s.context).toMatchObject([
|
||||
Expected.user("Interrupt tool settlement"),
|
||||
Expected.assistant({ finish: "error", error: { type: "aborted", message: "Step interrupted" } }, [
|
||||
Expected.reasoning("Plan before tool"),
|
||||
Expected.failedTool(
|
||||
{ id: "call-await-interrupt" },
|
||||
{ error: { type: "aborted", message: "Tool execution interrupted" } },
|
||||
@@ -5797,6 +5808,83 @@ describe("SessionRunnerLLM", () => {
|
||||
expect(eventTypes).not.toContain("session.step.ended.1")
|
||||
expect(eventTypes).not.toContain("session.retry.scheduled.1")
|
||||
expect(s.requests).toHaveLength(1)
|
||||
|
||||
yield* s.llm.push(TestLLM.text("Resumed with signatures intact", "resumed"))
|
||||
yield* s.runPrompt("Continue after cancelling tool")
|
||||
expect(s.requests[1]?.messages).toMatchObject([
|
||||
{ role: "user", content: [{ type: "text", text: "Interrupt tool settlement" }] },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "reasoning",
|
||||
text: "Plan before tool",
|
||||
providerMetadata: { openai: { signature: "sig_tool_interrupt" } },
|
||||
},
|
||||
{
|
||||
type: "tool-call",
|
||||
id: "call-await-interrupt",
|
||||
name: "echo",
|
||||
providerMetadata: { openai: { thoughtSignature: "tool_sig_1" } },
|
||||
},
|
||||
],
|
||||
},
|
||||
{ role: "tool", content: [{ type: "tool-result", id: "call-await-interrupt" }] },
|
||||
{ role: "user", content: [{ type: "text", text: "Continue after cancelling tool" }] },
|
||||
])
|
||||
})
|
||||
|
||||
scenario("keeps closed blocks and drops uncalled tool inputs when interrupted mid-stream", function* (s) {
|
||||
yield* s.admit("Interrupt mid-second-tool")
|
||||
const tools = yield* s.blockTools()
|
||||
const streamingSecond = yield* Deferred.make<void>()
|
||||
yield* s.llm.push(
|
||||
Stream.concat(
|
||||
Stream.fromIterable([
|
||||
LLMEvent.stepStart({ index: 0 }),
|
||||
LLMEvent.reasoningStart({ id: "r1" }),
|
||||
LLMEvent.reasoningDelta({ id: "r1", text: "Closed thought" }),
|
||||
LLMEvent.reasoningEnd({ id: "r1", providerMetadata: { openai: { signature: "sig_closed" } } }),
|
||||
LLMEvent.toolCall({
|
||||
id: "call-first",
|
||||
name: "echo",
|
||||
input: { text: "first" },
|
||||
providerMetadata: { openai: { thoughtSignature: "tool_sig_first" } },
|
||||
}),
|
||||
LLMEvent.toolInputStart({ id: "call-second-cut", name: "echo" }),
|
||||
LLMEvent.toolInputDelta({ id: "call-second-cut", name: "echo", text: '{"text":"half' }),
|
||||
]),
|
||||
Stream.fromEffect(Deferred.succeed(streamingSecond, undefined)).pipe(Stream.flatMap(() => Stream.never)),
|
||||
),
|
||||
)
|
||||
|
||||
const runner = yield* SessionRunner.Service
|
||||
const run = yield* runner.drain({ sessionID, force: true }).pipe(Effect.forkChild)
|
||||
yield* tools.started
|
||||
yield* Deferred.await(streamingSecond)
|
||||
yield* Fiber.interrupt(run)
|
||||
expect(Exit.isFailure(yield* Fiber.await(run))).toBe(true)
|
||||
|
||||
yield* s.llm.push(TestLLM.text("Continued cleanly", "continued"))
|
||||
yield* s.runPrompt("Proceed without second tool")
|
||||
expect(s.requests[1]?.messages).toMatchObject([
|
||||
{ role: "user", content: [{ type: "text", text: "Interrupt mid-second-tool" }] },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "reasoning", text: "Closed thought", providerMetadata: { openai: { signature: "sig_closed" } } },
|
||||
{
|
||||
type: "tool-call",
|
||||
id: "call-first",
|
||||
name: "echo",
|
||||
input: { text: "first" },
|
||||
providerMetadata: { openai: { thoughtSignature: "tool_sig_first" } },
|
||||
},
|
||||
],
|
||||
},
|
||||
{ role: "tool", content: [{ type: "tool-result", id: "call-first" }] },
|
||||
{ role: "user", content: [{ type: "text", text: "Proceed without second tool" }] },
|
||||
])
|
||||
})
|
||||
|
||||
scenario("forces a text response on an agent's configured final step", function* (s) {
|
||||
@@ -6321,7 +6409,7 @@ describe("SessionRunnerLLM", () => {
|
||||
})
|
||||
}
|
||||
|
||||
scenario("continues an incomplete stream after observable text", function* (s) {
|
||||
scenario("retries an incomplete stream cleanly from the prompt when no tool ran", function* (s) {
|
||||
const failure = incompleteStream()
|
||||
yield* s.admit("Continue partial output")
|
||||
yield* s.llm.push(
|
||||
@@ -6332,7 +6420,7 @@ describe("SessionRunnerLLM", () => {
|
||||
LLMEvent.textDelta({ id: "partial-rate-limit", text: "Partial" }),
|
||||
),
|
||||
)
|
||||
yield* s.llm.push(TestLLM.text(" continuation", "continued-text"))
|
||||
yield* s.llm.push(TestLLM.text("Recovered", "continued-text"))
|
||||
|
||||
const scheduled = yield* subscribeRetries(s)
|
||||
const run = yield* s.resume.pipe(Effect.forkChild)
|
||||
@@ -6341,38 +6429,24 @@ describe("SessionRunnerLLM", () => {
|
||||
yield* Fiber.join(run)
|
||||
|
||||
expect(s.requests).toHaveLength(2)
|
||||
expect(s.requests[1]?.messages.at(-2)).toMatchObject({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "Partial" }],
|
||||
})
|
||||
expect(s.requests[1]?.messages.at(-1)).toMatchObject({
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: INCOMPLETE_STREAM_CONTINUATION,
|
||||
},
|
||||
],
|
||||
})
|
||||
expect(s.requests[1]?.messages).toMatchObject([
|
||||
{ role: "user", content: [{ type: "text", text: "Continue partial output" }] },
|
||||
])
|
||||
const context = yield* s.context
|
||||
expect(context).toMatchObject([
|
||||
Expected.user("Continue partial output"),
|
||||
Expected.assistant({ finish: "error", error: { type: "provider.invalid-output" } }, [Expected.text("Partial")]),
|
||||
{
|
||||
type: "synthetic",
|
||||
text: INCOMPLETE_STREAM_CONTINUATION,
|
||||
},
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text(" continuation")]),
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
|
||||
])
|
||||
const assistants = context.filter((message) => message.type === "assistant")
|
||||
expect(new Set(assistants.map((message) => message.id)).size).toBe(2)
|
||||
expect(context.find((message) => message.type === "synthetic")?.description).toBeUndefined()
|
||||
expect(context.some((message) => message.type === "synthetic")).toBe(false)
|
||||
expect(yield* recordedEventTypes(sessionID)).toContain("session.retry.scheduled.1")
|
||||
yield* replaySessionProjection(sessionID)
|
||||
expect(yield* s.context).toMatchObject(context)
|
||||
})
|
||||
|
||||
scenario("continues an unknown finish after observable text", function* (s) {
|
||||
scenario("retries an unknown finish cleanly after observable text when no tool ran", function* (s) {
|
||||
yield* s.admit("Continue unknown finish")
|
||||
yield* s.llm.push([
|
||||
LLMEvent.stepStart({ index: 0 }),
|
||||
@@ -6382,7 +6456,7 @@ describe("SessionRunnerLLM", () => {
|
||||
LLMEvent.stepFinish({ index: 0, reason: { normalized: "unknown" } }),
|
||||
LLMEvent.finish({ reason: { normalized: "unknown" } }),
|
||||
])
|
||||
yield* s.llm.push(TestLLM.text(" continuation", "unknown-continuation"))
|
||||
yield* s.llm.push(TestLLM.text("Recovered", "unknown-continuation"))
|
||||
|
||||
const scheduled = yield* subscribeRetries(s)
|
||||
const run = yield* s.resume.pipe(Effect.forkChild)
|
||||
@@ -6391,19 +6465,17 @@ describe("SessionRunnerLLM", () => {
|
||||
yield* Fiber.join(run)
|
||||
|
||||
expect(s.requests).toHaveLength(2)
|
||||
expect(s.requests[1]?.messages.at(-1)).toMatchObject({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: INCOMPLETE_STREAM_CONTINUATION }],
|
||||
})
|
||||
expect(s.requests[1]?.messages).toMatchObject([
|
||||
{ role: "user", content: [{ type: "text", text: "Continue unknown finish" }] },
|
||||
])
|
||||
expect(yield* s.context).toMatchObject([
|
||||
{ type: "user" },
|
||||
Expected.assistant({ finish: "error" }, [Expected.text("Partial")]),
|
||||
{ type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text(" continuation")]),
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
|
||||
])
|
||||
})
|
||||
|
||||
scenario("continues after a mid-stream rate limit honoring retry-after", function* (s) {
|
||||
scenario("retries after a mid-stream rate limit honoring retry-after without partial text", function* (s) {
|
||||
yield* s.admit("Continue after rate limit")
|
||||
yield* s.llm.push(
|
||||
TestLLM.failAfter(
|
||||
@@ -6413,7 +6485,7 @@ describe("SessionRunnerLLM", () => {
|
||||
LLMEvent.textDelta({ id: "rate-limited-partial", text: "Partial" }),
|
||||
),
|
||||
)
|
||||
yield* s.llm.push(TestLLM.text(" continuation", "rate-limit-continuation"))
|
||||
yield* s.llm.push(TestLLM.text("Recovered", "rate-limit-continuation"))
|
||||
|
||||
const scheduled = yield* subscribeRetries(s)
|
||||
const run = yield* s.resume.pipe(Effect.forkChild)
|
||||
@@ -6424,24 +6496,18 @@ describe("SessionRunnerLLM", () => {
|
||||
yield* Fiber.join(run)
|
||||
|
||||
expect(s.requests).toHaveLength(2)
|
||||
expect(s.requests[1]?.messages.at(-2)).toMatchObject({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "Partial" }],
|
||||
})
|
||||
expect(s.requests[1]?.messages.at(-1)).toMatchObject({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: INCOMPLETE_STREAM_CONTINUATION }],
|
||||
})
|
||||
expect(s.requests[1]?.messages).toMatchObject([
|
||||
{ role: "user", content: [{ type: "text", text: "Continue after rate limit" }] },
|
||||
])
|
||||
expect(yield* recordedEventTypes(sessionID)).toContain("session.retry.scheduled.1")
|
||||
expect(yield* s.context).toMatchObject([
|
||||
Expected.user("Continue after rate limit"),
|
||||
Expected.assistant({ finish: "error", error: { type: "provider.rate-limit" } }, [Expected.text("Partial")]),
|
||||
{ type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text(" continuation")]),
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
|
||||
])
|
||||
})
|
||||
|
||||
scenario("continues after an unrecognized mid-stream provider failure", function* (s) {
|
||||
scenario("retries after an unrecognized mid-stream provider failure when no tool ran", function* (s) {
|
||||
const failure = new AIError({ reason: new UnknownProviderError({ message: "Provider returned error" }) })
|
||||
yield* s.admit("Continue after unknown failure")
|
||||
yield* s.llm.push(
|
||||
@@ -6452,7 +6518,7 @@ describe("SessionRunnerLLM", () => {
|
||||
LLMEvent.textDelta({ id: "unknown-failure-partial", text: "Partial" }),
|
||||
),
|
||||
)
|
||||
yield* s.llm.push(TestLLM.text(" continuation", "unknown-failure-continuation"))
|
||||
yield* s.llm.push(TestLLM.text("Recovered", "unknown-failure-continuation"))
|
||||
|
||||
const scheduled = yield* subscribeRetries(s)
|
||||
const run = yield* s.resume.pipe(Effect.forkChild)
|
||||
@@ -6461,19 +6527,17 @@ describe("SessionRunnerLLM", () => {
|
||||
yield* Fiber.join(run)
|
||||
|
||||
expect(s.requests).toHaveLength(2)
|
||||
expect(s.requests[1]?.messages.at(-1)).toMatchObject({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: INCOMPLETE_STREAM_CONTINUATION }],
|
||||
})
|
||||
expect(s.requests[1]?.messages).toMatchObject([
|
||||
{ role: "user", content: [{ type: "text", text: "Continue after unknown failure" }] },
|
||||
])
|
||||
expect(yield* s.context).toMatchObject([
|
||||
Expected.user("Continue after unknown failure"),
|
||||
Expected.assistant({ finish: "error", error: { type: "provider.unknown" } }, [Expected.text("Partial")]),
|
||||
{ type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text(" continuation")]),
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
|
||||
])
|
||||
})
|
||||
|
||||
scenario("drops interrupted reasoning before continuing an incomplete stream", function* (s) {
|
||||
scenario("drops interrupted reasoning and retries cleanly without continuation prompt", function* (s) {
|
||||
yield* s.admit("Continue interrupted reasoning")
|
||||
yield* s.llm.push(
|
||||
TestLLM.failAfter(
|
||||
@@ -6491,20 +6555,18 @@ describe("SessionRunnerLLM", () => {
|
||||
yield* TestClock.adjust("2400 millis")
|
||||
yield* Fiber.join(run)
|
||||
|
||||
expect(s.requests[1]?.messages.slice(-2)).toMatchObject([
|
||||
expect(s.requests[1]?.messages).toMatchObject([
|
||||
{ role: "user", content: [{ type: "text", text: "Continue interrupted reasoning" }] },
|
||||
{ role: "user", content: [{ type: "text", text: INCOMPLETE_STREAM_CONTINUATION }] },
|
||||
])
|
||||
expect(yield* s.context).toMatchObject([
|
||||
{ type: "user" },
|
||||
Expected.assistant({ finish: "error" }, [Expected.reasoning("Partial thought")]),
|
||||
{ type: "synthetic" },
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
|
||||
])
|
||||
})
|
||||
|
||||
for (const failure of [streamDisconnected, websocketDisconnected]) {
|
||||
scenario(`continues after ${failure.name} with durable reasoning state`, function* (s) {
|
||||
scenario(`retries cleanly after ${failure.name} during reasoning`, function* (s) {
|
||||
yield* s.admit("Recover disconnected reasoning")
|
||||
yield* s.llm.push(
|
||||
TestLLM.failAfter(
|
||||
@@ -6528,9 +6590,8 @@ describe("SessionRunnerLLM", () => {
|
||||
|
||||
expect(s.requests).toHaveLength(2)
|
||||
expect(yield* recordedEventTypes(sessionID)).toContain("session.retry.scheduled.1")
|
||||
expect(s.requests[1]?.messages.slice(-2)).toMatchObject([
|
||||
expect(s.requests[1]?.messages).toMatchObject([
|
||||
{ role: "user", content: [{ type: "text", text: "Recover disconnected reasoning" }] },
|
||||
{ role: "user", content: [{ type: "text", text: INCOMPLETE_STREAM_CONTINUATION }] },
|
||||
])
|
||||
expect(yield* s.context).toMatchObject([
|
||||
{ type: "user" },
|
||||
@@ -6541,13 +6602,12 @@ describe("SessionRunnerLLM", () => {
|
||||
native: { itemId: "rs_disconnected", reasoningEncryptedContent: "encrypted-state" },
|
||||
},
|
||||
]),
|
||||
{ type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
|
||||
])
|
||||
})
|
||||
}
|
||||
|
||||
scenario("continues an incomplete stream after settling a local tool", function* (s) {
|
||||
scenario("retries an incomplete stream from a settled local tool", function* (s) {
|
||||
yield* s.admit("Continue after tool")
|
||||
yield* s.llm.push(
|
||||
TestLLM.failAfter(
|
||||
@@ -6565,25 +6625,16 @@ describe("SessionRunnerLLM", () => {
|
||||
yield* Fiber.join(run)
|
||||
|
||||
expect(s.executions).toEqual(["settled"])
|
||||
expect(s.requests[1]?.messages.slice(-3)).toMatchObject([
|
||||
expect(s.requests[1]?.messages.slice(-2)).toMatchObject([
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "tool-call", id: "call-before-close", name: "echo", input: { text: "settled" } }],
|
||||
},
|
||||
{ role: "tool", content: [{ type: "tool-result", id: "call-before-close" }] },
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: INCOMPLETE_STREAM_CONTINUATION,
|
||||
},
|
||||
],
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
scenario("continues an incomplete stream after settling a local tool defect", function* (s) {
|
||||
scenario("retries an incomplete stream from a settled local tool defect", function* (s) {
|
||||
yield* s.admit("Continue after tool defect")
|
||||
yield* s.llm.push(
|
||||
TestLLM.failAfter(
|
||||
@@ -6600,7 +6651,7 @@ describe("SessionRunnerLLM", () => {
|
||||
yield* TestClock.adjust("2400 millis")
|
||||
yield* Fiber.join(run)
|
||||
|
||||
expect(messageRoles(s.requests[1])).toEqual(["user", "assistant", "tool", "user"])
|
||||
expect(messageRoles(s.requests[1])).toEqual(["user", "assistant", "tool"])
|
||||
expect(yield* s.context).toMatchObject([
|
||||
{ type: "user" },
|
||||
Expected.assistant({}, [
|
||||
@@ -6609,13 +6660,12 @@ describe("SessionRunnerLLM", () => {
|
||||
{ error: { type: "unknown", message: "unexpected tool defect" } },
|
||||
),
|
||||
]),
|
||||
{ type: "synthetic", text: INCOMPLETE_STREAM_CONTINUATION },
|
||||
Expected.assistant({ finish: "stop" }, [Expected.text("Recovered")]),
|
||||
])
|
||||
})
|
||||
|
||||
scenario(
|
||||
"shares retry accounting and assistant identity across transparent retries and partial continuations",
|
||||
"shares retry accounting and assistant identity across transparent retries and retries after output",
|
||||
function* (s) {
|
||||
const scheduled = yield* subscribeRetries(s)
|
||||
yield* s.admit("Mix retry paths")
|
||||
@@ -6626,7 +6676,7 @@ describe("SessionRunnerLLM", () => {
|
||||
LLMEvent.textStart({ id: "mixed-partial" }),
|
||||
LLMEvent.textDelta({ id: "mixed-partial", text: "Partial" }),
|
||||
)
|
||||
// Alternate transparent failures and partial continuations until the retry allowance is spent.
|
||||
// Alternate transparent failures and failures after output until the retry allowance is spent.
|
||||
const outcomes = RETRY_GAPS.map((_, index) => (index % 2 === 0 ? Stream.fail(failure) : partial))
|
||||
yield* s.llm.push(...outcomes, partial)
|
||||
const run = yield* s.resume.pipe(Effect.forkChild)
|
||||
@@ -6637,7 +6687,7 @@ describe("SessionRunnerLLM", () => {
|
||||
}
|
||||
expect(yield* Fiber.join(run).pipe(Effect.flip)).toBe(failure)
|
||||
expect(s.requests).toHaveLength(RETRY_GAPS.length + 1)
|
||||
// A transparent retry keeps the assistant identity; a partial continuation starts a new one.
|
||||
// A transparent retry keeps the assistant identity; a retry after output starts a new one.
|
||||
for (const [index, identity] of identities.entries()) {
|
||||
if (index === 0) continue
|
||||
if (index % 2 === 1) expect(identity).toBe(identities[index - 1])
|
||||
@@ -6646,14 +6696,14 @@ describe("SessionRunnerLLM", () => {
|
||||
const partials = outcomes.filter((outcome) => outcome === partial).length
|
||||
const messages = yield* s.context
|
||||
expect(messages.filter((message) => message.type === "assistant")).toHaveLength(partials + 1)
|
||||
expect(messages.filter((message) => message.type === "synthetic")).toHaveLength(partials)
|
||||
expect(messages.filter((message) => message.type === "synthetic")).toHaveLength(0)
|
||||
const events = yield* recordedEventTypes(sessionID)
|
||||
expect(events.filter((type) => type === "session.retry.scheduled.1")).toHaveLength(RETRY_GAPS.length)
|
||||
expect(events.filter((type) => type === "session.step.failed.1")).toHaveLength(partials + 1)
|
||||
},
|
||||
)
|
||||
|
||||
scenario("stops incomplete stream continuations once the retry allowance is spent", function* (s) {
|
||||
scenario("stops incomplete stream retries once the retry allowance is spent", function* (s) {
|
||||
yield* s.admit("Exhaust partial continuations")
|
||||
const failure = incompleteStream()
|
||||
yield* s.llm.always(
|
||||
@@ -6675,7 +6725,14 @@ describe("SessionRunnerLLM", () => {
|
||||
expect(s.requests).toHaveLength(RETRY_GAPS.length + 1)
|
||||
const context = yield* s.context
|
||||
expect(context.filter((message) => message.type === "assistant")).toHaveLength(RETRY_GAPS.length + 1)
|
||||
expect(context.filter((message) => message.type === "synthetic")).toHaveLength(RETRY_GAPS.length)
|
||||
expect(context.filter((message) => message.type === "synthetic")).toHaveLength(0)
|
||||
|
||||
yield* s.llm.always(TestLLM.text("Continued after exhaustion", "after-exhaustion"))
|
||||
yield* s.runPrompt("Continue please")
|
||||
expect(s.requests.at(-1)?.messages).toMatchObject([
|
||||
{ role: "user", content: [{ type: "text", text: "Exhaust partial continuations" }] },
|
||||
{ role: "user", content: [{ type: "text", text: "Continue please" }] },
|
||||
])
|
||||
})
|
||||
|
||||
scenario("stops once the retry allowance is spent", function* (s) {
|
||||
|
||||
Reference in New Issue
Block a user