diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 857347d1d46..de258fd3d16 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -40,6 +40,9 @@ const CONTINUE_AFTER_INCOMPLETE_STREAM = const CONTINUE_AFTER_OUTPUT_LIMIT = "Your last response hit the output token limit (stop reason: length). Do not apologize, recap, or repeat yourself. Break the remaining work into smaller pieces." +const CONTINUE_AFTER_EMPTY_RESPONSE = + "Your last response contained no answer or tool calls. Continue the task with the next action or a final answer." + type DrainInput = Parameters[0] const layer = Layer.effect( @@ -237,6 +240,7 @@ const layer = Layer.effect( let initial: SessionContext.Loaded | undefined = first let recoverOverflow = true let recoverContinuation = true + let recoverEmpty = true while (true) { // Reuse boundary preparation once; retries refresh context without delivering more input. const loaded = initial ?? (yield* prepareContext(sessionID).pipe(Effect.flatMap(context.load))) @@ -313,6 +317,15 @@ const layer = Layer.effect( ), }) const completed = yield* SessionStep.Outcome.$match(outcome, { + Empty: Effect.fnUntraced(function* () { + if (!recoverEmpty) + return yield* new StepFailedError({ + error: { type: "empty-response", message: "The model returned no answer or tool calls after one retry" }, + }) + recoverEmpty = false + yield* bus.publish(SessionEvent.Synthetic, { sessionID, text: CONTINUE_AFTER_EMPTY_RESPONSE }) + assistantMessageID = SessionMessage.ID.create() + }), Completed: Effect.fnUntraced(function* (outcome) { // The Session is about to go idle, which leaves the background summary time to finish. if (!outcome.needsContinuation) diff --git a/packages/core/src/session/runner/publish-llm-event.ts b/packages/core/src/session/runner/publish-llm-event.ts index 49ef5c7e22e..79558d0c550 100644 --- a/packages/core/src/session/runner/publish-llm-event.ts +++ b/packages/core/src/session/runner/publish-llm-event.ts @@ -60,6 +60,8 @@ const INPUT_FAILURES = { export interface StepRecord { /** The model produced visible output this attempt, which bars transparent retries and overflow recovery. */ readonly outputStarted: boolean + /** Reasoning and empty text blocks are not an answer. Tool input still follows tool recovery. */ + readonly empty: boolean readonly providerFailed: boolean /** The step's recorded assistant failure, if any. */ readonly failure?: SessionError.Error @@ -115,6 +117,7 @@ export const createLLMEventPublisher = (bus: Pick, inp let stepStarted = false let providerFailed = false let outputStarted = false + let hasText = false let stepFailure: SessionError.Error | undefined let stepSettlement: StepRecord["finish"] @@ -233,6 +236,7 @@ export const createLLMEventPublisher = (bus: Pick, inp name, (_id, value, ordinal, native) => Effect.gen(function* () { + if (name === "text" && value.trim()) hasText = true yield* bus.publish(events.Ended, { sessionID: input.sessionID, assistantMessageID: yield* currentAssistantMessageID(), @@ -591,6 +595,7 @@ export const createLLMEventPublisher = (bus: Pick, inp /** Immutable snapshot of everything recorded for this step so far. */ record: (): StepRecord => ({ outputStarted, + empty: !hasText && tools.size === 0, providerFailed, failure: stepFailure, finish: stepSettlement, diff --git a/packages/core/src/session/runner/step.ts b/packages/core/src/session/runner/step.ts index 9243c6702b5..66762f82517 100644 --- a/packages/core/src/session/runner/step.ts +++ b/packages/core/src/session/runner/step.ts @@ -29,6 +29,7 @@ import { SessionRunnerRetry } from "./retry.js" export type Outcome = Data.TaggedEnum<{ Completed: { readonly needsContinuation: boolean } + Empty: {} Retry: { readonly error: SessionError.Error; readonly decision: SessionRunnerRetry.Decision } Continue: { readonly error: SessionError.Error @@ -289,6 +290,7 @@ export const make = Effect.gen(function* () { if (record.failure) return yield* new StepFailedError({ error: record.failure }) const needsContinuation = input.prepared.request.toolChoice?.type !== "none" && record.needsContinuation if (record.finish?.finish === "length") return Outcome.OutputLimit({ needsContinuation }) + if (record.finish?.finish === "stop" && record.empty) return Outcome.Empty() return Outcome.Completed({ needsContinuation }) }), ) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 669988427da..9405ccd8542 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -846,10 +846,12 @@ const fragmentFixture = (kind: FragmentKind, id: string, chunks: readonly string completeEvents: [ ...partialEvents, LLMEvent.reasoningEnd({ id }), + LLMEvent.textStart({ id: "answer" }), + LLMEvent.textEnd({ id: "answer", text: "Done" }), LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" } }), LLMEvent.finish({ reason: { normalized: "stop" } }), ], - expectedAssistant: { type: "assistant", finish: "stop", content: [expectedContent] }, + expectedAssistant: { type: "assistant", finish: "stop", content: [expectedContent, Expected.text("Done")] }, expectedContent, } } @@ -1435,7 +1437,7 @@ describe("SessionRunnerLLM", () => { Stream.unwrap( Deferred.succeed(secondStarted, undefined).pipe( Effect.andThen(Deferred.await(releaseSecond)), - Effect.as(Stream.fromIterable(TestLLM.stop())), + Effect.as(Stream.fromIterable(TestLLM.text("Done", "done"))), ), ), Stream.fromIterable(TestLLM.text("Handled completion", "text-completion")), @@ -3656,8 +3658,8 @@ describe("SessionRunnerLLM", () => { TestLLM.failAfter(providerRejected({ message: "prompt too long", classification: "context-overflow" })), TestLLM.text("## Objective\n- Overflow summary", "overflow-summary"), TestLLM.text("Recovered", "overflow-recovered"), - TestLLM.stop(), - TestLLM.stop(), + TestLLM.text("Done", "done"), + TestLLM.text("Done", "done"), ) const first = yield* s.llm.gate() const run = yield* s.resume.pipe(Effect.forkChild) @@ -4062,7 +4064,7 @@ describe("SessionRunnerLLM", () => { scenario("reloads a model switch before a tool-driven continuation step", function* (s) { yield* s.admit("Echo this") - yield* s.llm.push(TestLLM.toolCall("call-echo", "echo", { text: "hello" }), TestLLM.stop()) + yield* s.llm.push(TestLLM.toolCall("call-echo", "echo", { text: "hello" }), TestLLM.text("Done", "done")) const tools = yield* s.blockTools() const run = yield* Effect.forkChild(s.resume) yield* tools.started @@ -4102,7 +4104,7 @@ describe("SessionRunnerLLM", () => { ), Stream.onEnd(Deferred.succeed(finished, undefined)), ), - TestLLM.stop(), + TestLLM.text("Done", "done"), ) const tools = yield* s.blockTools() const streamed = yield* s.bus.subscribe(SessionEvent.Step.Streamed).pipe( @@ -4173,6 +4175,8 @@ describe("SessionRunnerLLM", () => { anthropic: { ignored: true }, }, }), + LLMEvent.textStart({ id: "answer" }), + LLMEvent.textEnd({ id: "answer", text: "Done" }), ), ) yield* s.resume @@ -4191,6 +4195,7 @@ describe("SessionRunnerLLM", () => { text: "Encrypted thought", native: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" }, }, + Expected.text("Done"), ]), ]) @@ -4209,6 +4214,7 @@ describe("SessionRunnerLLM", () => { text: "Encrypted thought", providerMetadata: { openai: { itemId: "rs_1", reasoningEncryptedContent: "encrypted-state" } }, }, + { type: "text", text: "Done" }, ]) }) @@ -4451,7 +4457,7 @@ describe("SessionRunnerLLM", () => { scenario("steers an active step with newly recorded prompts", function* (s) { yield* s.admit("Start working") - yield* s.llm.push(TestLLM.stop(), TestLLM.stop()) + yield* s.llm.push(TestLLM.text("Done", "done"), TestLLM.text("Done", "done")) const first = yield* s.resumePaused yield* s.session.prompt({ sessionID, text: "Change direction" }) @@ -4467,7 +4473,11 @@ describe("SessionRunnerLLM", () => { scenario("promotes queued input after continuation ends", function* (s) { yield* s.admit("Start working") - yield* s.llm.push(TestLLM.toolCall("call-echo", "echo", { text: "hello" }), TestLLM.stop(), TestLLM.stop()) + yield* s.llm.push( + TestLLM.toolCall("call-echo", "echo", { text: "hello" }), + TestLLM.text("Done", "done"), + TestLLM.text("Done", "done"), + ) const first = yield* s.resumePaused yield* s.session.prompt({ @@ -4486,7 +4496,7 @@ describe("SessionRunnerLLM", () => { scenario("keeps queued input parked when a steer is cancelled during preparation", function* (s) { const runner = yield* SessionRunner.Service yield* s.admit("A") - yield* s.llm.push(TestLLM.stop(), TestLLM.stop(), TestLLM.stop()) + yield* s.llm.push(TestLLM.text("Done", "done"), TestLLM.text("Done", "done"), TestLLM.text("Done", "done")) const stream = yield* s.llm.gate() const run = yield* runner.drain({ sessionID, force: false }).pipe(Effect.forkChild) yield* stream.started @@ -4510,7 +4520,7 @@ describe("SessionRunnerLLM", () => { const location = Location.Ref.make({ directory: AbsolutePath.make("/moved") }) yield* s.admit("A") - yield* s.llm.push(TestLLM.stop(), TestLLM.stop()) + yield* s.llm.push(TestLLM.text("Done", "done"), TestLLM.text("Done", "done")) const stream = yield* s.llm.gate() const run = yield* runner.drain({ sessionID, force: false }).pipe(Effect.forkChild) yield* stream.started @@ -4544,7 +4554,7 @@ describe("SessionRunnerLLM", () => { scenario("preserves durable queued input for a later wake after interruption", function* (s) { yield* s.admit("Interrupt current work") - yield* s.llm.push([], TestLLM.stop()) + yield* s.llm.push([], TestLLM.text("Done", "done")) const stream = yield* s.llm.gate() const run = yield* s.resume.pipe(Effect.forkChild) @@ -4571,7 +4581,7 @@ describe("SessionRunnerLLM", () => { scenario("preserves durable steering input for a later resume after interruption", function* (s) { yield* s.admit("Interrupt current work") - yield* s.llm.push([], TestLLM.stop()) + yield* s.llm.push([], TestLLM.text("Done", "done")) const stream = yield* s.llm.gate() const run = yield* s.resume.pipe(Effect.forkChild) @@ -4598,7 +4608,7 @@ describe("SessionRunnerLLM", () => { scenario("promotes queued inputs one at a time in FIFO order", function* (s) { yield* s.admit("Start working") - yield* s.llm.push(TestLLM.stop(), TestLLM.stop(), TestLLM.stop()) + yield* s.llm.push(TestLLM.text("Done", "done"), TestLLM.text("Done", "done"), TestLLM.text("Done", "done")) const first = yield* s.resumePaused yield* s.session.prompt({ sessionID, text: "Queue first", delivery: "queue" }) @@ -4614,7 +4624,7 @@ describe("SessionRunnerLLM", () => { scenario("stops a steer-scoped drain before queued input", function* (s) { yield* s.session.prompt({ sessionID, text: "Queue for later", delivery: "queue", resume: false }) yield* s.session.prompt({ sessionID, text: "Steer now", resume: false }) - yield* s.llm.push(TestLLM.stop()) + yield* s.llm.push(TestLLM.text("Done", "done")) const runner = yield* SessionRunner.Service yield* runner.drain({ sessionID, force: false, promotable: "steer" }) @@ -4672,7 +4682,7 @@ describe("SessionRunnerLLM", () => { resume: false, }) - yield* s.llm.push(TestLLM.stop(), TestLLM.stop()) + yield* s.llm.push(TestLLM.text("Done", "done"), TestLLM.text("Done", "done")) yield* s.resume @@ -4684,7 +4694,12 @@ describe("SessionRunnerLLM", () => { scenario("promotes steers before the next queued input", function* (s) { yield* s.admit("Start working") - yield* s.llm.push(TestLLM.stop(), TestLLM.stop(), TestLLM.stop(), TestLLM.stop()) + yield* s.llm.push( + TestLLM.text("Done", "done"), + TestLLM.text("Done", "done"), + TestLLM.text("Done", "done"), + TestLLM.text("Done", "done"), + ) const firstStream = yield* s.llm.gate() const first = yield* s.resume.pipe(Effect.forkChild) @@ -4727,7 +4742,7 @@ describe("SessionRunnerLLM", () => { const execution = yield* SessionExecution.Service yield* s.admit("Start working") - yield* s.llm.push(TestLLM.stop(), TestLLM.stop()) + yield* s.llm.push(TestLLM.text("Done", "done"), TestLLM.text("Done", "done")) const first = yield* s.resumePaused yield* s.session.prompt({ sessionID, text: "First steer" }) @@ -5155,7 +5170,7 @@ describe("SessionRunnerLLM", () => { expect(yield* s.resume.pipe(Effect.catchDefect(Effect.succeed))).toBe(defect) fail = false s.requests.length = 0 - yield* s.llm.push(TestLLM.stop()) + yield* s.llm.push(TestLLM.text("Done", "done")) const stream = yield* s.llm.gate() yield* execution.wake(sessionID) @@ -5351,7 +5366,7 @@ describe("SessionRunnerLLM", () => { LLMEvent.toolCall({ id: "call-beta", name: "echo", input: { text: "beta-block" } }), LLMEvent.toolCall({ id: "call-ok", name: "echo", input: { text: "ok" } }), ), - TestLLM.stop(), + TestLLM.text("Done", "done"), ) yield* s.resume expect(calls).toEqual(["call-alpha", "call-beta", "call-ok"]) @@ -5388,7 +5403,7 @@ describe("SessionRunnerLLM", () => { ) yield* s.admit("Call blocked") - yield* s.llm.push(TestLLM.toolCall("call-blocked", "blocked", {}), TestLLM.stop()) + yield* s.llm.push(TestLLM.toolCall("call-blocked", "blocked", {}), TestLLM.text("Done", "done")) yield* s.resume @@ -5483,7 +5498,7 @@ describe("SessionRunnerLLM", () => { ) yield* s.admit("Call corrected") - yield* s.llm.push(TestLLM.toolCall("call-corrected", "corrected", {}), TestLLM.stop()) + yield* s.llm.push(TestLLM.toolCall("call-corrected", "corrected", {}), TestLLM.text("Done", "done")) yield* s.resume @@ -5501,10 +5516,7 @@ describe("SessionRunnerLLM", () => { const registry = yield* Tool.Service yield* transformTools(registry, { permissionfail: permissionFail }, { codemode: false }) yield* s.admit("Reject permission") - yield* s.llm.push(TestLLM.toolCall("call-permission", "permissionfail", {}), [ - LLMEvent.stepStart({ index: 0 }), - LLMEvent.stepFinish({ index: 0, reason: { normalized: "stop" } }), - ]) + yield* s.llm.push(TestLLM.toolCall("call-permission", "permissionfail", {}), TestLLM.text("Done", "done")) yield* s.resume @@ -5742,7 +5754,7 @@ describe("SessionRunnerLLM", () => { yield* s.llm.push( TestLLM.toolCall("call-before-steer", "echo", { text: "before" }), TestLLM.toolCall("call-after-steer", "echo", { text: "after" }), - TestLLM.stop(), + TestLLM.text("Done", "done"), ) const run = yield* s.resumePaused @@ -6027,6 +6039,95 @@ describe("SessionRunnerLLM", () => { ]) }) + scenario("continues an empty successful response after automatic compaction", function* (s) { + yield* s.llm.push(TestLLM.textWithUsage("Earlier answer", "earlier", 3_950)) + yield* s.runPrompt("Earlier question ".repeat(180)) + s.currentModel = compactModel + s.requests.length = 0 + yield* s.llm.push( + TestLLM.text("## Objective\n- Finish the task", "summary"), + TestLLM.complete({ reason: { normalized: "stop" } }), + TestLLM.text("Finished", "finished"), + ) + + yield* s.runPrompt("Recent exact request ".repeat(180)) + + expect(s.requests).toHaveLength(3) + expect(userTexts(s.requests[2]).at(-1)).toBe( + "Your last response contained no answer or tool calls. Continue the task with the next action or a final answer.", + ) + expect(yield* s.context).toMatchObject([ + { type: "compaction", status: "completed" }, + { type: "assistant", finish: "stop", content: [] }, + { type: "synthetic" }, + Expected.assistant({ finish: "stop" }, [Expected.text("Finished")]), + ]) + }) + + for (const content of ["none", "empty text", "whitespace", "reasoning"] as const) { + scenario(`bounds empty successful response recovery (${content})`, function* (s) { + const events = + content === "none" + ? [] + : content === "reasoning" + ? [ + LLMEvent.reasoningStart({ id: "thinking" }), + LLMEvent.reasoningDelta({ id: "thinking", text: "Thinking without an answer" }), + LLMEvent.reasoningEnd({ id: "thinking" }), + ] + : [ + LLMEvent.textStart({ id: "empty" }), + LLMEvent.textDelta({ id: "empty", text: content === "whitespace" ? " \n" : "" }), + LLMEvent.textEnd({ id: "empty" }), + ] + const empty = () => TestLLM.complete({ reason: { normalized: "stop" } }, ...events) + yield* s.llm.push(empty(), empty()) + + expect(yield* s.runPrompt("Finish the task").pipe(Effect.flip)).toMatchObject({ + error: { type: "empty-response" }, + }) + expect(s.requests).toHaveLength(2) + expect((yield* s.messages).filter((message) => message.type === "synthetic")).toHaveLength(1) + + yield* s.llm.push(empty(), TestLLM.text("Finished", "finished")) + yield* s.runPrompt("Try again") + expect(s.requests).toHaveLength(4) + expect((yield* s.context).at(-1)).toMatchObject(Expected.assistant({}, [Expected.text("Finished")])) + }) + } + + scenario("accepts text supplied only at text end without empty response recovery", function* (s) { + yield* s.llm.push( + TestLLM.complete( + { reason: { normalized: "stop" } }, + LLMEvent.textStart({ id: "answer" }), + LLMEvent.textEnd({ id: "answer", text: "Finished" }), + ), + ) + yield* s.runPrompt("Finish the task") + expect(s.requests).toHaveLength(1) + expect((yield* s.context).at(-1)).toMatchObject(Expected.assistant({}, [Expected.text("Finished")])) + }) + + scenario("does not recover an empty response when the stream fails after its finish event", function* (s) { + const failure = invalidRequest() + yield* s.llm.push(TestLLM.failAfter(failure, ...TestLLM.stop())) + + expect(yield* s.runPrompt("Finish the task").pipe(Effect.flip)).toBe(failure) + expect(s.requests).toHaveLength(1) + expect((yield* s.messages).filter((message) => message.type === "synthetic")).toHaveLength(0) + }) + + scenario("does not recover an empty response when the stream is interrupted", function* (s) { + yield* s.llm.push(Stream.fromIterable(TestLLM.stop()).pipe(Stream.concat(Stream.fromEffect(Effect.interrupt)))) + + const exit = yield* s.runPrompt("Finish the task").pipe(Effect.exit) + expect(exit._tag).toBe("Failure") + if (Exit.isFailure(exit)) expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true) + expect(s.requests).toHaveLength(1) + expect((yield* s.messages).filter((message) => message.type === "synthetic")).toHaveLength(0) + }) + scenario("uses a larger provider retry-after delay", function* (s) { yield* s.admit("Retry rate limit") yield* s.llm.push(Stream.fail(rateLimited(5_000))) @@ -6577,7 +6678,7 @@ describe("SessionRunnerLLM", () => { yield* s.admit("Retry without consuming a step") const failure = providerUnavailable() yield* s.llm.push(Stream.fail(failure)) - yield* s.llm.push(TestLLM.toolCall("call-after-retry", "echo", { text: "recovered" }), TestLLM.stop()) + yield* s.llm.push(TestLLM.toolCall("call-after-retry", "echo", { text: "recovered" }), TestLLM.text("Done", "done")) const scheduled = yield* subscribeRetries(s) const run = yield* s.resume.pipe(Effect.forkChild) @@ -6671,7 +6772,7 @@ describe("SessionRunnerLLM", () => { expect(yield* s.runPrompt("Call a malformed tool").pipe(Effect.flip)).toBe(failure) const assistant = requireAssistant(yield* s.context) - yield* s.llm.push(TestLLM.stop()) + yield* s.llm.push(TestLLM.text("Done", "done")) yield* s.runPrompt("Continue") expect(yield* recordedStepSettlementEvents(sessionID, assistant.id)).toMatchObject([ @@ -6698,7 +6799,7 @@ describe("SessionRunnerLLM", () => { LLMEvent.toolInputStart({ id: "call-incomplete", name: "echo" }), LLMEvent.toolInputDelta({ id: "call-incomplete", name: "echo", text: '{"text":"partial' }), ), - TestLLM.stop(), + TestLLM.text("Done", "done"), ) yield* s.runPrompt("Recover unfinished tool input") @@ -6764,7 +6865,7 @@ describe("SessionRunnerLLM", () => { raw, }), ), - TestLLM.stop(), + TestLLM.text("Done", "done"), ) yield* s.runPrompt("Recover malformed tool input") @@ -6853,7 +6954,7 @@ describe("SessionRunnerLLM", () => { raw: '{"text":"partial', }), ), - TestLLM.stop(), + TestLLM.text("Done", "done"), ) const run = yield* s.resume.pipe(Effect.forkChild) @@ -6970,7 +7071,7 @@ describe("SessionRunnerLLM", () => { malformed("call-first"), TestLLM.toolCall("call-valid-between", "echo", { text: "valid" }), malformed("call-second"), - TestLLM.stop(), + TestLLM.text("Done", "done"), ) yield* s.runPrompt("Keep producing malformed tools")