From b910054418aa11bc1002593cb51fbfa788cabd36 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Thu, 5 Nov 2026 00:48:05 -0800 Subject: [PATCH] fix(core): share recovered subagent settlement lifecycle --- .../core/src/session/execution/restart.ts | 82 +++----- packages/core/src/session/subagent-job.ts | 123 ++++++++---- packages/core/test/session-execution.test.ts | 178 +++++++++--------- packages/core/test/subagent-lifetime.test.ts | 148 ++++++++++++++- 4 files changed, 351 insertions(+), 180 deletions(-) diff --git a/packages/core/src/session/execution/restart.ts b/packages/core/src/session/execution/restart.ts index bdf24c6c1c6..37861f01900 100644 --- a/packages/core/src/session/execution/restart.ts +++ b/packages/core/src/session/execution/restart.ts @@ -11,7 +11,7 @@ import { SessionSchema } from "../schema.js" import { SessionStore } from "../store.js" import { ImageNotice } from "../../image/notice.js" import { ShellResult } from "../../shell/result.js" -import { SubagentCompletion } from "../subagent-completion.js" +import { SubagentJob } from "../subagent-job.js" import { SubagentMetadata } from "../subagent-metadata.js" const CONTINUE_AFTER_SERVER_RESTART = @@ -73,6 +73,7 @@ export const layer = (options?: Options) => const bus = yield* Bus.Service const jobs = yield* Job.Service const sessions = yield* Session.Service + const subagents = yield* SubagentJob.Service const scope = yield* Effect.scope const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS @@ -166,6 +167,7 @@ export const layer = (options?: Options) => background: Job.Background, recovery: Extract, suspended: ReadonlySet, + completions: Array>, ): Effect.fn.Return<"stale" | "owned" | "reported" | "exhausted" | "resume"> { const child = yield* store.get(recovery.childSessionID) if (!child || child.parentID !== recovery.parentSessionID || !(yield* store.get(recovery.parentSessionID))) { @@ -174,20 +176,23 @@ export const layer = (options?: Options) => } if ((yield* jobs.get(background.id))?.status === "running") return "owned" - // A silent child reports nothing, whether its settle ran or restart replays it, so only the - // durable marker is left to clear. const silent = !SubagentMetadata.notify(child.metadata) const notify = Effect.fnUntraced(function* (result: Pick) { + const completion = subagents + .complete({ + ...result, + recovery, + notificationID: background.notificationID, + resume: suspended.has(recovery.parentSessionID) ? false : undefined, + }) + .pipe(Effect.orDie) if (silent) { - yield* jobs.completeBackground(background.notificationID) + // Silent completion wakes a held parent. Defer it until parent accounting and + // recovered job ownership are established, just like a running child's observer. + completions.push(completion) return } - yield* SubagentCompletion.deliver(sessions, jobs, { - ...result, - recovery, - notificationID: background.notificationID, - resume: suspended.has(recovery.parentSessionID) ? false : undefined, - }).pipe(Effect.orDie) + yield* completion }) // Finished before the crash: no settle survives to report it, so this replay is the report. @@ -205,42 +210,6 @@ export const layer = (options?: Options) => return "resume" }) - const resumeChild = Effect.fnUntraced(function* ( - background: Job.Background, - recovery: Extract, - ) { - yield* jobs.start({ - id: background.id, - type: "subagent", - title: recovery.description, - notificationID: background.notificationID, - recovery, - run: execution.resume(recovery.childSessionID).pipe( - Effect.andThen(store.context(recovery.childSessionID)), - Effect.map((messages) => { - const assistant = messages.findLast( - (message) => - message.type === "assistant" && message.time.completed !== undefined && message.error === undefined, - ) - return SubagentCompletion.text(assistant) - }), - ), - }) - yield* jobs.background(background.id) - // The child's settle admits its report before the job persists a terminal row, so only a - // durably terminal row may be cleared. Shutdown cancellation is never persisted — the row - // stays running for the next sweep to resume — and a silent child leaves only this marker. - yield* jobs.wait({ id: background.id }).pipe( - Effect.andThen(jobs.pendingBackground), - Effect.flatMap((pending) => - pending.some((row) => row.notificationID === background.notificationID && row.status !== "running") - ? jobs.completeBackground(background.notificationID) - : Effect.void, - ), - Effect.forkIn(scope), - ) - }) - return Service.of({ resumeSuspendedSessions: Effect.gen(function* () { const active = yield* execution.active @@ -258,6 +227,7 @@ export const layer = (options?: Options) => const owned = new Set() const exhausted = new Set() const reported = new Set() + const completions: Array> = [] const resumable: Array<{ readonly background: Job.Background readonly recovery: Extract @@ -271,7 +241,7 @@ export const layer = (options?: Options) => if ((yield* jobs.get(background.id))?.status === "running") return return yield* recoverToolJob(background, recovery) } - const result = yield* recoverSubagent(background, recovery, suspended) + const result = yield* recoverSubagent(background, recovery, suspended, completions) if (result === "owned" || result === "resume") owned.add(recovery.childSessionID) if (result === "exhausted") exhausted.add(recovery.childSessionID) if (result === "reported") reported.add(background.notificationID) @@ -292,9 +262,19 @@ export const layer = (options?: Options) => ) // A resumed child's settle wakes its parent, so children start only once every suspended // parent has been counted and notified above. - yield* Effect.forEach(resumable, ({ background, recovery }) => resumeChild(background, recovery), { - discard: true, - }) + yield* Effect.forEach( + resumable, + ({ background, recovery }) => + subagents.recover({ + id: background.id, + notificationID: background.notificationID, + recovery, + }), + { + discard: true, + }, + ) + yield* Effect.forEach(completions, (completion) => completion.pipe(Effect.forkIn(scope)), { discard: true }) // Held direct prompts have no job marker. Wake them only after cancellations are admitted, // parents are counted, and recovered jobs own their reports. A wake preserves the held // answer boundary without retrying an interrupted model execution. @@ -324,5 +304,5 @@ export const layer = (options?: Options) => export const node = makeGlobalNode({ service: Service, layer: layer(), - deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node], + deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node, SubagentJob.node], }) diff --git a/packages/core/src/session/subagent-job.ts b/packages/core/src/session/subagent-job.ts index 5c01066973b..b6a69e0e475 100644 --- a/packages/core/src/session/subagent-job.ts +++ b/packages/core/src/session/subagent-job.ts @@ -22,8 +22,19 @@ export type Input = { readonly notify: boolean } +type Completion = Pick & { + readonly recovery: Recovery + readonly resume?: boolean +} + export interface Interface { start: (input: Input) => Effect.Effect + recover: (input: { + recovery: Recovery + id: string + notificationID: Job.Background["notificationID"] + }) => Effect.Effect + complete: (input: Completion) => Effect.Effect background: (input: Input) => Effect.Effect notify: (recovery: Recovery, startedAt: number) => Effect.Effect } @@ -69,6 +80,7 @@ const make: Effect.Effect< // Resume once for launch, then wait until the child is idle with no background work or input // left. Never resume just to inspect settlement: resuming an idle child runs another model step. const settle = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) { + if (!(yield* store.get(sessionID))) return yield* new Session.NotFoundError({ sessionID }) const outcomes = [ SessionEvent.Execution.Succeeded, SessionEvent.Execution.Failed, @@ -90,12 +102,13 @@ const make: Effect.Effect< if (event.data.reason === "shutdown") return yield* Effect.never return yield* Effect.interrupt }) - yield* sessions.resume(sessionID) + yield* execution.resume(sessionID) while (true) { - yield* sessions.wait(sessionID) + if (!(yield* store.get(sessionID))) return yield* new Session.NotFoundError({ sessionID }) + yield* execution.awaitIdle(sessionID) // Read active last: input may have been promoted while the pending-work queries ran. const pending = yield* SessionBackground.unsettled(jobs, store, db, sessionID) - const active = (yield* sessions.active).has(sessionID) + const active = yield* execution.isActive(sessionID) const event = yield* Queue.poll(settled) if (Option.isSome(event)) { yield* terminal(event.value) @@ -106,32 +119,53 @@ const make: Effect.Effect< } }, Effect.scoped) - const notify = Effect.fn("SubagentJob.notify")(function* (recovery: Recovery, startedAt: number) { - const key = `${recovery.childSessionID}:${startedAt}` + const complete = Effect.fn("SubagentJob.complete")(function* (input: Completion) { + const metadata = (yield* store.get(input.recovery.childSessionID))?.metadata + if (!SubagentMetadata.notify(metadata)) { + if (input.notificationID) yield* jobs.completeBackground(input.notificationID) + // A settlement may have seen this job still running. Clear its marker before waking + // the idle parent, so a held report gets another chance to release without a model step. + const parentID = input.recovery.parentSessionID + if (!(yield* store.get(parentID))) return + yield* execution.awaitIdle(parentID) + yield* execution.wake(parentID) + return + } + yield* SubagentCompletion.deliver( + { + synthetic: (notice) => + Effect.gen(function* () { + yield* sessions.synthetic({ ...notice, resume: false }) + if (notice.resume !== false && !(yield* sessions.get(notice.sessionID)).revert) + yield* execution.wake(notice.sessionID) + }).pipe(Effect.uninterruptible), + }, + jobs, + input, + ) + }) + + const notify = Effect.fn("SubagentJob.notify")(function* ( + recovery: Recovery, + startedAt: number, + id: string = recovery.childSessionID, + ) { + const key = `${id}:${startedAt}` if (notifications.has(key)) return notifications.add(key) yield* Effect.gen(function* () { - const info = (yield* jobs.wait({ id: recovery.childSessionID })).info + const info = (yield* jobs.wait({ id })).info if (!info) return - // The preference belongs to the spawn call, so read it after the child settles. A silent - // child still clears its durable marker rather than leave a row that restart recovery - // would replay as a report the parent asked not to receive. - const metadata = yield* sessions.get(recovery.childSessionID).pipe( - Effect.map((session) => session.metadata), - Effect.catchTag("Session.NotFoundError", () => Effect.succeed(undefined)), + // Scope shutdown cancels only the in-memory job. Its running durable row must survive + // for the next process, unlike an explicit cancellation persisted by Job.cancel. + if ( + info.status === "cancelled" && + (yield* jobs.pendingBackground).some( + (row) => row.status === "running" && (row.notificationID === info.notificationID || row.id === id), + ) ) - if (!SubagentMetadata.notify(metadata)) { - if (info.notificationID) yield* jobs.completeBackground(info.notificationID) - // Nothing reaches the parent, so wake it once idle: a parent whose report waited on this job - // settles again and reports its final state, without running a model step. Waiting for idle - // rather than skipping an active parent covers a settlement that read this job as running. - const parentID = recovery.parentSessionID - if (!(yield* store.get(parentID))) return - yield* execution.awaitIdle(parentID) - yield* execution.wake(parentID) return - } - yield* SubagentCompletion.deliver(sessions, jobs, { ...info, recovery }) + yield* complete({ ...info, recovery }) }).pipe( Effect.ensuring(Effect.sync(() => notifications.delete(key))), Effect.forkIn(scope, { startImmediately: true }), @@ -139,7 +173,32 @@ const make: Effect.Effect< ) }) + const run = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) { + yield* settle(sessionID) + const messages = yield* store.messages({ sessionID, order: "desc", limit: 20 }) + return SubagentCompletion.text( + messages.find( + (message) => + message.type === "assistant" && message.time.completed !== undefined && message.error === undefined, + ), + ) + }) + return Service.of({ + recover: (input) => + Effect.gen(function* () { + const info = yield* jobs.start({ + id: input.id, + type: "subagent", + title: input.recovery.description, + notificationID: input.notificationID, + recovery: input.recovery, + run: run(input.recovery.childSessionID), + }) + yield* jobs.background(input.id) + yield* notify(input.recovery, info.started_at, input.id) + }).pipe(Effect.setContext(context)), + complete, start: (input: Input) => Effect.gen(function* () { yield* writeNotify(input.recovery, input.notify) @@ -149,19 +208,7 @@ const make: Effect.Effect< title: input.recovery.description, metadata: {}, recovery: input.recovery, - run: Effect.gen(function* () { - yield* settle(input.recovery.childSessionID) - const messages = yield* sessions.messages({ - sessionID: input.recovery.childSessionID, - order: "desc", - limit: 20, - }) - const assistant = messages.find( - (message) => - message.type === "assistant" && message.time.completed !== undefined && message.error === undefined, - ) - return SubagentCompletion.text(assistant) - }), + run: run(input.recovery.childSessionID), }) }).pipe(Effect.setContext(context)), background: Effect.fn("SubagentJob.background")(function* (input: Input) { @@ -173,8 +220,10 @@ const make: Effect.Effect< }) }) +export const layer = Layer.effect(Service, make) + export const node: LayerNode.Provider = Node.makeGlobalNode({ service: Service, - layer: Layer.effect(Service, make), + layer, deps: [Session.node, SessionExecution.node, Job.node, Bus.node, SessionStore.node, Database.node], }) diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index 3bfebc4aff7..b4964f9bb2c 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -28,6 +28,7 @@ import { SessionMessage } from "@opencode/core/session/message" import { SessionRunner } from "@opencode/core/session/runner/index" import { SessionInboxTable, SessionTable } from "@opencode/core/session/sql" import { SessionStore } from "@opencode/core/session/store" +import { SubagentJob } from "@opencode/core/session/subagent-job" import { Cause, Context, Deferred, Effect, Exit, Fiber, Layer, LayerMap, Scope } from "effect" import { eq } from "drizzle-orm" import { testEffect } from "./lib/effect" @@ -1024,16 +1025,21 @@ describe("SessionRestart background recovery", () => { yield* Deferred.succeed(release, undefined) yield* Deferred.await(parentWoken) yield* Effect.forEach([parent, child], execution.awaitIdle, { discard: true }) - expect(drained).toEqual(resumeAttempts < 2 ? [child, parent] : [parent]) + // Core settlement and the job observer both admit the same notification. Their + // wakes may coalesce or run a second no-input drain; the child resumes only once. + expect([...new Set(drained)]).toEqual(resumeAttempts < 2 ? [child, parent] : [parent]) + expect(drained.filter((id) => id === child)).toHaveLength(resumeAttempts < 2 ? 1 : 0) expect((yield* store.context(parent)).filter((message) => message.type === "synthetic")).toMatchObject([ { metadata: { source: "subagent", childID: child, state: resumeAttempts < 2 ? "completed" : "error" } }, ]) expect(yield* claims(database)).toEqual({ [parent]: false, [child]: false }) expect(yield* attempts(database, child)).toBe(0) yield* pendingEmpty(restarted) + yield* execution.awaitIdle(parent) + const count = drained.length yield* restart.resumeSuspendedSessions - expect(drained).toHaveLength(resumeAttempts < 2 ? 2 : 1) + expect(drained).toHaveLength(count) }), ) } @@ -1519,54 +1525,82 @@ describe("SessionRestart background recovery", () => { }), ) - it.effect("clears a finished silent child's marker without reporting it", () => - Effect.gen(function* () { - const database = yield* Database.Service - const store = yield* SessionStore.Service - const jobs = yield* Job.Service - const parent = Session.ID.make("ses_subagent_silent_parent") - const child = Session.ID.make("ses_subagent_silent_child") - yield* seedSessions(database, [parent]) - yield* seedSessions(database, [child], { parent_id: parent, metadata: { "subagent.notify": false } }) - yield* store.holdReport(child, { notificationID: SessionMessage.ID.create(), since: undefined }) - const complete = yield* Deferred.make() - yield* jobs.start({ - id: child, - type: "subagent", - recovery: { - kind: "subagent", - parentSessionID: parent, - childSessionID: child, - agent: "explore", - description: "Silent inspection", - }, - run: Deferred.await(complete), - }) - yield* jobs.background(child) - yield* Deferred.succeed(complete, "Silent result") - yield* jobs.wait({ id: child }) - expect(yield* jobs.pendingBackground).toHaveLength(1) + for (const state of ["finished", "exhausted"] as const) { + it.effect(`a ${state} silent child releases its parent's held report on restart`, () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const grandparent = Session.ID.make("ses_subagent_silent_grandparent") + const parent = Session.ID.make("ses_subagent_silent_parent") + const child = Session.ID.make("ses_subagent_silent_child") + yield* seedSessions(database, [grandparent]) + yield* seedSessions(database, [parent], { parent_id: grandparent }) + yield* seedSessions(database, [child], { + parent_id: parent, + metadata: { "subagent.notify": false }, + resume_attempts: state === "exhausted" ? 10 : 0, + }) + yield* answer(bus, parent, "Held parent answer") + const held = { notificationID: SessionMessage.ID.create(), since: undefined } + yield* store.holdReport(parent, held) + yield* store.holdReport(child, { notificationID: SessionMessage.ID.create(), since: undefined }) + const complete = yield* Deferred.make() + yield* jobs.start({ + id: child, + type: "subagent", + recovery: { + kind: "subagent", + parentSessionID: parent, + childSessionID: child, + agent: "explore", + description: "Silent inspection", + }, + run: Deferred.await(complete), + }) + yield* jobs.background(child) + if (state === "finished") { + yield* Deferred.succeed(complete, "Silent result") + yield* jobs.wait({ id: child }) + } + expect(yield* jobs.pendingBackground).toHaveLength(1) - const drained: Session.ID[] = [] - const scope = yield* Scope.make() - yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) - const restarted = yield* Job.make.pipe(Scope.provide(scope)) - const context = yield* buildExecution( - scope, - ({ sessionID }) => Effect.sync(() => void drained.push(sessionID)), - undefined, - restarted, - ) - yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + const drained: Session.ID[] = [] + const woken = yield* Deferred.make() + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Scope.provide(scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => + Effect.gen(function* () { + drained.push(sessionID) + if (sessionID === grandparent) yield* Deferred.succeed(woken, undefined) + }), + undefined, + restarted, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + yield* Deferred.await(woken) + yield* Context.get(context, SessionExecution.Service).awaitIdle(parent) + yield* Context.get(context, SessionExecution.Service).awaitIdle(grandparent) - expect(drained).toEqual([]) - expect(yield* SessionInbox.list(database.db, parent)).toEqual([]) - expect(yield* restarted.pendingBackground).toEqual([]) - expect(yield* store.listHeldChildren()).toEqual([]) - }), - ) + expect(drained).not.toContain(child) + expect(yield* SessionInbox.list(database.db, parent)).toEqual([]) + expect(yield* restarted.pendingBackground).toEqual([]) + expect(yield* store.listHeldChildren()).toEqual([]) + expect(yield* SessionInbox.list(database.db, grandparent)).toMatchObject([ + { id: held.notificationID, payload: { text: expect.stringContaining("Held parent answer") } }, + ]) + const count = drained.length + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + expect(drained).toHaveLength(count) + }), + ) + } - it.effect("resumes a silent child after restart without reporting or waking its parent", () => + it.effect("resumes a silent child after restart and wakes its parent without reporting", () => Effect.gen(function* () { const database = yield* Database.Service const jobs = yield* Job.Service @@ -1596,6 +1630,7 @@ describe("SessionRestart background recovery", () => { if (!marker) return yield* Effect.die("background record missing") const answered = yield* Deferred.make() + const woken = yield* Deferred.make() const drained: Session.ID[] = [] const scope = yield* Scope.make() yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) @@ -1605,7 +1640,7 @@ describe("SessionRestart background recovery", () => { ({ sessionID }) => sessionID === child ? answer(bus, child, "Silent resumed result").pipe(Effect.andThen(Deferred.succeed(answered, undefined))) - : Effect.sync(() => void drained.push(sessionID)), + : Effect.sync(() => void drained.push(sessionID)).pipe(Effect.andThen(Deferred.succeed(woken, undefined))), undefined, restarted, ) @@ -1613,8 +1648,10 @@ describe("SessionRestart background recovery", () => { yield* restart.resumeSuspendedSessions yield* Deferred.await(answered) yield* pendingEmpty(restarted) + yield* Deferred.await(woken) + yield* Context.get(context, SessionExecution.Service).awaitIdle(parent) - expect(drained).toEqual([]) + expect(drained).toEqual([parent]) expect(yield* SessionInbox.list(database.db, parent)).toEqual([]) expect(yield* restarted.pendingBackground).toEqual([]) yield* restart.resumeSuspendedSessions @@ -1622,48 +1659,6 @@ describe("SessionRestart background recovery", () => { }), ) - it.effect("clears a silent child's marker when its resume budget is exhausted", () => - Effect.gen(function* () { - const database = yield* Database.Service - const jobs = yield* Job.Service - const parent = Session.ID.make("ses_subagent_silent_exhausted_parent") - const child = Session.ID.make("ses_subagent_silent_exhausted_child") - yield* seedSessions(database, [parent]) - yield* seedSessions(database, [child], { - parent_id: parent, - resume_attempts: 10, - metadata: { "subagent.notify": false }, - }) - yield* jobs.start({ - id: child, - type: "subagent", - recovery: { - kind: "subagent", - parentSessionID: parent, - childSessionID: child, - agent: "explore", - description: "Silent exhausted inspection", - }, - run: Effect.never, - }) - yield* jobs.background(child) - - const scope = yield* Scope.make() - yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) - const restarted = yield* Job.make.pipe(Scope.provide(scope)) - const context = yield* buildExecution( - scope, - () => Effect.die("A silent child must not wake its parent"), - undefined, - restarted, - ) - yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions - - expect(yield* SessionInbox.list(database.db, parent)).toEqual([]) - expect(yield* restarted.pendingBackground).toEqual([]) - }), - ) - it.effect("retains a subagent completion marker when synthetic admission conflicts", () => Effect.gen(function* () { const database = yield* Database.Service @@ -2160,6 +2155,7 @@ function buildExecution( ) return yield* Layer.buildWithScope( SessionRestart.layer(options).pipe( + Layer.provideMerge(Layer.fresh(SubagentJob.layer)), Layer.provideMerge(sessionLayer), Layer.provideMerge(Layer.fresh(SessionExecution.layer)), Layer.provide(Layer.succeed(Database.Service, database)), diff --git a/packages/core/test/subagent-lifetime.test.ts b/packages/core/test/subagent-lifetime.test.ts index f6516a11c9e..f5d4161f02a 100644 --- a/packages/core/test/subagent-lifetime.test.ts +++ b/packages/core/test/subagent-lifetime.test.ts @@ -417,7 +417,7 @@ describe("subagent observer lifetime", () => { yield* Effect.addFinalizer(() => Scope.close(before, Exit.void)) const oldJobs = yield* Job.make.pipe(Scope.provide(before)) const old = yield* Layer.buildWithScope( - SessionExecution.layer.pipe(Layer.provide(Layer.succeed(Job.Service, oldJobs))), + SessionExecution.layer.pipe(Layer.provide(Layer.succeed(Job.Service, oldJobs)), Layer.fresh), before, ) const oldExecution = Context.get(old, SessionExecution.Service) @@ -443,8 +443,10 @@ describe("subagent observer lifetime", () => { const jobs = yield* Job.make.pipe(Scope.provide(after)) const context = yield* Layer.buildWithScope( SessionRestart.layer().pipe( + Layer.provideMerge(SubagentJob.layer), Layer.provideMerge(SessionExecution.layer), Layer.provide(Layer.succeed(Job.Service, jobs)), + Layer.fresh, ), after, ) @@ -470,6 +472,150 @@ describe("subagent observer lifetime", () => { }), ) + for (const outcome of ["complete", "shutdown"] as const) { + it.live(`restart retains a child's job through its silent grandchild's settlement (${outcome})`, () => + Effect.gen(function* () { + const store = yield* SessionStore.Service + const llm = yield* TestLLM.Test + const f = yield* family() + const build = (scope: Scope.Closeable, jobs: Job.Interface) => + Layer.buildWithScope( + SessionRestart.layer().pipe( + Layer.provideMerge(SubagentJob.layer), + Layer.provideMerge(SessionExecution.layer), + Layer.provide(Layer.succeed(Job.Service, jobs)), + Layer.fresh, + ), + scope, + ) + const before = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(before, Exit.void)) + const oldJobs = yield* Job.make.pipe(Scope.provide(before)) + const old = yield* build(before, oldJobs) + const oldSubagents = Context.get(old, SubagentJob.Service) + const oldExecution = Context.get(old, SessionExecution.Service) + const child = { + recovery: { + kind: "subagent" as const, + parentSessionID: f.parent.id, + childSessionID: f.child.id, + agent: "reviewer", + description: "Review fixture", + }, + notify: true, + } + const descendant = { + recovery: { + ...child.recovery, + parentSessionID: f.child.id, + childSessionID: f.descendant.id, + description: "Check the tests", + }, + notify: false, + } + yield* llm.push(Stream.never, TestLLM.text("before restart", "before")) + yield* f.sessions.prompt({ sessionID: f.descendant.id, text: "Check the tests", resume: false }) + yield* oldSubagents.start(descendant) + yield* oldSubagents.background(descendant) + yield* llm.wait(1) + yield* f.sessions.prompt({ sessionID: f.child.id, text: "Review fixture", resume: false }) + yield* oldSubagents.start(child) + yield* oldSubagents.background(child) + yield* llm.wait(2) + yield* oldExecution.awaitIdle(f.child.id) + const held = yield* store.heldReport(f.child.id) + const markers = yield* oldJobs.pendingBackground + expect(held).toBeDefined() + expect(markers).toHaveLength(2) + expect(markers.every((row) => row.status === "running")).toBe(true) + yield* Scope.close(before, Exit.void) + expect(yield* oldJobs.pendingBackground).toEqual(markers) + expect(yield* store.heldReport(f.child.id)).toEqual(held) + expect(yield* reports(f.parent.id)).toEqual([]) + + const after = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(after, Exit.void)) + const jobs = yield* Job.make.pipe(Scope.provide(after)) + const go = yield* Deferred.make() + const settled = yield* Deferred.make() + const finish = yield* Deferred.make() + const cleared = yield* Deferred.make() + const context = yield* build(after, { + ...jobs, + // Hold the real job just after D's execution settles: C's wake must observe D + // still running and hold again before its completion observer gets to clear D. + start: (input) => + jobs.start({ + ...input, + run: + input.id !== f.descendant.id + ? input.run + : input.run.pipe( + Effect.tap(() => + Deferred.succeed(settled, undefined).pipe(Effect.andThen(Deferred.await(finish))), + ), + ), + }), + completeBackground: (id) => + jobs + .completeBackground(id) + .pipe( + Effect.tap(() => (id === held?.notificationID ? Deferred.succeed(cleared, undefined) : Effect.void)), + ), + }) + const restart = Context.get(context, SessionRestart.Service) + const execution = Context.get(context, SessionExecution.Service) + yield* llm.push( + Stream.unwrap( + Deferred.await(go).pipe(Effect.as(Stream.fromIterable(TestLLM.text("grandchild done", "grandchild")))), + ), + TestLLM.text("after restart", "after"), + ) + yield* restart.resumeSuspendedSessions + yield* llm.wait(4) + yield* execution.awaitIdle(f.child.id) + expect(yield* store.heldReport(f.child.id)).toEqual(held) + expect((yield* jobs.get(f.child.id))?.status).toBe("running") + expect(yield* reports(f.parent.id)).toEqual([]) + if (outcome === "shutdown") { + yield* Scope.close(after, Exit.void) + expect(yield* jobs.pendingBackground).toEqual(markers) + expect(yield* store.heldReport(f.child.id)).toEqual(held) + expect(yield* reports(f.parent.id)).toEqual([]) + expect((yield* reports(f.child.id)).filter((message) => message.metadata?.source === "subagent")).toEqual([]) + return + } + yield* Deferred.succeed(go, undefined) + yield* Deferred.await(settled) + yield* execution.awaitIdle(f.child.id) + expect((yield* jobs.get(f.descendant.id))?.status).toBe("running") + expect((yield* jobs.get(f.child.id))?.status).toBe("running") + expect(yield* store.heldReport(f.child.id)).toEqual(held) + expect(yield* reports(f.parent.id)).toEqual([]) + + yield* Deferred.succeed(finish, undefined) + yield* Deferred.await(cleared) + yield* llm.wait(5) + yield* execution.awaitIdle(f.parent.id) + const notices = yield* reports(f.parent.id) + expect(notices).toHaveLength(1) + expect(notices[0]).toMatchObject({ + id: held?.notificationID, + metadata: { source: "subagent", childID: f.child.id, state: "completed" }, + }) + expect(notices[0]?.text).toContain("after restart") + expect(notices[0]?.text).not.toContain("grandchild done") + expect(notices[0]?.text).not.toContain("before restart") + expect(yield* jobs.pendingBackground).toEqual([]) + expect(yield* store.listHeldChildren()).toEqual([]) + expect(yield* llm.requests()).toHaveLength(5) + yield* restart.resumeSuspendedSessions + expect(yield* reports(f.parent.id)).toEqual(notices) + expect(yield* llm.requests()).toHaveLength(5) + }), + ) + } + it.live("a silent background grandchild job releases its parent's held report", () => Effect.gen(function* () { const jobs = yield* Job.Service