fix(core): share recovered subagent settlement lifecycle

This commit is contained in:
Kit Langton
2026-11-05 00:48:05 -08:00
Unverified
parent e406fd65a5
commit b910054418
4 changed files with 351 additions and 180 deletions
+31 -51
View File
@@ -11,7 +11,7 @@ import { SessionSchema } from "../schema.js"
import { SessionStore } from "../store.js"
import { ImageNotice } from "../../image/notice.js"
import { ShellResult } from "../../shell/result.js"
import { SubagentCompletion } from "../subagent-completion.js"
import { SubagentJob } from "../subagent-job.js"
import { SubagentMetadata } from "../subagent-metadata.js"
const CONTINUE_AFTER_SERVER_RESTART =
@@ -73,6 +73,7 @@ export const layer = (options?: Options) =>
const bus = yield* Bus.Service
const jobs = yield* Job.Service
const sessions = yield* Session.Service
const subagents = yield* SubagentJob.Service
const scope = yield* Effect.scope
const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS
@@ -166,6 +167,7 @@ export const layer = (options?: Options) =>
background: Job.Background,
recovery: Extract<Job.Recovery, { kind: "subagent" }>,
suspended: ReadonlySet<SessionSchema.ID>,
completions: Array<Effect.Effect<void>>,
): Effect.fn.Return<"stale" | "owned" | "reported" | "exhausted" | "resume"> {
const child = yield* store.get(recovery.childSessionID)
if (!child || child.parentID !== recovery.parentSessionID || !(yield* store.get(recovery.parentSessionID))) {
@@ -174,20 +176,23 @@ export const layer = (options?: Options) =>
}
if ((yield* jobs.get(background.id))?.status === "running") return "owned"
// A silent child reports nothing, whether its settle ran or restart replays it, so only the
// durable marker is left to clear.
const silent = !SubagentMetadata.notify(child.metadata)
const notify = Effect.fnUntraced(function* (result: Pick<Job.Background, "status" | "output" | "error">) {
const completion = subagents
.complete({
...result,
recovery,
notificationID: background.notificationID,
resume: suspended.has(recovery.parentSessionID) ? false : undefined,
})
.pipe(Effect.orDie)
if (silent) {
yield* jobs.completeBackground(background.notificationID)
// Silent completion wakes a held parent. Defer it until parent accounting and
// recovered job ownership are established, just like a running child's observer.
completions.push(completion)
return
}
yield* SubagentCompletion.deliver(sessions, jobs, {
...result,
recovery,
notificationID: background.notificationID,
resume: suspended.has(recovery.parentSessionID) ? false : undefined,
}).pipe(Effect.orDie)
yield* completion
})
// Finished before the crash: no settle survives to report it, so this replay is the report.
@@ -205,42 +210,6 @@ export const layer = (options?: Options) =>
return "resume"
})
const resumeChild = Effect.fnUntraced(function* (
background: Job.Background,
recovery: Extract<Job.Recovery, { kind: "subagent" }>,
) {
yield* jobs.start({
id: background.id,
type: "subagent",
title: recovery.description,
notificationID: background.notificationID,
recovery,
run: execution.resume(recovery.childSessionID).pipe(
Effect.andThen(store.context(recovery.childSessionID)),
Effect.map((messages) => {
const assistant = messages.findLast(
(message) =>
message.type === "assistant" && message.time.completed !== undefined && message.error === undefined,
)
return SubagentCompletion.text(assistant)
}),
),
})
yield* jobs.background(background.id)
// The child's settle admits its report before the job persists a terminal row, so only a
// durably terminal row may be cleared. Shutdown cancellation is never persisted — the row
// stays running for the next sweep to resume — and a silent child leaves only this marker.
yield* jobs.wait({ id: background.id }).pipe(
Effect.andThen(jobs.pendingBackground),
Effect.flatMap((pending) =>
pending.some((row) => row.notificationID === background.notificationID && row.status !== "running")
? jobs.completeBackground(background.notificationID)
: Effect.void,
),
Effect.forkIn(scope),
)
})
return Service.of({
resumeSuspendedSessions: Effect.gen(function* () {
const active = yield* execution.active
@@ -258,6 +227,7 @@ export const layer = (options?: Options) =>
const owned = new Set<SessionSchema.ID>()
const exhausted = new Set<SessionSchema.ID>()
const reported = new Set<string>()
const completions: Array<Effect.Effect<void>> = []
const resumable: Array<{
readonly background: Job.Background
readonly recovery: Extract<Job.Recovery, { kind: "subagent" }>
@@ -271,7 +241,7 @@ export const layer = (options?: Options) =>
if ((yield* jobs.get(background.id))?.status === "running") return
return yield* recoverToolJob(background, recovery)
}
const result = yield* recoverSubagent(background, recovery, suspended)
const result = yield* recoverSubagent(background, recovery, suspended, completions)
if (result === "owned" || result === "resume") owned.add(recovery.childSessionID)
if (result === "exhausted") exhausted.add(recovery.childSessionID)
if (result === "reported") reported.add(background.notificationID)
@@ -292,9 +262,19 @@ export const layer = (options?: Options) =>
)
// A resumed child's settle wakes its parent, so children start only once every suspended
// parent has been counted and notified above.
yield* Effect.forEach(resumable, ({ background, recovery }) => resumeChild(background, recovery), {
discard: true,
})
yield* Effect.forEach(
resumable,
({ background, recovery }) =>
subagents.recover({
id: background.id,
notificationID: background.notificationID,
recovery,
}),
{
discard: true,
},
)
yield* Effect.forEach(completions, (completion) => completion.pipe(Effect.forkIn(scope)), { discard: true })
// Held direct prompts have no job marker. Wake them only after cancellations are admitted,
// parents are counted, and recovered jobs own their reports. A wake preserves the held
// answer boundary without retrying an interrupted model execution.
@@ -324,5 +304,5 @@ export const layer = (options?: Options) =>
export const node = makeGlobalNode({
service: Service,
layer: layer(),
deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node],
deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node, SubagentJob.node],
})
+86 -37
View File
@@ -22,8 +22,19 @@ export type Input = {
readonly notify: boolean
}
type Completion = Pick<Job.Info, "status" | "output" | "error" | "notificationID"> & {
readonly recovery: Recovery
readonly resume?: boolean
}
export interface Interface {
start: (input: Input) => Effect.Effect<Job.Info>
recover: (input: {
recovery: Recovery
id: string
notificationID: Job.Background["notificationID"]
}) => Effect.Effect<void>
complete: (input: Completion) => Effect.Effect<void, Session.NotFoundError | Session.SyntheticConflictError>
background: (input: Input) => Effect.Effect<void>
notify: (recovery: Recovery, startedAt: number) => Effect.Effect<void>
}
@@ -69,6 +80,7 @@ const make: Effect.Effect<
// Resume once for launch, then wait until the child is idle with no background work or input
// left. Never resume just to inspect settlement: resuming an idle child runs another model step.
const settle = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) {
if (!(yield* store.get(sessionID))) return yield* new Session.NotFoundError({ sessionID })
const outcomes = [
SessionEvent.Execution.Succeeded,
SessionEvent.Execution.Failed,
@@ -90,12 +102,13 @@ const make: Effect.Effect<
if (event.data.reason === "shutdown") return yield* Effect.never
return yield* Effect.interrupt
})
yield* sessions.resume(sessionID)
yield* execution.resume(sessionID)
while (true) {
yield* sessions.wait(sessionID)
if (!(yield* store.get(sessionID))) return yield* new Session.NotFoundError({ sessionID })
yield* execution.awaitIdle(sessionID)
// Read active last: input may have been promoted while the pending-work queries ran.
const pending = yield* SessionBackground.unsettled(jobs, store, db, sessionID)
const active = (yield* sessions.active).has(sessionID)
const active = yield* execution.isActive(sessionID)
const event = yield* Queue.poll(settled)
if (Option.isSome(event)) {
yield* terminal(event.value)
@@ -106,32 +119,53 @@ const make: Effect.Effect<
}
}, Effect.scoped)
const notify = Effect.fn("SubagentJob.notify")(function* (recovery: Recovery, startedAt: number) {
const key = `${recovery.childSessionID}:${startedAt}`
const complete = Effect.fn("SubagentJob.complete")(function* (input: Completion) {
const metadata = (yield* store.get(input.recovery.childSessionID))?.metadata
if (!SubagentMetadata.notify(metadata)) {
if (input.notificationID) yield* jobs.completeBackground(input.notificationID)
// A settlement may have seen this job still running. Clear its marker before waking
// the idle parent, so a held report gets another chance to release without a model step.
const parentID = input.recovery.parentSessionID
if (!(yield* store.get(parentID))) return
yield* execution.awaitIdle(parentID)
yield* execution.wake(parentID)
return
}
yield* SubagentCompletion.deliver(
{
synthetic: (notice) =>
Effect.gen(function* () {
yield* sessions.synthetic({ ...notice, resume: false })
if (notice.resume !== false && !(yield* sessions.get(notice.sessionID)).revert)
yield* execution.wake(notice.sessionID)
}).pipe(Effect.uninterruptible),
},
jobs,
input,
)
})
const notify = Effect.fn("SubagentJob.notify")(function* (
recovery: Recovery,
startedAt: number,
id: string = recovery.childSessionID,
) {
const key = `${id}:${startedAt}`
if (notifications.has(key)) return
notifications.add(key)
yield* Effect.gen(function* () {
const info = (yield* jobs.wait({ id: recovery.childSessionID })).info
const info = (yield* jobs.wait({ id })).info
if (!info) return
// The preference belongs to the spawn call, so read it after the child settles. A silent
// child still clears its durable marker rather than leave a row that restart recovery
// would replay as a report the parent asked not to receive.
const metadata = yield* sessions.get(recovery.childSessionID).pipe(
Effect.map((session) => session.metadata),
Effect.catchTag("Session.NotFoundError", () => Effect.succeed(undefined)),
// Scope shutdown cancels only the in-memory job. Its running durable row must survive
// for the next process, unlike an explicit cancellation persisted by Job.cancel.
if (
info.status === "cancelled" &&
(yield* jobs.pendingBackground).some(
(row) => row.status === "running" && (row.notificationID === info.notificationID || row.id === id),
)
)
if (!SubagentMetadata.notify(metadata)) {
if (info.notificationID) yield* jobs.completeBackground(info.notificationID)
// Nothing reaches the parent, so wake it once idle: a parent whose report waited on this job
// settles again and reports its final state, without running a model step. Waiting for idle
// rather than skipping an active parent covers a settlement that read this job as running.
const parentID = recovery.parentSessionID
if (!(yield* store.get(parentID))) return
yield* execution.awaitIdle(parentID)
yield* execution.wake(parentID)
return
}
yield* SubagentCompletion.deliver(sessions, jobs, { ...info, recovery })
yield* complete({ ...info, recovery })
}).pipe(
Effect.ensuring(Effect.sync(() => notifications.delete(key))),
Effect.forkIn(scope, { startImmediately: true }),
@@ -139,7 +173,32 @@ const make: Effect.Effect<
)
})
const run = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) {
yield* settle(sessionID)
const messages = yield* store.messages({ sessionID, order: "desc", limit: 20 })
return SubagentCompletion.text(
messages.find(
(message) =>
message.type === "assistant" && message.time.completed !== undefined && message.error === undefined,
),
)
})
return Service.of({
recover: (input) =>
Effect.gen(function* () {
const info = yield* jobs.start({
id: input.id,
type: "subagent",
title: input.recovery.description,
notificationID: input.notificationID,
recovery: input.recovery,
run: run(input.recovery.childSessionID),
})
yield* jobs.background(input.id)
yield* notify(input.recovery, info.started_at, input.id)
}).pipe(Effect.setContext(context)),
complete,
start: (input: Input) =>
Effect.gen(function* () {
yield* writeNotify(input.recovery, input.notify)
@@ -149,19 +208,7 @@ const make: Effect.Effect<
title: input.recovery.description,
metadata: {},
recovery: input.recovery,
run: Effect.gen(function* () {
yield* settle(input.recovery.childSessionID)
const messages = yield* sessions.messages({
sessionID: input.recovery.childSessionID,
order: "desc",
limit: 20,
})
const assistant = messages.find(
(message) =>
message.type === "assistant" && message.time.completed !== undefined && message.error === undefined,
)
return SubagentCompletion.text(assistant)
}),
run: run(input.recovery.childSessionID),
})
}).pipe(Effect.setContext(context)),
background: Effect.fn("SubagentJob.background")(function* (input: Input) {
@@ -173,8 +220,10 @@ const make: Effect.Effect<
})
})
export const layer = Layer.effect(Service, make)
export const node: LayerNode.Provider<Service, never, typeof Node.tags.values.global> = Node.makeGlobalNode({
service: Service,
layer: Layer.effect(Service, make),
layer,
deps: [Session.node, SessionExecution.node, Job.node, Bus.node, SessionStore.node, Database.node],
})
+87 -91
View File
@@ -28,6 +28,7 @@ import { SessionMessage } from "@opencode/core/session/message"
import { SessionRunner } from "@opencode/core/session/runner/index"
import { SessionInboxTable, SessionTable } from "@opencode/core/session/sql"
import { SessionStore } from "@opencode/core/session/store"
import { SubagentJob } from "@opencode/core/session/subagent-job"
import { Cause, Context, Deferred, Effect, Exit, Fiber, Layer, LayerMap, Scope } from "effect"
import { eq } from "drizzle-orm"
import { testEffect } from "./lib/effect"
@@ -1024,16 +1025,21 @@ describe("SessionRestart background recovery", () => {
yield* Deferred.succeed(release, undefined)
yield* Deferred.await(parentWoken)
yield* Effect.forEach([parent, child], execution.awaitIdle, { discard: true })
expect(drained).toEqual(resumeAttempts < 2 ? [child, parent] : [parent])
// Core settlement and the job observer both admit the same notification. Their
// wakes may coalesce or run a second no-input drain; the child resumes only once.
expect([...new Set(drained)]).toEqual(resumeAttempts < 2 ? [child, parent] : [parent])
expect(drained.filter((id) => id === child)).toHaveLength(resumeAttempts < 2 ? 1 : 0)
expect((yield* store.context(parent)).filter((message) => message.type === "synthetic")).toMatchObject([
{ metadata: { source: "subagent", childID: child, state: resumeAttempts < 2 ? "completed" : "error" } },
])
expect(yield* claims(database)).toEqual({ [parent]: false, [child]: false })
expect(yield* attempts(database, child)).toBe(0)
yield* pendingEmpty(restarted)
yield* execution.awaitIdle(parent)
const count = drained.length
yield* restart.resumeSuspendedSessions
expect(drained).toHaveLength(resumeAttempts < 2 ? 2 : 1)
expect(drained).toHaveLength(count)
}),
)
}
@@ -1519,54 +1525,82 @@ describe("SessionRestart background recovery", () => {
}),
)
it.effect("clears a finished silent child's marker without reporting it", () =>
Effect.gen(function* () {
const database = yield* Database.Service
const store = yield* SessionStore.Service
const jobs = yield* Job.Service
const parent = Session.ID.make("ses_subagent_silent_parent")
const child = Session.ID.make("ses_subagent_silent_child")
yield* seedSessions(database, [parent])
yield* seedSessions(database, [child], { parent_id: parent, metadata: { "subagent.notify": false } })
yield* store.holdReport(child, { notificationID: SessionMessage.ID.create(), since: undefined })
const complete = yield* Deferred.make<string>()
yield* jobs.start({
id: child,
type: "subagent",
recovery: {
kind: "subagent",
parentSessionID: parent,
childSessionID: child,
agent: "explore",
description: "Silent inspection",
},
run: Deferred.await(complete),
})
yield* jobs.background(child)
yield* Deferred.succeed(complete, "Silent result")
yield* jobs.wait({ id: child })
expect(yield* jobs.pendingBackground).toHaveLength(1)
for (const state of ["finished", "exhausted"] as const) {
it.effect(`a ${state} silent child releases its parent's held report on restart`, () =>
Effect.gen(function* () {
const database = yield* Database.Service
const store = yield* SessionStore.Service
const jobs = yield* Job.Service
const bus = yield* Bus.Service
const grandparent = Session.ID.make("ses_subagent_silent_grandparent")
const parent = Session.ID.make("ses_subagent_silent_parent")
const child = Session.ID.make("ses_subagent_silent_child")
yield* seedSessions(database, [grandparent])
yield* seedSessions(database, [parent], { parent_id: grandparent })
yield* seedSessions(database, [child], {
parent_id: parent,
metadata: { "subagent.notify": false },
resume_attempts: state === "exhausted" ? 10 : 0,
})
yield* answer(bus, parent, "Held parent answer")
const held = { notificationID: SessionMessage.ID.create(), since: undefined }
yield* store.holdReport(parent, held)
yield* store.holdReport(child, { notificationID: SessionMessage.ID.create(), since: undefined })
const complete = yield* Deferred.make<string>()
yield* jobs.start({
id: child,
type: "subagent",
recovery: {
kind: "subagent",
parentSessionID: parent,
childSessionID: child,
agent: "explore",
description: "Silent inspection",
},
run: Deferred.await(complete),
})
yield* jobs.background(child)
if (state === "finished") {
yield* Deferred.succeed(complete, "Silent result")
yield* jobs.wait({ id: child })
}
expect(yield* jobs.pendingBackground).toHaveLength(1)
const drained: Session.ID[] = []
const scope = yield* Scope.make()
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
const restarted = yield* Job.make.pipe(Scope.provide(scope))
const context = yield* buildExecution(
scope,
({ sessionID }) => Effect.sync(() => void drained.push(sessionID)),
undefined,
restarted,
)
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
const drained: Session.ID[] = []
const woken = yield* Deferred.make<void>()
const scope = yield* Scope.make()
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
const restarted = yield* Job.make.pipe(Scope.provide(scope))
const context = yield* buildExecution(
scope,
({ sessionID }) =>
Effect.gen(function* () {
drained.push(sessionID)
if (sessionID === grandparent) yield* Deferred.succeed(woken, undefined)
}),
undefined,
restarted,
)
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
yield* Deferred.await(woken)
yield* Context.get(context, SessionExecution.Service).awaitIdle(parent)
yield* Context.get(context, SessionExecution.Service).awaitIdle(grandparent)
expect(drained).toEqual([])
expect(yield* SessionInbox.list(database.db, parent)).toEqual([])
expect(yield* restarted.pendingBackground).toEqual([])
expect(yield* store.listHeldChildren()).toEqual([])
}),
)
expect(drained).not.toContain(child)
expect(yield* SessionInbox.list(database.db, parent)).toEqual([])
expect(yield* restarted.pendingBackground).toEqual([])
expect(yield* store.listHeldChildren()).toEqual([])
expect(yield* SessionInbox.list(database.db, grandparent)).toMatchObject([
{ id: held.notificationID, payload: { text: expect.stringContaining("Held parent answer") } },
])
const count = drained.length
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
expect(drained).toHaveLength(count)
}),
)
}
it.effect("resumes a silent child after restart without reporting or waking its parent", () =>
it.effect("resumes a silent child after restart and wakes its parent without reporting", () =>
Effect.gen(function* () {
const database = yield* Database.Service
const jobs = yield* Job.Service
@@ -1596,6 +1630,7 @@ describe("SessionRestart background recovery", () => {
if (!marker) return yield* Effect.die("background record missing")
const answered = yield* Deferred.make<void>()
const woken = yield* Deferred.make<void>()
const drained: Session.ID[] = []
const scope = yield* Scope.make()
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
@@ -1605,7 +1640,7 @@ describe("SessionRestart background recovery", () => {
({ sessionID }) =>
sessionID === child
? answer(bus, child, "Silent resumed result").pipe(Effect.andThen(Deferred.succeed(answered, undefined)))
: Effect.sync(() => void drained.push(sessionID)),
: Effect.sync(() => void drained.push(sessionID)).pipe(Effect.andThen(Deferred.succeed(woken, undefined))),
undefined,
restarted,
)
@@ -1613,8 +1648,10 @@ describe("SessionRestart background recovery", () => {
yield* restart.resumeSuspendedSessions
yield* Deferred.await(answered)
yield* pendingEmpty(restarted)
yield* Deferred.await(woken)
yield* Context.get(context, SessionExecution.Service).awaitIdle(parent)
expect(drained).toEqual([])
expect(drained).toEqual([parent])
expect(yield* SessionInbox.list(database.db, parent)).toEqual([])
expect(yield* restarted.pendingBackground).toEqual([])
yield* restart.resumeSuspendedSessions
@@ -1622,48 +1659,6 @@ describe("SessionRestart background recovery", () => {
}),
)
it.effect("clears a silent child's marker when its resume budget is exhausted", () =>
Effect.gen(function* () {
const database = yield* Database.Service
const jobs = yield* Job.Service
const parent = Session.ID.make("ses_subagent_silent_exhausted_parent")
const child = Session.ID.make("ses_subagent_silent_exhausted_child")
yield* seedSessions(database, [parent])
yield* seedSessions(database, [child], {
parent_id: parent,
resume_attempts: 10,
metadata: { "subagent.notify": false },
})
yield* jobs.start({
id: child,
type: "subagent",
recovery: {
kind: "subagent",
parentSessionID: parent,
childSessionID: child,
agent: "explore",
description: "Silent exhausted inspection",
},
run: Effect.never,
})
yield* jobs.background(child)
const scope = yield* Scope.make()
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
const restarted = yield* Job.make.pipe(Scope.provide(scope))
const context = yield* buildExecution(
scope,
() => Effect.die("A silent child must not wake its parent"),
undefined,
restarted,
)
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
expect(yield* SessionInbox.list(database.db, parent)).toEqual([])
expect(yield* restarted.pendingBackground).toEqual([])
}),
)
it.effect("retains a subagent completion marker when synthetic admission conflicts", () =>
Effect.gen(function* () {
const database = yield* Database.Service
@@ -2160,6 +2155,7 @@ function buildExecution(
)
return yield* Layer.buildWithScope(
SessionRestart.layer(options).pipe(
Layer.provideMerge(Layer.fresh(SubagentJob.layer)),
Layer.provideMerge(sessionLayer),
Layer.provideMerge(Layer.fresh(SessionExecution.layer)),
Layer.provide(Layer.succeed(Database.Service, database)),
+147 -1
View File
@@ -417,7 +417,7 @@ describe("subagent observer lifetime", () => {
yield* Effect.addFinalizer(() => Scope.close(before, Exit.void))
const oldJobs = yield* Job.make.pipe(Scope.provide(before))
const old = yield* Layer.buildWithScope(
SessionExecution.layer.pipe(Layer.provide(Layer.succeed(Job.Service, oldJobs))),
SessionExecution.layer.pipe(Layer.provide(Layer.succeed(Job.Service, oldJobs)), Layer.fresh),
before,
)
const oldExecution = Context.get(old, SessionExecution.Service)
@@ -443,8 +443,10 @@ describe("subagent observer lifetime", () => {
const jobs = yield* Job.make.pipe(Scope.provide(after))
const context = yield* Layer.buildWithScope(
SessionRestart.layer().pipe(
Layer.provideMerge(SubagentJob.layer),
Layer.provideMerge(SessionExecution.layer),
Layer.provide(Layer.succeed(Job.Service, jobs)),
Layer.fresh,
),
after,
)
@@ -470,6 +472,150 @@ describe("subagent observer lifetime", () => {
}),
)
for (const outcome of ["complete", "shutdown"] as const) {
it.live(`restart retains a child's job through its silent grandchild's settlement (${outcome})`, () =>
Effect.gen(function* () {
const store = yield* SessionStore.Service
const llm = yield* TestLLM.Test
const f = yield* family()
const build = (scope: Scope.Closeable, jobs: Job.Interface) =>
Layer.buildWithScope(
SessionRestart.layer().pipe(
Layer.provideMerge(SubagentJob.layer),
Layer.provideMerge(SessionExecution.layer),
Layer.provide(Layer.succeed(Job.Service, jobs)),
Layer.fresh,
),
scope,
)
const before = yield* Scope.make()
yield* Effect.addFinalizer(() => Scope.close(before, Exit.void))
const oldJobs = yield* Job.make.pipe(Scope.provide(before))
const old = yield* build(before, oldJobs)
const oldSubagents = Context.get(old, SubagentJob.Service)
const oldExecution = Context.get(old, SessionExecution.Service)
const child = {
recovery: {
kind: "subagent" as const,
parentSessionID: f.parent.id,
childSessionID: f.child.id,
agent: "reviewer",
description: "Review fixture",
},
notify: true,
}
const descendant = {
recovery: {
...child.recovery,
parentSessionID: f.child.id,
childSessionID: f.descendant.id,
description: "Check the tests",
},
notify: false,
}
yield* llm.push(Stream.never, TestLLM.text("before restart", "before"))
yield* f.sessions.prompt({ sessionID: f.descendant.id, text: "Check the tests", resume: false })
yield* oldSubagents.start(descendant)
yield* oldSubagents.background(descendant)
yield* llm.wait(1)
yield* f.sessions.prompt({ sessionID: f.child.id, text: "Review fixture", resume: false })
yield* oldSubagents.start(child)
yield* oldSubagents.background(child)
yield* llm.wait(2)
yield* oldExecution.awaitIdle(f.child.id)
const held = yield* store.heldReport(f.child.id)
const markers = yield* oldJobs.pendingBackground
expect(held).toBeDefined()
expect(markers).toHaveLength(2)
expect(markers.every((row) => row.status === "running")).toBe(true)
yield* Scope.close(before, Exit.void)
expect(yield* oldJobs.pendingBackground).toEqual(markers)
expect(yield* store.heldReport(f.child.id)).toEqual(held)
expect(yield* reports(f.parent.id)).toEqual([])
const after = yield* Scope.make()
yield* Effect.addFinalizer(() => Scope.close(after, Exit.void))
const jobs = yield* Job.make.pipe(Scope.provide(after))
const go = yield* Deferred.make<void>()
const settled = yield* Deferred.make<void>()
const finish = yield* Deferred.make<void>()
const cleared = yield* Deferred.make<void>()
const context = yield* build(after, {
...jobs,
// Hold the real job just after D's execution settles: C's wake must observe D
// still running and hold again before its completion observer gets to clear D.
start: (input) =>
jobs.start({
...input,
run:
input.id !== f.descendant.id
? input.run
: input.run.pipe(
Effect.tap(() =>
Deferred.succeed(settled, undefined).pipe(Effect.andThen(Deferred.await(finish))),
),
),
}),
completeBackground: (id) =>
jobs
.completeBackground(id)
.pipe(
Effect.tap(() => (id === held?.notificationID ? Deferred.succeed(cleared, undefined) : Effect.void)),
),
})
const restart = Context.get(context, SessionRestart.Service)
const execution = Context.get(context, SessionExecution.Service)
yield* llm.push(
Stream.unwrap(
Deferred.await(go).pipe(Effect.as(Stream.fromIterable(TestLLM.text("grandchild done", "grandchild")))),
),
TestLLM.text("after restart", "after"),
)
yield* restart.resumeSuspendedSessions
yield* llm.wait(4)
yield* execution.awaitIdle(f.child.id)
expect(yield* store.heldReport(f.child.id)).toEqual(held)
expect((yield* jobs.get(f.child.id))?.status).toBe("running")
expect(yield* reports(f.parent.id)).toEqual([])
if (outcome === "shutdown") {
yield* Scope.close(after, Exit.void)
expect(yield* jobs.pendingBackground).toEqual(markers)
expect(yield* store.heldReport(f.child.id)).toEqual(held)
expect(yield* reports(f.parent.id)).toEqual([])
expect((yield* reports(f.child.id)).filter((message) => message.metadata?.source === "subagent")).toEqual([])
return
}
yield* Deferred.succeed(go, undefined)
yield* Deferred.await(settled)
yield* execution.awaitIdle(f.child.id)
expect((yield* jobs.get(f.descendant.id))?.status).toBe("running")
expect((yield* jobs.get(f.child.id))?.status).toBe("running")
expect(yield* store.heldReport(f.child.id)).toEqual(held)
expect(yield* reports(f.parent.id)).toEqual([])
yield* Deferred.succeed(finish, undefined)
yield* Deferred.await(cleared)
yield* llm.wait(5)
yield* execution.awaitIdle(f.parent.id)
const notices = yield* reports(f.parent.id)
expect(notices).toHaveLength(1)
expect(notices[0]).toMatchObject({
id: held?.notificationID,
metadata: { source: "subagent", childID: f.child.id, state: "completed" },
})
expect(notices[0]?.text).toContain("after restart")
expect(notices[0]?.text).not.toContain("grandchild done")
expect(notices[0]?.text).not.toContain("before restart")
expect(yield* jobs.pendingBackground).toEqual([])
expect(yield* store.listHeldChildren()).toEqual([])
expect(yield* llm.requests()).toHaveLength(5)
yield* restart.resumeSuspendedSessions
expect(yield* reports(f.parent.id)).toEqual(notices)
expect(yield* llm.requests()).toHaveLength(5)
}),
)
}
it.live("a silent background grandchild job releases its parent's held report", () =>
Effect.gen(function* () {
const jobs = yield* Job.Service