mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-21 04:12:08 +03:00
fix(core): share recovered subagent settlement lifecycle
This commit is contained in:
@@ -11,7 +11,7 @@ import { SessionSchema } from "../schema.js"
|
||||
import { SessionStore } from "../store.js"
|
||||
import { ImageNotice } from "../../image/notice.js"
|
||||
import { ShellResult } from "../../shell/result.js"
|
||||
import { SubagentCompletion } from "../subagent-completion.js"
|
||||
import { SubagentJob } from "../subagent-job.js"
|
||||
import { SubagentMetadata } from "../subagent-metadata.js"
|
||||
|
||||
const CONTINUE_AFTER_SERVER_RESTART =
|
||||
@@ -73,6 +73,7 @@ export const layer = (options?: Options) =>
|
||||
const bus = yield* Bus.Service
|
||||
const jobs = yield* Job.Service
|
||||
const sessions = yield* Session.Service
|
||||
const subagents = yield* SubagentJob.Service
|
||||
const scope = yield* Effect.scope
|
||||
const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS
|
||||
|
||||
@@ -166,6 +167,7 @@ export const layer = (options?: Options) =>
|
||||
background: Job.Background,
|
||||
recovery: Extract<Job.Recovery, { kind: "subagent" }>,
|
||||
suspended: ReadonlySet<SessionSchema.ID>,
|
||||
completions: Array<Effect.Effect<void>>,
|
||||
): Effect.fn.Return<"stale" | "owned" | "reported" | "exhausted" | "resume"> {
|
||||
const child = yield* store.get(recovery.childSessionID)
|
||||
if (!child || child.parentID !== recovery.parentSessionID || !(yield* store.get(recovery.parentSessionID))) {
|
||||
@@ -174,20 +176,23 @@ export const layer = (options?: Options) =>
|
||||
}
|
||||
if ((yield* jobs.get(background.id))?.status === "running") return "owned"
|
||||
|
||||
// A silent child reports nothing, whether its settle ran or restart replays it, so only the
|
||||
// durable marker is left to clear.
|
||||
const silent = !SubagentMetadata.notify(child.metadata)
|
||||
const notify = Effect.fnUntraced(function* (result: Pick<Job.Background, "status" | "output" | "error">) {
|
||||
const completion = subagents
|
||||
.complete({
|
||||
...result,
|
||||
recovery,
|
||||
notificationID: background.notificationID,
|
||||
resume: suspended.has(recovery.parentSessionID) ? false : undefined,
|
||||
})
|
||||
.pipe(Effect.orDie)
|
||||
if (silent) {
|
||||
yield* jobs.completeBackground(background.notificationID)
|
||||
// Silent completion wakes a held parent. Defer it until parent accounting and
|
||||
// recovered job ownership are established, just like a running child's observer.
|
||||
completions.push(completion)
|
||||
return
|
||||
}
|
||||
yield* SubagentCompletion.deliver(sessions, jobs, {
|
||||
...result,
|
||||
recovery,
|
||||
notificationID: background.notificationID,
|
||||
resume: suspended.has(recovery.parentSessionID) ? false : undefined,
|
||||
}).pipe(Effect.orDie)
|
||||
yield* completion
|
||||
})
|
||||
|
||||
// Finished before the crash: no settle survives to report it, so this replay is the report.
|
||||
@@ -205,42 +210,6 @@ export const layer = (options?: Options) =>
|
||||
return "resume"
|
||||
})
|
||||
|
||||
const resumeChild = Effect.fnUntraced(function* (
|
||||
background: Job.Background,
|
||||
recovery: Extract<Job.Recovery, { kind: "subagent" }>,
|
||||
) {
|
||||
yield* jobs.start({
|
||||
id: background.id,
|
||||
type: "subagent",
|
||||
title: recovery.description,
|
||||
notificationID: background.notificationID,
|
||||
recovery,
|
||||
run: execution.resume(recovery.childSessionID).pipe(
|
||||
Effect.andThen(store.context(recovery.childSessionID)),
|
||||
Effect.map((messages) => {
|
||||
const assistant = messages.findLast(
|
||||
(message) =>
|
||||
message.type === "assistant" && message.time.completed !== undefined && message.error === undefined,
|
||||
)
|
||||
return SubagentCompletion.text(assistant)
|
||||
}),
|
||||
),
|
||||
})
|
||||
yield* jobs.background(background.id)
|
||||
// The child's settle admits its report before the job persists a terminal row, so only a
|
||||
// durably terminal row may be cleared. Shutdown cancellation is never persisted — the row
|
||||
// stays running for the next sweep to resume — and a silent child leaves only this marker.
|
||||
yield* jobs.wait({ id: background.id }).pipe(
|
||||
Effect.andThen(jobs.pendingBackground),
|
||||
Effect.flatMap((pending) =>
|
||||
pending.some((row) => row.notificationID === background.notificationID && row.status !== "running")
|
||||
? jobs.completeBackground(background.notificationID)
|
||||
: Effect.void,
|
||||
),
|
||||
Effect.forkIn(scope),
|
||||
)
|
||||
})
|
||||
|
||||
return Service.of({
|
||||
resumeSuspendedSessions: Effect.gen(function* () {
|
||||
const active = yield* execution.active
|
||||
@@ -258,6 +227,7 @@ export const layer = (options?: Options) =>
|
||||
const owned = new Set<SessionSchema.ID>()
|
||||
const exhausted = new Set<SessionSchema.ID>()
|
||||
const reported = new Set<string>()
|
||||
const completions: Array<Effect.Effect<void>> = []
|
||||
const resumable: Array<{
|
||||
readonly background: Job.Background
|
||||
readonly recovery: Extract<Job.Recovery, { kind: "subagent" }>
|
||||
@@ -271,7 +241,7 @@ export const layer = (options?: Options) =>
|
||||
if ((yield* jobs.get(background.id))?.status === "running") return
|
||||
return yield* recoverToolJob(background, recovery)
|
||||
}
|
||||
const result = yield* recoverSubagent(background, recovery, suspended)
|
||||
const result = yield* recoverSubagent(background, recovery, suspended, completions)
|
||||
if (result === "owned" || result === "resume") owned.add(recovery.childSessionID)
|
||||
if (result === "exhausted") exhausted.add(recovery.childSessionID)
|
||||
if (result === "reported") reported.add(background.notificationID)
|
||||
@@ -292,9 +262,19 @@ export const layer = (options?: Options) =>
|
||||
)
|
||||
// A resumed child's settle wakes its parent, so children start only once every suspended
|
||||
// parent has been counted and notified above.
|
||||
yield* Effect.forEach(resumable, ({ background, recovery }) => resumeChild(background, recovery), {
|
||||
discard: true,
|
||||
})
|
||||
yield* Effect.forEach(
|
||||
resumable,
|
||||
({ background, recovery }) =>
|
||||
subagents.recover({
|
||||
id: background.id,
|
||||
notificationID: background.notificationID,
|
||||
recovery,
|
||||
}),
|
||||
{
|
||||
discard: true,
|
||||
},
|
||||
)
|
||||
yield* Effect.forEach(completions, (completion) => completion.pipe(Effect.forkIn(scope)), { discard: true })
|
||||
// Held direct prompts have no job marker. Wake them only after cancellations are admitted,
|
||||
// parents are counted, and recovered jobs own their reports. A wake preserves the held
|
||||
// answer boundary without retrying an interrupted model execution.
|
||||
@@ -324,5 +304,5 @@ export const layer = (options?: Options) =>
|
||||
export const node = makeGlobalNode({
|
||||
service: Service,
|
||||
layer: layer(),
|
||||
deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node],
|
||||
deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node, SubagentJob.node],
|
||||
})
|
||||
|
||||
@@ -22,8 +22,19 @@ export type Input = {
|
||||
readonly notify: boolean
|
||||
}
|
||||
|
||||
type Completion = Pick<Job.Info, "status" | "output" | "error" | "notificationID"> & {
|
||||
readonly recovery: Recovery
|
||||
readonly resume?: boolean
|
||||
}
|
||||
|
||||
export interface Interface {
|
||||
start: (input: Input) => Effect.Effect<Job.Info>
|
||||
recover: (input: {
|
||||
recovery: Recovery
|
||||
id: string
|
||||
notificationID: Job.Background["notificationID"]
|
||||
}) => Effect.Effect<void>
|
||||
complete: (input: Completion) => Effect.Effect<void, Session.NotFoundError | Session.SyntheticConflictError>
|
||||
background: (input: Input) => Effect.Effect<void>
|
||||
notify: (recovery: Recovery, startedAt: number) => Effect.Effect<void>
|
||||
}
|
||||
@@ -69,6 +80,7 @@ const make: Effect.Effect<
|
||||
// Resume once for launch, then wait until the child is idle with no background work or input
|
||||
// left. Never resume just to inspect settlement: resuming an idle child runs another model step.
|
||||
const settle = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) {
|
||||
if (!(yield* store.get(sessionID))) return yield* new Session.NotFoundError({ sessionID })
|
||||
const outcomes = [
|
||||
SessionEvent.Execution.Succeeded,
|
||||
SessionEvent.Execution.Failed,
|
||||
@@ -90,12 +102,13 @@ const make: Effect.Effect<
|
||||
if (event.data.reason === "shutdown") return yield* Effect.never
|
||||
return yield* Effect.interrupt
|
||||
})
|
||||
yield* sessions.resume(sessionID)
|
||||
yield* execution.resume(sessionID)
|
||||
while (true) {
|
||||
yield* sessions.wait(sessionID)
|
||||
if (!(yield* store.get(sessionID))) return yield* new Session.NotFoundError({ sessionID })
|
||||
yield* execution.awaitIdle(sessionID)
|
||||
// Read active last: input may have been promoted while the pending-work queries ran.
|
||||
const pending = yield* SessionBackground.unsettled(jobs, store, db, sessionID)
|
||||
const active = (yield* sessions.active).has(sessionID)
|
||||
const active = yield* execution.isActive(sessionID)
|
||||
const event = yield* Queue.poll(settled)
|
||||
if (Option.isSome(event)) {
|
||||
yield* terminal(event.value)
|
||||
@@ -106,32 +119,53 @@ const make: Effect.Effect<
|
||||
}
|
||||
}, Effect.scoped)
|
||||
|
||||
const notify = Effect.fn("SubagentJob.notify")(function* (recovery: Recovery, startedAt: number) {
|
||||
const key = `${recovery.childSessionID}:${startedAt}`
|
||||
const complete = Effect.fn("SubagentJob.complete")(function* (input: Completion) {
|
||||
const metadata = (yield* store.get(input.recovery.childSessionID))?.metadata
|
||||
if (!SubagentMetadata.notify(metadata)) {
|
||||
if (input.notificationID) yield* jobs.completeBackground(input.notificationID)
|
||||
// A settlement may have seen this job still running. Clear its marker before waking
|
||||
// the idle parent, so a held report gets another chance to release without a model step.
|
||||
const parentID = input.recovery.parentSessionID
|
||||
if (!(yield* store.get(parentID))) return
|
||||
yield* execution.awaitIdle(parentID)
|
||||
yield* execution.wake(parentID)
|
||||
return
|
||||
}
|
||||
yield* SubagentCompletion.deliver(
|
||||
{
|
||||
synthetic: (notice) =>
|
||||
Effect.gen(function* () {
|
||||
yield* sessions.synthetic({ ...notice, resume: false })
|
||||
if (notice.resume !== false && !(yield* sessions.get(notice.sessionID)).revert)
|
||||
yield* execution.wake(notice.sessionID)
|
||||
}).pipe(Effect.uninterruptible),
|
||||
},
|
||||
jobs,
|
||||
input,
|
||||
)
|
||||
})
|
||||
|
||||
const notify = Effect.fn("SubagentJob.notify")(function* (
|
||||
recovery: Recovery,
|
||||
startedAt: number,
|
||||
id: string = recovery.childSessionID,
|
||||
) {
|
||||
const key = `${id}:${startedAt}`
|
||||
if (notifications.has(key)) return
|
||||
notifications.add(key)
|
||||
yield* Effect.gen(function* () {
|
||||
const info = (yield* jobs.wait({ id: recovery.childSessionID })).info
|
||||
const info = (yield* jobs.wait({ id })).info
|
||||
if (!info) return
|
||||
// The preference belongs to the spawn call, so read it after the child settles. A silent
|
||||
// child still clears its durable marker rather than leave a row that restart recovery
|
||||
// would replay as a report the parent asked not to receive.
|
||||
const metadata = yield* sessions.get(recovery.childSessionID).pipe(
|
||||
Effect.map((session) => session.metadata),
|
||||
Effect.catchTag("Session.NotFoundError", () => Effect.succeed(undefined)),
|
||||
// Scope shutdown cancels only the in-memory job. Its running durable row must survive
|
||||
// for the next process, unlike an explicit cancellation persisted by Job.cancel.
|
||||
if (
|
||||
info.status === "cancelled" &&
|
||||
(yield* jobs.pendingBackground).some(
|
||||
(row) => row.status === "running" && (row.notificationID === info.notificationID || row.id === id),
|
||||
)
|
||||
)
|
||||
if (!SubagentMetadata.notify(metadata)) {
|
||||
if (info.notificationID) yield* jobs.completeBackground(info.notificationID)
|
||||
// Nothing reaches the parent, so wake it once idle: a parent whose report waited on this job
|
||||
// settles again and reports its final state, without running a model step. Waiting for idle
|
||||
// rather than skipping an active parent covers a settlement that read this job as running.
|
||||
const parentID = recovery.parentSessionID
|
||||
if (!(yield* store.get(parentID))) return
|
||||
yield* execution.awaitIdle(parentID)
|
||||
yield* execution.wake(parentID)
|
||||
return
|
||||
}
|
||||
yield* SubagentCompletion.deliver(sessions, jobs, { ...info, recovery })
|
||||
yield* complete({ ...info, recovery })
|
||||
}).pipe(
|
||||
Effect.ensuring(Effect.sync(() => notifications.delete(key))),
|
||||
Effect.forkIn(scope, { startImmediately: true }),
|
||||
@@ -139,7 +173,32 @@ const make: Effect.Effect<
|
||||
)
|
||||
})
|
||||
|
||||
const run = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) {
|
||||
yield* settle(sessionID)
|
||||
const messages = yield* store.messages({ sessionID, order: "desc", limit: 20 })
|
||||
return SubagentCompletion.text(
|
||||
messages.find(
|
||||
(message) =>
|
||||
message.type === "assistant" && message.time.completed !== undefined && message.error === undefined,
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
return Service.of({
|
||||
recover: (input) =>
|
||||
Effect.gen(function* () {
|
||||
const info = yield* jobs.start({
|
||||
id: input.id,
|
||||
type: "subagent",
|
||||
title: input.recovery.description,
|
||||
notificationID: input.notificationID,
|
||||
recovery: input.recovery,
|
||||
run: run(input.recovery.childSessionID),
|
||||
})
|
||||
yield* jobs.background(input.id)
|
||||
yield* notify(input.recovery, info.started_at, input.id)
|
||||
}).pipe(Effect.setContext(context)),
|
||||
complete,
|
||||
start: (input: Input) =>
|
||||
Effect.gen(function* () {
|
||||
yield* writeNotify(input.recovery, input.notify)
|
||||
@@ -149,19 +208,7 @@ const make: Effect.Effect<
|
||||
title: input.recovery.description,
|
||||
metadata: {},
|
||||
recovery: input.recovery,
|
||||
run: Effect.gen(function* () {
|
||||
yield* settle(input.recovery.childSessionID)
|
||||
const messages = yield* sessions.messages({
|
||||
sessionID: input.recovery.childSessionID,
|
||||
order: "desc",
|
||||
limit: 20,
|
||||
})
|
||||
const assistant = messages.find(
|
||||
(message) =>
|
||||
message.type === "assistant" && message.time.completed !== undefined && message.error === undefined,
|
||||
)
|
||||
return SubagentCompletion.text(assistant)
|
||||
}),
|
||||
run: run(input.recovery.childSessionID),
|
||||
})
|
||||
}).pipe(Effect.setContext(context)),
|
||||
background: Effect.fn("SubagentJob.background")(function* (input: Input) {
|
||||
@@ -173,8 +220,10 @@ const make: Effect.Effect<
|
||||
})
|
||||
})
|
||||
|
||||
export const layer = Layer.effect(Service, make)
|
||||
|
||||
export const node: LayerNode.Provider<Service, never, typeof Node.tags.values.global> = Node.makeGlobalNode({
|
||||
service: Service,
|
||||
layer: Layer.effect(Service, make),
|
||||
layer,
|
||||
deps: [Session.node, SessionExecution.node, Job.node, Bus.node, SessionStore.node, Database.node],
|
||||
})
|
||||
|
||||
@@ -28,6 +28,7 @@ import { SessionMessage } from "@opencode/core/session/message"
|
||||
import { SessionRunner } from "@opencode/core/session/runner/index"
|
||||
import { SessionInboxTable, SessionTable } from "@opencode/core/session/sql"
|
||||
import { SessionStore } from "@opencode/core/session/store"
|
||||
import { SubagentJob } from "@opencode/core/session/subagent-job"
|
||||
import { Cause, Context, Deferred, Effect, Exit, Fiber, Layer, LayerMap, Scope } from "effect"
|
||||
import { eq } from "drizzle-orm"
|
||||
import { testEffect } from "./lib/effect"
|
||||
@@ -1024,16 +1025,21 @@ describe("SessionRestart background recovery", () => {
|
||||
yield* Deferred.succeed(release, undefined)
|
||||
yield* Deferred.await(parentWoken)
|
||||
yield* Effect.forEach([parent, child], execution.awaitIdle, { discard: true })
|
||||
expect(drained).toEqual(resumeAttempts < 2 ? [child, parent] : [parent])
|
||||
// Core settlement and the job observer both admit the same notification. Their
|
||||
// wakes may coalesce or run a second no-input drain; the child resumes only once.
|
||||
expect([...new Set(drained)]).toEqual(resumeAttempts < 2 ? [child, parent] : [parent])
|
||||
expect(drained.filter((id) => id === child)).toHaveLength(resumeAttempts < 2 ? 1 : 0)
|
||||
expect((yield* store.context(parent)).filter((message) => message.type === "synthetic")).toMatchObject([
|
||||
{ metadata: { source: "subagent", childID: child, state: resumeAttempts < 2 ? "completed" : "error" } },
|
||||
])
|
||||
expect(yield* claims(database)).toEqual({ [parent]: false, [child]: false })
|
||||
expect(yield* attempts(database, child)).toBe(0)
|
||||
yield* pendingEmpty(restarted)
|
||||
yield* execution.awaitIdle(parent)
|
||||
|
||||
const count = drained.length
|
||||
yield* restart.resumeSuspendedSessions
|
||||
expect(drained).toHaveLength(resumeAttempts < 2 ? 2 : 1)
|
||||
expect(drained).toHaveLength(count)
|
||||
}),
|
||||
)
|
||||
}
|
||||
@@ -1519,54 +1525,82 @@ describe("SessionRestart background recovery", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("clears a finished silent child's marker without reporting it", () =>
|
||||
Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
const store = yield* SessionStore.Service
|
||||
const jobs = yield* Job.Service
|
||||
const parent = Session.ID.make("ses_subagent_silent_parent")
|
||||
const child = Session.ID.make("ses_subagent_silent_child")
|
||||
yield* seedSessions(database, [parent])
|
||||
yield* seedSessions(database, [child], { parent_id: parent, metadata: { "subagent.notify": false } })
|
||||
yield* store.holdReport(child, { notificationID: SessionMessage.ID.create(), since: undefined })
|
||||
const complete = yield* Deferred.make<string>()
|
||||
yield* jobs.start({
|
||||
id: child,
|
||||
type: "subagent",
|
||||
recovery: {
|
||||
kind: "subagent",
|
||||
parentSessionID: parent,
|
||||
childSessionID: child,
|
||||
agent: "explore",
|
||||
description: "Silent inspection",
|
||||
},
|
||||
run: Deferred.await(complete),
|
||||
})
|
||||
yield* jobs.background(child)
|
||||
yield* Deferred.succeed(complete, "Silent result")
|
||||
yield* jobs.wait({ id: child })
|
||||
expect(yield* jobs.pendingBackground).toHaveLength(1)
|
||||
for (const state of ["finished", "exhausted"] as const) {
|
||||
it.effect(`a ${state} silent child releases its parent's held report on restart`, () =>
|
||||
Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
const store = yield* SessionStore.Service
|
||||
const jobs = yield* Job.Service
|
||||
const bus = yield* Bus.Service
|
||||
const grandparent = Session.ID.make("ses_subagent_silent_grandparent")
|
||||
const parent = Session.ID.make("ses_subagent_silent_parent")
|
||||
const child = Session.ID.make("ses_subagent_silent_child")
|
||||
yield* seedSessions(database, [grandparent])
|
||||
yield* seedSessions(database, [parent], { parent_id: grandparent })
|
||||
yield* seedSessions(database, [child], {
|
||||
parent_id: parent,
|
||||
metadata: { "subagent.notify": false },
|
||||
resume_attempts: state === "exhausted" ? 10 : 0,
|
||||
})
|
||||
yield* answer(bus, parent, "Held parent answer")
|
||||
const held = { notificationID: SessionMessage.ID.create(), since: undefined }
|
||||
yield* store.holdReport(parent, held)
|
||||
yield* store.holdReport(child, { notificationID: SessionMessage.ID.create(), since: undefined })
|
||||
const complete = yield* Deferred.make<string>()
|
||||
yield* jobs.start({
|
||||
id: child,
|
||||
type: "subagent",
|
||||
recovery: {
|
||||
kind: "subagent",
|
||||
parentSessionID: parent,
|
||||
childSessionID: child,
|
||||
agent: "explore",
|
||||
description: "Silent inspection",
|
||||
},
|
||||
run: Deferred.await(complete),
|
||||
})
|
||||
yield* jobs.background(child)
|
||||
if (state === "finished") {
|
||||
yield* Deferred.succeed(complete, "Silent result")
|
||||
yield* jobs.wait({ id: child })
|
||||
}
|
||||
expect(yield* jobs.pendingBackground).toHaveLength(1)
|
||||
|
||||
const drained: Session.ID[] = []
|
||||
const scope = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
|
||||
const restarted = yield* Job.make.pipe(Scope.provide(scope))
|
||||
const context = yield* buildExecution(
|
||||
scope,
|
||||
({ sessionID }) => Effect.sync(() => void drained.push(sessionID)),
|
||||
undefined,
|
||||
restarted,
|
||||
)
|
||||
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
|
||||
const drained: Session.ID[] = []
|
||||
const woken = yield* Deferred.make<void>()
|
||||
const scope = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
|
||||
const restarted = yield* Job.make.pipe(Scope.provide(scope))
|
||||
const context = yield* buildExecution(
|
||||
scope,
|
||||
({ sessionID }) =>
|
||||
Effect.gen(function* () {
|
||||
drained.push(sessionID)
|
||||
if (sessionID === grandparent) yield* Deferred.succeed(woken, undefined)
|
||||
}),
|
||||
undefined,
|
||||
restarted,
|
||||
)
|
||||
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
|
||||
yield* Deferred.await(woken)
|
||||
yield* Context.get(context, SessionExecution.Service).awaitIdle(parent)
|
||||
yield* Context.get(context, SessionExecution.Service).awaitIdle(grandparent)
|
||||
|
||||
expect(drained).toEqual([])
|
||||
expect(yield* SessionInbox.list(database.db, parent)).toEqual([])
|
||||
expect(yield* restarted.pendingBackground).toEqual([])
|
||||
expect(yield* store.listHeldChildren()).toEqual([])
|
||||
}),
|
||||
)
|
||||
expect(drained).not.toContain(child)
|
||||
expect(yield* SessionInbox.list(database.db, parent)).toEqual([])
|
||||
expect(yield* restarted.pendingBackground).toEqual([])
|
||||
expect(yield* store.listHeldChildren()).toEqual([])
|
||||
expect(yield* SessionInbox.list(database.db, grandparent)).toMatchObject([
|
||||
{ id: held.notificationID, payload: { text: expect.stringContaining("Held parent answer") } },
|
||||
])
|
||||
const count = drained.length
|
||||
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
|
||||
expect(drained).toHaveLength(count)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
it.effect("resumes a silent child after restart without reporting or waking its parent", () =>
|
||||
it.effect("resumes a silent child after restart and wakes its parent without reporting", () =>
|
||||
Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
const jobs = yield* Job.Service
|
||||
@@ -1596,6 +1630,7 @@ describe("SessionRestart background recovery", () => {
|
||||
if (!marker) return yield* Effect.die("background record missing")
|
||||
|
||||
const answered = yield* Deferred.make<void>()
|
||||
const woken = yield* Deferred.make<void>()
|
||||
const drained: Session.ID[] = []
|
||||
const scope = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
|
||||
@@ -1605,7 +1640,7 @@ describe("SessionRestart background recovery", () => {
|
||||
({ sessionID }) =>
|
||||
sessionID === child
|
||||
? answer(bus, child, "Silent resumed result").pipe(Effect.andThen(Deferred.succeed(answered, undefined)))
|
||||
: Effect.sync(() => void drained.push(sessionID)),
|
||||
: Effect.sync(() => void drained.push(sessionID)).pipe(Effect.andThen(Deferred.succeed(woken, undefined))),
|
||||
undefined,
|
||||
restarted,
|
||||
)
|
||||
@@ -1613,8 +1648,10 @@ describe("SessionRestart background recovery", () => {
|
||||
yield* restart.resumeSuspendedSessions
|
||||
yield* Deferred.await(answered)
|
||||
yield* pendingEmpty(restarted)
|
||||
yield* Deferred.await(woken)
|
||||
yield* Context.get(context, SessionExecution.Service).awaitIdle(parent)
|
||||
|
||||
expect(drained).toEqual([])
|
||||
expect(drained).toEqual([parent])
|
||||
expect(yield* SessionInbox.list(database.db, parent)).toEqual([])
|
||||
expect(yield* restarted.pendingBackground).toEqual([])
|
||||
yield* restart.resumeSuspendedSessions
|
||||
@@ -1622,48 +1659,6 @@ describe("SessionRestart background recovery", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("clears a silent child's marker when its resume budget is exhausted", () =>
|
||||
Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
const jobs = yield* Job.Service
|
||||
const parent = Session.ID.make("ses_subagent_silent_exhausted_parent")
|
||||
const child = Session.ID.make("ses_subagent_silent_exhausted_child")
|
||||
yield* seedSessions(database, [parent])
|
||||
yield* seedSessions(database, [child], {
|
||||
parent_id: parent,
|
||||
resume_attempts: 10,
|
||||
metadata: { "subagent.notify": false },
|
||||
})
|
||||
yield* jobs.start({
|
||||
id: child,
|
||||
type: "subagent",
|
||||
recovery: {
|
||||
kind: "subagent",
|
||||
parentSessionID: parent,
|
||||
childSessionID: child,
|
||||
agent: "explore",
|
||||
description: "Silent exhausted inspection",
|
||||
},
|
||||
run: Effect.never,
|
||||
})
|
||||
yield* jobs.background(child)
|
||||
|
||||
const scope = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void))
|
||||
const restarted = yield* Job.make.pipe(Scope.provide(scope))
|
||||
const context = yield* buildExecution(
|
||||
scope,
|
||||
() => Effect.die("A silent child must not wake its parent"),
|
||||
undefined,
|
||||
restarted,
|
||||
)
|
||||
yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions
|
||||
|
||||
expect(yield* SessionInbox.list(database.db, parent)).toEqual([])
|
||||
expect(yield* restarted.pendingBackground).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("retains a subagent completion marker when synthetic admission conflicts", () =>
|
||||
Effect.gen(function* () {
|
||||
const database = yield* Database.Service
|
||||
@@ -2160,6 +2155,7 @@ function buildExecution(
|
||||
)
|
||||
return yield* Layer.buildWithScope(
|
||||
SessionRestart.layer(options).pipe(
|
||||
Layer.provideMerge(Layer.fresh(SubagentJob.layer)),
|
||||
Layer.provideMerge(sessionLayer),
|
||||
Layer.provideMerge(Layer.fresh(SessionExecution.layer)),
|
||||
Layer.provide(Layer.succeed(Database.Service, database)),
|
||||
|
||||
@@ -417,7 +417,7 @@ describe("subagent observer lifetime", () => {
|
||||
yield* Effect.addFinalizer(() => Scope.close(before, Exit.void))
|
||||
const oldJobs = yield* Job.make.pipe(Scope.provide(before))
|
||||
const old = yield* Layer.buildWithScope(
|
||||
SessionExecution.layer.pipe(Layer.provide(Layer.succeed(Job.Service, oldJobs))),
|
||||
SessionExecution.layer.pipe(Layer.provide(Layer.succeed(Job.Service, oldJobs)), Layer.fresh),
|
||||
before,
|
||||
)
|
||||
const oldExecution = Context.get(old, SessionExecution.Service)
|
||||
@@ -443,8 +443,10 @@ describe("subagent observer lifetime", () => {
|
||||
const jobs = yield* Job.make.pipe(Scope.provide(after))
|
||||
const context = yield* Layer.buildWithScope(
|
||||
SessionRestart.layer().pipe(
|
||||
Layer.provideMerge(SubagentJob.layer),
|
||||
Layer.provideMerge(SessionExecution.layer),
|
||||
Layer.provide(Layer.succeed(Job.Service, jobs)),
|
||||
Layer.fresh,
|
||||
),
|
||||
after,
|
||||
)
|
||||
@@ -470,6 +472,150 @@ describe("subagent observer lifetime", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
for (const outcome of ["complete", "shutdown"] as const) {
|
||||
it.live(`restart retains a child's job through its silent grandchild's settlement (${outcome})`, () =>
|
||||
Effect.gen(function* () {
|
||||
const store = yield* SessionStore.Service
|
||||
const llm = yield* TestLLM.Test
|
||||
const f = yield* family()
|
||||
const build = (scope: Scope.Closeable, jobs: Job.Interface) =>
|
||||
Layer.buildWithScope(
|
||||
SessionRestart.layer().pipe(
|
||||
Layer.provideMerge(SubagentJob.layer),
|
||||
Layer.provideMerge(SessionExecution.layer),
|
||||
Layer.provide(Layer.succeed(Job.Service, jobs)),
|
||||
Layer.fresh,
|
||||
),
|
||||
scope,
|
||||
)
|
||||
const before = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(before, Exit.void))
|
||||
const oldJobs = yield* Job.make.pipe(Scope.provide(before))
|
||||
const old = yield* build(before, oldJobs)
|
||||
const oldSubagents = Context.get(old, SubagentJob.Service)
|
||||
const oldExecution = Context.get(old, SessionExecution.Service)
|
||||
const child = {
|
||||
recovery: {
|
||||
kind: "subagent" as const,
|
||||
parentSessionID: f.parent.id,
|
||||
childSessionID: f.child.id,
|
||||
agent: "reviewer",
|
||||
description: "Review fixture",
|
||||
},
|
||||
notify: true,
|
||||
}
|
||||
const descendant = {
|
||||
recovery: {
|
||||
...child.recovery,
|
||||
parentSessionID: f.child.id,
|
||||
childSessionID: f.descendant.id,
|
||||
description: "Check the tests",
|
||||
},
|
||||
notify: false,
|
||||
}
|
||||
yield* llm.push(Stream.never, TestLLM.text("before restart", "before"))
|
||||
yield* f.sessions.prompt({ sessionID: f.descendant.id, text: "Check the tests", resume: false })
|
||||
yield* oldSubagents.start(descendant)
|
||||
yield* oldSubagents.background(descendant)
|
||||
yield* llm.wait(1)
|
||||
yield* f.sessions.prompt({ sessionID: f.child.id, text: "Review fixture", resume: false })
|
||||
yield* oldSubagents.start(child)
|
||||
yield* oldSubagents.background(child)
|
||||
yield* llm.wait(2)
|
||||
yield* oldExecution.awaitIdle(f.child.id)
|
||||
const held = yield* store.heldReport(f.child.id)
|
||||
const markers = yield* oldJobs.pendingBackground
|
||||
expect(held).toBeDefined()
|
||||
expect(markers).toHaveLength(2)
|
||||
expect(markers.every((row) => row.status === "running")).toBe(true)
|
||||
yield* Scope.close(before, Exit.void)
|
||||
expect(yield* oldJobs.pendingBackground).toEqual(markers)
|
||||
expect(yield* store.heldReport(f.child.id)).toEqual(held)
|
||||
expect(yield* reports(f.parent.id)).toEqual([])
|
||||
|
||||
const after = yield* Scope.make()
|
||||
yield* Effect.addFinalizer(() => Scope.close(after, Exit.void))
|
||||
const jobs = yield* Job.make.pipe(Scope.provide(after))
|
||||
const go = yield* Deferred.make<void>()
|
||||
const settled = yield* Deferred.make<void>()
|
||||
const finish = yield* Deferred.make<void>()
|
||||
const cleared = yield* Deferred.make<void>()
|
||||
const context = yield* build(after, {
|
||||
...jobs,
|
||||
// Hold the real job just after D's execution settles: C's wake must observe D
|
||||
// still running and hold again before its completion observer gets to clear D.
|
||||
start: (input) =>
|
||||
jobs.start({
|
||||
...input,
|
||||
run:
|
||||
input.id !== f.descendant.id
|
||||
? input.run
|
||||
: input.run.pipe(
|
||||
Effect.tap(() =>
|
||||
Deferred.succeed(settled, undefined).pipe(Effect.andThen(Deferred.await(finish))),
|
||||
),
|
||||
),
|
||||
}),
|
||||
completeBackground: (id) =>
|
||||
jobs
|
||||
.completeBackground(id)
|
||||
.pipe(
|
||||
Effect.tap(() => (id === held?.notificationID ? Deferred.succeed(cleared, undefined) : Effect.void)),
|
||||
),
|
||||
})
|
||||
const restart = Context.get(context, SessionRestart.Service)
|
||||
const execution = Context.get(context, SessionExecution.Service)
|
||||
yield* llm.push(
|
||||
Stream.unwrap(
|
||||
Deferred.await(go).pipe(Effect.as(Stream.fromIterable(TestLLM.text("grandchild done", "grandchild")))),
|
||||
),
|
||||
TestLLM.text("after restart", "after"),
|
||||
)
|
||||
yield* restart.resumeSuspendedSessions
|
||||
yield* llm.wait(4)
|
||||
yield* execution.awaitIdle(f.child.id)
|
||||
expect(yield* store.heldReport(f.child.id)).toEqual(held)
|
||||
expect((yield* jobs.get(f.child.id))?.status).toBe("running")
|
||||
expect(yield* reports(f.parent.id)).toEqual([])
|
||||
if (outcome === "shutdown") {
|
||||
yield* Scope.close(after, Exit.void)
|
||||
expect(yield* jobs.pendingBackground).toEqual(markers)
|
||||
expect(yield* store.heldReport(f.child.id)).toEqual(held)
|
||||
expect(yield* reports(f.parent.id)).toEqual([])
|
||||
expect((yield* reports(f.child.id)).filter((message) => message.metadata?.source === "subagent")).toEqual([])
|
||||
return
|
||||
}
|
||||
yield* Deferred.succeed(go, undefined)
|
||||
yield* Deferred.await(settled)
|
||||
yield* execution.awaitIdle(f.child.id)
|
||||
expect((yield* jobs.get(f.descendant.id))?.status).toBe("running")
|
||||
expect((yield* jobs.get(f.child.id))?.status).toBe("running")
|
||||
expect(yield* store.heldReport(f.child.id)).toEqual(held)
|
||||
expect(yield* reports(f.parent.id)).toEqual([])
|
||||
|
||||
yield* Deferred.succeed(finish, undefined)
|
||||
yield* Deferred.await(cleared)
|
||||
yield* llm.wait(5)
|
||||
yield* execution.awaitIdle(f.parent.id)
|
||||
const notices = yield* reports(f.parent.id)
|
||||
expect(notices).toHaveLength(1)
|
||||
expect(notices[0]).toMatchObject({
|
||||
id: held?.notificationID,
|
||||
metadata: { source: "subagent", childID: f.child.id, state: "completed" },
|
||||
})
|
||||
expect(notices[0]?.text).toContain("after restart")
|
||||
expect(notices[0]?.text).not.toContain("grandchild done")
|
||||
expect(notices[0]?.text).not.toContain("before restart")
|
||||
expect(yield* jobs.pendingBackground).toEqual([])
|
||||
expect(yield* store.listHeldChildren()).toEqual([])
|
||||
expect(yield* llm.requests()).toHaveLength(5)
|
||||
yield* restart.resumeSuspendedSessions
|
||||
expect(yield* reports(f.parent.id)).toEqual(notices)
|
||||
expect(yield* llm.requests()).toHaveLength(5)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
it.live("a silent background grandchild job releases its parent's held report", () =>
|
||||
Effect.gen(function* () {
|
||||
const jobs = yield* Job.Service
|
||||
|
||||
Reference in New Issue
Block a user