fix(cli): bound server shutdown with a 10 s deadline (#58817)

This commit is contained in:
Kit Langton
2026-10-25 07:47:11 -07:00
committed by GitHub
Unverified
parent 1564686791
commit b6cbbd7646
8 changed files with 278 additions and 8 deletions
+2
View File
@@ -17,6 +17,7 @@ import { Npm } from "@opencode/util/npm"
import { EffectFlock } from "@opencode/util/effect-flock"
import { Heap } from "./heap"
import { CpuProfile } from "./cpu-profile"
import { ShutdownDeadline } from "./shutdown-deadline"
ensurePluginRuntime()
@@ -85,6 +86,7 @@ const Handlers = Runtime.handlers(Commands, {
Effect.gen(function* () {
yield* Heap.listen
yield* CpuProfile.listen
yield* ShutdownDeadline.install()
const runFork = Effect.runForkWith(yield* Effect.context<never>())
const uncaughtException = (cause: Error, origin: "uncaughtException" | "unhandledRejection") => {
runFork(Effect.logError("uncaught exception", { cause, origin }))
+65
View File
@@ -0,0 +1,65 @@
import { Teardown } from "@opencode/util/teardown"
import { Duration, Effect } from "effect"
/** After SIGTERM or SIGINT, teardown gets this long before the process reports what it is stuck on and exits. */
export const DEADLINE = Duration.seconds(10)
// The file logger batches writes for up to a second.
const LOG_FLUSH = Duration.millis(1_500)
/**
* Bounds signal-initiated teardown. NodeRuntime.runMain interrupts the main fiber on SIGTERM or SIGINT and then waits
* for every finalizer, so a finalizer that never finishes would keep the process alive until something kills it.
* A second signal during teardown reports and exits at once.
*/
export const install = Effect.fnUntraced(function* (input: Duration.Input = DEADLINE) {
const deadline = Duration.fromInputUnsafe(input)
const fork = Effect.runForkWith(yield* Effect.context<never>())
let started = false
const expire = (signal: NodeJS.Signals) =>
Effect.gen(function* () {
yield* Effect.logInfo("shutdown started", { signal })
yield* Effect.sleep(deadline)
const pending = Teardown.pending()
process.stderr.write(report(`shutdown did not finish within ${Duration.format(deadline)}`, pending))
yield* Effect.logError("shutdown deadline exceeded", {
deadline: Duration.format(deadline),
pending: pending.map((item) => `${item.label} for ${item.ms}ms`),
})
yield* Effect.sleep(LOG_FLUSH)
yield* Effect.sync(() => process.exit(1))
})
const onSignal = (signal: NodeJS.Signals) => {
if (started) {
process.stderr.write(report(`received ${signal} again during shutdown`, Teardown.pending()))
process.exit(1)
}
started = true
fork(expire(signal))
}
const off = () => {
process.off("SIGTERM", onSignal)
process.off("SIGINT", onSignal)
}
yield* Effect.acquireRelease(
Effect.sync(() => {
process.on("SIGTERM", onSignal)
process.on("SIGINT", onSignal)
}),
// Once shutdown has started, keep listening: this release can run while a later finalizer is still stuck.
() =>
Effect.sync(() => {
if (!started) off()
}),
)
})
function report(reason: string, pending: ReturnType<typeof Teardown.pending>) {
const lines = pending.map((item) => ` ${(item.ms / 1000).toFixed(1)}s ${item.label}`)
return [
`${reason}; exiting. Still pending:`,
...(lines.length > 0 ? lines : [" nothing tracked; the stuck finalizer is outside any LayerNode layer"]),
"",
].join("\n")
}
export * as ShutdownDeadline from "./shutdown-deadline"
@@ -0,0 +1,29 @@
import { NodeRuntime } from "@effect/platform-node"
import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner"
import { LayerNode } from "@opencode/util/effect/layer-node"
import { Context, Effect, Layer } from "effect"
import { ChildProcess } from "effect/process"
import { ShutdownDeadline } from "../../src/shutdown-deadline"
class Hung extends Context.Service<Hung, {}>()("test/Hung") {}
// Owns a running child, then a finalizer that never finishes, so releasing the layer hangs before the child stops.
const hung = LayerNode.make({
service: Hung,
layer: Layer.effect(
Hung,
Effect.gen(function* () {
const child = yield* ChildProcess.make(process.execPath, ["-e", "setInterval(() => {}, 1000)"])
yield* Effect.addFinalizer(() => Effect.never)
console.log(`SHUTDOWN_READY ${child.pid}`)
return Hung.of({})
}),
),
deps: [CrossSpawnSpawner.node],
})
Effect.gen(function* () {
yield* ShutdownDeadline.install(Number(process.argv[2]))
yield* Layer.build(LayerNode.compile(hung))
return yield* Effect.never
}).pipe(Effect.scoped, NodeRuntime.runMain)
@@ -0,0 +1,74 @@
import { expect, test } from "bun:test"
import path from "node:path"
// Windows cannot catch SIGTERM: the process ends at once, so there is no teardown to bound.
const posix = test.skipIf(process.platform === "win32")
posix("exits when teardown outlives the deadline and names what is pending", async () => {
await hang(200, async (proc, pid) => {
proc.kill("SIGTERM")
const stderr = await new Response(proc.stderr).text()
expect(stderr).toContain("shutdown did not finish within 200ms")
expect(stderr).toContain("layer test/Hung releasing")
expect(stderr).toContain(`child ${pid} running: ${process.execPath} -e setInterval`)
expect(await proc.exited).toBe(1)
})
})
posix("a second signal during teardown reports and exits without waiting for the deadline", async () => {
await hang(60_000, async (proc, pid) => {
const start = Date.now()
proc.kill("SIGTERM")
await Bun.sleep(200)
proc.kill("SIGINT")
const stderr = await new Response(proc.stderr).text()
expect(stderr).toContain("received SIGINT again during shutdown")
expect(stderr).toContain("layer test/Hung releasing")
expect(stderr).toContain(`child ${pid} running:`)
expect(await proc.exited).toBe(1)
expect(Date.now() - start).toBeLessThan(10_000)
})
})
async function hang(
deadline: number,
body: (proc: Bun.Subprocess<"ignore", "pipe", "pipe">, pid: number) => Promise<void>,
) {
const proc = Bun.spawn([process.execPath, path.join(import.meta.dir, "fixture/shutdown-hang.ts"), String(deadline)], {
cwd: path.join(import.meta.dir, ".."),
stdin: "ignore",
stdout: "pipe",
stderr: "pipe",
})
const pid = Number((await readLine(proc.stdout, "SHUTDOWN_READY "))?.split(" ")[1])
try {
expect(pid).toBeGreaterThan(0)
await body(proc, pid)
} finally {
proc.kill("SIGKILL")
if (running(pid)) process.kill(pid, "SIGKILL")
}
}
async function readLine(stream: ReadableStream<Uint8Array>, prefix: string) {
const decoder = new TextDecoder()
const chunks: string[] = []
for await (const chunk of stream) {
chunks.push(decoder.decode(chunk, { stream: true }))
const line = chunks
.join("")
.split("\n")
.find((line) => line.startsWith(prefix))
if (line) return line
}
}
function running(pid: number) {
if (!Number.isSafeInteger(pid) || pid <= 0) return false
try {
process.kill(pid, 0)
return true
} catch {
return false
}
}
@@ -6,6 +6,7 @@ import { Deferred, Effect, Exit, PlatformError, Stream } from "effect"
import { ChildProcess, ChildProcessSpawner } from "effect/process"
import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner"
import { LayerNode } from "@opencode/util/effect/layer-node"
import { Teardown } from "@opencode/util/teardown"
import { testEffect } from "../lib/effect"
const live = LayerNode.compile(CrossSpawnSpawner.node)
@@ -301,6 +302,39 @@ describe("cross-spawn spawner", () => {
)
}
fx.live(
"stops tracking a released child while a grandchild holds stdio",
Effect.gen(function* () {
const tmp = yield* Effect.acquireRelease(
Effect.promise(() => tmpdir()),
(tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()),
)
const pidFile = path.join(tmp.path, "child.pid")
yield* Effect.addFinalizer(() =>
Effect.tryPromise(async () => process.kill(Number(await fs.readFile(pidFile, "utf8")), "SIGKILL")).pipe(
Effect.ignore,
),
)
const label = yield* Effect.scoped(
Effect.gen(function* () {
const handle = yield* ChildProcess.make(
"node",
[path.join(import.meta.dir, "../fixture/held-stdio.cjs"), "exit", pidFile],
{ stdin: "ignore" },
)
const label = `child ${handle.pid} running:`
expect(Teardown.pending().some((item) => item.label.startsWith(label))).toBe(true)
yield* handle.exitCode
return label
}),
)
// Checked before yielding to the event loop: release itself settles the entry, not a later "close" event.
expect(Teardown.pending().some((item) => item.label.startsWith(label))).toBe(false)
expect(alive(Number(yield* Effect.promise(() => fs.readFile(pidFile, "utf8"))))).toBe(true)
}),
10_000,
)
fx.effect(
"kills a running process",
Effect.gen(function* () {
+27 -6
View File
@@ -18,6 +18,10 @@ import { PassThrough } from "node:stream"
import launch from "cross-spawn"
import { makeGlobalNode } from "./effect/app-node.js"
import { filesystem, path } from "./effect/app-node-platform.js"
import { Teardown } from "./teardown.js"
/** Closing a child's scope escalates to SIGKILL after this, unless the command sets its own `forceKillAfter`. */
export const RELEASE_FORCE_KILL_AFTER = "3 seconds"
const toError = (err: unknown): Error => (err instanceof globalThis.Error ? err : new globalThis.Error(String(err)))
@@ -95,7 +99,12 @@ export class KilledBySignal extends Error {
}
type ExitSignal = Deferred.Deferred<readonly [code: number | null, signal: NodeJS.Signals | null]>
type Spawned = readonly [process: NodeChildProcess.ChildProcess, closed: ExitSignal, exited: ExitSignal]
type Spawned = readonly [
process: NodeChildProcess.ChildProcess,
closed: ExitSignal,
exited: ExitSignal,
settle: () => void,
]
const makeCrossSpawnSpawner = Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem
@@ -281,6 +290,7 @@ const makeCrossSpawnSpawner = Effect.gen(function* () {
const closed = Deferred.makeUnsafe<readonly [code: number | null, signal: NodeJS.Signals | null]>()
const exited = Deferred.makeUnsafe<readonly [code: number | null, signal: NodeJS.Signals | null]>()
const proc = launch(command.command, command.args, opts)
let settle = () => {}
let end = false
let exit: readonly [code: number | null, signal: NodeJS.Signals | null] | undefined
proc.on("error", (err) => {
@@ -293,11 +303,15 @@ const makeCrossSpawnSpawner = Effect.gen(function* () {
proc.on("close", (...args) => {
if (end) return
end = true
settle()
Deferred.doneUnsafe(exited, Exit.succeed(exit ?? args))
Deferred.doneUnsafe(closed, Exit.succeed(exit ?? args))
})
proc.on("spawn", () => {
resume(Effect.succeed([proc, closed, exited]))
settle = Teardown.track(
`child ${proc.pid} running: ${[command.command, ...command.args].join(" ").slice(0, 200)}`,
)
resume(Effect.succeed([proc, closed, exited, () => settle()]))
})
return Effect.sync(() => {
proc.kill("SIGTERM")
@@ -309,7 +323,7 @@ const makeCrossSpawnSpawner = Effect.gen(function* () {
opts: NodeChildProcess.SpawnOptions,
) {
yield* Effect.logInfo("spawning process", { command: command.command, args: command.args, cwd: opts.cwd })
const [proc, closed, exited] = yield* launchProcess(command, opts)
const [proc, closed, exited, settle] = yield* launchProcess(command, opts)
const stopOutput = yield* Deferred.make<void>()
// Register before process release so the deadline remains active during termination.
yield* Effect.forkScoped(
@@ -325,7 +339,7 @@ const makeCrossSpawnSpawner = Effect.gen(function* () {
discard(proc.stderr)
}),
)
return [proc, closed, exited, stopOutput] as const
return [proc, closed, exited, stopOutput, settle] as const
})
const discard = (readable: NodeChildProcess.ChildProcess["stdout"]) => {
@@ -445,12 +459,19 @@ const makeCrossSpawnSpawner = Effect.gen(function* () {
return yield* Effect.ignore(killGroup(command, proc, command.options.killSignal ?? "SIGTERM"))
}
}
yield* Effect.ignore(stop(command, proc, closed, stopOutput, command.options))
yield* Effect.ignore(
stop(command, proc, closed, stopOutput, {
killSignal: command.options.killSignal,
forceKillAfter: command.options.forceKillAfter ?? RELEASE_FORCE_KILL_AFTER,
}),
)
},
(effect, [proc, , , stopOutput]) =>
(effect, [proc, , , stopOutput, settle]) =>
effect.pipe(
Effect.ensuring(
Effect.gen(function* () {
// A detached descendant holding the pipes can keep "close" from ever firing.
settle()
yield* Deferred.succeed(stopOutput, undefined)
proc.stdout?.destroy()
proc.stderr?.destroy()
+28 -2
View File
@@ -1,4 +1,5 @@
import { Brand, Context, Effect, Layer, type Scope } from "effect"
import { Brand, Context, Effect, Layer, Scope } from "effect"
import { Teardown } from "../teardown.js"
export * as LayerNode from "./layer-node.js"
@@ -233,6 +234,7 @@ export interface CompileOptions {
}
type Resolved = {
readonly name: string
readonly implementation: RuntimeLayer
readonly dependencies: readonly Resolved[]
readonly shared: boolean
@@ -274,6 +276,7 @@ export function compile<A, E>(root: Graph<A, E>, options: CompileOptions = {}):
return definition.dependencies.flatMap((dependency) => resolve(dependency, isShared))
if (definition.kind === "unbound") throw new Error(`Unbound layer node: ${definition.name}`)
const node: Resolved = {
name: definition.name,
implementation: definition.implementation,
dependencies: definition.dependencies.flatMap((dependency) => resolve(dependency, isShared)),
shared: isShared,
@@ -310,7 +313,9 @@ export function compile<A, E>(root: Graph<A, E>, options: CompileOptions = {}):
const dependencies = node.dependencies.map(build)
// Acquisition uses the selected cache, while lazy LayerMaps inherit the enclosing shared cache.
const implementation = Layer.suspend(() => node.implementation.pipe(Layer.provide([ambient, ...dependencies])))
const layer = Layer.fromBuild((_, scope) => buildContext(implementation, node.shared ? memoMap : local, scope))
const layer = Layer.fromBuild((_, scope) =>
trackRelease(node.name, scope, () => buildContext(implementation, node.shared ? memoMap : local, scope)),
)
layers.set(node, layer)
return layer
}
@@ -322,6 +327,27 @@ export function compile<A, E>(root: Graph<A, E>, options: CompileOptions = {}):
}) as Layer.Layer<A, E>
}
// The node's scope releases sequentially in reverse order, so a finalizer added before the build runs last and one
// added after it runs first: together they bracket the release of everything this node acquired. The build must
// not start before the first finalizer is added; Layer builds fork their scopes as soon as they are called.
function trackRelease<A, E, R>(name: string, scope: Scope.Scope, build: () => Effect.Effect<A, E, R>) {
return Effect.gen(function* () {
let settle = () => {}
yield* Scope.addFinalizer(
scope,
Effect.sync(() => settle()),
)
const result = yield* build()
yield* Scope.addFinalizer(
scope,
Effect.sync(() => {
settle = Teardown.track(`layer ${name} releasing`)
}),
)
return result
})
}
class BuildResult extends Context.Service<BuildResult, Context.Context<never>>()("@opencode/LayerNode/BuildResult") {}
function buildContext(layer: RuntimeLayer, memoMap: Layer.MemoMap, scope: Scope.Scope) {
+19
View File
@@ -0,0 +1,19 @@
export * as Teardown from "./teardown.js"
// Work that shutdown may be waiting on: layers releasing their resources and child processes that have not
// closed. Read when a shutdown deadline expires, to name what kept the process alive.
const entries = new Set<{ readonly label: string; readonly since: number }>()
/** Record pending work; call the returned function once it settles. */
export function track(label: string) {
const entry = { label, since: Date.now() }
entries.add(entry)
return () => {
entries.delete(entry)
}
}
/** Pending work, oldest first, with how long each has been pending. */
export function pending(now = Date.now()) {
return Array.from(entries, (entry) => ({ label: entry.label, ms: now - entry.since }))
}