diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index a81de93cbf3..ef7004815c4 100755 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -17,6 +17,7 @@ import { Npm } from "@opencode/util/npm" import { EffectFlock } from "@opencode/util/effect-flock" import { Heap } from "./heap" import { CpuProfile } from "./cpu-profile" +import { ShutdownDeadline } from "./shutdown-deadline" ensurePluginRuntime() @@ -85,6 +86,7 @@ const Handlers = Runtime.handlers(Commands, { Effect.gen(function* () { yield* Heap.listen yield* CpuProfile.listen + yield* ShutdownDeadline.install() const runFork = Effect.runForkWith(yield* Effect.context()) const uncaughtException = (cause: Error, origin: "uncaughtException" | "unhandledRejection") => { runFork(Effect.logError("uncaught exception", { cause, origin })) diff --git a/packages/cli/src/shutdown-deadline.ts b/packages/cli/src/shutdown-deadline.ts new file mode 100644 index 00000000000..6d42987ce72 --- /dev/null +++ b/packages/cli/src/shutdown-deadline.ts @@ -0,0 +1,65 @@ +import { Teardown } from "@opencode/util/teardown" +import { Duration, Effect } from "effect" + +/** After SIGTERM or SIGINT, teardown gets this long before the process reports what it is stuck on and exits. */ +export const DEADLINE = Duration.seconds(10) +// The file logger batches writes for up to a second. +const LOG_FLUSH = Duration.millis(1_500) + +/** + * Bounds signal-initiated teardown. NodeRuntime.runMain interrupts the main fiber on SIGTERM or SIGINT and then waits + * for every finalizer, so a finalizer that never finishes would keep the process alive until something kills it. + * A second signal during teardown reports and exits at once. + */ +export const install = Effect.fnUntraced(function* (input: Duration.Input = DEADLINE) { + const deadline = Duration.fromInputUnsafe(input) + const fork = Effect.runForkWith(yield* Effect.context()) + let started = false + const expire = (signal: NodeJS.Signals) => + Effect.gen(function* () { + yield* Effect.logInfo("shutdown started", { signal }) + yield* Effect.sleep(deadline) + const pending = Teardown.pending() + process.stderr.write(report(`shutdown did not finish within ${Duration.format(deadline)}`, pending)) + yield* Effect.logError("shutdown deadline exceeded", { + deadline: Duration.format(deadline), + pending: pending.map((item) => `${item.label} for ${item.ms}ms`), + }) + yield* Effect.sleep(LOG_FLUSH) + yield* Effect.sync(() => process.exit(1)) + }) + const onSignal = (signal: NodeJS.Signals) => { + if (started) { + process.stderr.write(report(`received ${signal} again during shutdown`, Teardown.pending())) + process.exit(1) + } + started = true + fork(expire(signal)) + } + const off = () => { + process.off("SIGTERM", onSignal) + process.off("SIGINT", onSignal) + } + yield* Effect.acquireRelease( + Effect.sync(() => { + process.on("SIGTERM", onSignal) + process.on("SIGINT", onSignal) + }), + // Once shutdown has started, keep listening: this release can run while a later finalizer is still stuck. + () => + Effect.sync(() => { + if (!started) off() + }), + ) +}) + +function report(reason: string, pending: ReturnType) { + const lines = pending.map((item) => ` ${(item.ms / 1000).toFixed(1)}s ${item.label}`) + return [ + `${reason}; exiting. Still pending:`, + ...(lines.length > 0 ? lines : [" nothing tracked; the stuck finalizer is outside any LayerNode layer"]), + "", + ].join("\n") +} + +export * as ShutdownDeadline from "./shutdown-deadline" diff --git a/packages/cli/test/fixture/shutdown-hang.ts b/packages/cli/test/fixture/shutdown-hang.ts new file mode 100644 index 00000000000..db187b3fd15 --- /dev/null +++ b/packages/cli/test/fixture/shutdown-hang.ts @@ -0,0 +1,29 @@ +import { NodeRuntime } from "@effect/platform-node" +import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { Context, Effect, Layer } from "effect" +import { ChildProcess } from "effect/process" +import { ShutdownDeadline } from "../../src/shutdown-deadline" + +class Hung extends Context.Service()("test/Hung") {} + +// Owns a running child, then a finalizer that never finishes, so releasing the layer hangs before the child stops. +const hung = LayerNode.make({ + service: Hung, + layer: Layer.effect( + Hung, + Effect.gen(function* () { + const child = yield* ChildProcess.make(process.execPath, ["-e", "setInterval(() => {}, 1000)"]) + yield* Effect.addFinalizer(() => Effect.never) + console.log(`SHUTDOWN_READY ${child.pid}`) + return Hung.of({}) + }), + ), + deps: [CrossSpawnSpawner.node], +}) + +Effect.gen(function* () { + yield* ShutdownDeadline.install(Number(process.argv[2])) + yield* Layer.build(LayerNode.compile(hung)) + return yield* Effect.never +}).pipe(Effect.scoped, NodeRuntime.runMain) diff --git a/packages/cli/test/shutdown-deadline.test.ts b/packages/cli/test/shutdown-deadline.test.ts new file mode 100644 index 00000000000..8b9f0b870b8 --- /dev/null +++ b/packages/cli/test/shutdown-deadline.test.ts @@ -0,0 +1,74 @@ +import { expect, test } from "bun:test" +import path from "node:path" + +// Windows cannot catch SIGTERM: the process ends at once, so there is no teardown to bound. +const posix = test.skipIf(process.platform === "win32") + +posix("exits when teardown outlives the deadline and names what is pending", async () => { + await hang(200, async (proc, pid) => { + proc.kill("SIGTERM") + const stderr = await new Response(proc.stderr).text() + expect(stderr).toContain("shutdown did not finish within 200ms") + expect(stderr).toContain("layer test/Hung releasing") + expect(stderr).toContain(`child ${pid} running: ${process.execPath} -e setInterval`) + expect(await proc.exited).toBe(1) + }) +}) + +posix("a second signal during teardown reports and exits without waiting for the deadline", async () => { + await hang(60_000, async (proc, pid) => { + const start = Date.now() + proc.kill("SIGTERM") + await Bun.sleep(200) + proc.kill("SIGINT") + const stderr = await new Response(proc.stderr).text() + expect(stderr).toContain("received SIGINT again during shutdown") + expect(stderr).toContain("layer test/Hung releasing") + expect(stderr).toContain(`child ${pid} running:`) + expect(await proc.exited).toBe(1) + expect(Date.now() - start).toBeLessThan(10_000) + }) +}) + +async function hang( + deadline: number, + body: (proc: Bun.Subprocess<"ignore", "pipe", "pipe">, pid: number) => Promise, +) { + const proc = Bun.spawn([process.execPath, path.join(import.meta.dir, "fixture/shutdown-hang.ts"), String(deadline)], { + cwd: path.join(import.meta.dir, ".."), + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }) + const pid = Number((await readLine(proc.stdout, "SHUTDOWN_READY "))?.split(" ")[1]) + try { + expect(pid).toBeGreaterThan(0) + await body(proc, pid) + } finally { + proc.kill("SIGKILL") + if (running(pid)) process.kill(pid, "SIGKILL") + } +} + +async function readLine(stream: ReadableStream, prefix: string) { + const decoder = new TextDecoder() + const chunks: string[] = [] + for await (const chunk of stream) { + chunks.push(decoder.decode(chunk, { stream: true })) + const line = chunks + .join("") + .split("\n") + .find((line) => line.startsWith(prefix)) + if (line) return line + } +} + +function running(pid: number) { + if (!Number.isSafeInteger(pid) || pid <= 0) return false + try { + process.kill(pid, 0) + return true + } catch { + return false + } +} diff --git a/packages/core/test/effect/cross-spawn-spawner.test.ts b/packages/core/test/effect/cross-spawn-spawner.test.ts index bd265a5a711..d4b70701320 100644 --- a/packages/core/test/effect/cross-spawn-spawner.test.ts +++ b/packages/core/test/effect/cross-spawn-spawner.test.ts @@ -6,6 +6,7 @@ import { Deferred, Effect, Exit, PlatformError, Stream } from "effect" import { ChildProcess, ChildProcessSpawner } from "effect/process" import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner" import { LayerNode } from "@opencode/util/effect/layer-node" +import { Teardown } from "@opencode/util/teardown" import { testEffect } from "../lib/effect" const live = LayerNode.compile(CrossSpawnSpawner.node) @@ -301,6 +302,39 @@ describe("cross-spawn spawner", () => { ) } + fx.live( + "stops tracking a released child while a grandchild holds stdio", + Effect.gen(function* () { + const tmp = yield* Effect.acquireRelease( + Effect.promise(() => tmpdir()), + (tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()), + ) + const pidFile = path.join(tmp.path, "child.pid") + yield* Effect.addFinalizer(() => + Effect.tryPromise(async () => process.kill(Number(await fs.readFile(pidFile, "utf8")), "SIGKILL")).pipe( + Effect.ignore, + ), + ) + const label = yield* Effect.scoped( + Effect.gen(function* () { + const handle = yield* ChildProcess.make( + "node", + [path.join(import.meta.dir, "../fixture/held-stdio.cjs"), "exit", pidFile], + { stdin: "ignore" }, + ) + const label = `child ${handle.pid} running:` + expect(Teardown.pending().some((item) => item.label.startsWith(label))).toBe(true) + yield* handle.exitCode + return label + }), + ) + // Checked before yielding to the event loop: release itself settles the entry, not a later "close" event. + expect(Teardown.pending().some((item) => item.label.startsWith(label))).toBe(false) + expect(alive(Number(yield* Effect.promise(() => fs.readFile(pidFile, "utf8"))))).toBe(true) + }), + 10_000, + ) + fx.effect( "kills a running process", Effect.gen(function* () { diff --git a/packages/util/src/cross-spawn-spawner.ts b/packages/util/src/cross-spawn-spawner.ts index 8df615c8fbd..48a3d225bc4 100644 --- a/packages/util/src/cross-spawn-spawner.ts +++ b/packages/util/src/cross-spawn-spawner.ts @@ -18,6 +18,10 @@ import { PassThrough } from "node:stream" import launch from "cross-spawn" import { makeGlobalNode } from "./effect/app-node.js" import { filesystem, path } from "./effect/app-node-platform.js" +import { Teardown } from "./teardown.js" + +/** Closing a child's scope escalates to SIGKILL after this, unless the command sets its own `forceKillAfter`. */ +export const RELEASE_FORCE_KILL_AFTER = "3 seconds" const toError = (err: unknown): Error => (err instanceof globalThis.Error ? err : new globalThis.Error(String(err))) @@ -95,7 +99,12 @@ export class KilledBySignal extends Error { } type ExitSignal = Deferred.Deferred -type Spawned = readonly [process: NodeChildProcess.ChildProcess, closed: ExitSignal, exited: ExitSignal] +type Spawned = readonly [ + process: NodeChildProcess.ChildProcess, + closed: ExitSignal, + exited: ExitSignal, + settle: () => void, +] const makeCrossSpawnSpawner = Effect.gen(function* () { const fs = yield* FileSystem.FileSystem @@ -281,6 +290,7 @@ const makeCrossSpawnSpawner = Effect.gen(function* () { const closed = Deferred.makeUnsafe() const exited = Deferred.makeUnsafe() const proc = launch(command.command, command.args, opts) + let settle = () => {} let end = false let exit: readonly [code: number | null, signal: NodeJS.Signals | null] | undefined proc.on("error", (err) => { @@ -293,11 +303,15 @@ const makeCrossSpawnSpawner = Effect.gen(function* () { proc.on("close", (...args) => { if (end) return end = true + settle() Deferred.doneUnsafe(exited, Exit.succeed(exit ?? args)) Deferred.doneUnsafe(closed, Exit.succeed(exit ?? args)) }) proc.on("spawn", () => { - resume(Effect.succeed([proc, closed, exited])) + settle = Teardown.track( + `child ${proc.pid} running: ${[command.command, ...command.args].join(" ").slice(0, 200)}`, + ) + resume(Effect.succeed([proc, closed, exited, () => settle()])) }) return Effect.sync(() => { proc.kill("SIGTERM") @@ -309,7 +323,7 @@ const makeCrossSpawnSpawner = Effect.gen(function* () { opts: NodeChildProcess.SpawnOptions, ) { yield* Effect.logInfo("spawning process", { command: command.command, args: command.args, cwd: opts.cwd }) - const [proc, closed, exited] = yield* launchProcess(command, opts) + const [proc, closed, exited, settle] = yield* launchProcess(command, opts) const stopOutput = yield* Deferred.make() // Register before process release so the deadline remains active during termination. yield* Effect.forkScoped( @@ -325,7 +339,7 @@ const makeCrossSpawnSpawner = Effect.gen(function* () { discard(proc.stderr) }), ) - return [proc, closed, exited, stopOutput] as const + return [proc, closed, exited, stopOutput, settle] as const }) const discard = (readable: NodeChildProcess.ChildProcess["stdout"]) => { @@ -445,12 +459,19 @@ const makeCrossSpawnSpawner = Effect.gen(function* () { return yield* Effect.ignore(killGroup(command, proc, command.options.killSignal ?? "SIGTERM")) } } - yield* Effect.ignore(stop(command, proc, closed, stopOutput, command.options)) + yield* Effect.ignore( + stop(command, proc, closed, stopOutput, { + killSignal: command.options.killSignal, + forceKillAfter: command.options.forceKillAfter ?? RELEASE_FORCE_KILL_AFTER, + }), + ) }, - (effect, [proc, , , stopOutput]) => + (effect, [proc, , , stopOutput, settle]) => effect.pipe( Effect.ensuring( Effect.gen(function* () { + // A detached descendant holding the pipes can keep "close" from ever firing. + settle() yield* Deferred.succeed(stopOutput, undefined) proc.stdout?.destroy() proc.stderr?.destroy() diff --git a/packages/util/src/effect/layer-node.ts b/packages/util/src/effect/layer-node.ts index 0ed19e2a5bb..90241bc66b0 100644 --- a/packages/util/src/effect/layer-node.ts +++ b/packages/util/src/effect/layer-node.ts @@ -1,4 +1,5 @@ -import { Brand, Context, Effect, Layer, type Scope } from "effect" +import { Brand, Context, Effect, Layer, Scope } from "effect" +import { Teardown } from "../teardown.js" export * as LayerNode from "./layer-node.js" @@ -233,6 +234,7 @@ export interface CompileOptions { } type Resolved = { + readonly name: string readonly implementation: RuntimeLayer readonly dependencies: readonly Resolved[] readonly shared: boolean @@ -274,6 +276,7 @@ export function compile(root: Graph, options: CompileOptions = {}): return definition.dependencies.flatMap((dependency) => resolve(dependency, isShared)) if (definition.kind === "unbound") throw new Error(`Unbound layer node: ${definition.name}`) const node: Resolved = { + name: definition.name, implementation: definition.implementation, dependencies: definition.dependencies.flatMap((dependency) => resolve(dependency, isShared)), shared: isShared, @@ -310,7 +313,9 @@ export function compile(root: Graph, options: CompileOptions = {}): const dependencies = node.dependencies.map(build) // Acquisition uses the selected cache, while lazy LayerMaps inherit the enclosing shared cache. const implementation = Layer.suspend(() => node.implementation.pipe(Layer.provide([ambient, ...dependencies]))) - const layer = Layer.fromBuild((_, scope) => buildContext(implementation, node.shared ? memoMap : local, scope)) + const layer = Layer.fromBuild((_, scope) => + trackRelease(node.name, scope, () => buildContext(implementation, node.shared ? memoMap : local, scope)), + ) layers.set(node, layer) return layer } @@ -322,6 +327,27 @@ export function compile(root: Graph, options: CompileOptions = {}): }) as Layer.Layer } +// The node's scope releases sequentially in reverse order, so a finalizer added before the build runs last and one +// added after it runs first: together they bracket the release of everything this node acquired. The build must +// not start before the first finalizer is added; Layer builds fork their scopes as soon as they are called. +function trackRelease(name: string, scope: Scope.Scope, build: () => Effect.Effect) { + return Effect.gen(function* () { + let settle = () => {} + yield* Scope.addFinalizer( + scope, + Effect.sync(() => settle()), + ) + const result = yield* build() + yield* Scope.addFinalizer( + scope, + Effect.sync(() => { + settle = Teardown.track(`layer ${name} releasing`) + }), + ) + return result + }) +} + class BuildResult extends Context.Service>()("@opencode/LayerNode/BuildResult") {} function buildContext(layer: RuntimeLayer, memoMap: Layer.MemoMap, scope: Scope.Scope) { diff --git a/packages/util/src/teardown.ts b/packages/util/src/teardown.ts new file mode 100644 index 00000000000..178e76ae1c9 --- /dev/null +++ b/packages/util/src/teardown.ts @@ -0,0 +1,19 @@ +export * as Teardown from "./teardown.js" + +// Work that shutdown may be waiting on: layers releasing their resources and child processes that have not +// closed. Read when a shutdown deadline expires, to name what kept the process alive. +const entries = new Set<{ readonly label: string; readonly since: number }>() + +/** Record pending work; call the returned function once it settles. */ +export function track(label: string) { + const entry = { label, since: Date.now() } + entries.add(entry) + return () => { + entries.delete(entry) + } +} + +/** Pending work, oldest first, with how long each has been pending. */ +export function pending(now = Date.now()) { + return Array.from(entries, (entry) => ({ label: entry.label, ms: now - entry.since })) +}