mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-19 08:39:02 +03:00
fix(cli): bound server shutdown with a 10 s deadline (#58817)
This commit is contained in:
committed by
GitHub
Unverified
parent
1564686791
commit
b6cbbd7646
@@ -17,6 +17,7 @@ import { Npm } from "@opencode/util/npm"
|
||||
import { EffectFlock } from "@opencode/util/effect-flock"
|
||||
import { Heap } from "./heap"
|
||||
import { CpuProfile } from "./cpu-profile"
|
||||
import { ShutdownDeadline } from "./shutdown-deadline"
|
||||
|
||||
ensurePluginRuntime()
|
||||
|
||||
@@ -85,6 +86,7 @@ const Handlers = Runtime.handlers(Commands, {
|
||||
Effect.gen(function* () {
|
||||
yield* Heap.listen
|
||||
yield* CpuProfile.listen
|
||||
yield* ShutdownDeadline.install()
|
||||
const runFork = Effect.runForkWith(yield* Effect.context<never>())
|
||||
const uncaughtException = (cause: Error, origin: "uncaughtException" | "unhandledRejection") => {
|
||||
runFork(Effect.logError("uncaught exception", { cause, origin }))
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { Teardown } from "@opencode/util/teardown"
|
||||
import { Duration, Effect } from "effect"
|
||||
|
||||
/** After SIGTERM or SIGINT, teardown gets this long before the process reports what it is stuck on and exits. */
|
||||
export const DEADLINE = Duration.seconds(10)
|
||||
// The file logger batches writes for up to a second.
|
||||
const LOG_FLUSH = Duration.millis(1_500)
|
||||
|
||||
/**
|
||||
* Bounds signal-initiated teardown. NodeRuntime.runMain interrupts the main fiber on SIGTERM or SIGINT and then waits
|
||||
* for every finalizer, so a finalizer that never finishes would keep the process alive until something kills it.
|
||||
* A second signal during teardown reports and exits at once.
|
||||
*/
|
||||
export const install = Effect.fnUntraced(function* (input: Duration.Input = DEADLINE) {
|
||||
const deadline = Duration.fromInputUnsafe(input)
|
||||
const fork = Effect.runForkWith(yield* Effect.context<never>())
|
||||
let started = false
|
||||
const expire = (signal: NodeJS.Signals) =>
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.logInfo("shutdown started", { signal })
|
||||
yield* Effect.sleep(deadline)
|
||||
const pending = Teardown.pending()
|
||||
process.stderr.write(report(`shutdown did not finish within ${Duration.format(deadline)}`, pending))
|
||||
yield* Effect.logError("shutdown deadline exceeded", {
|
||||
deadline: Duration.format(deadline),
|
||||
pending: pending.map((item) => `${item.label} for ${item.ms}ms`),
|
||||
})
|
||||
yield* Effect.sleep(LOG_FLUSH)
|
||||
yield* Effect.sync(() => process.exit(1))
|
||||
})
|
||||
const onSignal = (signal: NodeJS.Signals) => {
|
||||
if (started) {
|
||||
process.stderr.write(report(`received ${signal} again during shutdown`, Teardown.pending()))
|
||||
process.exit(1)
|
||||
}
|
||||
started = true
|
||||
fork(expire(signal))
|
||||
}
|
||||
const off = () => {
|
||||
process.off("SIGTERM", onSignal)
|
||||
process.off("SIGINT", onSignal)
|
||||
}
|
||||
yield* Effect.acquireRelease(
|
||||
Effect.sync(() => {
|
||||
process.on("SIGTERM", onSignal)
|
||||
process.on("SIGINT", onSignal)
|
||||
}),
|
||||
// Once shutdown has started, keep listening: this release can run while a later finalizer is still stuck.
|
||||
() =>
|
||||
Effect.sync(() => {
|
||||
if (!started) off()
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
function report(reason: string, pending: ReturnType<typeof Teardown.pending>) {
|
||||
const lines = pending.map((item) => ` ${(item.ms / 1000).toFixed(1)}s ${item.label}`)
|
||||
return [
|
||||
`${reason}; exiting. Still pending:`,
|
||||
...(lines.length > 0 ? lines : [" nothing tracked; the stuck finalizer is outside any LayerNode layer"]),
|
||||
"",
|
||||
].join("\n")
|
||||
}
|
||||
|
||||
export * as ShutdownDeadline from "./shutdown-deadline"
|
||||
@@ -0,0 +1,29 @@
|
||||
import { NodeRuntime } from "@effect/platform-node"
|
||||
import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner"
|
||||
import { LayerNode } from "@opencode/util/effect/layer-node"
|
||||
import { Context, Effect, Layer } from "effect"
|
||||
import { ChildProcess } from "effect/process"
|
||||
import { ShutdownDeadline } from "../../src/shutdown-deadline"
|
||||
|
||||
class Hung extends Context.Service<Hung, {}>()("test/Hung") {}
|
||||
|
||||
// Owns a running child, then a finalizer that never finishes, so releasing the layer hangs before the child stops.
|
||||
const hung = LayerNode.make({
|
||||
service: Hung,
|
||||
layer: Layer.effect(
|
||||
Hung,
|
||||
Effect.gen(function* () {
|
||||
const child = yield* ChildProcess.make(process.execPath, ["-e", "setInterval(() => {}, 1000)"])
|
||||
yield* Effect.addFinalizer(() => Effect.never)
|
||||
console.log(`SHUTDOWN_READY ${child.pid}`)
|
||||
return Hung.of({})
|
||||
}),
|
||||
),
|
||||
deps: [CrossSpawnSpawner.node],
|
||||
})
|
||||
|
||||
Effect.gen(function* () {
|
||||
yield* ShutdownDeadline.install(Number(process.argv[2]))
|
||||
yield* Layer.build(LayerNode.compile(hung))
|
||||
return yield* Effect.never
|
||||
}).pipe(Effect.scoped, NodeRuntime.runMain)
|
||||
@@ -0,0 +1,74 @@
|
||||
import { expect, test } from "bun:test"
|
||||
import path from "node:path"
|
||||
|
||||
// Windows cannot catch SIGTERM: the process ends at once, so there is no teardown to bound.
|
||||
const posix = test.skipIf(process.platform === "win32")
|
||||
|
||||
posix("exits when teardown outlives the deadline and names what is pending", async () => {
|
||||
await hang(200, async (proc, pid) => {
|
||||
proc.kill("SIGTERM")
|
||||
const stderr = await new Response(proc.stderr).text()
|
||||
expect(stderr).toContain("shutdown did not finish within 200ms")
|
||||
expect(stderr).toContain("layer test/Hung releasing")
|
||||
expect(stderr).toContain(`child ${pid} running: ${process.execPath} -e setInterval`)
|
||||
expect(await proc.exited).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
posix("a second signal during teardown reports and exits without waiting for the deadline", async () => {
|
||||
await hang(60_000, async (proc, pid) => {
|
||||
const start = Date.now()
|
||||
proc.kill("SIGTERM")
|
||||
await Bun.sleep(200)
|
||||
proc.kill("SIGINT")
|
||||
const stderr = await new Response(proc.stderr).text()
|
||||
expect(stderr).toContain("received SIGINT again during shutdown")
|
||||
expect(stderr).toContain("layer test/Hung releasing")
|
||||
expect(stderr).toContain(`child ${pid} running:`)
|
||||
expect(await proc.exited).toBe(1)
|
||||
expect(Date.now() - start).toBeLessThan(10_000)
|
||||
})
|
||||
})
|
||||
|
||||
async function hang(
|
||||
deadline: number,
|
||||
body: (proc: Bun.Subprocess<"ignore", "pipe", "pipe">, pid: number) => Promise<void>,
|
||||
) {
|
||||
const proc = Bun.spawn([process.execPath, path.join(import.meta.dir, "fixture/shutdown-hang.ts"), String(deadline)], {
|
||||
cwd: path.join(import.meta.dir, ".."),
|
||||
stdin: "ignore",
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
})
|
||||
const pid = Number((await readLine(proc.stdout, "SHUTDOWN_READY "))?.split(" ")[1])
|
||||
try {
|
||||
expect(pid).toBeGreaterThan(0)
|
||||
await body(proc, pid)
|
||||
} finally {
|
||||
proc.kill("SIGKILL")
|
||||
if (running(pid)) process.kill(pid, "SIGKILL")
|
||||
}
|
||||
}
|
||||
|
||||
async function readLine(stream: ReadableStream<Uint8Array>, prefix: string) {
|
||||
const decoder = new TextDecoder()
|
||||
const chunks: string[] = []
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(decoder.decode(chunk, { stream: true }))
|
||||
const line = chunks
|
||||
.join("")
|
||||
.split("\n")
|
||||
.find((line) => line.startsWith(prefix))
|
||||
if (line) return line
|
||||
}
|
||||
}
|
||||
|
||||
function running(pid: number) {
|
||||
if (!Number.isSafeInteger(pid) || pid <= 0) return false
|
||||
try {
|
||||
process.kill(pid, 0)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import { Deferred, Effect, Exit, PlatformError, Stream } from "effect"
|
||||
import { ChildProcess, ChildProcessSpawner } from "effect/process"
|
||||
import { CrossSpawnSpawner } from "@opencode/util/cross-spawn-spawner"
|
||||
import { LayerNode } from "@opencode/util/effect/layer-node"
|
||||
import { Teardown } from "@opencode/util/teardown"
|
||||
import { testEffect } from "../lib/effect"
|
||||
|
||||
const live = LayerNode.compile(CrossSpawnSpawner.node)
|
||||
@@ -301,6 +302,39 @@ describe("cross-spawn spawner", () => {
|
||||
)
|
||||
}
|
||||
|
||||
fx.live(
|
||||
"stops tracking a released child while a grandchild holds stdio",
|
||||
Effect.gen(function* () {
|
||||
const tmp = yield* Effect.acquireRelease(
|
||||
Effect.promise(() => tmpdir()),
|
||||
(tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()),
|
||||
)
|
||||
const pidFile = path.join(tmp.path, "child.pid")
|
||||
yield* Effect.addFinalizer(() =>
|
||||
Effect.tryPromise(async () => process.kill(Number(await fs.readFile(pidFile, "utf8")), "SIGKILL")).pipe(
|
||||
Effect.ignore,
|
||||
),
|
||||
)
|
||||
const label = yield* Effect.scoped(
|
||||
Effect.gen(function* () {
|
||||
const handle = yield* ChildProcess.make(
|
||||
"node",
|
||||
[path.join(import.meta.dir, "../fixture/held-stdio.cjs"), "exit", pidFile],
|
||||
{ stdin: "ignore" },
|
||||
)
|
||||
const label = `child ${handle.pid} running:`
|
||||
expect(Teardown.pending().some((item) => item.label.startsWith(label))).toBe(true)
|
||||
yield* handle.exitCode
|
||||
return label
|
||||
}),
|
||||
)
|
||||
// Checked before yielding to the event loop: release itself settles the entry, not a later "close" event.
|
||||
expect(Teardown.pending().some((item) => item.label.startsWith(label))).toBe(false)
|
||||
expect(alive(Number(yield* Effect.promise(() => fs.readFile(pidFile, "utf8"))))).toBe(true)
|
||||
}),
|
||||
10_000,
|
||||
)
|
||||
|
||||
fx.effect(
|
||||
"kills a running process",
|
||||
Effect.gen(function* () {
|
||||
|
||||
@@ -18,6 +18,10 @@ import { PassThrough } from "node:stream"
|
||||
import launch from "cross-spawn"
|
||||
import { makeGlobalNode } from "./effect/app-node.js"
|
||||
import { filesystem, path } from "./effect/app-node-platform.js"
|
||||
import { Teardown } from "./teardown.js"
|
||||
|
||||
/** Closing a child's scope escalates to SIGKILL after this, unless the command sets its own `forceKillAfter`. */
|
||||
export const RELEASE_FORCE_KILL_AFTER = "3 seconds"
|
||||
|
||||
const toError = (err: unknown): Error => (err instanceof globalThis.Error ? err : new globalThis.Error(String(err)))
|
||||
|
||||
@@ -95,7 +99,12 @@ export class KilledBySignal extends Error {
|
||||
}
|
||||
|
||||
type ExitSignal = Deferred.Deferred<readonly [code: number | null, signal: NodeJS.Signals | null]>
|
||||
type Spawned = readonly [process: NodeChildProcess.ChildProcess, closed: ExitSignal, exited: ExitSignal]
|
||||
type Spawned = readonly [
|
||||
process: NodeChildProcess.ChildProcess,
|
||||
closed: ExitSignal,
|
||||
exited: ExitSignal,
|
||||
settle: () => void,
|
||||
]
|
||||
|
||||
const makeCrossSpawnSpawner = Effect.gen(function* () {
|
||||
const fs = yield* FileSystem.FileSystem
|
||||
@@ -281,6 +290,7 @@ const makeCrossSpawnSpawner = Effect.gen(function* () {
|
||||
const closed = Deferred.makeUnsafe<readonly [code: number | null, signal: NodeJS.Signals | null]>()
|
||||
const exited = Deferred.makeUnsafe<readonly [code: number | null, signal: NodeJS.Signals | null]>()
|
||||
const proc = launch(command.command, command.args, opts)
|
||||
let settle = () => {}
|
||||
let end = false
|
||||
let exit: readonly [code: number | null, signal: NodeJS.Signals | null] | undefined
|
||||
proc.on("error", (err) => {
|
||||
@@ -293,11 +303,15 @@ const makeCrossSpawnSpawner = Effect.gen(function* () {
|
||||
proc.on("close", (...args) => {
|
||||
if (end) return
|
||||
end = true
|
||||
settle()
|
||||
Deferred.doneUnsafe(exited, Exit.succeed(exit ?? args))
|
||||
Deferred.doneUnsafe(closed, Exit.succeed(exit ?? args))
|
||||
})
|
||||
proc.on("spawn", () => {
|
||||
resume(Effect.succeed([proc, closed, exited]))
|
||||
settle = Teardown.track(
|
||||
`child ${proc.pid} running: ${[command.command, ...command.args].join(" ").slice(0, 200)}`,
|
||||
)
|
||||
resume(Effect.succeed([proc, closed, exited, () => settle()]))
|
||||
})
|
||||
return Effect.sync(() => {
|
||||
proc.kill("SIGTERM")
|
||||
@@ -309,7 +323,7 @@ const makeCrossSpawnSpawner = Effect.gen(function* () {
|
||||
opts: NodeChildProcess.SpawnOptions,
|
||||
) {
|
||||
yield* Effect.logInfo("spawning process", { command: command.command, args: command.args, cwd: opts.cwd })
|
||||
const [proc, closed, exited] = yield* launchProcess(command, opts)
|
||||
const [proc, closed, exited, settle] = yield* launchProcess(command, opts)
|
||||
const stopOutput = yield* Deferred.make<void>()
|
||||
// Register before process release so the deadline remains active during termination.
|
||||
yield* Effect.forkScoped(
|
||||
@@ -325,7 +339,7 @@ const makeCrossSpawnSpawner = Effect.gen(function* () {
|
||||
discard(proc.stderr)
|
||||
}),
|
||||
)
|
||||
return [proc, closed, exited, stopOutput] as const
|
||||
return [proc, closed, exited, stopOutput, settle] as const
|
||||
})
|
||||
|
||||
const discard = (readable: NodeChildProcess.ChildProcess["stdout"]) => {
|
||||
@@ -445,12 +459,19 @@ const makeCrossSpawnSpawner = Effect.gen(function* () {
|
||||
return yield* Effect.ignore(killGroup(command, proc, command.options.killSignal ?? "SIGTERM"))
|
||||
}
|
||||
}
|
||||
yield* Effect.ignore(stop(command, proc, closed, stopOutput, command.options))
|
||||
yield* Effect.ignore(
|
||||
stop(command, proc, closed, stopOutput, {
|
||||
killSignal: command.options.killSignal,
|
||||
forceKillAfter: command.options.forceKillAfter ?? RELEASE_FORCE_KILL_AFTER,
|
||||
}),
|
||||
)
|
||||
},
|
||||
(effect, [proc, , , stopOutput]) =>
|
||||
(effect, [proc, , , stopOutput, settle]) =>
|
||||
effect.pipe(
|
||||
Effect.ensuring(
|
||||
Effect.gen(function* () {
|
||||
// A detached descendant holding the pipes can keep "close" from ever firing.
|
||||
settle()
|
||||
yield* Deferred.succeed(stopOutput, undefined)
|
||||
proc.stdout?.destroy()
|
||||
proc.stderr?.destroy()
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Brand, Context, Effect, Layer, type Scope } from "effect"
|
||||
import { Brand, Context, Effect, Layer, Scope } from "effect"
|
||||
import { Teardown } from "../teardown.js"
|
||||
|
||||
export * as LayerNode from "./layer-node.js"
|
||||
|
||||
@@ -233,6 +234,7 @@ export interface CompileOptions {
|
||||
}
|
||||
|
||||
type Resolved = {
|
||||
readonly name: string
|
||||
readonly implementation: RuntimeLayer
|
||||
readonly dependencies: readonly Resolved[]
|
||||
readonly shared: boolean
|
||||
@@ -274,6 +276,7 @@ export function compile<A, E>(root: Graph<A, E>, options: CompileOptions = {}):
|
||||
return definition.dependencies.flatMap((dependency) => resolve(dependency, isShared))
|
||||
if (definition.kind === "unbound") throw new Error(`Unbound layer node: ${definition.name}`)
|
||||
const node: Resolved = {
|
||||
name: definition.name,
|
||||
implementation: definition.implementation,
|
||||
dependencies: definition.dependencies.flatMap((dependency) => resolve(dependency, isShared)),
|
||||
shared: isShared,
|
||||
@@ -310,7 +313,9 @@ export function compile<A, E>(root: Graph<A, E>, options: CompileOptions = {}):
|
||||
const dependencies = node.dependencies.map(build)
|
||||
// Acquisition uses the selected cache, while lazy LayerMaps inherit the enclosing shared cache.
|
||||
const implementation = Layer.suspend(() => node.implementation.pipe(Layer.provide([ambient, ...dependencies])))
|
||||
const layer = Layer.fromBuild((_, scope) => buildContext(implementation, node.shared ? memoMap : local, scope))
|
||||
const layer = Layer.fromBuild((_, scope) =>
|
||||
trackRelease(node.name, scope, () => buildContext(implementation, node.shared ? memoMap : local, scope)),
|
||||
)
|
||||
layers.set(node, layer)
|
||||
return layer
|
||||
}
|
||||
@@ -322,6 +327,27 @@ export function compile<A, E>(root: Graph<A, E>, options: CompileOptions = {}):
|
||||
}) as Layer.Layer<A, E>
|
||||
}
|
||||
|
||||
// The node's scope releases sequentially in reverse order, so a finalizer added before the build runs last and one
|
||||
// added after it runs first: together they bracket the release of everything this node acquired. The build must
|
||||
// not start before the first finalizer is added; Layer builds fork their scopes as soon as they are called.
|
||||
function trackRelease<A, E, R>(name: string, scope: Scope.Scope, build: () => Effect.Effect<A, E, R>) {
|
||||
return Effect.gen(function* () {
|
||||
let settle = () => {}
|
||||
yield* Scope.addFinalizer(
|
||||
scope,
|
||||
Effect.sync(() => settle()),
|
||||
)
|
||||
const result = yield* build()
|
||||
yield* Scope.addFinalizer(
|
||||
scope,
|
||||
Effect.sync(() => {
|
||||
settle = Teardown.track(`layer ${name} releasing`)
|
||||
}),
|
||||
)
|
||||
return result
|
||||
})
|
||||
}
|
||||
|
||||
class BuildResult extends Context.Service<BuildResult, Context.Context<never>>()("@opencode/LayerNode/BuildResult") {}
|
||||
|
||||
function buildContext(layer: RuntimeLayer, memoMap: Layer.MemoMap, scope: Scope.Scope) {
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
export * as Teardown from "./teardown.js"
|
||||
|
||||
// Work that shutdown may be waiting on: layers releasing their resources and child processes that have not
|
||||
// closed. Read when a shutdown deadline expires, to name what kept the process alive.
|
||||
const entries = new Set<{ readonly label: string; readonly since: number }>()
|
||||
|
||||
/** Record pending work; call the returned function once it settles. */
|
||||
export function track(label: string) {
|
||||
const entry = { label, since: Date.now() }
|
||||
entries.add(entry)
|
||||
return () => {
|
||||
entries.delete(entry)
|
||||
}
|
||||
}
|
||||
|
||||
/** Pending work, oldest first, with how long each has been pending. */
|
||||
export function pending(now = Date.now()) {
|
||||
return Array.from(entries, (entry) => ({ label: entry.label, ms: now - entry.since }))
|
||||
}
|
||||
Reference in New Issue
Block a user