fix(client): wait for service shutdown before restart (#50042)

This commit is contained in:
Kit Langton
2026-10-04 20:59:19 -07:00
committed by GitHub
Unverified
parent ae57dd0cf5
commit 25c54cbdf7
6 changed files with 161 additions and 14 deletions
+4 -6
View File
@@ -157,10 +157,8 @@ export const stop = Effect.fn("service.stop")(function* (options: StopOptions =
options.pty === "handoff" && info !== undefined
? PtyHandoff.prepare(options.file ?? fallback(), info, defaultEnsureTiming.requestTimeout)
: PtyHandoff.clear(options.file ?? fallback()),
).pipe(
Effect.catch((cause) => Effect.logWarning("Failed to prepare persistent terminals for replacement", cause)),
)
if (info !== undefined) yield* terminate(info, options, defaultEnsureTiming)
).pipe(Effect.catch((cause) => Effect.logWarning("Failed to prepare persistent terminals for replacement", cause)))
if (info !== undefined) yield* terminate(info, options, ensureTiming(options))
})
function fallback() {
@@ -298,9 +296,9 @@ const terminate = Effect.fnUntraced(function* (info: Info, options: { readonly f
if (current === undefined || !same(current, info)) return
yield* signal(info.pid, "SIGTERM")
const done = yield* stopped(info.pid).pipe(Effect.retry(poll(timing)), Effect.option)
// The registration can disappear or change hands before this process exits. Only the PID we
// signalled can tell us whether it has stopped, so escalate based on that process.
if (Option.isNone(done)) {
const latest = yield* read(options.file)
if (latest === undefined || !same(latest, info)) return
yield* signal(info.pid, "SIGKILL")
yield* stopped(info.pid).pipe(Effect.retry(poll(timing)))
}
+3 -3
View File
@@ -129,7 +129,7 @@ export async function stop(options: StopOptions = {}) {
? PtyHandoff.prepare(options.file ?? fallback(), info, defaultEnsureTiming.requestTimeout)
: PtyHandoff.clear(options.file ?? fallback())
).catch((cause: unknown) => console.warn("Failed to prepare persistent terminals for replacement", cause))
if (info !== undefined) await terminate(info, options, defaultEnsureTiming)
if (info !== undefined) await terminate(info, options, ensureTiming(options))
}
function fallback() {
@@ -258,9 +258,9 @@ async function terminate(info: Info, options: { readonly file?: string }, timing
const current = await read(options.file)
if (current === undefined || !same(current, info)) return
signal(info.pid, "SIGTERM")
// The registration can disappear or change hands before this process exits. Only the PID we
// signalled can tell us whether it has stopped, so escalate based on that process.
if (!(await waitUntilStopped(info.pid, timing))) {
const latest = await read(options.file)
if (latest === undefined || !same(latest, info)) return
signal(info.pid, "SIGKILL")
if (!(await waitUntilStopped(info.pid, timing))) throw new Error(`Server process ${info.pid} is still running`)
}
@@ -25,6 +25,16 @@ export async function serviceFixture() {
processes.push(subprocess)
return subprocess
},
// The service's parent execs `sleep`, which never reaps it, so SIGKILL leaves a zombie that
// still answers `kill(pid, 0)`. Terminating the returned parent lets init reap the service.
spawnUnreaped(mode: string, ...args: string[]) {
const subprocess = Bun.spawn(["sh", "-c", '"$@" & exec sleep 60', "sh", ...command(mode, ...args)], {
stdout: "ignore",
stderr: "inherit",
})
processes.push(subprocess)
return subprocess
},
// Service.ensure detaches contenders; track the elected process before asserting.
track(pid: number) {
pids.add(pid)
@@ -54,3 +64,8 @@ export async function serviceFixture() {
},
}
}
export async function expectPortAvailable(url: string) {
const server = Bun.serve({ port: Number(new URL(url).port), fetch: () => new Response() })
await server.stop(true)
}
+11 -3
View File
@@ -1,4 +1,4 @@
import { appendFile, rename, writeFile } from "node:fs/promises"
import { appendFile, rename, rm, writeFile } from "node:fs/promises"
const [registration, mode, delay] = process.argv.slice(2)
if (registration === undefined || mode === undefined) throw new Error("Missing service fixture arguments")
@@ -91,6 +91,10 @@ const server = Bun.serve({
},
})
// Install handlers before publishing: a test may signal as soon as the registration appears.
process.on("SIGTERM", () => void shutdown("SIGTERM"))
process.on("SIGINT", () => void shutdown("SIGINT"))
await writeFile(
registration + ".tmp",
JSON.stringify({
@@ -106,8 +110,12 @@ await rename(registration + ".tmp", registration)
async function shutdown(signal?: NodeJS.Signals) {
if (signal !== undefined) await writeFile(registration + ".signal", signal)
// A lingering server unregisters on SIGTERM but keeps running, and holds its port, until killed.
if (mode === "lingering") {
await rm(registration, { force: true })
await writeFile(registration + ".unregistered", "")
await Bun.sleep(Number(delay))
}
server.stop(true)
process.exit()
}
process.on("SIGTERM", () => void shutdown("SIGTERM"))
process.on("SIGINT", () => void shutdown("SIGINT"))
+65 -1
View File
@@ -1,9 +1,10 @@
import { expect, test } from "bun:test"
import { Service, type EnsureReason } from "../src/promise/service"
import { serviceFixture } from "./fixture/service-fixture"
import { expectPortAvailable, serviceFixture } from "./fixture/service-fixture"
import { accelerate } from "./fixture/service-timing"
const ensure = accelerate(Service.ensure)
const stop = accelerate(Service.stop)
test("discovers a registered service", async () => {
await using fixture = await serviceFixture()
@@ -171,3 +172,66 @@ test("signals the registered service process", async () => {
expect(await Bun.file(registration + ".signal").text()).toBe("SIGTERM")
expect(await Bun.file(registration).exists()).toBe(false)
})
test("stop escalates when the registration disappears before the process exits", async () => {
await using fixture = await serviceFixture()
const registration = fixture.registration
const existing = fixture.spawn("lingering", "5000")
await fixture.waitForFile()
const original = await Bun.file(registration).json()
await stop({ file: registration })
expect(await Bun.file(registration + ".signal").text()).toBe("SIGTERM")
expect(() => process.kill(original.pid, 0)).toThrow()
await expectPortAvailable(original.url)
expect(await Bun.file(registration).exists()).toBe(false)
await existing.exited
}, 15_000)
test.skipIf(process.platform === "win32")(
"stop fails when the process survives SIGKILL",
async () => {
await using fixture = await serviceFixture()
const registration = fixture.registration
fixture.spawnUnreaped("lingering", "60000")
await fixture.waitForFile()
const original = await Bun.file(registration).json()
await expect(stop({ file: registration })).rejects.toThrow(`Server process ${original.pid} is still running`)
expect(await Bun.file(registration + ".signal").text()).toBe("SIGTERM")
},
15_000,
)
test("stop waits for the original process while preserving a newly registered successor", async () => {
await using fixture = await serviceFixture()
const registration = fixture.registration
const existing = fixture.spawn("lingering", "15000")
await fixture.waitForFile()
const original = await Bun.file(registration).json()
// Default timing keeps the grace period open long enough for the successor to register first.
const stopping = Service.stop({ file: registration })
try {
await fixture.waitForFile(registration + ".unregistered")
const successor = fixture.spawn("graceful")
await fixture.waitForFile()
const replacement = await Bun.file(registration).json()
expect(existing.exitCode).toBe(null)
expect(replacement.pid).toBe(successor.pid)
await stopping
expect(() => process.kill(original.pid, 0)).toThrow()
await expectPortAvailable(original.url)
expect(await Bun.file(registration).json()).toEqual(replacement)
expect(await fetch(new URL("/api/info", replacement.url)).then((response) => response.json())).toMatchObject({
pid: successor.pid,
})
expect(successor.exitCode).toBe(null)
} finally {
existing.kill("SIGKILL")
await stopping
}
}, 20_000)
+63 -1
View File
@@ -3,10 +3,11 @@ import { expect, test } from "bun:test"
import { Effect, FileSystem } from "effect"
import { writeFile } from "node:fs/promises"
import { Service, type EnsureReason } from "../src/effect/service"
import { serviceFixture } from "./fixture/service-fixture"
import { expectPortAvailable, serviceFixture } from "./fixture/service-fixture"
import { accelerate } from "./fixture/service-timing"
const ensure = accelerate(Service.ensure)
const stop = accelerate(Service.stop)
test("a concurrent same-version start cannot invalidate a resolved endpoint", async () => {
await using fixture = await serviceFixture()
@@ -161,6 +162,67 @@ test("signals an unresponsive registered service process", async () => {
expect(await Bun.file(registration).exists()).toBe(false)
})
test("stop escalates when the registration disappears before the process exits", async () => {
await using fixture = await serviceFixture()
const registration = fixture.registration
const existing = fixture.spawn("lingering", "5000")
await fixture.waitForFile()
const original = await Bun.file(registration).json()
await run(stop({ file: registration }))
expect(await Bun.file(registration + ".signal").text()).toBe("SIGTERM")
expect(() => process.kill(original.pid, 0)).toThrow()
await expectPortAvailable(original.url)
expect(await Bun.file(registration).exists()).toBe(false)
await existing.exited
}, 15_000)
test.skipIf(process.platform === "win32")(
"stop fails when the process survives SIGKILL",
async () => {
await using fixture = await serviceFixture()
const registration = fixture.registration
fixture.spawnUnreaped("lingering", "60000")
await fixture.waitForFile()
const original = await Bun.file(registration).json()
await expect(run(stop({ file: registration }))).rejects.toThrow(`Server process ${original.pid} is still running`)
expect(await Bun.file(registration + ".signal").text()).toBe("SIGTERM")
},
15_000,
)
test("stop waits for the original process while preserving a newly registered successor", async () => {
await using fixture = await serviceFixture()
const registration = fixture.registration
const existing = fixture.spawn("lingering", "15000")
await fixture.waitForFile()
const original = await Bun.file(registration).json()
// Default timing keeps the grace period open long enough for the successor to register first.
const stopping = run(Service.stop({ file: registration }))
try {
await fixture.waitForFile(registration + ".unregistered")
const successor = fixture.spawn("graceful")
await fixture.waitForFile()
const replacement = await Bun.file(registration).json()
expect(existing.exitCode).toBe(null)
expect(replacement.pid).toBe(successor.pid)
await stopping
expect(() => process.kill(original.pid, 0)).toThrow()
await expectPortAvailable(original.url)
expect(await Bun.file(registration).json()).toEqual(replacement)
expect(await status(replacement.url)).toMatchObject({ pid: successor.pid })
expect(successor.exitCode).toBe(null)
} finally {
existing.kill("SIGKILL")
await stopping
}
}, 20_000)
test("signals an incompatible service before starting its replacement", async () => {
await using fixture = await serviceFixture()
const registration = fixture.registration