From 25c54cbdf7559fa066544045a2b77633e5e89d02 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sun, 4 Oct 2026 20:59:19 -0700 Subject: [PATCH] fix(client): wait for service shutdown before restart (#50042) --- packages/client/src/effect/service.ts | 10 ++- packages/client/src/promise/service.ts | 6 +- .../client/test/fixture/service-fixture.ts | 15 +++++ packages/client/test/fixture/service.ts | 14 +++- packages/client/test/promise-service.test.ts | 66 ++++++++++++++++++- packages/client/test/service.test.ts | 64 +++++++++++++++++- 6 files changed, 161 insertions(+), 14 deletions(-) diff --git a/packages/client/src/effect/service.ts b/packages/client/src/effect/service.ts index af398009fb1..4c95f3972a5 100644 --- a/packages/client/src/effect/service.ts +++ b/packages/client/src/effect/service.ts @@ -157,10 +157,8 @@ export const stop = Effect.fn("service.stop")(function* (options: StopOptions = options.pty === "handoff" && info !== undefined ? PtyHandoff.prepare(options.file ?? fallback(), info, defaultEnsureTiming.requestTimeout) : PtyHandoff.clear(options.file ?? fallback()), - ).pipe( - Effect.catch((cause) => Effect.logWarning("Failed to prepare persistent terminals for replacement", cause)), - ) - if (info !== undefined) yield* terminate(info, options, defaultEnsureTiming) + ).pipe(Effect.catch((cause) => Effect.logWarning("Failed to prepare persistent terminals for replacement", cause))) + if (info !== undefined) yield* terminate(info, options, ensureTiming(options)) }) function fallback() { @@ -298,9 +296,9 @@ const terminate = Effect.fnUntraced(function* (info: Info, options: { readonly f if (current === undefined || !same(current, info)) return yield* signal(info.pid, "SIGTERM") const done = yield* stopped(info.pid).pipe(Effect.retry(poll(timing)), Effect.option) + // The registration can disappear or change hands before this process exits. Only the PID we + // signalled can tell us whether it has stopped, so escalate based on that process. if (Option.isNone(done)) { - const latest = yield* read(options.file) - if (latest === undefined || !same(latest, info)) return yield* signal(info.pid, "SIGKILL") yield* stopped(info.pid).pipe(Effect.retry(poll(timing))) } diff --git a/packages/client/src/promise/service.ts b/packages/client/src/promise/service.ts index 1fe02a97d0f..860465e874b 100644 --- a/packages/client/src/promise/service.ts +++ b/packages/client/src/promise/service.ts @@ -129,7 +129,7 @@ export async function stop(options: StopOptions = {}) { ? PtyHandoff.prepare(options.file ?? fallback(), info, defaultEnsureTiming.requestTimeout) : PtyHandoff.clear(options.file ?? fallback()) ).catch((cause: unknown) => console.warn("Failed to prepare persistent terminals for replacement", cause)) - if (info !== undefined) await terminate(info, options, defaultEnsureTiming) + if (info !== undefined) await terminate(info, options, ensureTiming(options)) } function fallback() { @@ -258,9 +258,9 @@ async function terminate(info: Info, options: { readonly file?: string }, timing const current = await read(options.file) if (current === undefined || !same(current, info)) return signal(info.pid, "SIGTERM") + // The registration can disappear or change hands before this process exits. Only the PID we + // signalled can tell us whether it has stopped, so escalate based on that process. if (!(await waitUntilStopped(info.pid, timing))) { - const latest = await read(options.file) - if (latest === undefined || !same(latest, info)) return signal(info.pid, "SIGKILL") if (!(await waitUntilStopped(info.pid, timing))) throw new Error(`Server process ${info.pid} is still running`) } diff --git a/packages/client/test/fixture/service-fixture.ts b/packages/client/test/fixture/service-fixture.ts index a8dac655d77..f3b7094caef 100644 --- a/packages/client/test/fixture/service-fixture.ts +++ b/packages/client/test/fixture/service-fixture.ts @@ -25,6 +25,16 @@ export async function serviceFixture() { processes.push(subprocess) return subprocess }, + // The service's parent execs `sleep`, which never reaps it, so SIGKILL leaves a zombie that + // still answers `kill(pid, 0)`. Terminating the returned parent lets init reap the service. + spawnUnreaped(mode: string, ...args: string[]) { + const subprocess = Bun.spawn(["sh", "-c", '"$@" & exec sleep 60', "sh", ...command(mode, ...args)], { + stdout: "ignore", + stderr: "inherit", + }) + processes.push(subprocess) + return subprocess + }, // Service.ensure detaches contenders; track the elected process before asserting. track(pid: number) { pids.add(pid) @@ -54,3 +64,8 @@ export async function serviceFixture() { }, } } + +export async function expectPortAvailable(url: string) { + const server = Bun.serve({ port: Number(new URL(url).port), fetch: () => new Response() }) + await server.stop(true) +} diff --git a/packages/client/test/fixture/service.ts b/packages/client/test/fixture/service.ts index fa70fdc4fa5..4d88f86b724 100644 --- a/packages/client/test/fixture/service.ts +++ b/packages/client/test/fixture/service.ts @@ -1,4 +1,4 @@ -import { appendFile, rename, writeFile } from "node:fs/promises" +import { appendFile, rename, rm, writeFile } from "node:fs/promises" const [registration, mode, delay] = process.argv.slice(2) if (registration === undefined || mode === undefined) throw new Error("Missing service fixture arguments") @@ -91,6 +91,10 @@ const server = Bun.serve({ }, }) +// Install handlers before publishing: a test may signal as soon as the registration appears. +process.on("SIGTERM", () => void shutdown("SIGTERM")) +process.on("SIGINT", () => void shutdown("SIGINT")) + await writeFile( registration + ".tmp", JSON.stringify({ @@ -106,8 +110,12 @@ await rename(registration + ".tmp", registration) async function shutdown(signal?: NodeJS.Signals) { if (signal !== undefined) await writeFile(registration + ".signal", signal) + // A lingering server unregisters on SIGTERM but keeps running, and holds its port, until killed. + if (mode === "lingering") { + await rm(registration, { force: true }) + await writeFile(registration + ".unregistered", "") + await Bun.sleep(Number(delay)) + } server.stop(true) process.exit() } -process.on("SIGTERM", () => void shutdown("SIGTERM")) -process.on("SIGINT", () => void shutdown("SIGINT")) diff --git a/packages/client/test/promise-service.test.ts b/packages/client/test/promise-service.test.ts index 2ff7fe7eee1..9859dcf47b0 100644 --- a/packages/client/test/promise-service.test.ts +++ b/packages/client/test/promise-service.test.ts @@ -1,9 +1,10 @@ import { expect, test } from "bun:test" import { Service, type EnsureReason } from "../src/promise/service" -import { serviceFixture } from "./fixture/service-fixture" +import { expectPortAvailable, serviceFixture } from "./fixture/service-fixture" import { accelerate } from "./fixture/service-timing" const ensure = accelerate(Service.ensure) +const stop = accelerate(Service.stop) test("discovers a registered service", async () => { await using fixture = await serviceFixture() @@ -171,3 +172,66 @@ test("signals the registered service process", async () => { expect(await Bun.file(registration + ".signal").text()).toBe("SIGTERM") expect(await Bun.file(registration).exists()).toBe(false) }) + +test("stop escalates when the registration disappears before the process exits", async () => { + await using fixture = await serviceFixture() + const registration = fixture.registration + const existing = fixture.spawn("lingering", "5000") + await fixture.waitForFile() + const original = await Bun.file(registration).json() + + await stop({ file: registration }) + + expect(await Bun.file(registration + ".signal").text()).toBe("SIGTERM") + expect(() => process.kill(original.pid, 0)).toThrow() + await expectPortAvailable(original.url) + expect(await Bun.file(registration).exists()).toBe(false) + await existing.exited +}, 15_000) + +test.skipIf(process.platform === "win32")( + "stop fails when the process survives SIGKILL", + async () => { + await using fixture = await serviceFixture() + const registration = fixture.registration + fixture.spawnUnreaped("lingering", "60000") + await fixture.waitForFile() + const original = await Bun.file(registration).json() + + await expect(stop({ file: registration })).rejects.toThrow(`Server process ${original.pid} is still running`) + expect(await Bun.file(registration + ".signal").text()).toBe("SIGTERM") + }, + 15_000, +) + +test("stop waits for the original process while preserving a newly registered successor", async () => { + await using fixture = await serviceFixture() + const registration = fixture.registration + const existing = fixture.spawn("lingering", "15000") + await fixture.waitForFile() + const original = await Bun.file(registration).json() + // Default timing keeps the grace period open long enough for the successor to register first. + const stopping = Service.stop({ file: registration }) + + try { + await fixture.waitForFile(registration + ".unregistered") + const successor = fixture.spawn("graceful") + await fixture.waitForFile() + const replacement = await Bun.file(registration).json() + expect(existing.exitCode).toBe(null) + expect(replacement.pid).toBe(successor.pid) + + await stopping + + expect(() => process.kill(original.pid, 0)).toThrow() + await expectPortAvailable(original.url) + expect(await Bun.file(registration).json()).toEqual(replacement) + expect(await fetch(new URL("/api/info", replacement.url)).then((response) => response.json())).toMatchObject({ + pid: successor.pid, + }) + expect(successor.exitCode).toBe(null) + } finally { + existing.kill("SIGKILL") + await stopping + } +}, 20_000) diff --git a/packages/client/test/service.test.ts b/packages/client/test/service.test.ts index 83c754a50dd..d4c18dabcf6 100644 --- a/packages/client/test/service.test.ts +++ b/packages/client/test/service.test.ts @@ -3,10 +3,11 @@ import { expect, test } from "bun:test" import { Effect, FileSystem } from "effect" import { writeFile } from "node:fs/promises" import { Service, type EnsureReason } from "../src/effect/service" -import { serviceFixture } from "./fixture/service-fixture" +import { expectPortAvailable, serviceFixture } from "./fixture/service-fixture" import { accelerate } from "./fixture/service-timing" const ensure = accelerate(Service.ensure) +const stop = accelerate(Service.stop) test("a concurrent same-version start cannot invalidate a resolved endpoint", async () => { await using fixture = await serviceFixture() @@ -161,6 +162,67 @@ test("signals an unresponsive registered service process", async () => { expect(await Bun.file(registration).exists()).toBe(false) }) +test("stop escalates when the registration disappears before the process exits", async () => { + await using fixture = await serviceFixture() + const registration = fixture.registration + const existing = fixture.spawn("lingering", "5000") + await fixture.waitForFile() + const original = await Bun.file(registration).json() + + await run(stop({ file: registration })) + + expect(await Bun.file(registration + ".signal").text()).toBe("SIGTERM") + expect(() => process.kill(original.pid, 0)).toThrow() + await expectPortAvailable(original.url) + expect(await Bun.file(registration).exists()).toBe(false) + await existing.exited +}, 15_000) + +test.skipIf(process.platform === "win32")( + "stop fails when the process survives SIGKILL", + async () => { + await using fixture = await serviceFixture() + const registration = fixture.registration + fixture.spawnUnreaped("lingering", "60000") + await fixture.waitForFile() + const original = await Bun.file(registration).json() + + await expect(run(stop({ file: registration }))).rejects.toThrow(`Server process ${original.pid} is still running`) + expect(await Bun.file(registration + ".signal").text()).toBe("SIGTERM") + }, + 15_000, +) + +test("stop waits for the original process while preserving a newly registered successor", async () => { + await using fixture = await serviceFixture() + const registration = fixture.registration + const existing = fixture.spawn("lingering", "15000") + await fixture.waitForFile() + const original = await Bun.file(registration).json() + // Default timing keeps the grace period open long enough for the successor to register first. + const stopping = run(Service.stop({ file: registration })) + + try { + await fixture.waitForFile(registration + ".unregistered") + const successor = fixture.spawn("graceful") + await fixture.waitForFile() + const replacement = await Bun.file(registration).json() + expect(existing.exitCode).toBe(null) + expect(replacement.pid).toBe(successor.pid) + + await stopping + + expect(() => process.kill(original.pid, 0)).toThrow() + await expectPortAvailable(original.url) + expect(await Bun.file(registration).json()).toEqual(replacement) + expect(await status(replacement.url)).toMatchObject({ pid: successor.pid }) + expect(successor.exitCode).toBe(null) + } finally { + existing.kill("SIGKILL") + await stopping + } +}, 20_000) + test("signals an incompatible service before starting its replacement", async () => { await using fixture = await serviceFixture() const registration = fixture.registration