From e2d540042ad39339e4024d71b4b86556b7e7df43 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Mon, 5 Oct 2026 08:18:09 -0700 Subject: [PATCH] refactor(client): share startup attempt bookkeeping between clients (#53240) --- packages/client/src/effect/service.ts | 55 ++------ packages/client/src/promise/service.ts | 55 ++------ packages/client/src/service-contender.ts | 66 ++++++++- .../test/service-contender-pool.test.ts | 126 ++++++++++++++++++ 4 files changed, 217 insertions(+), 85 deletions(-) create mode 100644 packages/client/test/service-contender-pool.test.ts diff --git a/packages/client/src/effect/service.ts b/packages/client/src/effect/service.ts index 5b9ef75ac2b..39d20eb4537 100644 --- a/packages/client/src/effect/service.ts +++ b/packages/client/src/effect/service.ts @@ -1,11 +1,6 @@ import { Effect, FileSystem, Option, Schedule, Schema } from "effect" import type { DiscoverOptions, EnsureOptions, StopOptions } from "../service.js" -import { - contenderFailure, - contenderFinished, - type ServiceContender, - spawnServiceContender, -} from "../service-contender.js" +import { contenderPool, spawnServiceContender } from "../service-contender.js" import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js" import { matchesVersion } from "../service-version.js" import { PtyHandoff } from "../pty-handoff.js" @@ -53,12 +48,9 @@ export const incumbent = Effect.fn("service.incumbent")(function* ( /** Ensure a healthy, compatible local service is running. */ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOptions = {}) { const timing = ensureTiming(options) - const contenders = new Set() + const pool = contenderPool(timing) let timeouts: { readonly info: Info; readonly count: number } | undefined let announced = false - let lastSpawn = 0 - let spawnDelay = timing.spawnDelay - let failure: Error | undefined const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => Effect.sync(() => { if (announced) return @@ -90,19 +82,13 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti yield* Effect.logWarning("Background service is unresponsive; recovery cannot preserve persistent terminals") yield* Effect.tryPromise(() => PtyHandoff.clear(options.file ?? fallback())) yield* terminate(info, options, timing) - for (const item of contenders) { - if (item.child.pid === info.pid || contenderFinished(item)) { - item.release() - contenders.delete(item) - } - } - failure = undefined + pool.evict(info.pid) + pool.recruitNow() timeouts = undefined - lastSpawn = Date.now() - spawnDelay } } else timeouts = undefined if (service !== undefined) { - spawnDelay = timing.spawnDelay + pool.serviceAnswered() const versionMatches = matchesVersion(service.version, options) const compatible = service.compatible && versionMatches if (!service.compatible && versionMatches) @@ -125,30 +111,15 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti file: options.file, pty: service.state === "ready" ? "handoff" : "clear", }).pipe(Effect.ignore) - for (const item of contenders) { - if (item.child.pid === service.info.pid || contenderFinished(item)) { - item.release() - contenders.delete(item) - } - } - failure = undefined - lastSpawn = 0 + pool.evict(service.info.pid) return Option.none() - } else if (lastSpawn === 0 && info !== undefined) lastSpawn = Date.now() - - const finished = [...contenders].filter(contenderFinished) - failure ??= finished.map(contenderFailure).find((error): error is Error => error !== undefined) - if (finished.some((item) => item.child.exitCode === 0)) { - spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay) } - finished.forEach((item) => contenders.delete(item)) - if (failure !== undefined && contenders.size === 0) return yield* Effect.fail(failure) - // Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the - // survivors finish without recruiting replacements that could hide the error indefinitely. - if (failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay) { + + const failed = pool.reap() + if (failed !== undefined) return yield* Effect.fail(failed) + if (pool.shouldRecruit(info !== undefined)) { yield* announce("missing") - contenders.add(yield* spawnContender) - lastSpawn = Date.now() + pool.add(yield* spawnContender) } return Option.none() }).pipe( @@ -158,10 +129,10 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti // like the Promise variant rather than by attempt count. schedule: Schedule.spaced(timing.pollInterval).pipe(Schedule.upTo({ duration: timing.promiseTimeout })), }), - Effect.ensuring(Effect.sync(() => contenders.forEach((contender) => contender.release()))), + Effect.ensuring(Effect.sync(() => pool.releaseAll())), ) if (Option.isNone(found)) - return yield* Effect.fail(failure ?? new Error("Timed out waiting for the background service to start")) + return yield* Effect.fail(pool.failure() ?? new Error("Timed out waiting for the background service to start")) return found.value.endpoint }) diff --git a/packages/client/src/promise/service.ts b/packages/client/src/promise/service.ts index 293d506354c..f564096f60c 100644 --- a/packages/client/src/promise/service.ts +++ b/packages/client/src/promise/service.ts @@ -1,11 +1,6 @@ import { readFile, rm } from "node:fs/promises" import type { DiscoverOptions, Endpoint, Info, EnsureOptions, StopOptions } from "../service.js" -import { - contenderFailure, - contenderFinished, - type ServiceContender, - spawnServiceContender, -} from "../service-contender.js" +import { contenderPool, spawnServiceContender } from "../service-contender.js" import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js" import { matchesVersion } from "../service-version.js" import { PtyHandoff } from "../pty-handoff.js" @@ -33,12 +28,9 @@ export async function discover(options: DiscoverOptions = {}) { export async function ensure(options: EnsureOptions = {}): Promise { const timing = ensureTiming(options) const deadline = Date.now() + timing.promiseTimeout - const contenders = new Set() + const pool = contenderPool(timing) let timeouts: { readonly info: Info; readonly count: number } | undefined let announced = false - let lastSpawn = 0 - let spawnDelay = timing.spawnDelay - let failure: Error | undefined const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => { if (announced) return @@ -57,7 +49,8 @@ export async function ensure(options: EnsureOptions = {}): Promise { try { while (true) { - if (Date.now() >= deadline) throw failure ?? new Error("Timed out waiting for the background service to start") + if (Date.now() >= deadline) + throw pool.failure() ?? new Error("Timed out waiting for the background service to start") const registration = await registered(options.file, timing.requestTimeout) if (registration.timedOut && registration.info !== undefined) { timeouts = { @@ -69,20 +62,14 @@ export async function ensure(options: EnsureOptions = {}): Promise { console.warn("Background service is unresponsive; recovery cannot preserve persistent terminals") await PtyHandoff.clear(options.file ?? fallback()) await terminate(registration.info, options, timing) - for (const item of contenders) { - if (item.child.pid === registration.info.pid || contenderFinished(item)) { - item.release() - contenders.delete(item) - } - } - failure = undefined + pool.evict(registration.info.pid) + pool.recruitNow() timeouts = undefined - lastSpawn = Date.now() - spawnDelay } } else timeouts = undefined if (registration.service !== undefined) { - spawnDelay = timing.spawnDelay + pool.serviceAnswered() const service = registration.service const versionMatches = matchesVersion(service.version, options) const compatible = service.compatible && versionMatches @@ -103,36 +90,20 @@ export async function ensure(options: EnsureOptions = {}): Promise { file: options.file, pty: service.state === "ready" ? "handoff" : "clear", }).catch(() => undefined) - for (const item of contenders) { - if (item.child.pid === service.info.pid || contenderFinished(item)) { - item.release() - contenders.delete(item) - } - } - failure = undefined - lastSpawn = 0 + pool.evict(service.info.pid) } } else { - if (lastSpawn === 0 && registration.info !== undefined) lastSpawn = Date.now() - const finished = [...contenders].filter(contenderFinished) - failure ??= finished.map(contenderFailure).find((error) => error !== undefined) - if (finished.some((item) => item.child.exitCode === 0)) { - spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay) - } - finished.forEach((item) => contenders.delete(item)) - if (failure !== undefined && contenders.size === 0) throw failure - // Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the - // survivors finish without recruiting replacements that could hide the error indefinitely. - if (failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay) { + const failed = pool.reap() + if (failed !== undefined) throw failed + if (pool.shouldRecruit(registration.info !== undefined)) { announce("missing") - contenders.add(await spawnContender()) - lastSpawn = Date.now() + pool.add(await spawnContender()) } } await delay(timing.pollInterval) } } finally { - contenders.forEach((contender) => contender.release()) + pool.releaseAll() } } diff --git a/packages/client/src/service-contender.ts b/packages/client/src/service-contender.ts index 593eaa1050c..79542c30a44 100644 --- a/packages/client/src/service-contender.ts +++ b/packages/client/src/service-contender.ts @@ -1,4 +1,5 @@ import { spawn, type ChildProcess } from "node:child_process" +import type { EnsureTiming } from "./service-timing.js" export type ServiceContender = { readonly child: ChildProcess @@ -53,7 +54,70 @@ export function spawnServiceContender( } } -export function contenderFailure(contender: ServiceContender) { +/** + * The startup attempts of one `ensure()` call. It keeps at most two attempts alive, remembers + * the first startup failure, and backs off when attempts exit cleanly because another one won. + */ +export function contenderPool(timing: EnsureTiming) { + const contenders = new Set() + let failure: Error | undefined + let spawnDelay = timing.spawnDelay + let lastSpawn = 0 + return { + /** The first startup failure seen since the last eviction. */ + failure: () => failure, + /** A registered service answered, so the next attempt waits the base delay again. */ + serviceAnswered() { + spawnDelay = timing.spawnDelay + }, + /** + * The owner `pid` was replaced. Drop its attempt and any finished ones, forget their + * failure, and restart the spawn clock. + */ + evict(pid: number) { + for (const item of contenders) { + if (item.child.pid === pid || contenderFinished(item)) { + item.release() + contenders.delete(item) + } + } + failure = undefined + lastSpawn = 0 + }, + /** Let the next attempt start without waiting a spawn delay. */ + recruitNow() { + lastSpawn = Date.now() - spawnDelay + }, + /** Collect finished attempts. Returns the startup failure once no attempt is left alive. */ + reap() { + const finished = [...contenders].filter(contenderFinished) + failure ??= finished.map(contenderFailure).find((error) => error !== undefined) + if (finished.some((item) => item.child.exitCode === 0)) + spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay) + finished.forEach((item) => contenders.delete(item)) + return contenders.size === 0 ? failure : undefined + }, + /** + * Whether to start another attempt now. A registration that has not answered yet gets one + * spawn delay before an attempt competes with it. + */ + shouldRecruit(registered: boolean) { + if (lastSpawn === 0 && registered) lastSpawn = Date.now() + // Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the + // survivors finish without recruiting replacements that could hide the error indefinitely. + return failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay + }, + add(contender: ServiceContender) { + contenders.add(contender) + lastSpawn = Date.now() + }, + releaseAll() { + contenders.forEach((contender) => contender.release()) + }, + } +} + +function contenderFailure(contender: ServiceContender) { const error = contender.error() if (error !== undefined) return error if (contender.child.exitCode !== null && contender.child.exitCode !== 0) diff --git a/packages/client/test/service-contender-pool.test.ts b/packages/client/test/service-contender-pool.test.ts new file mode 100644 index 00000000000..9bfc62732bf --- /dev/null +++ b/packages/client/test/service-contender-pool.test.ts @@ -0,0 +1,126 @@ +import { afterEach, expect, test } from "bun:test" +import { + contenderFinished, + contenderPool, + type ServiceContender, + spawnServiceContender, +} from "../src/service-contender" +import { defaultEnsureTiming } from "../src/service-timing" + +const spawned: ServiceContender[] = [] +afterEach(() => { + spawned.splice(0).forEach((contender) => { + contender.release() + contender.child.kill("SIGKILL") + }) +}) + +test("remembers the first startup failure and reports it once no attempt is left", async () => { + const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 0 }) + const survivor = running() + const failed = failing("first failure") + pool.add(survivor) + pool.add(failed) + await finished(failed) + + expect(pool.reap()).toBeUndefined() + expect(pool.failure()?.message).toContain("first failure") + expect(pool.shouldRecruit(false)).toBe(false) + + survivor.child.kill("SIGKILL") + await finished(survivor) + expect(pool.reap()?.message).toContain("first failure") +}) + +test("evicting a replaced owner forgets its failure and restarts the spawn clock", async () => { + const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 60_000 }) + const owner = running() + const failed = failing("old failure") + pool.add(owner) + pool.add(failed) + await finished(failed) + expect(pool.reap()).toBeUndefined() + expect(pool.failure()).toBeDefined() + + pool.evict(owner.child.pid!) + + expect(pool.failure()).toBeUndefined() + expect(pool.reap()).toBeUndefined() + expect(pool.shouldRecruit(false)).toBe(true) + expect(owner.child.exitCode).toBeNull() +}) + +test("an unanswered registration gets one spawn delay before an attempt competes", () => { + const timing = { ...defaultEnsureTiming, spawnDelay: 60_000 } + expect(contenderPool(timing).shouldRecruit(false)).toBe(true) + + const registered = contenderPool(timing) + expect(registered.shouldRecruit(true)).toBe(false) + registered.recruitNow() + expect(registered.shouldRecruit(true)).toBe(true) +}) + +test("keeps at most two attempts alive", async () => { + const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 0 }) + const exiting = clean() + pool.add(running()) + pool.add(exiting) + expect(pool.shouldRecruit(false)).toBe(false) + + await finished(exiting) + expect(pool.reap()).toBeUndefined() + expect(pool.shouldRecruit(false)).toBe(true) +}) + +test("clean exits back off up to the maximum spawn delay", async () => { + const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 200, maxSpawnDelay: 300 }) + const exiting = clean() + pool.add(exiting) + const start = Date.now() + await finished(exiting) + expect(pool.reap()).toBeUndefined() + + // The base delay (200 ms) has passed, but the backed-off delay (300 ms) has not, unless a + // loaded machine overslept past it. + await Bun.sleep(Math.max(0, 250 - (Date.now() - start))) + if (Date.now() - start < 290) expect(pool.shouldRecruit(false)).toBe(false) + // Doubling would give 400 ms; the cap allows an attempt at 300 ms. + await Bun.sleep(Math.max(0, 320 - (Date.now() - start))) + expect(pool.shouldRecruit(false)).toBe(true) +}) + +test("an answering service resets the backoff to the base spawn delay", async () => { + const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 200, maxSpawnDelay: 1_000 }) + const exiting = clean() + pool.add(exiting) + const start = Date.now() + await finished(exiting) + expect(pool.reap()).toBeUndefined() + + pool.serviceAnswered() + // Backed off, the next attempt would wait 400 ms; after the reset it waits the base 200 ms. + await Bun.sleep(Math.max(0, 250 - (Date.now() - start))) + expect(pool.shouldRecruit(false)).toBe(true) +}) + +function spawn(code: string) { + const contender = spawnServiceContender(process.execPath, ["-e", code]) + spawned.push(contender) + return contender +} + +function running() { + return spawn("setTimeout(() => {}, 60_000)") +} + +function failing(message: string) { + return spawn(`console.error(${JSON.stringify(message)}); process.exit(1)`) +} + +function clean() { + return spawn("process.exit(0)") +} + +async function finished(contender: ServiceContender) { + while (!contenderFinished(contender)) await Bun.sleep(5) +}