refactor(client): share startup attempt bookkeeping between clients (#53240)

This commit is contained in:
Kit Langton
2026-10-05 08:18:09 -07:00
committed by GitHub
Unverified
parent ef07075156
commit e2d540042a
4 changed files with 217 additions and 85 deletions
+13 -42
View File
@@ -1,11 +1,6 @@
import { Effect, FileSystem, Option, Schedule, Schema } from "effect"
import type { DiscoverOptions, EnsureOptions, StopOptions } from "../service.js"
import {
contenderFailure,
contenderFinished,
type ServiceContender,
spawnServiceContender,
} from "../service-contender.js"
import { contenderPool, spawnServiceContender } from "../service-contender.js"
import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js"
import { matchesVersion } from "../service-version.js"
import { PtyHandoff } from "../pty-handoff.js"
@@ -53,12 +48,9 @@ export const incumbent = Effect.fn("service.incumbent")(function* (
/** Ensure a healthy, compatible local service is running. */
export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOptions = {}) {
const timing = ensureTiming(options)
const contenders = new Set<ServiceContender>()
const pool = contenderPool(timing)
let timeouts: { readonly info: Info; readonly count: number } | undefined
let announced = false
let lastSpawn = 0
let spawnDelay = timing.spawnDelay
let failure: Error | undefined
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) =>
Effect.sync(() => {
if (announced) return
@@ -90,19 +82,13 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
yield* Effect.logWarning("Background service is unresponsive; recovery cannot preserve persistent terminals")
yield* Effect.tryPromise(() => PtyHandoff.clear(options.file ?? fallback()))
yield* terminate(info, options, timing)
for (const item of contenders) {
if (item.child.pid === info.pid || contenderFinished(item)) {
item.release()
contenders.delete(item)
}
}
failure = undefined
pool.evict(info.pid)
pool.recruitNow()
timeouts = undefined
lastSpawn = Date.now() - spawnDelay
}
} else timeouts = undefined
if (service !== undefined) {
spawnDelay = timing.spawnDelay
pool.serviceAnswered()
const versionMatches = matchesVersion(service.version, options)
const compatible = service.compatible && versionMatches
if (!service.compatible && versionMatches)
@@ -125,30 +111,15 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
file: options.file,
pty: service.state === "ready" ? "handoff" : "clear",
}).pipe(Effect.ignore)
for (const item of contenders) {
if (item.child.pid === service.info.pid || contenderFinished(item)) {
item.release()
contenders.delete(item)
}
}
failure = undefined
lastSpawn = 0
pool.evict(service.info.pid)
return Option.none<LocalService>()
} else if (lastSpawn === 0 && info !== undefined) lastSpawn = Date.now()
const finished = [...contenders].filter(contenderFinished)
failure ??= finished.map(contenderFailure).find((error): error is Error => error !== undefined)
if (finished.some((item) => item.child.exitCode === 0)) {
spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay)
}
finished.forEach((item) => contenders.delete(item))
if (failure !== undefined && contenders.size === 0) return yield* Effect.fail(failure)
// Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the
// survivors finish without recruiting replacements that could hide the error indefinitely.
if (failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay) {
const failed = pool.reap()
if (failed !== undefined) return yield* Effect.fail(failed)
if (pool.shouldRecruit(info !== undefined)) {
yield* announce("missing")
contenders.add(yield* spawnContender)
lastSpawn = Date.now()
pool.add(yield* spawnContender)
}
return Option.none<LocalService>()
}).pipe(
@@ -158,10 +129,10 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
// like the Promise variant rather than by attempt count.
schedule: Schedule.spaced(timing.pollInterval).pipe(Schedule.upTo({ duration: timing.promiseTimeout })),
}),
Effect.ensuring(Effect.sync(() => contenders.forEach((contender) => contender.release()))),
Effect.ensuring(Effect.sync(() => pool.releaseAll())),
)
if (Option.isNone(found))
return yield* Effect.fail(failure ?? new Error("Timed out waiting for the background service to start"))
return yield* Effect.fail(pool.failure() ?? new Error("Timed out waiting for the background service to start"))
return found.value.endpoint
})
+13 -42
View File
@@ -1,11 +1,6 @@
import { readFile, rm } from "node:fs/promises"
import type { DiscoverOptions, Endpoint, Info, EnsureOptions, StopOptions } from "../service.js"
import {
contenderFailure,
contenderFinished,
type ServiceContender,
spawnServiceContender,
} from "../service-contender.js"
import { contenderPool, spawnServiceContender } from "../service-contender.js"
import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js"
import { matchesVersion } from "../service-version.js"
import { PtyHandoff } from "../pty-handoff.js"
@@ -33,12 +28,9 @@ export async function discover(options: DiscoverOptions = {}) {
export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
const timing = ensureTiming(options)
const deadline = Date.now() + timing.promiseTimeout
const contenders = new Set<ServiceContender>()
const pool = contenderPool(timing)
let timeouts: { readonly info: Info; readonly count: number } | undefined
let announced = false
let lastSpawn = 0
let spawnDelay = timing.spawnDelay
let failure: Error | undefined
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => {
if (announced) return
@@ -57,7 +49,8 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
try {
while (true) {
if (Date.now() >= deadline) throw failure ?? new Error("Timed out waiting for the background service to start")
if (Date.now() >= deadline)
throw pool.failure() ?? new Error("Timed out waiting for the background service to start")
const registration = await registered(options.file, timing.requestTimeout)
if (registration.timedOut && registration.info !== undefined) {
timeouts = {
@@ -69,20 +62,14 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
console.warn("Background service is unresponsive; recovery cannot preserve persistent terminals")
await PtyHandoff.clear(options.file ?? fallback())
await terminate(registration.info, options, timing)
for (const item of contenders) {
if (item.child.pid === registration.info.pid || contenderFinished(item)) {
item.release()
contenders.delete(item)
}
}
failure = undefined
pool.evict(registration.info.pid)
pool.recruitNow()
timeouts = undefined
lastSpawn = Date.now() - spawnDelay
}
} else timeouts = undefined
if (registration.service !== undefined) {
spawnDelay = timing.spawnDelay
pool.serviceAnswered()
const service = registration.service
const versionMatches = matchesVersion(service.version, options)
const compatible = service.compatible && versionMatches
@@ -103,36 +90,20 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
file: options.file,
pty: service.state === "ready" ? "handoff" : "clear",
}).catch(() => undefined)
for (const item of contenders) {
if (item.child.pid === service.info.pid || contenderFinished(item)) {
item.release()
contenders.delete(item)
}
}
failure = undefined
lastSpawn = 0
pool.evict(service.info.pid)
}
} else {
if (lastSpawn === 0 && registration.info !== undefined) lastSpawn = Date.now()
const finished = [...contenders].filter(contenderFinished)
failure ??= finished.map(contenderFailure).find((error) => error !== undefined)
if (finished.some((item) => item.child.exitCode === 0)) {
spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay)
}
finished.forEach((item) => contenders.delete(item))
if (failure !== undefined && contenders.size === 0) throw failure
// Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the
// survivors finish without recruiting replacements that could hide the error indefinitely.
if (failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay) {
const failed = pool.reap()
if (failed !== undefined) throw failed
if (pool.shouldRecruit(registration.info !== undefined)) {
announce("missing")
contenders.add(await spawnContender())
lastSpawn = Date.now()
pool.add(await spawnContender())
}
}
await delay(timing.pollInterval)
}
} finally {
contenders.forEach((contender) => contender.release())
pool.releaseAll()
}
}
+65 -1
View File
@@ -1,4 +1,5 @@
import { spawn, type ChildProcess } from "node:child_process"
import type { EnsureTiming } from "./service-timing.js"
export type ServiceContender = {
readonly child: ChildProcess
@@ -53,7 +54,70 @@ export function spawnServiceContender(
}
}
export function contenderFailure(contender: ServiceContender) {
/**
* The startup attempts of one `ensure()` call. It keeps at most two attempts alive, remembers
* the first startup failure, and backs off when attempts exit cleanly because another one won.
*/
export function contenderPool(timing: EnsureTiming) {
const contenders = new Set<ServiceContender>()
let failure: Error | undefined
let spawnDelay = timing.spawnDelay
let lastSpawn = 0
return {
/** The first startup failure seen since the last eviction. */
failure: () => failure,
/** A registered service answered, so the next attempt waits the base delay again. */
serviceAnswered() {
spawnDelay = timing.spawnDelay
},
/**
* The owner `pid` was replaced. Drop its attempt and any finished ones, forget their
* failure, and restart the spawn clock.
*/
evict(pid: number) {
for (const item of contenders) {
if (item.child.pid === pid || contenderFinished(item)) {
item.release()
contenders.delete(item)
}
}
failure = undefined
lastSpawn = 0
},
/** Let the next attempt start without waiting a spawn delay. */
recruitNow() {
lastSpawn = Date.now() - spawnDelay
},
/** Collect finished attempts. Returns the startup failure once no attempt is left alive. */
reap() {
const finished = [...contenders].filter(contenderFinished)
failure ??= finished.map(contenderFailure).find((error) => error !== undefined)
if (finished.some((item) => item.child.exitCode === 0))
spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay)
finished.forEach((item) => contenders.delete(item))
return contenders.size === 0 ? failure : undefined
},
/**
* Whether to start another attempt now. A registration that has not answered yet gets one
* spawn delay before an attempt competes with it.
*/
shouldRecruit(registered: boolean) {
if (lastSpawn === 0 && registered) lastSpawn = Date.now()
// Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the
// survivors finish without recruiting replacements that could hide the error indefinitely.
return failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay
},
add(contender: ServiceContender) {
contenders.add(contender)
lastSpawn = Date.now()
},
releaseAll() {
contenders.forEach((contender) => contender.release())
},
}
}
function contenderFailure(contender: ServiceContender) {
const error = contender.error()
if (error !== undefined) return error
if (contender.child.exitCode !== null && contender.child.exitCode !== 0)
@@ -0,0 +1,126 @@
import { afterEach, expect, test } from "bun:test"
import {
contenderFinished,
contenderPool,
type ServiceContender,
spawnServiceContender,
} from "../src/service-contender"
import { defaultEnsureTiming } from "../src/service-timing"
const spawned: ServiceContender[] = []
afterEach(() => {
spawned.splice(0).forEach((contender) => {
contender.release()
contender.child.kill("SIGKILL")
})
})
test("remembers the first startup failure and reports it once no attempt is left", async () => {
const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 0 })
const survivor = running()
const failed = failing("first failure")
pool.add(survivor)
pool.add(failed)
await finished(failed)
expect(pool.reap()).toBeUndefined()
expect(pool.failure()?.message).toContain("first failure")
expect(pool.shouldRecruit(false)).toBe(false)
survivor.child.kill("SIGKILL")
await finished(survivor)
expect(pool.reap()?.message).toContain("first failure")
})
test("evicting a replaced owner forgets its failure and restarts the spawn clock", async () => {
const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 60_000 })
const owner = running()
const failed = failing("old failure")
pool.add(owner)
pool.add(failed)
await finished(failed)
expect(pool.reap()).toBeUndefined()
expect(pool.failure()).toBeDefined()
pool.evict(owner.child.pid!)
expect(pool.failure()).toBeUndefined()
expect(pool.reap()).toBeUndefined()
expect(pool.shouldRecruit(false)).toBe(true)
expect(owner.child.exitCode).toBeNull()
})
test("an unanswered registration gets one spawn delay before an attempt competes", () => {
const timing = { ...defaultEnsureTiming, spawnDelay: 60_000 }
expect(contenderPool(timing).shouldRecruit(false)).toBe(true)
const registered = contenderPool(timing)
expect(registered.shouldRecruit(true)).toBe(false)
registered.recruitNow()
expect(registered.shouldRecruit(true)).toBe(true)
})
test("keeps at most two attempts alive", async () => {
const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 0 })
const exiting = clean()
pool.add(running())
pool.add(exiting)
expect(pool.shouldRecruit(false)).toBe(false)
await finished(exiting)
expect(pool.reap()).toBeUndefined()
expect(pool.shouldRecruit(false)).toBe(true)
})
test("clean exits back off up to the maximum spawn delay", async () => {
const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 200, maxSpawnDelay: 300 })
const exiting = clean()
pool.add(exiting)
const start = Date.now()
await finished(exiting)
expect(pool.reap()).toBeUndefined()
// The base delay (200 ms) has passed, but the backed-off delay (300 ms) has not, unless a
// loaded machine overslept past it.
await Bun.sleep(Math.max(0, 250 - (Date.now() - start)))
if (Date.now() - start < 290) expect(pool.shouldRecruit(false)).toBe(false)
// Doubling would give 400 ms; the cap allows an attempt at 300 ms.
await Bun.sleep(Math.max(0, 320 - (Date.now() - start)))
expect(pool.shouldRecruit(false)).toBe(true)
})
test("an answering service resets the backoff to the base spawn delay", async () => {
const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 200, maxSpawnDelay: 1_000 })
const exiting = clean()
pool.add(exiting)
const start = Date.now()
await finished(exiting)
expect(pool.reap()).toBeUndefined()
pool.serviceAnswered()
// Backed off, the next attempt would wait 400 ms; after the reset it waits the base 200 ms.
await Bun.sleep(Math.max(0, 250 - (Date.now() - start)))
expect(pool.shouldRecruit(false)).toBe(true)
})
function spawn(code: string) {
const contender = spawnServiceContender(process.execPath, ["-e", code])
spawned.push(contender)
return contender
}
function running() {
return spawn("setTimeout(() => {}, 60_000)")
}
function failing(message: string) {
return spawn(`console.error(${JSON.stringify(message)}); process.exit(1)`)
}
function clean() {
return spawn("process.exit(0)")
}
async function finished(contender: ServiceContender) {
while (!contenderFinished(contender)) await Bun.sleep(5)
}