mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-19 17:01:37 +03:00
refactor(client): share startup attempt bookkeeping between clients (#53240)
This commit is contained in:
committed by
GitHub
Unverified
parent
ef07075156
commit
e2d540042a
@@ -1,11 +1,6 @@
|
||||
import { Effect, FileSystem, Option, Schedule, Schema } from "effect"
|
||||
import type { DiscoverOptions, EnsureOptions, StopOptions } from "../service.js"
|
||||
import {
|
||||
contenderFailure,
|
||||
contenderFinished,
|
||||
type ServiceContender,
|
||||
spawnServiceContender,
|
||||
} from "../service-contender.js"
|
||||
import { contenderPool, spawnServiceContender } from "../service-contender.js"
|
||||
import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js"
|
||||
import { matchesVersion } from "../service-version.js"
|
||||
import { PtyHandoff } from "../pty-handoff.js"
|
||||
@@ -53,12 +48,9 @@ export const incumbent = Effect.fn("service.incumbent")(function* (
|
||||
/** Ensure a healthy, compatible local service is running. */
|
||||
export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOptions = {}) {
|
||||
const timing = ensureTiming(options)
|
||||
const contenders = new Set<ServiceContender>()
|
||||
const pool = contenderPool(timing)
|
||||
let timeouts: { readonly info: Info; readonly count: number } | undefined
|
||||
let announced = false
|
||||
let lastSpawn = 0
|
||||
let spawnDelay = timing.spawnDelay
|
||||
let failure: Error | undefined
|
||||
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) =>
|
||||
Effect.sync(() => {
|
||||
if (announced) return
|
||||
@@ -90,19 +82,13 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
|
||||
yield* Effect.logWarning("Background service is unresponsive; recovery cannot preserve persistent terminals")
|
||||
yield* Effect.tryPromise(() => PtyHandoff.clear(options.file ?? fallback()))
|
||||
yield* terminate(info, options, timing)
|
||||
for (const item of contenders) {
|
||||
if (item.child.pid === info.pid || contenderFinished(item)) {
|
||||
item.release()
|
||||
contenders.delete(item)
|
||||
}
|
||||
}
|
||||
failure = undefined
|
||||
pool.evict(info.pid)
|
||||
pool.recruitNow()
|
||||
timeouts = undefined
|
||||
lastSpawn = Date.now() - spawnDelay
|
||||
}
|
||||
} else timeouts = undefined
|
||||
if (service !== undefined) {
|
||||
spawnDelay = timing.spawnDelay
|
||||
pool.serviceAnswered()
|
||||
const versionMatches = matchesVersion(service.version, options)
|
||||
const compatible = service.compatible && versionMatches
|
||||
if (!service.compatible && versionMatches)
|
||||
@@ -125,30 +111,15 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
|
||||
file: options.file,
|
||||
pty: service.state === "ready" ? "handoff" : "clear",
|
||||
}).pipe(Effect.ignore)
|
||||
for (const item of contenders) {
|
||||
if (item.child.pid === service.info.pid || contenderFinished(item)) {
|
||||
item.release()
|
||||
contenders.delete(item)
|
||||
}
|
||||
}
|
||||
failure = undefined
|
||||
lastSpawn = 0
|
||||
pool.evict(service.info.pid)
|
||||
return Option.none<LocalService>()
|
||||
} else if (lastSpawn === 0 && info !== undefined) lastSpawn = Date.now()
|
||||
|
||||
const finished = [...contenders].filter(contenderFinished)
|
||||
failure ??= finished.map(contenderFailure).find((error): error is Error => error !== undefined)
|
||||
if (finished.some((item) => item.child.exitCode === 0)) {
|
||||
spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay)
|
||||
}
|
||||
finished.forEach((item) => contenders.delete(item))
|
||||
if (failure !== undefined && contenders.size === 0) return yield* Effect.fail(failure)
|
||||
// Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the
|
||||
// survivors finish without recruiting replacements that could hide the error indefinitely.
|
||||
if (failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay) {
|
||||
|
||||
const failed = pool.reap()
|
||||
if (failed !== undefined) return yield* Effect.fail(failed)
|
||||
if (pool.shouldRecruit(info !== undefined)) {
|
||||
yield* announce("missing")
|
||||
contenders.add(yield* spawnContender)
|
||||
lastSpawn = Date.now()
|
||||
pool.add(yield* spawnContender)
|
||||
}
|
||||
return Option.none<LocalService>()
|
||||
}).pipe(
|
||||
@@ -158,10 +129,10 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
|
||||
// like the Promise variant rather than by attempt count.
|
||||
schedule: Schedule.spaced(timing.pollInterval).pipe(Schedule.upTo({ duration: timing.promiseTimeout })),
|
||||
}),
|
||||
Effect.ensuring(Effect.sync(() => contenders.forEach((contender) => contender.release()))),
|
||||
Effect.ensuring(Effect.sync(() => pool.releaseAll())),
|
||||
)
|
||||
if (Option.isNone(found))
|
||||
return yield* Effect.fail(failure ?? new Error("Timed out waiting for the background service to start"))
|
||||
return yield* Effect.fail(pool.failure() ?? new Error("Timed out waiting for the background service to start"))
|
||||
return found.value.endpoint
|
||||
})
|
||||
|
||||
|
||||
@@ -1,11 +1,6 @@
|
||||
import { readFile, rm } from "node:fs/promises"
|
||||
import type { DiscoverOptions, Endpoint, Info, EnsureOptions, StopOptions } from "../service.js"
|
||||
import {
|
||||
contenderFailure,
|
||||
contenderFinished,
|
||||
type ServiceContender,
|
||||
spawnServiceContender,
|
||||
} from "../service-contender.js"
|
||||
import { contenderPool, spawnServiceContender } from "../service-contender.js"
|
||||
import { defaultEnsureTiming, ensureTiming, type EnsureTiming } from "../service-timing.js"
|
||||
import { matchesVersion } from "../service-version.js"
|
||||
import { PtyHandoff } from "../pty-handoff.js"
|
||||
@@ -33,12 +28,9 @@ export async function discover(options: DiscoverOptions = {}) {
|
||||
export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
const timing = ensureTiming(options)
|
||||
const deadline = Date.now() + timing.promiseTimeout
|
||||
const contenders = new Set<ServiceContender>()
|
||||
const pool = contenderPool(timing)
|
||||
let timeouts: { readonly info: Info; readonly count: number } | undefined
|
||||
let announced = false
|
||||
let lastSpawn = 0
|
||||
let spawnDelay = timing.spawnDelay
|
||||
let failure: Error | undefined
|
||||
|
||||
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => {
|
||||
if (announced) return
|
||||
@@ -57,7 +49,8 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
if (Date.now() >= deadline) throw failure ?? new Error("Timed out waiting for the background service to start")
|
||||
if (Date.now() >= deadline)
|
||||
throw pool.failure() ?? new Error("Timed out waiting for the background service to start")
|
||||
const registration = await registered(options.file, timing.requestTimeout)
|
||||
if (registration.timedOut && registration.info !== undefined) {
|
||||
timeouts = {
|
||||
@@ -69,20 +62,14 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
console.warn("Background service is unresponsive; recovery cannot preserve persistent terminals")
|
||||
await PtyHandoff.clear(options.file ?? fallback())
|
||||
await terminate(registration.info, options, timing)
|
||||
for (const item of contenders) {
|
||||
if (item.child.pid === registration.info.pid || contenderFinished(item)) {
|
||||
item.release()
|
||||
contenders.delete(item)
|
||||
}
|
||||
}
|
||||
failure = undefined
|
||||
pool.evict(registration.info.pid)
|
||||
pool.recruitNow()
|
||||
timeouts = undefined
|
||||
lastSpawn = Date.now() - spawnDelay
|
||||
}
|
||||
} else timeouts = undefined
|
||||
|
||||
if (registration.service !== undefined) {
|
||||
spawnDelay = timing.spawnDelay
|
||||
pool.serviceAnswered()
|
||||
const service = registration.service
|
||||
const versionMatches = matchesVersion(service.version, options)
|
||||
const compatible = service.compatible && versionMatches
|
||||
@@ -103,36 +90,20 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
file: options.file,
|
||||
pty: service.state === "ready" ? "handoff" : "clear",
|
||||
}).catch(() => undefined)
|
||||
for (const item of contenders) {
|
||||
if (item.child.pid === service.info.pid || contenderFinished(item)) {
|
||||
item.release()
|
||||
contenders.delete(item)
|
||||
}
|
||||
}
|
||||
failure = undefined
|
||||
lastSpawn = 0
|
||||
pool.evict(service.info.pid)
|
||||
}
|
||||
} else {
|
||||
if (lastSpawn === 0 && registration.info !== undefined) lastSpawn = Date.now()
|
||||
const finished = [...contenders].filter(contenderFinished)
|
||||
failure ??= finished.map(contenderFailure).find((error) => error !== undefined)
|
||||
if (finished.some((item) => item.child.exitCode === 0)) {
|
||||
spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay)
|
||||
}
|
||||
finished.forEach((item) => contenders.delete(item))
|
||||
if (failure !== undefined && contenders.size === 0) throw failure
|
||||
// Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the
|
||||
// survivors finish without recruiting replacements that could hide the error indefinitely.
|
||||
if (failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay) {
|
||||
const failed = pool.reap()
|
||||
if (failed !== undefined) throw failed
|
||||
if (pool.shouldRecruit(registration.info !== undefined)) {
|
||||
announce("missing")
|
||||
contenders.add(await spawnContender())
|
||||
lastSpawn = Date.now()
|
||||
pool.add(await spawnContender())
|
||||
}
|
||||
}
|
||||
await delay(timing.pollInterval)
|
||||
}
|
||||
} finally {
|
||||
contenders.forEach((contender) => contender.release())
|
||||
pool.releaseAll()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { spawn, type ChildProcess } from "node:child_process"
|
||||
import type { EnsureTiming } from "./service-timing.js"
|
||||
|
||||
export type ServiceContender = {
|
||||
readonly child: ChildProcess
|
||||
@@ -53,7 +54,70 @@ export function spawnServiceContender(
|
||||
}
|
||||
}
|
||||
|
||||
export function contenderFailure(contender: ServiceContender) {
|
||||
/**
|
||||
* The startup attempts of one `ensure()` call. It keeps at most two attempts alive, remembers
|
||||
* the first startup failure, and backs off when attempts exit cleanly because another one won.
|
||||
*/
|
||||
export function contenderPool(timing: EnsureTiming) {
|
||||
const contenders = new Set<ServiceContender>()
|
||||
let failure: Error | undefined
|
||||
let spawnDelay = timing.spawnDelay
|
||||
let lastSpawn = 0
|
||||
return {
|
||||
/** The first startup failure seen since the last eviction. */
|
||||
failure: () => failure,
|
||||
/** A registered service answered, so the next attempt waits the base delay again. */
|
||||
serviceAnswered() {
|
||||
spawnDelay = timing.spawnDelay
|
||||
},
|
||||
/**
|
||||
* The owner `pid` was replaced. Drop its attempt and any finished ones, forget their
|
||||
* failure, and restart the spawn clock.
|
||||
*/
|
||||
evict(pid: number) {
|
||||
for (const item of contenders) {
|
||||
if (item.child.pid === pid || contenderFinished(item)) {
|
||||
item.release()
|
||||
contenders.delete(item)
|
||||
}
|
||||
}
|
||||
failure = undefined
|
||||
lastSpawn = 0
|
||||
},
|
||||
/** Let the next attempt start without waiting a spawn delay. */
|
||||
recruitNow() {
|
||||
lastSpawn = Date.now() - spawnDelay
|
||||
},
|
||||
/** Collect finished attempts. Returns the startup failure once no attempt is left alive. */
|
||||
reap() {
|
||||
const finished = [...contenders].filter(contenderFinished)
|
||||
failure ??= finished.map(contenderFailure).find((error) => error !== undefined)
|
||||
if (finished.some((item) => item.child.exitCode === 0))
|
||||
spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay)
|
||||
finished.forEach((item) => contenders.delete(item))
|
||||
return contenders.size === 0 ? failure : undefined
|
||||
},
|
||||
/**
|
||||
* Whether to start another attempt now. A registration that has not answered yet gets one
|
||||
* spawn delay before an attempt competes with it.
|
||||
*/
|
||||
shouldRecruit(registered: boolean) {
|
||||
if (lastSpawn === 0 && registered) lastSpawn = Date.now()
|
||||
// Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the
|
||||
// survivors finish without recruiting replacements that could hide the error indefinitely.
|
||||
return failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay
|
||||
},
|
||||
add(contender: ServiceContender) {
|
||||
contenders.add(contender)
|
||||
lastSpawn = Date.now()
|
||||
},
|
||||
releaseAll() {
|
||||
contenders.forEach((contender) => contender.release())
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function contenderFailure(contender: ServiceContender) {
|
||||
const error = contender.error()
|
||||
if (error !== undefined) return error
|
||||
if (contender.child.exitCode !== null && contender.child.exitCode !== 0)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import { afterEach, expect, test } from "bun:test"
|
||||
import {
|
||||
contenderFinished,
|
||||
contenderPool,
|
||||
type ServiceContender,
|
||||
spawnServiceContender,
|
||||
} from "../src/service-contender"
|
||||
import { defaultEnsureTiming } from "../src/service-timing"
|
||||
|
||||
const spawned: ServiceContender[] = []
|
||||
afterEach(() => {
|
||||
spawned.splice(0).forEach((contender) => {
|
||||
contender.release()
|
||||
contender.child.kill("SIGKILL")
|
||||
})
|
||||
})
|
||||
|
||||
test("remembers the first startup failure and reports it once no attempt is left", async () => {
|
||||
const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 0 })
|
||||
const survivor = running()
|
||||
const failed = failing("first failure")
|
||||
pool.add(survivor)
|
||||
pool.add(failed)
|
||||
await finished(failed)
|
||||
|
||||
expect(pool.reap()).toBeUndefined()
|
||||
expect(pool.failure()?.message).toContain("first failure")
|
||||
expect(pool.shouldRecruit(false)).toBe(false)
|
||||
|
||||
survivor.child.kill("SIGKILL")
|
||||
await finished(survivor)
|
||||
expect(pool.reap()?.message).toContain("first failure")
|
||||
})
|
||||
|
||||
test("evicting a replaced owner forgets its failure and restarts the spawn clock", async () => {
|
||||
const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 60_000 })
|
||||
const owner = running()
|
||||
const failed = failing("old failure")
|
||||
pool.add(owner)
|
||||
pool.add(failed)
|
||||
await finished(failed)
|
||||
expect(pool.reap()).toBeUndefined()
|
||||
expect(pool.failure()).toBeDefined()
|
||||
|
||||
pool.evict(owner.child.pid!)
|
||||
|
||||
expect(pool.failure()).toBeUndefined()
|
||||
expect(pool.reap()).toBeUndefined()
|
||||
expect(pool.shouldRecruit(false)).toBe(true)
|
||||
expect(owner.child.exitCode).toBeNull()
|
||||
})
|
||||
|
||||
test("an unanswered registration gets one spawn delay before an attempt competes", () => {
|
||||
const timing = { ...defaultEnsureTiming, spawnDelay: 60_000 }
|
||||
expect(contenderPool(timing).shouldRecruit(false)).toBe(true)
|
||||
|
||||
const registered = contenderPool(timing)
|
||||
expect(registered.shouldRecruit(true)).toBe(false)
|
||||
registered.recruitNow()
|
||||
expect(registered.shouldRecruit(true)).toBe(true)
|
||||
})
|
||||
|
||||
test("keeps at most two attempts alive", async () => {
|
||||
const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 0 })
|
||||
const exiting = clean()
|
||||
pool.add(running())
|
||||
pool.add(exiting)
|
||||
expect(pool.shouldRecruit(false)).toBe(false)
|
||||
|
||||
await finished(exiting)
|
||||
expect(pool.reap()).toBeUndefined()
|
||||
expect(pool.shouldRecruit(false)).toBe(true)
|
||||
})
|
||||
|
||||
test("clean exits back off up to the maximum spawn delay", async () => {
|
||||
const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 200, maxSpawnDelay: 300 })
|
||||
const exiting = clean()
|
||||
pool.add(exiting)
|
||||
const start = Date.now()
|
||||
await finished(exiting)
|
||||
expect(pool.reap()).toBeUndefined()
|
||||
|
||||
// The base delay (200 ms) has passed, but the backed-off delay (300 ms) has not, unless a
|
||||
// loaded machine overslept past it.
|
||||
await Bun.sleep(Math.max(0, 250 - (Date.now() - start)))
|
||||
if (Date.now() - start < 290) expect(pool.shouldRecruit(false)).toBe(false)
|
||||
// Doubling would give 400 ms; the cap allows an attempt at 300 ms.
|
||||
await Bun.sleep(Math.max(0, 320 - (Date.now() - start)))
|
||||
expect(pool.shouldRecruit(false)).toBe(true)
|
||||
})
|
||||
|
||||
test("an answering service resets the backoff to the base spawn delay", async () => {
|
||||
const pool = contenderPool({ ...defaultEnsureTiming, spawnDelay: 200, maxSpawnDelay: 1_000 })
|
||||
const exiting = clean()
|
||||
pool.add(exiting)
|
||||
const start = Date.now()
|
||||
await finished(exiting)
|
||||
expect(pool.reap()).toBeUndefined()
|
||||
|
||||
pool.serviceAnswered()
|
||||
// Backed off, the next attempt would wait 400 ms; after the reset it waits the base 200 ms.
|
||||
await Bun.sleep(Math.max(0, 250 - (Date.now() - start)))
|
||||
expect(pool.shouldRecruit(false)).toBe(true)
|
||||
})
|
||||
|
||||
function spawn(code: string) {
|
||||
const contender = spawnServiceContender(process.execPath, ["-e", code])
|
||||
spawned.push(contender)
|
||||
return contender
|
||||
}
|
||||
|
||||
function running() {
|
||||
return spawn("setTimeout(() => {}, 60_000)")
|
||||
}
|
||||
|
||||
function failing(message: string) {
|
||||
return spawn(`console.error(${JSON.stringify(message)}); process.exit(1)`)
|
||||
}
|
||||
|
||||
function clean() {
|
||||
return spawn("process.exit(0)")
|
||||
}
|
||||
|
||||
async function finished(contender: ServiceContender) {
|
||||
while (!contenderFinished(contender)) await Bun.sleep(5)
|
||||
}
|
||||
Reference in New Issue
Block a user