mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-19 00:03:13 +03:00
fix(client): preserve background service startup failures (#50784)
This commit is contained in:
committed by
GitHub
Unverified
parent
515b3022ac
commit
999097bdac
@@ -58,6 +58,7 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
|
||||
let announced = false
|
||||
let lastSpawn = 0
|
||||
let spawnDelay = timing.spawnDelay
|
||||
let failure: Error | undefined
|
||||
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) =>
|
||||
Effect.sync(() => {
|
||||
if (announced) return
|
||||
@@ -89,6 +90,13 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
|
||||
yield* Effect.logWarning("Background service is unresponsive; recovery cannot preserve persistent terminals")
|
||||
yield* Effect.tryPromise(() => PtyHandoff.clear(options.file ?? fallback()))
|
||||
yield* terminate(info, options, timing)
|
||||
for (const item of contenders) {
|
||||
if (item.child.pid === info.pid || contenderFinished(item)) {
|
||||
item.release()
|
||||
contenders.delete(item)
|
||||
}
|
||||
}
|
||||
failure = undefined
|
||||
timeouts = undefined
|
||||
lastSpawn = Date.now() - spawnDelay
|
||||
}
|
||||
@@ -117,19 +125,27 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
|
||||
file: options.file,
|
||||
pty: service.state === "ready" ? "handoff" : "clear",
|
||||
}).pipe(Effect.ignore)
|
||||
for (const item of contenders) {
|
||||
if (item.child.pid === service.info.pid || contenderFinished(item)) {
|
||||
item.release()
|
||||
contenders.delete(item)
|
||||
}
|
||||
}
|
||||
failure = undefined
|
||||
lastSpawn = 0
|
||||
return Option.none<LocalService>()
|
||||
} else if (lastSpawn === 0 && info !== undefined) lastSpawn = Date.now()
|
||||
|
||||
const finished = [...contenders].filter(contenderFinished)
|
||||
const failure = finished.map(contenderFailure).find((error): error is Error => error !== undefined)
|
||||
failure ??= finished.map(contenderFailure).find((error): error is Error => error !== undefined)
|
||||
if (finished.some((item) => item.child.exitCode === 0)) {
|
||||
spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay)
|
||||
}
|
||||
finished.forEach((item) => contenders.delete(item))
|
||||
if (failure !== undefined && contenders.size === 0) return yield* Effect.fail(failure)
|
||||
// Keep one candidate plus one lock probe so a pre-lock stall cannot block recovery.
|
||||
if (contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay) {
|
||||
// Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the
|
||||
// survivors finish without recruiting replacements that could hide the error indefinitely.
|
||||
if (failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay) {
|
||||
yield* announce("missing")
|
||||
contenders.add(yield* spawnContender)
|
||||
lastSpawn = Date.now()
|
||||
@@ -145,7 +161,7 @@ export const ensure = Effect.fn("service.ensure")(function* (options: EnsureOpti
|
||||
Effect.ensuring(Effect.sync(() => contenders.forEach((contender) => contender.release()))),
|
||||
)
|
||||
if (Option.isNone(found))
|
||||
return yield* Effect.fail(new Error("Timed out waiting for the background service to start"))
|
||||
return yield* Effect.fail(failure ?? new Error("Timed out waiting for the background service to start"))
|
||||
return found.value.endpoint
|
||||
})
|
||||
|
||||
|
||||
@@ -38,6 +38,7 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
let announced = false
|
||||
let lastSpawn = 0
|
||||
let spawnDelay = timing.spawnDelay
|
||||
let failure: Error | undefined
|
||||
|
||||
const announce = (reason: "missing" | "version-mismatch", previousVersion?: string) => {
|
||||
if (announced) return
|
||||
@@ -56,7 +57,7 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
if (Date.now() >= deadline) throw new Error("Timed out waiting for the background service to start")
|
||||
if (Date.now() >= deadline) throw failure ?? new Error("Timed out waiting for the background service to start")
|
||||
const registration = await registered(options.file, timing.requestTimeout)
|
||||
if (registration.timedOut && registration.info !== undefined) {
|
||||
timeouts = {
|
||||
@@ -68,6 +69,13 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
console.warn("Background service is unresponsive; recovery cannot preserve persistent terminals")
|
||||
await PtyHandoff.clear(options.file ?? fallback())
|
||||
await terminate(registration.info, options, timing)
|
||||
for (const item of contenders) {
|
||||
if (item.child.pid === registration.info.pid || contenderFinished(item)) {
|
||||
item.release()
|
||||
contenders.delete(item)
|
||||
}
|
||||
}
|
||||
failure = undefined
|
||||
timeouts = undefined
|
||||
lastSpawn = Date.now() - spawnDelay
|
||||
}
|
||||
@@ -95,19 +103,27 @@ export async function ensure(options: EnsureOptions = {}): Promise<Endpoint> {
|
||||
file: options.file,
|
||||
pty: service.state === "ready" ? "handoff" : "clear",
|
||||
}).catch(() => undefined)
|
||||
for (const item of contenders) {
|
||||
if (item.child.pid === service.info.pid || contenderFinished(item)) {
|
||||
item.release()
|
||||
contenders.delete(item)
|
||||
}
|
||||
}
|
||||
failure = undefined
|
||||
lastSpawn = 0
|
||||
}
|
||||
} else {
|
||||
if (lastSpawn === 0 && registration.info !== undefined) lastSpawn = Date.now()
|
||||
const finished = [...contenders].filter(contenderFinished)
|
||||
const failure = finished.map(contenderFailure).find((error) => error !== undefined)
|
||||
failure ??= finished.map(contenderFailure).find((error) => error !== undefined)
|
||||
if (finished.some((item) => item.child.exitCode === 0)) {
|
||||
spawnDelay = Math.min(spawnDelay * 2, timing.maxSpawnDelay)
|
||||
}
|
||||
finished.forEach((item) => contenders.delete(item))
|
||||
if (failure !== undefined && contenders.size === 0) throw failure
|
||||
// Keep one candidate plus one lock probe so a pre-lock stall cannot block recovery.
|
||||
if (contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay) {
|
||||
// Keep one candidate plus one lock probe for pre-lock stalls. After a failure, let the
|
||||
// survivors finish without recruiting replacements that could hide the error indefinitely.
|
||||
if (failure === undefined && contenders.size < 2 && Date.now() - lastSpawn >= spawnDelay) {
|
||||
announce("missing")
|
||||
contenders.add(await spawnContender())
|
||||
lastSpawn = Date.now()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises"
|
||||
import { mkdtemp, rename, rm } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
import { waitForExit } from "./service-timing"
|
||||
@@ -8,6 +8,16 @@ export async function serviceFixture() {
|
||||
const registration = join(directory, "service.json")
|
||||
const processes: Bun.Subprocess[] = []
|
||||
const pids = new Set<number>()
|
||||
const starts = async () =>
|
||||
(
|
||||
await Bun.file(registration + ".starts")
|
||||
.text()
|
||||
.catch(() => "")
|
||||
)
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map(Number)
|
||||
const command = (mode: string, ...args: string[]) => [
|
||||
process.execPath,
|
||||
join(import.meta.dir, "service.ts"),
|
||||
@@ -20,6 +30,21 @@ export async function serviceFixture() {
|
||||
directory,
|
||||
registration,
|
||||
command,
|
||||
starts,
|
||||
async waitForStarts(count: number) {
|
||||
for (let attempt = 0; attempt < 600; attempt++) {
|
||||
const result = await starts()
|
||||
if (result.length >= count) return result
|
||||
await Bun.sleep(5)
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${count} contenders`)
|
||||
},
|
||||
async release(pid: number, action: "fail" | "hang" | "ready") {
|
||||
const file = registration + `.release-${pid}`
|
||||
await Bun.write(file + ".tmp", action)
|
||||
await rename(file + ".tmp", file)
|
||||
if (action === "fail") await waitForExit(pid)
|
||||
},
|
||||
spawn(mode: string, ...args: string[]) {
|
||||
const subprocess = Bun.spawn(command(mode, ...args), { stdout: "ignore", stderr: "inherit" })
|
||||
processes.push(subprocess)
|
||||
@@ -47,6 +72,8 @@ export async function serviceFixture() {
|
||||
throw new Error(`Timed out waiting for ${file}`)
|
||||
},
|
||||
async [Symbol.asyncDispose]() {
|
||||
// Include detached contenders that never registered, even when an assertion fails.
|
||||
for (const pid of await starts()) pids.add(pid)
|
||||
await Promise.all([
|
||||
...processes.map(async (subprocess) => {
|
||||
subprocess.kill("SIGTERM")
|
||||
|
||||
@@ -17,6 +17,18 @@ if (mode === "environment") {
|
||||
}
|
||||
if (mode === "signal") process.kill(process.pid, process.platform === "win32" ? "SIGTERM" : "SIGKILL")
|
||||
|
||||
let controlled = ""
|
||||
if (mode === "controlled") {
|
||||
await appendFile(registration + ".starts", process.pid + "\n")
|
||||
const release = registration + `.release-${process.pid}`
|
||||
while (!(await Bun.file(release).exists())) await Bun.sleep(5)
|
||||
controlled = await Bun.file(release).text()
|
||||
if (controlled === "fail") {
|
||||
process.stderr.write("actionable startup failure: storage initialization denied\n")
|
||||
process.exit(23)
|
||||
}
|
||||
}
|
||||
|
||||
if (mode === "delayed" || mode === "delayed-failed" || mode === "coordinated" || mode === "coordinated-failed-loser") {
|
||||
await appendFile(registration + ".starts", process.pid + "\n")
|
||||
const owner = await writeFile(registration + ".owner", String(process.pid), { flag: "wx" })
|
||||
@@ -63,7 +75,7 @@ const server = Bun.serve({
|
||||
return new Response(null, { status: 404 })
|
||||
requests += 1
|
||||
if (mode === "starting") await writeFile(registration + ".status-request", "")
|
||||
if (mode === "hanging") {
|
||||
if (mode === "hanging" || controlled === "hang") {
|
||||
await appendFile(registration + ".requests", process.pid + "\n")
|
||||
return new Promise<Response>(() => {})
|
||||
}
|
||||
@@ -92,8 +104,10 @@ const server = Bun.serve({
|
||||
})
|
||||
|
||||
// Install handlers before publishing: a test may signal as soon as the registration appears.
|
||||
process.on("SIGTERM", () => void shutdown("SIGTERM"))
|
||||
process.on("SIGINT", () => void shutdown("SIGINT"))
|
||||
if (controlled !== "hang") {
|
||||
process.on("SIGTERM", () => void shutdown("SIGTERM"))
|
||||
process.on("SIGINT", () => void shutdown("SIGINT"))
|
||||
}
|
||||
|
||||
await writeFile(
|
||||
registration + ".tmp",
|
||||
|
||||
@@ -112,6 +112,76 @@ test("reports a failed registered service", async () => {
|
||||
)
|
||||
})
|
||||
|
||||
test("reports overlapping contender failures without recruiting replacements", async () => {
|
||||
await using fixture = await serviceFixture()
|
||||
const started = Date.now()
|
||||
const pending = ensure({ file: fixture.registration, version: "test", command: fixture.command("controlled") }).catch(
|
||||
(error: unknown) => error,
|
||||
)
|
||||
const [first, second] = await fixture.waitForStarts(2)
|
||||
await fixture.release(first, "fail")
|
||||
// Let discovery observe the exit across two accelerated spawn windows before the survivor exits.
|
||||
await Bun.sleep(450)
|
||||
await fixture.release(second, "fail")
|
||||
const error = await pending
|
||||
|
||||
expect(Date.now() - started).toBeLessThan(3_000)
|
||||
expect(error).toBeInstanceOf(Error)
|
||||
if (!(error instanceof Error)) throw error
|
||||
expect(error.message).toContain("Server process exited with code 23")
|
||||
expect(error.message).toContain("storage initialization denied")
|
||||
expect(await fixture.starts()).toHaveLength(2)
|
||||
})
|
||||
|
||||
test("retains a contender failure until the deadline while its survivor stalls", async () => {
|
||||
await using fixture = await serviceFixture()
|
||||
const started = Date.now()
|
||||
const pending = ensure({ file: fixture.registration, version: "test", command: fixture.command("controlled") }).catch(
|
||||
(error: unknown) => error,
|
||||
)
|
||||
const [first, second] = await fixture.waitForStarts(2)
|
||||
await fixture.release(first, "fail")
|
||||
const error = await pending
|
||||
|
||||
expect(Date.now() - started).toBeGreaterThanOrEqual(3_000)
|
||||
expect(error).toBeInstanceOf(Error)
|
||||
if (!(error instanceof Error)) throw error
|
||||
expect(error.message).toContain("Server process exited with code 23")
|
||||
expect(error.message).toContain("storage initialization denied")
|
||||
expect(await fixture.starts()).toHaveLength(2)
|
||||
expect(() => process.kill(second, 0)).not.toThrow()
|
||||
})
|
||||
|
||||
test("accepts a surviving contender after a failure without recruiting replacements", async () => {
|
||||
await using fixture = await serviceFixture()
|
||||
const pending = ensure({ file: fixture.registration, version: "test", command: fixture.command("controlled") })
|
||||
const [first, second] = await fixture.waitForStarts(2)
|
||||
await fixture.release(first, "fail")
|
||||
await Bun.sleep(450)
|
||||
await fixture.release(second, "ready")
|
||||
const endpoint = await pending
|
||||
|
||||
expect((await Service.discover({ file: fixture.registration, version: "test" }))?.url).toBe(endpoint.url)
|
||||
expect((await Bun.file(fixture.registration).json()).pid).toBe(second)
|
||||
expect(await fixture.starts()).toHaveLength(2)
|
||||
})
|
||||
|
||||
test("recovers when an unresponsive contender is evicted after a prior failure", async () => {
|
||||
await using fixture = await serviceFixture()
|
||||
const pending = ensure({ file: fixture.registration, version: "test", command: fixture.command("controlled") })
|
||||
const [first, second] = await fixture.waitForStarts(2)
|
||||
await fixture.release(second, "hang")
|
||||
await fixture.waitForFile()
|
||||
await fixture.release(first, "fail")
|
||||
const [, , third] = await fixture.waitForStarts(3)
|
||||
await fixture.release(third, "ready")
|
||||
const endpoint = await pending
|
||||
fixture.track(third)
|
||||
|
||||
expect((await Service.discover({ file: fixture.registration, version: "test" }))?.url).toBe(endpoint.url)
|
||||
expect((await Bun.file(fixture.registration).json()).pid).toBe(third)
|
||||
})
|
||||
|
||||
test("reports a bounded contender stderr tail with native promises", async () => {
|
||||
await using fixture = await serviceFixture()
|
||||
const registration = fixture.registration
|
||||
|
||||
@@ -290,6 +290,76 @@ test("reports a contender that fails to start", async () => {
|
||||
).rejects.toThrow("Server process exited with code 1")
|
||||
})
|
||||
|
||||
test("reports overlapping contender failures without recruiting replacements", async () => {
|
||||
await using fixture = await serviceFixture()
|
||||
const started = Date.now()
|
||||
const pending = run(
|
||||
ensure({ file: fixture.registration, version: "test", command: fixture.command("controlled") }),
|
||||
).catch((error: unknown) => error)
|
||||
const [first, second] = await fixture.waitForStarts(2)
|
||||
await fixture.release(first, "fail")
|
||||
// Let discovery observe the exit across two accelerated spawn windows before the survivor exits.
|
||||
await Bun.sleep(450)
|
||||
await fixture.release(second, "fail")
|
||||
const error = await pending
|
||||
|
||||
expect(Date.now() - started).toBeLessThan(3_000)
|
||||
expect(error).toBeInstanceOf(Error)
|
||||
if (!(error instanceof Error)) throw error
|
||||
expect(error.message).toContain("Server process exited with code 23")
|
||||
expect(error.message).toContain("storage initialization denied")
|
||||
expect(await fixture.starts()).toHaveLength(2)
|
||||
})
|
||||
|
||||
test("retains a contender failure until the deadline while its survivor stalls", async () => {
|
||||
await using fixture = await serviceFixture()
|
||||
const started = Date.now()
|
||||
const pending = run(
|
||||
ensure({ file: fixture.registration, version: "test", command: fixture.command("controlled") }),
|
||||
).catch((error: unknown) => error)
|
||||
const [first, second] = await fixture.waitForStarts(2)
|
||||
await fixture.release(first, "fail")
|
||||
const error = await pending
|
||||
|
||||
expect(Date.now() - started).toBeGreaterThanOrEqual(3_000)
|
||||
expect(error).toBeInstanceOf(Error)
|
||||
if (!(error instanceof Error)) throw error
|
||||
expect(error.message).toContain("Server process exited with code 23")
|
||||
expect(error.message).toContain("storage initialization denied")
|
||||
expect(await fixture.starts()).toHaveLength(2)
|
||||
expect(() => process.kill(second, 0)).not.toThrow()
|
||||
})
|
||||
|
||||
test("accepts a surviving contender after a failure without recruiting replacements", async () => {
|
||||
await using fixture = await serviceFixture()
|
||||
const pending = run(ensure({ file: fixture.registration, version: "test", command: fixture.command("controlled") }))
|
||||
const [first, second] = await fixture.waitForStarts(2)
|
||||
await fixture.release(first, "fail")
|
||||
await Bun.sleep(450)
|
||||
await fixture.release(second, "ready")
|
||||
const endpoint = await pending
|
||||
|
||||
expect((await run(Service.discover({ file: fixture.registration, version: "test" })))?.url).toBe(endpoint.url)
|
||||
expect((await Bun.file(fixture.registration).json()).pid).toBe(second)
|
||||
expect(await fixture.starts()).toHaveLength(2)
|
||||
})
|
||||
|
||||
test("recovers when an unresponsive contender is evicted after a prior failure", async () => {
|
||||
await using fixture = await serviceFixture()
|
||||
const pending = run(ensure({ file: fixture.registration, version: "test", command: fixture.command("controlled") }))
|
||||
const [first, second] = await fixture.waitForStarts(2)
|
||||
await fixture.release(second, "hang")
|
||||
await fixture.waitForFile()
|
||||
await fixture.release(first, "fail")
|
||||
const [, , third] = await fixture.waitForStarts(3)
|
||||
await fixture.release(third, "ready")
|
||||
const endpoint = await pending
|
||||
fixture.track(third)
|
||||
|
||||
expect((await run(Service.discover({ file: fixture.registration, version: "test" })))?.url).toBe(endpoint.url)
|
||||
expect((await Bun.file(fixture.registration).json()).pid).toBe(third)
|
||||
})
|
||||
|
||||
test("reports a bounded contender stderr tail", async () => {
|
||||
await using fixture = await serviceFixture()
|
||||
const registration = fixture.registration
|
||||
|
||||
Reference in New Issue
Block a user