fix(desktop): retry failed storage writes and refuse writes after close (#61804)

This commit is contained in:
Luke Parker
2026-11-04 16:25:22 -06:00
committed by GitHub
Unverified
parent 2e5d757adc
commit 1a2505118f
6 changed files with 484 additions and 40 deletions
@@ -93,6 +93,16 @@ describe("state store", () => {
])
})
// StorageUpdate broadcasts and acknowledges the revision `update` returns, so a closed store must refuse first.
test("refuses updates after close without handing out a revision", () => {
const { store } = open()
expect(store.update("w", { tabs: "[]" }, [])).toBe(1)
store.close()
expect(() => store.update("w", {}, [])).toThrow("Write-behind buffer is closed")
expect(() => store.set("w", "tabs", "[1]")).toThrow("Write-behind buffer is closed")
expect(store.items("w").revision).toBe(1)
})
test("survives close and reopen on disk", async () => {
const root = await mkdtemp(path.join(tmpdir(), "opencode-state-"))
roots.push(root)
+4 -1
View File
@@ -9,7 +9,7 @@ type Row = { name: string; key: string; value: string | null }
// Reads hit SQLite directly: they happen at mount time and a point lookup on the primary key
// costs microseconds, so a second in-memory copy would only duplicate the renderer's cache.
export function createStateStore(db: Database, input: { onError?: (error: unknown) => void } = {}) {
export function createStateStore(db: Database, input: { onError?: (cause: unknown) => void } = {}) {
// Prepared once; the flush loop then only binds values instead of rebuilding SQL per row.
const byKey = and(eq(state.name, sql.placeholder("name")), eq(state.key, sql.placeholder("key")))
const read = db.select({ value: state.value }).from(state).where(byKey).prepare()
@@ -83,6 +83,9 @@ export function createStateStore(db: Database, input: { onError?: (error: unknow
return { items, revision }
},
update(name: string, insert: Record<string, string>, removed: readonly string[]) {
// A revision tells the renderer its rows are saved, so none is handed out once writes are refused.
if (writer.closed()) throw new Error("Write-behind buffer is closed")
for (const [key, value] of Object.entries(insert)) set(name, key, value)
for (const key of removed) unset(name, key)
@@ -0,0 +1,135 @@
import { describe, expect, test } from "bun:test"
import {
initialWriteBehindState,
isWriteBehindClosed,
selectPendingRows,
transitionWriteBehind,
type WriteBehindCommand,
type WriteBehindEvent,
type WriteBehindState,
} from "./write-behind-machine"
const delay = 10
const error = new Error("SQLITE_BUSY")
// Folds events through the machine and keeps the commands of each step.
function replay(events: readonly WriteBehindEvent<number>[], from = initialWriteBehindState<number>()) {
return events.reduce<{ state: WriteBehindState<number>; commands: (readonly WriteBehindCommand<number>[])[] }>(
(result, event) => {
const next = transitionWriteBehind(result.state, event, delay)
return { state: next.state, commands: [...result.commands, next.commands] }
},
{ state: from, commands: [] },
)
}
const set = (key: string, value: number): WriteBehindEvent<number> => ({ type: "set", key, value })
const timer = (attempt: number): WriteBehindEvent<number> => ({ type: "timer.fired", attempt })
const failed = (attempt: number): WriteBehindEvent<number> => ({ type: "write.failed", attempt, error })
const succeeded = (attempt: number): WriteBehindEvent<number> => ({ type: "write.succeeded", attempt })
describe("write-behind machine", () => {
test("schedules a retry for a failed write instead of leaving the rows waiting for another set", () => {
const result = replay([set("a", 1), timer(1), failed(2)])
expect(result.state).toEqual({ status: "scheduled", pending: new Map([["a", 1]]), attempt: 3, failures: 1 })
expect(result.commands).toEqual([
[{ type: "start-timer", attempt: 1, delayMs: delay }],
[{ type: "cancel-timer" }, { type: "write", attempt: 2, batch: new Map([["a", 1]]) }],
[
{ type: "start-timer", attempt: 3, delayMs: delay },
{ type: "report-error", error },
],
])
})
test("backs off up to 64 times the delay, then resets after a success", () => {
const failures = Array.from({ length: 8 }, (_, index) => [timer(1 + index * 2), failed(2 + index * 2)]).flat()
const result = replay([set("a", 1), ...failures, timer(17), succeeded(18), set("b", 1)])
const delays = result.commands
.flat()
.flatMap((command) => (command.type === "start-timer" ? [command.delayMs / delay] : []))
expect(delays).toEqual([1, 1, 2, 4, 8, 16, 32, 64, 64, 1])
})
test.each([
["refuses a set", set("a", 1)],
["ignores a flush", { type: "flush.requested" }],
] satisfies [string, WriteBehindEvent<number>][])("%s after close", (_name, event) => {
const closed = replay([{ type: "close.requested" }]).state
const result = transitionWriteBehind(closed, event, delay)
expect(isWriteBehindClosed(closed)).toBe(true)
expect(result.state).toBe(closed)
expect(result.commands).toEqual(event.type === "set" ? [{ type: "reject-closed" }] : [])
})
test("writes queued rows on close and refuses rows set while that write runs", () => {
const closing = replay([set("a", 1), { type: "close.requested" }])
const late = transitionWriteBehind(closing.state, set("b", 2), delay)
expect(closing.commands.at(-1)).toEqual([
{ type: "cancel-timer" },
{ type: "write", attempt: 2, batch: new Map([["a", 1]]) },
])
expect(isWriteBehindClosed(closing.state)).toBe(true)
expect(late).toEqual({ state: closing.state, commands: [{ type: "reject-closed" }] })
})
test("closes without a retry when the last write fails", () => {
const result = replay([set("a", 1), { type: "close.requested" }, failed(2)])
expect(result.state.status).toBe("closed")
expect(result.commands.at(-1)).toEqual([{ type: "report-error", error }])
})
test("ignores a timer or write result from another attempt", () => {
const scheduled = replay([set("a", 1)]).state
const flushing = replay([{ type: "flush.requested" }], scheduled).state
expect(transitionWriteBehind(scheduled, timer(0), delay).state).toBe(scheduled)
expect(transitionWriteBehind(flushing, succeeded(1), delay).state).toBe(flushing)
expect(transitionWriteBehind(flushing, failed(3), delay).state).toBe(flushing)
expect(transitionWriteBehind(flushing, timer(1), delay).state).toBe(flushing)
})
test("a row set during a write wins over the failed one and is written next", () => {
const result = replay([set("a", 1), set("b", 1), timer(1), set("a", 2), failed(2)])
expect([...selectPendingRows(result.state)]).toEqual([
["a", 2],
["b", 1],
])
})
test("queues rows set during a successful write for the next flush", () => {
const result = replay([set("a", 1), timer(1), set("b", 2), succeeded(2)])
expect(result.state).toEqual({ status: "scheduled", pending: new Map([["b", 2]]), attempt: 3, failures: 0 })
expect(result.commands.at(-1)).toEqual([{ type: "start-timer", attempt: 3, delayMs: delay }])
})
test("stops the timer when a drop clears every queued row", () => {
const partial = replay([set("a", 1), set("b", 2), { type: "drop", predicate: (value) => value === 1 }])
const all = replay([{ type: "drop", predicate: () => true }], partial.state)
expect([...selectPendingRows(partial.state)]).toEqual([["b", 2]])
expect(partial.commands.at(-1)).toEqual([])
expect(all.state.status).toBe("idle")
expect(all.commands).toEqual([[{ type: "cancel-timer" }]])
})
test("a row dropped during a write is not written again when it fails", () => {
const result = replay([set("a", 1), timer(1), { type: "drop", predicate: () => true }, failed(2)])
expect(result.state.status).toBe("idle")
expect(result.commands.at(-1)).toEqual([{ type: "report-error", error }])
})
})
@@ -0,0 +1,214 @@
/** A run of failed writes backs off up to `2 ** 6 = 64` times the base delay. */
export const MAX_RETRY_EXPONENT = 6
/**
* A write-behind buffer: rows wait in `scheduled` for the timer, are written in one batch in `flushing`, and go back
* to `scheduled` with a longer delay when the write fails, so no acknowledged row is left without a pending write.
* `attempt` names the current timer and write; a result for another attempt is stale.
*/
export type WriteBehindState<T> =
| { readonly status: "idle" }
| {
readonly status: "scheduled"
readonly pending: ReadonlyMap<string, T>
readonly attempt: number
/** Consecutive failed writes, for the retry backoff. */
readonly failures: number
}
| {
readonly status: "flushing"
readonly inFlight: ReadonlyMap<string, T>
/** Rows set while the batch is written; they win over `inFlight` for the same key. */
readonly queued: ReadonlyMap<string, T>
readonly attempt: number
readonly failures: number
/** The buffer closes once this write settles. */
readonly closing: boolean
}
| { readonly status: "closed" }
export type WriteBehindEvent<T> =
| { readonly type: "set"; readonly key: string; readonly value: T }
| { readonly type: "drop"; readonly predicate: (value: T) => boolean }
| { readonly type: "timer.fired"; readonly attempt: number }
| { readonly type: "flush.requested" }
| { readonly type: "close.requested" }
| { readonly type: "write.succeeded"; readonly attempt: number }
| { readonly type: "write.failed"; readonly attempt: number; readonly error: unknown }
export type WriteBehindCommand<T> =
| { readonly type: "start-timer"; readonly attempt: number; readonly delayMs: number }
| { readonly type: "cancel-timer" }
| { readonly type: "write"; readonly attempt: number; readonly batch: Map<string, T> }
| { readonly type: "report-error"; readonly error: unknown }
| { readonly type: "reject-closed" }
export type WriteBehindTransition<T> = {
readonly state: WriteBehindState<T>
readonly commands: readonly WriteBehindCommand<T>[]
}
const IDLE = Object.freeze({ status: "idle" as const })
const CLOSED = Object.freeze({ status: "closed" as const })
const EMPTY: ReadonlyMap<string, never> = new Map<string, never>()
export function initialWriteBehindState<T>(): WriteBehindState<T> {
return IDLE
}
export function transitionWriteBehind<T>(
state: WriteBehindState<T>,
event: WriteBehindEvent<T>,
delay: number,
): WriteBehindTransition<T> {
if (state.status === "closed")
return event.type === "set" ? unchanged(state, [{ type: "reject-closed" }]) : unchanged(state)
if (state.status === "idle") {
if (event.type === "close.requested") return { state: CLOSED, commands: [] }
if (event.type !== "set") return unchanged(state)
return {
state: { status: "scheduled", pending: new Map([[event.key, event.value]]), attempt: 1, failures: 0 },
commands: [{ type: "start-timer", attempt: 1, delayMs: delay }],
}
}
if (state.status === "scheduled") return scheduled(state, event)
return flushing(state, event, delay)
}
function scheduled<T>(
state: Extract<WriteBehindState<T>, { status: "scheduled" }>,
event: WriteBehindEvent<T>,
): WriteBehindTransition<T> {
if (event.type === "set")
return { state: { ...state, pending: put(state.pending, event.key, event.value) }, commands: [] }
if (event.type === "drop") {
const pending = without(state.pending, event.predicate)
if (pending === state.pending) return unchanged(state)
if (pending.size === 0) return { state: IDLE, commands: [{ type: "cancel-timer" }] }
return { state: { ...state, pending }, commands: [] }
}
if (event.type === "timer.fired" && event.attempt !== state.attempt) return unchanged(state)
if (event.type === "timer.fired" || event.type === "flush.requested" || event.type === "close.requested") {
const attempt = state.attempt + 1
return {
state: {
status: "flushing",
inFlight: state.pending,
queued: EMPTY,
attempt,
failures: state.failures,
closing: event.type === "close.requested",
},
commands: [{ type: "cancel-timer" }, { type: "write", attempt, batch: new Map(state.pending) }],
}
}
return unchanged(state)
}
function flushing<T>(
state: Extract<WriteBehindState<T>, { status: "flushing" }>,
event: WriteBehindEvent<T>,
delay: number,
): WriteBehindTransition<T> {
if (event.type === "set") {
if (state.closing) return unchanged(state, [{ type: "reject-closed" }])
return { state: { ...state, queued: put(state.queued, event.key, event.value) }, commands: [] }
}
// A cleared row must not come back when a failed batch is queued again.
if (event.type === "drop") {
const inFlight = without(state.inFlight, event.predicate)
const queued = without(state.queued, event.predicate)
if (inFlight === state.inFlight && queued === state.queued) return unchanged(state)
return { state: { ...state, inFlight, queued }, commands: [] }
}
if (event.type === "close.requested")
return state.closing ? unchanged(state) : { state: { ...state, closing: true }, commands: [] }
if (event.type !== "write.succeeded" && event.type !== "write.failed") return unchanged(state)
if (event.attempt !== state.attempt) return unchanged(state)
const attempt = state.attempt + 1
if (event.type === "write.succeeded") {
if (state.closing) return { state: CLOSED, commands: [] }
if (state.queued.size === 0) return { state: IDLE, commands: [] }
return {
state: { status: "scheduled", pending: state.queued, attempt, failures: 0 },
commands: [{ type: "start-timer", attempt, delayMs: delay }],
}
}
const report = { type: "report-error", error: event.error } as const
// The database is closing with the app; there is no later write to retry with.
if (state.closing) return { state: CLOSED, commands: [report] }
// The renderer already saw these rows saved, so they are written again; rows set since then win.
const pending = new Map([...state.inFlight, ...state.queued])
if (pending.size === 0) return { state: IDLE, commands: [report] }
// The retry is scheduled before the error is reported, so a report that throws cannot leave the rows unscheduled.
return {
state: { status: "scheduled", pending, attempt, failures: state.failures + 1 },
commands: [
{ type: "start-timer", attempt, delayMs: delay * 2 ** Math.min(state.failures, MAX_RETRY_EXPONENT) },
report,
],
}
}
function put<T>(map: ReadonlyMap<string, T>, key: string, value: T) {
return new Map(map).set(key, value)
}
function without<T>(map: ReadonlyMap<string, T>, predicate: (value: T) => boolean) {
const kept = new Map([...map].filter(([, value]) => !predicate(value)))
return kept.size === map.size ? map : kept
}
function unchanged<T>(
state: WriteBehindState<T>,
commands: readonly WriteBehindCommand<T>[] = [],
): WriteBehindTransition<T> {
return { state, commands }
}
/** New rows are refused: the buffer closed, or closes once its last write settles. */
export function isWriteBehindClosed<T>(state: WriteBehindState<T>) {
return state.status === "closed" || (state.status === "flushing" && state.closing)
}
/** The rows not yet written, by key; a row set during a write wins over the one being written. */
export function selectPendingRows<T>(state: WriteBehindState<T>): ReadonlyMap<string, T> {
if (state.status === "scheduled") return state.pending
if (state.status === "flushing")
return state.queued.size === 0 ? state.inFlight : new Map([...state.inFlight, ...state.queued])
return EMPTY
}
@@ -1,10 +1,16 @@
import { describe, expect, test } from "bun:test"
import { afterEach, beforeEach, describe, expect, jest, test } from "bun:test"
import { createWriteBehind } from "./write-behind"
const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
describe("write-behind", () => {
test("coalesces a burst into one batch with the latest value per key", async () => {
beforeEach(() => {
jest.useFakeTimers()
})
afterEach(() => {
jest.useRealTimers()
})
test("coalesces a burst into one batch with the latest value per key", () => {
const batches: Map<string, number>[] = []
const writer = createWriteBehind<number>({ delay: 10, write: (batch) => batches.push(batch) })
writer.set("a", 1)
@@ -12,7 +18,7 @@ describe("write-behind", () => {
writer.set("a", 2)
expect(writer.get("a")).toBe(2)
expect(batches).toHaveLength(0)
await wait(30)
jest.advanceTimersByTime(10)
expect(batches).toHaveLength(1)
expect([...batches[0]!]).toEqual([
["a", 2],
@@ -21,7 +27,7 @@ describe("write-behind", () => {
expect(writer.has("a")).toBe(false)
})
test("flush writes immediately and close stops accepting writes", () => {
test("flush writes immediately and close refuses later writes", () => {
const batches: Map<string, number>[] = []
const writer = createWriteBehind<number>({ delay: 1_000, write: (batch) => batches.push(batch) })
writer.set("a", 1)
@@ -30,8 +36,67 @@ describe("write-behind", () => {
writer.set("b", 2)
writer.close()
expect(batches).toHaveLength(2)
writer.set("c", 3)
expect(writer.closed()).toBe(true)
expect(() => writer.set("c", 3)).toThrow("Write-behind buffer is closed")
writer.flush()
expect(batches).toHaveLength(2)
expect(jest.getTimerCount()).toBe(0)
})
test("retries a failed write on its own, backing off until it succeeds", () => {
const errors: unknown[] = []
const batches: Map<string, number>[] = []
const failures = { left: 2 }
const writer = createWriteBehind<number>({
delay: 10,
onError: (error) => errors.push(error),
write: (batch) => {
if (failures.left-- > 0) throw new Error("SQLITE_BUSY")
batches.push(batch)
},
})
writer.set("a", 1)
jest.advanceTimersByTime(10)
expect(errors).toHaveLength(1)
expect(writer.get("a")).toBe(1)
// The first retry waits the base delay, the second twice that.
jest.advanceTimersByTime(10)
expect(errors).toHaveLength(2)
jest.advanceTimersByTime(19)
expect(batches).toEqual([])
jest.advanceTimersByTime(1)
expect(batches).toEqual([new Map([["a", 1]])])
expect(writer.has("a")).toBe(false)
expect(jest.getTimerCount()).toBe(0)
})
test("keeps the retry scheduled when reporting the failure throws", () => {
const batches: Map<string, number>[] = []
const failures = { left: 1 }
const writer = createWriteBehind<number>({
delay: 10,
write: (batch) => {
if (failures.left-- > 0) throw new Error("SQLITE_BUSY")
batches.push(batch)
},
})
writer.set("a", 1)
expect(() => writer.flush()).toThrow("SQLITE_BUSY")
jest.advanceTimersByTime(10)
expect(batches).toEqual([new Map([["a", 1]])])
})
test("stops the timer when every queued row is dropped", () => {
const batches: Map<string, number>[] = []
const writer = createWriteBehind<number>({ delay: 10, write: (batch) => batches.push(batch) })
writer.set("a", 1)
writer.drop(() => true)
expect(jest.getTimerCount()).toBe(0)
jest.advanceTimersByTime(10)
expect(batches).toEqual([])
})
})
@@ -1,50 +1,67 @@
import {
initialWriteBehindState,
isWriteBehindClosed,
selectPendingRows,
transitionWriteBehind,
type WriteBehindCommand,
type WriteBehindEvent,
} from "./write-behind-machine"
// Coalesces a burst of writes into one batch. The latest value per key wins, so a store that
// re-serializes on every mutation costs one row write per flush instead of one per change.
export function createWriteBehind<T>(input: {
delay: number
write: (batch: Map<string, T>) => void
onError?: (error: unknown) => void
onError?: (cause: unknown) => void
}) {
const pending = new Map<string, T>()
let timer: ReturnType<typeof setTimeout> | undefined
let closed = false
const current = { state: initialWriteBehindState<T>() }
// The running flush timer; there is at most one.
const timers = new Set<ReturnType<typeof setTimeout>>()
const flush = () => {
clearTimeout(timer)
timer = undefined
const dispatch = (event: WriteBehindEvent<T>) => {
const next = transitionWriteBehind(current.state, event, input.delay)
current.state = next.state
next.commands.forEach(perform)
}
if (closed || pending.size === 0) return
const batch = new Map(pending)
pending.clear()
const perform = (command: WriteBehindCommand<T>) => {
if (command.type === "write") return dispatch(write(command.attempt, command.batch))
if (command.type === "report-error") {
if (!input.onError) throw command.error
return input.onError(command.error)
}
if (command.type === "reject-closed") throw new Error("Write-behind buffer is closed")
timers.forEach(clearTimeout)
timers.clear()
if (command.type === "start-timer")
timers.add(setTimeout(() => dispatch({ type: "timer.fired", attempt: command.attempt }), command.delayMs))
}
// The write is the store's own SQLite transaction; a failure is a result the machine retries, not a crash.
const write = (attempt: number, batch: Map<string, T>): WriteBehindEvent<T> => {
try {
input.write(batch)
} catch (error) {
// The renderer already saw these writes succeed. Keep them queued so the next flush retries
// them; anything written for the same key since then takes precedence.
for (const [key, value] of batch) if (!pending.has(key)) pending.set(key, value)
if (!input.onError) throw error
input.onError(error)
return { type: "write.succeeded", attempt }
} catch (error) {
return { type: "write.failed", attempt, error }
}
}
return {
get: (key: string) => pending.get(key),
has: (key: string) => pending.has(key),
entries: () => pending.values(),
set(key: string, value: T) {
if (closed) return
pending.set(key, value)
timer ??= setTimeout(flush, input.delay)
},
drop(predicate: (value: T) => boolean) {
for (const [key, value] of pending) if (predicate(value)) pending.delete(key)
},
flush,
close() {
flush()
closed = true
},
get: (key: string) => selectPendingRows(current.state).get(key),
has: (key: string) => selectPendingRows(current.state).has(key),
entries: () => selectPendingRows(current.state).values(),
/** Refuses new rows once closing, so a caller never acknowledges a write that will not happen. */
closed: () => isWriteBehindClosed(current.state),
set: (key: string, value: T) => dispatch({ type: "set", key, value }),
drop: (predicate: (value: T) => boolean) => dispatch({ type: "drop", predicate }),
flush: () => dispatch({ type: "flush.requested" }),
close: () => dispatch({ type: "close.requested" }),
}
}