diff --git a/packages/desktop/src/main/storage/state.test.ts b/packages/desktop/src/main/storage/state.test.ts index b97b8ecaa2e..9d27ca9a7eb 100644 --- a/packages/desktop/src/main/storage/state.test.ts +++ b/packages/desktop/src/main/storage/state.test.ts @@ -93,6 +93,16 @@ describe("state store", () => { ]) }) + // StorageUpdate broadcasts and acknowledges the revision `update` returns, so a closed store must refuse first. + test("refuses updates after close without handing out a revision", () => { + const { store } = open() + expect(store.update("w", { tabs: "[]" }, [])).toBe(1) + store.close() + expect(() => store.update("w", {}, [])).toThrow("Write-behind buffer is closed") + expect(() => store.set("w", "tabs", "[1]")).toThrow("Write-behind buffer is closed") + expect(store.items("w").revision).toBe(1) + }) + test("survives close and reopen on disk", async () => { const root = await mkdtemp(path.join(tmpdir(), "opencode-state-")) roots.push(root) diff --git a/packages/desktop/src/main/storage/state.ts b/packages/desktop/src/main/storage/state.ts index bd175fd073a..3dbdd0d6948 100644 --- a/packages/desktop/src/main/storage/state.ts +++ b/packages/desktop/src/main/storage/state.ts @@ -9,7 +9,7 @@ type Row = { name: string; key: string; value: string | null } // Reads hit SQLite directly: they happen at mount time and a point lookup on the primary key // costs microseconds, so a second in-memory copy would only duplicate the renderer's cache. -export function createStateStore(db: Database, input: { onError?: (error: unknown) => void } = {}) { +export function createStateStore(db: Database, input: { onError?: (cause: unknown) => void } = {}) { // Prepared once; the flush loop then only binds values instead of rebuilding SQL per row. const byKey = and(eq(state.name, sql.placeholder("name")), eq(state.key, sql.placeholder("key"))) const read = db.select({ value: state.value }).from(state).where(byKey).prepare() @@ -83,6 +83,9 @@ export function createStateStore(db: Database, input: { onError?: (error: unknow return { items, revision } }, update(name: string, insert: Record, removed: readonly string[]) { + // A revision tells the renderer its rows are saved, so none is handed out once writes are refused. + if (writer.closed()) throw new Error("Write-behind buffer is closed") + for (const [key, value] of Object.entries(insert)) set(name, key, value) for (const key of removed) unset(name, key) diff --git a/packages/desktop/src/main/storage/write-behind-machine.test.ts b/packages/desktop/src/main/storage/write-behind-machine.test.ts new file mode 100644 index 00000000000..901944a8778 --- /dev/null +++ b/packages/desktop/src/main/storage/write-behind-machine.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, test } from "bun:test" +import { + initialWriteBehindState, + isWriteBehindClosed, + selectPendingRows, + transitionWriteBehind, + type WriteBehindCommand, + type WriteBehindEvent, + type WriteBehindState, +} from "./write-behind-machine" + +const delay = 10 + +const error = new Error("SQLITE_BUSY") + +// Folds events through the machine and keeps the commands of each step. +function replay(events: readonly WriteBehindEvent[], from = initialWriteBehindState()) { + return events.reduce<{ state: WriteBehindState; commands: (readonly WriteBehindCommand[])[] }>( + (result, event) => { + const next = transitionWriteBehind(result.state, event, delay) + + return { state: next.state, commands: [...result.commands, next.commands] } + }, + { state: from, commands: [] }, + ) +} + +const set = (key: string, value: number): WriteBehindEvent => ({ type: "set", key, value }) + +const timer = (attempt: number): WriteBehindEvent => ({ type: "timer.fired", attempt }) + +const failed = (attempt: number): WriteBehindEvent => ({ type: "write.failed", attempt, error }) + +const succeeded = (attempt: number): WriteBehindEvent => ({ type: "write.succeeded", attempt }) + +describe("write-behind machine", () => { + test("schedules a retry for a failed write instead of leaving the rows waiting for another set", () => { + const result = replay([set("a", 1), timer(1), failed(2)]) + + expect(result.state).toEqual({ status: "scheduled", pending: new Map([["a", 1]]), attempt: 3, failures: 1 }) + expect(result.commands).toEqual([ + [{ type: "start-timer", attempt: 1, delayMs: delay }], + [{ type: "cancel-timer" }, { type: "write", attempt: 2, batch: new Map([["a", 1]]) }], + [ + { type: "start-timer", attempt: 3, delayMs: delay }, + { type: "report-error", error }, + ], + ]) + }) + + test("backs off up to 64 times the delay, then resets after a success", () => { + const failures = Array.from({ length: 8 }, (_, index) => [timer(1 + index * 2), failed(2 + index * 2)]).flat() + const result = replay([set("a", 1), ...failures, timer(17), succeeded(18), set("b", 1)]) + + const delays = result.commands + .flat() + .flatMap((command) => (command.type === "start-timer" ? [command.delayMs / delay] : [])) + + expect(delays).toEqual([1, 1, 2, 4, 8, 16, 32, 64, 64, 1]) + }) + + test.each([ + ["refuses a set", set("a", 1)], + ["ignores a flush", { type: "flush.requested" }], + ] satisfies [string, WriteBehindEvent][])("%s after close", (_name, event) => { + const closed = replay([{ type: "close.requested" }]).state + const result = transitionWriteBehind(closed, event, delay) + + expect(isWriteBehindClosed(closed)).toBe(true) + expect(result.state).toBe(closed) + expect(result.commands).toEqual(event.type === "set" ? [{ type: "reject-closed" }] : []) + }) + + test("writes queued rows on close and refuses rows set while that write runs", () => { + const closing = replay([set("a", 1), { type: "close.requested" }]) + const late = transitionWriteBehind(closing.state, set("b", 2), delay) + + expect(closing.commands.at(-1)).toEqual([ + { type: "cancel-timer" }, + { type: "write", attempt: 2, batch: new Map([["a", 1]]) }, + ]) + expect(isWriteBehindClosed(closing.state)).toBe(true) + expect(late).toEqual({ state: closing.state, commands: [{ type: "reject-closed" }] }) + }) + + test("closes without a retry when the last write fails", () => { + const result = replay([set("a", 1), { type: "close.requested" }, failed(2)]) + + expect(result.state.status).toBe("closed") + expect(result.commands.at(-1)).toEqual([{ type: "report-error", error }]) + }) + + test("ignores a timer or write result from another attempt", () => { + const scheduled = replay([set("a", 1)]).state + const flushing = replay([{ type: "flush.requested" }], scheduled).state + + expect(transitionWriteBehind(scheduled, timer(0), delay).state).toBe(scheduled) + expect(transitionWriteBehind(flushing, succeeded(1), delay).state).toBe(flushing) + expect(transitionWriteBehind(flushing, failed(3), delay).state).toBe(flushing) + expect(transitionWriteBehind(flushing, timer(1), delay).state).toBe(flushing) + }) + + test("a row set during a write wins over the failed one and is written next", () => { + const result = replay([set("a", 1), set("b", 1), timer(1), set("a", 2), failed(2)]) + + expect([...selectPendingRows(result.state)]).toEqual([ + ["a", 2], + ["b", 1], + ]) + }) + + test("queues rows set during a successful write for the next flush", () => { + const result = replay([set("a", 1), timer(1), set("b", 2), succeeded(2)]) + + expect(result.state).toEqual({ status: "scheduled", pending: new Map([["b", 2]]), attempt: 3, failures: 0 }) + expect(result.commands.at(-1)).toEqual([{ type: "start-timer", attempt: 3, delayMs: delay }]) + }) + + test("stops the timer when a drop clears every queued row", () => { + const partial = replay([set("a", 1), set("b", 2), { type: "drop", predicate: (value) => value === 1 }]) + const all = replay([{ type: "drop", predicate: () => true }], partial.state) + + expect([...selectPendingRows(partial.state)]).toEqual([["b", 2]]) + expect(partial.commands.at(-1)).toEqual([]) + expect(all.state.status).toBe("idle") + expect(all.commands).toEqual([[{ type: "cancel-timer" }]]) + }) + + test("a row dropped during a write is not written again when it fails", () => { + const result = replay([set("a", 1), timer(1), { type: "drop", predicate: () => true }, failed(2)]) + + expect(result.state.status).toBe("idle") + expect(result.commands.at(-1)).toEqual([{ type: "report-error", error }]) + }) +}) diff --git a/packages/desktop/src/main/storage/write-behind-machine.ts b/packages/desktop/src/main/storage/write-behind-machine.ts new file mode 100644 index 00000000000..a77cee08647 --- /dev/null +++ b/packages/desktop/src/main/storage/write-behind-machine.ts @@ -0,0 +1,214 @@ +/** A run of failed writes backs off up to `2 ** 6 = 64` times the base delay. */ +export const MAX_RETRY_EXPONENT = 6 + +/** + * A write-behind buffer: rows wait in `scheduled` for the timer, are written in one batch in `flushing`, and go back + * to `scheduled` with a longer delay when the write fails, so no acknowledged row is left without a pending write. + * `attempt` names the current timer and write; a result for another attempt is stale. + */ +export type WriteBehindState = + | { readonly status: "idle" } + | { + readonly status: "scheduled" + readonly pending: ReadonlyMap + readonly attempt: number + /** Consecutive failed writes, for the retry backoff. */ + readonly failures: number + } + | { + readonly status: "flushing" + readonly inFlight: ReadonlyMap + /** Rows set while the batch is written; they win over `inFlight` for the same key. */ + readonly queued: ReadonlyMap + readonly attempt: number + readonly failures: number + /** The buffer closes once this write settles. */ + readonly closing: boolean + } + | { readonly status: "closed" } + +export type WriteBehindEvent = + | { readonly type: "set"; readonly key: string; readonly value: T } + | { readonly type: "drop"; readonly predicate: (value: T) => boolean } + | { readonly type: "timer.fired"; readonly attempt: number } + | { readonly type: "flush.requested" } + | { readonly type: "close.requested" } + | { readonly type: "write.succeeded"; readonly attempt: number } + | { readonly type: "write.failed"; readonly attempt: number; readonly error: unknown } + +export type WriteBehindCommand = + | { readonly type: "start-timer"; readonly attempt: number; readonly delayMs: number } + | { readonly type: "cancel-timer" } + | { readonly type: "write"; readonly attempt: number; readonly batch: Map } + | { readonly type: "report-error"; readonly error: unknown } + | { readonly type: "reject-closed" } + +export type WriteBehindTransition = { + readonly state: WriteBehindState + readonly commands: readonly WriteBehindCommand[] +} + +const IDLE = Object.freeze({ status: "idle" as const }) + +const CLOSED = Object.freeze({ status: "closed" as const }) + +const EMPTY: ReadonlyMap = new Map() + +export function initialWriteBehindState(): WriteBehindState { + return IDLE +} + +export function transitionWriteBehind( + state: WriteBehindState, + event: WriteBehindEvent, + delay: number, +): WriteBehindTransition { + if (state.status === "closed") + return event.type === "set" ? unchanged(state, [{ type: "reject-closed" }]) : unchanged(state) + + if (state.status === "idle") { + if (event.type === "close.requested") return { state: CLOSED, commands: [] } + + if (event.type !== "set") return unchanged(state) + + return { + state: { status: "scheduled", pending: new Map([[event.key, event.value]]), attempt: 1, failures: 0 }, + commands: [{ type: "start-timer", attempt: 1, delayMs: delay }], + } + } + + if (state.status === "scheduled") return scheduled(state, event) + + return flushing(state, event, delay) +} + +function scheduled( + state: Extract, { status: "scheduled" }>, + event: WriteBehindEvent, +): WriteBehindTransition { + if (event.type === "set") + return { state: { ...state, pending: put(state.pending, event.key, event.value) }, commands: [] } + + if (event.type === "drop") { + const pending = without(state.pending, event.predicate) + + if (pending === state.pending) return unchanged(state) + + if (pending.size === 0) return { state: IDLE, commands: [{ type: "cancel-timer" }] } + + return { state: { ...state, pending }, commands: [] } + } + + if (event.type === "timer.fired" && event.attempt !== state.attempt) return unchanged(state) + + if (event.type === "timer.fired" || event.type === "flush.requested" || event.type === "close.requested") { + const attempt = state.attempt + 1 + + return { + state: { + status: "flushing", + inFlight: state.pending, + queued: EMPTY, + attempt, + failures: state.failures, + closing: event.type === "close.requested", + }, + commands: [{ type: "cancel-timer" }, { type: "write", attempt, batch: new Map(state.pending) }], + } + } + + return unchanged(state) +} + +function flushing( + state: Extract, { status: "flushing" }>, + event: WriteBehindEvent, + delay: number, +): WriteBehindTransition { + if (event.type === "set") { + if (state.closing) return unchanged(state, [{ type: "reject-closed" }]) + + return { state: { ...state, queued: put(state.queued, event.key, event.value) }, commands: [] } + } + + // A cleared row must not come back when a failed batch is queued again. + if (event.type === "drop") { + const inFlight = without(state.inFlight, event.predicate) + const queued = without(state.queued, event.predicate) + + if (inFlight === state.inFlight && queued === state.queued) return unchanged(state) + + return { state: { ...state, inFlight, queued }, commands: [] } + } + + if (event.type === "close.requested") + return state.closing ? unchanged(state) : { state: { ...state, closing: true }, commands: [] } + + if (event.type !== "write.succeeded" && event.type !== "write.failed") return unchanged(state) + + if (event.attempt !== state.attempt) return unchanged(state) + + const attempt = state.attempt + 1 + + if (event.type === "write.succeeded") { + if (state.closing) return { state: CLOSED, commands: [] } + + if (state.queued.size === 0) return { state: IDLE, commands: [] } + + return { + state: { status: "scheduled", pending: state.queued, attempt, failures: 0 }, + commands: [{ type: "start-timer", attempt, delayMs: delay }], + } + } + + const report = { type: "report-error", error: event.error } as const + + // The database is closing with the app; there is no later write to retry with. + if (state.closing) return { state: CLOSED, commands: [report] } + + // The renderer already saw these rows saved, so they are written again; rows set since then win. + const pending = new Map([...state.inFlight, ...state.queued]) + + if (pending.size === 0) return { state: IDLE, commands: [report] } + + // The retry is scheduled before the error is reported, so a report that throws cannot leave the rows unscheduled. + return { + state: { status: "scheduled", pending, attempt, failures: state.failures + 1 }, + commands: [ + { type: "start-timer", attempt, delayMs: delay * 2 ** Math.min(state.failures, MAX_RETRY_EXPONENT) }, + report, + ], + } +} + +function put(map: ReadonlyMap, key: string, value: T) { + return new Map(map).set(key, value) +} + +function without(map: ReadonlyMap, predicate: (value: T) => boolean) { + const kept = new Map([...map].filter(([, value]) => !predicate(value))) + + return kept.size === map.size ? map : kept +} + +function unchanged( + state: WriteBehindState, + commands: readonly WriteBehindCommand[] = [], +): WriteBehindTransition { + return { state, commands } +} + +/** New rows are refused: the buffer closed, or closes once its last write settles. */ +export function isWriteBehindClosed(state: WriteBehindState) { + return state.status === "closed" || (state.status === "flushing" && state.closing) +} + +/** The rows not yet written, by key; a row set during a write wins over the one being written. */ +export function selectPendingRows(state: WriteBehindState): ReadonlyMap { + if (state.status === "scheduled") return state.pending + + if (state.status === "flushing") + return state.queued.size === 0 ? state.inFlight : new Map([...state.inFlight, ...state.queued]) + + return EMPTY +} diff --git a/packages/desktop/src/main/storage/write-behind.test.ts b/packages/desktop/src/main/storage/write-behind.test.ts index bc02c9e0677..877707d3384 100644 --- a/packages/desktop/src/main/storage/write-behind.test.ts +++ b/packages/desktop/src/main/storage/write-behind.test.ts @@ -1,10 +1,16 @@ -import { describe, expect, test } from "bun:test" +import { afterEach, beforeEach, describe, expect, jest, test } from "bun:test" import { createWriteBehind } from "./write-behind" -const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) - describe("write-behind", () => { - test("coalesces a burst into one batch with the latest value per key", async () => { + beforeEach(() => { + jest.useFakeTimers() + }) + + afterEach(() => { + jest.useRealTimers() + }) + + test("coalesces a burst into one batch with the latest value per key", () => { const batches: Map[] = [] const writer = createWriteBehind({ delay: 10, write: (batch) => batches.push(batch) }) writer.set("a", 1) @@ -12,7 +18,7 @@ describe("write-behind", () => { writer.set("a", 2) expect(writer.get("a")).toBe(2) expect(batches).toHaveLength(0) - await wait(30) + jest.advanceTimersByTime(10) expect(batches).toHaveLength(1) expect([...batches[0]!]).toEqual([ ["a", 2], @@ -21,7 +27,7 @@ describe("write-behind", () => { expect(writer.has("a")).toBe(false) }) - test("flush writes immediately and close stops accepting writes", () => { + test("flush writes immediately and close refuses later writes", () => { const batches: Map[] = [] const writer = createWriteBehind({ delay: 1_000, write: (batch) => batches.push(batch) }) writer.set("a", 1) @@ -30,8 +36,67 @@ describe("write-behind", () => { writer.set("b", 2) writer.close() expect(batches).toHaveLength(2) - writer.set("c", 3) + expect(writer.closed()).toBe(true) + expect(() => writer.set("c", 3)).toThrow("Write-behind buffer is closed") writer.flush() expect(batches).toHaveLength(2) + expect(jest.getTimerCount()).toBe(0) + }) + + test("retries a failed write on its own, backing off until it succeeds", () => { + const errors: unknown[] = [] + const batches: Map[] = [] + const failures = { left: 2 } + + const writer = createWriteBehind({ + delay: 10, + onError: (error) => errors.push(error), + write: (batch) => { + if (failures.left-- > 0) throw new Error("SQLITE_BUSY") + batches.push(batch) + }, + }) + + writer.set("a", 1) + jest.advanceTimersByTime(10) + expect(errors).toHaveLength(1) + expect(writer.get("a")).toBe(1) + // The first retry waits the base delay, the second twice that. + jest.advanceTimersByTime(10) + expect(errors).toHaveLength(2) + jest.advanceTimersByTime(19) + expect(batches).toEqual([]) + jest.advanceTimersByTime(1) + expect(batches).toEqual([new Map([["a", 1]])]) + expect(writer.has("a")).toBe(false) + expect(jest.getTimerCount()).toBe(0) + }) + + test("keeps the retry scheduled when reporting the failure throws", () => { + const batches: Map[] = [] + const failures = { left: 1 } + + const writer = createWriteBehind({ + delay: 10, + write: (batch) => { + if (failures.left-- > 0) throw new Error("SQLITE_BUSY") + batches.push(batch) + }, + }) + + writer.set("a", 1) + expect(() => writer.flush()).toThrow("SQLITE_BUSY") + jest.advanceTimersByTime(10) + expect(batches).toEqual([new Map([["a", 1]])]) + }) + + test("stops the timer when every queued row is dropped", () => { + const batches: Map[] = [] + const writer = createWriteBehind({ delay: 10, write: (batch) => batches.push(batch) }) + writer.set("a", 1) + writer.drop(() => true) + expect(jest.getTimerCount()).toBe(0) + jest.advanceTimersByTime(10) + expect(batches).toEqual([]) }) }) diff --git a/packages/desktop/src/main/storage/write-behind.ts b/packages/desktop/src/main/storage/write-behind.ts index 66aa5a85e88..55d304d102d 100644 --- a/packages/desktop/src/main/storage/write-behind.ts +++ b/packages/desktop/src/main/storage/write-behind.ts @@ -1,50 +1,67 @@ +import { + initialWriteBehindState, + isWriteBehindClosed, + selectPendingRows, + transitionWriteBehind, + type WriteBehindCommand, + type WriteBehindEvent, +} from "./write-behind-machine" + // Coalesces a burst of writes into one batch. The latest value per key wins, so a store that // re-serializes on every mutation costs one row write per flush instead of one per change. export function createWriteBehind(input: { delay: number write: (batch: Map) => void - onError?: (error: unknown) => void + onError?: (cause: unknown) => void }) { - const pending = new Map() - let timer: ReturnType | undefined - let closed = false + const current = { state: initialWriteBehindState() } + // The running flush timer; there is at most one. + const timers = new Set>() - const flush = () => { - clearTimeout(timer) - timer = undefined + const dispatch = (event: WriteBehindEvent) => { + const next = transitionWriteBehind(current.state, event, input.delay) + current.state = next.state + next.commands.forEach(perform) + } - if (closed || pending.size === 0) return - const batch = new Map(pending) - pending.clear() + const perform = (command: WriteBehindCommand) => { + if (command.type === "write") return dispatch(write(command.attempt, command.batch)) + if (command.type === "report-error") { + if (!input.onError) throw command.error + + return input.onError(command.error) + } + + if (command.type === "reject-closed") throw new Error("Write-behind buffer is closed") + + timers.forEach(clearTimeout) + timers.clear() + + if (command.type === "start-timer") + timers.add(setTimeout(() => dispatch({ type: "timer.fired", attempt: command.attempt }), command.delayMs)) + } + + // The write is the store's own SQLite transaction; a failure is a result the machine retries, not a crash. + const write = (attempt: number, batch: Map): WriteBehindEvent => { try { input.write(batch) - } catch (error) { - // The renderer already saw these writes succeed. Keep them queued so the next flush retries - // them; anything written for the same key since then takes precedence. - for (const [key, value] of batch) if (!pending.has(key)) pending.set(key, value) - if (!input.onError) throw error - input.onError(error) + return { type: "write.succeeded", attempt } + } catch (error) { + return { type: "write.failed", attempt, error } } } return { - get: (key: string) => pending.get(key), - has: (key: string) => pending.has(key), - entries: () => pending.values(), - set(key: string, value: T) { - if (closed) return - pending.set(key, value) - timer ??= setTimeout(flush, input.delay) - }, - drop(predicate: (value: T) => boolean) { - for (const [key, value] of pending) if (predicate(value)) pending.delete(key) - }, - flush, - close() { - flush() - closed = true - }, + get: (key: string) => selectPendingRows(current.state).get(key), + has: (key: string) => selectPendingRows(current.state).has(key), + entries: () => selectPendingRows(current.state).values(), + /** Refuses new rows once closing, so a caller never acknowledges a write that will not happen. */ + closed: () => isWriteBehindClosed(current.state), + set: (key: string, value: T) => dispatch({ type: "set", key, value }), + drop: (predicate: (value: T) => boolean) => dispatch({ type: "drop", predicate }), + flush: () => dispatch({ type: "flush.requested" }), + close: () => dispatch({ type: "close.requested" }), } }