From 5d971e5272ca3c3992a7e0443cafcbcfdfc15e6d Mon Sep 17 00:00:00 2001 From: Luke Parker <10430890+Hona@users.noreply.github.com> Date: Mon, 7 Sep 2026 14:17:34 +1000 Subject: [PATCH] perf(app): externalize large draft text into content-addressed chunks (#47706) --- .../app/src/runtime/persistence/README.md | 25 ++ .../src/runtime/persistence/drafts.test.ts | 232 ++++++++++++++++++ .../app/src/runtime/persistence/drafts.ts | 222 +++++++++++++++-- .../app/src/runtime/persistence/persist.ts | 3 + .../app/src/runtime/persistence/storage.ts | 17 +- .../test-browser/draft-history-cache.test.ts | 7 +- .../test-browser/prompt-persistence.test.ts | 27 +- .../desktop/src/main/ipc-handlers/storage.ts | 4 +- .../desktop/src/main/storage/drafts.test.ts | 100 +++++++- packages/desktop/src/main/storage/drafts.ts | 101 ++++++-- .../desktop/src/main/storage/migration.gen.ts | 4 + .../20260907031611_blob-touched/migration.sql | 1 + .../20260907031611_blob-touched/snapshot.json | 141 +++++++++++ packages/desktop/src/main/storage/schema.ts | 4 +- packages/desktop/src/renderer/api-types.ts | 2 +- packages/desktop/src/renderer/api.ts | 2 +- .../desktop/src/shared/ipc-rpc/storage.ts | 3 +- 17 files changed, 842 insertions(+), 53 deletions(-) create mode 100644 packages/app/src/runtime/persistence/drafts.test.ts create mode 100644 packages/desktop/src/main/storage/migration/20260907031611_blob-touched/migration.sql create mode 100644 packages/desktop/src/main/storage/migration/20260907031611_blob-touched/snapshot.json diff --git a/packages/app/src/runtime/persistence/README.md b/packages/app/src/runtime/persistence/README.md index f074056e86a..63df55436d9 100644 --- a/packages/app/src/runtime/persistence/README.md +++ b/packages/app/src/runtime/persistence/README.md @@ -108,3 +108,28 @@ Storage-key relocation (`previousKey`, workspace aliases, draft storage moves) remains separate from schema migration. Draft blob externalization and hydration also remain in the storage adapter: composer codecs receive hydrated references, not raw ID-only blob documents. + +## Large draft content + +Draft documents never carry large text inline. Any string of `draftTextThreshold` +characters or more is split into `draftTextChunk`-sized content-addressed blobs and +stored as `{ blob: { kind: "text", ids: [...] } }`; reads join the chunks again. A +content-keyed cache means unchanged chunks are not hashed or sent on later saves, so +typing after a large paste uploads one chunk per save rather than the paste. Chunk +boundaries never split a surrogate pair, and a failed upload is evicted so the next save +retries it. + +Blob collection is made safe by validation on write, not by timing. A strict document write +is refused while it references blob ids the store does not hold, so the previous document +stays visible; the renderer uploads the missing bytes again from the chunk text or the image +`Blob` it still holds, renames the references to the ids the uploads returned (content hashes +normally, fresh ids on a store without WebCrypto), and publishes. This covers a chunk another +tab collected, and an image the composer kept in its history long after its blob was +collected. The desktop host additionally refreshes `touched_at` for every blob a written +document references and collects only blobs unreferenced and untouched for `blobGrace`, so +repairs stay rare. +`persisted()` hands the draft store the encoded document (`setDocument`) rather than +a serialized string, so the store does not re-parse the full document to externalize +it. Both blob collectors (desktop SQL, browser IndexedDB) keep chunk ids alive. This +follows VS Code's rule that editor content lives in per-resource backups, not in the +state database. diff --git a/packages/app/src/runtime/persistence/drafts.test.ts b/packages/app/src/runtime/persistence/drafts.test.ts new file mode 100644 index 00000000000..d611015da9d --- /dev/null +++ b/packages/app/src/runtime/persistence/drafts.test.ts @@ -0,0 +1,232 @@ +import { describe, expect, test } from "bun:test" +import { createDraftStore, draftTextChunk, draftTextThreshold } from "./drafts" + +function memoryDriver() { + const documents = new Map() + const blobs = new Map() + let puts = 0 + return { + documents, + blobs, + puts: () => puts, + driver: { + get: async (key: string) => documents.get(key) ?? null, + // Like the real stores: report referenced blobs that are not held; a strict write with any + // missing is refused. + set: async (key: string, value: string, strict: boolean) => { + const ids = new Set() + JSON.parse(value, (_key, item) => { + if (item?.blob && typeof item.blob.id === "string") ids.add(item.blob.id) + if (item?.blob && Array.isArray(item.blob.ids)) item.blob.ids.forEach((id: unknown) => ids.add(String(id))) + return item + }) + const missing = [...ids].filter((id) => !blobs.has(id)) + if (!strict || missing.length === 0) documents.set(key, value) + return missing + }, + remove: async (key: string) => void documents.delete(key), + putBlob: async (blob: Blob) => { + puts++ + const id = `blob-${await blob.text().then((text) => Bun.hash(text).toString(16))}` + blobs.set(id, blob) + return id + }, + getBlob: async (id: string) => blobs.get(id) ?? null, + }, + } +} + +const large = "x".repeat(draftTextThreshold) +const paste = Array.from({ length: 3 * draftTextChunk }, (_, i) => String.fromCharCode(97 + (i % 26))).join("") + +describe("draft store text externalization", () => { + test("large strings become chunk lists and small ones stay inline", async () => { + const memory = memoryDriver() + const store = createDraftStore(memory.driver) + await store.setDocument("doc", { + prompt: [ + { type: "text", content: paste }, + { type: "text", content: "hi" }, + ], + }) + const stored = JSON.parse(memory.documents.get("doc")!) + expect(stored.prompt[0].content.blob.kind).toBe("text") + expect(stored.prompt[0].content.blob.ids).toHaveLength(3) + expect(stored.prompt[1].content).toBe("hi") + expect(memory.documents.get("doc")!.length).toBeLessThan(400) + const chunks = await Promise.all( + stored.prompt[0].content.blob.ids.map((id: string) => memory.blobs.get(id)!.text()), + ) + expect(chunks.join("")).toBe(paste) + }) + + test("appending to a large string re-uploads only the final chunk", async () => { + const memory = memoryDriver() + const store = createDraftStore(memory.driver) + await store.setDocument("doc", { prompt: [{ type: "text", content: paste }] }) + expect(memory.puts()).toBe(3) + await store.setDocument("doc", { prompt: [{ type: "text", content: `${paste}!` }] }) + expect(memory.puts()).toBe(4) + await store.setDocument("doc", { prompt: [{ type: "text", content: `${paste}!` }], cursor: 1 }) + expect(memory.puts()).toBe(4) + }) + + test("reads join the chunks again and reuse cached content", async () => { + const memory = memoryDriver() + const store = createDraftStore(memory.driver) + await store.setDocument("doc", { prompt: [{ type: "text", content: paste }] }) + const fresh = createDraftStore(memory.driver) + expect(JSON.parse((await fresh.getItem("doc"))!)).toEqual({ prompt: [{ type: "text", content: paste }] }) + memory.blobs.clear() + expect(JSON.parse((await fresh.getItem("doc"))!)).toEqual({ prompt: [{ type: "text", content: paste }] }) + }) + + test("a missing chunk decodes to empty text instead of failing the document", async () => { + const memory = memoryDriver() + memory.documents.set( + "doc", + JSON.stringify({ prompt: [{ type: "text", content: { blob: { kind: "text", ids: ["gone"] } } }] }), + ) + const store = createDraftStore(memory.driver) + expect(JSON.parse((await store.getItem("doc"))!)).toEqual({ prompt: [{ type: "text", content: "" }] }) + }) + + test("a cached chunk id the store no longer holds is uploaded again on the next save", async () => { + const memory = memoryDriver() + const store = createDraftStore(memory.driver) + await store.setDocument("doc", { prompt: [{ type: "text", content: large }] }) + const [id] = JSON.parse(memory.documents.get("doc")!).prompt[0].content.blob.ids + await store.setDocument("doc", { prompt: [{ type: "text", content: `${large}!` }] }) + // Another tab collected the chunk for `large` while this tab still caches its id. + memory.blobs.clear() + // Undo republishes the cached id; the write reports it missing and the chunk is uploaded again. + await store.setDocument("doc", { prompt: [{ type: "text", content: large }] }) + expect(memory.puts()).toBe(3) + const fresh = createDraftStore(memory.driver) + expect(JSON.parse((await fresh.getItem("doc"))!).prompt[0].content).toBe(large) + expect(memory.blobs.has(id)).toBe(true) + }) + + test("an image reference whose blob was collected is restored from its object url", async () => { + const memory = memoryDriver() + const store = createDraftStore(memory.driver) + const image = await store.putBlob(new Blob([new Uint8Array([1, 2, 3])], { type: "image/png" })) + // The composer kept this reference (for example in its history) while the store collected the bytes. + memory.blobs.clear() + await store.setDocument("doc", { prompt: [{ type: "image", blob: { id: image.id, url: image.url } }] }) + expect(memory.blobs.has(image.id)).toBe(true) + expect(new Uint8Array(await memory.blobs.get(image.id)!.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3])) + }) + + test("the previous document stays visible until missing blobs are restored", async () => { + const memory = memoryDriver() + const store = createDraftStore(memory.driver) + await store.setDocument("doc", { prompt: [{ type: "text", content: large }] }) + await store.setDocument("doc", { prompt: [{ type: "text", content: `${large}!` }] }) + const before = memory.documents.get("doc") + memory.blobs.clear() + // Hold the repair upload: while it is pending, another reader must still see the old document. + const gate = Promise.withResolvers() + const putBlob = memory.driver.putBlob + memory.driver.putBlob = async (blob) => { + await gate.promise + return putBlob(blob) + } + const saving = store.setDocument("doc", { prompt: [{ type: "text", content: large }] }) + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(memory.documents.get("doc")).toBe(before) + gate.resolve() + await saving + expect(JSON.parse(memory.documents.get("doc")!).prompt[0].content.blob.ids).toHaveLength(1) + const fresh = createDraftStore(memory.driver) + expect(JSON.parse((await fresh.getItem("doc"))!).prompt[0].content).toBe(large) + }) + + test("references are renamed when a restored blob comes back under a different id", async () => { + const memory = memoryDriver() + // A store without WebCrypto assigns a fresh id to every upload. + let counter = 0 + memory.driver.putBlob = async (blob) => { + const id = `random-${counter++}` + memory.blobs.set(id, blob) + return id + } + const store = createDraftStore(memory.driver) + const image = await store.putBlob(new Blob([new Uint8Array([7])], { type: "image/png" })) + await store.setDocument("doc", { + prompt: [ + { type: "text", content: paste }, + { type: "image", blob: image }, + ], + }) + const original = JSON.parse(memory.documents.get("doc")!) + memory.blobs.clear() + await store.setDocument("doc", { + prompt: [ + { type: "text", content: paste }, + { type: "image", blob: image }, + ], + }) + const restored = JSON.parse(memory.documents.get("doc")!) + expect(restored.prompt[0].content.blob.ids).not.toEqual(original.prompt[0].content.blob.ids) + expect(restored.prompt[1].blob.id).not.toBe(original.prompt[1].blob.id) + for (const id of [...restored.prompt[0].content.blob.ids, restored.prompt[1].blob.id]) + expect(memory.blobs.has(id)).toBe(true) + const fresh = createDraftStore(memory.driver) + const read = JSON.parse((await fresh.getItem("doc"))!) + expect(read.prompt[0].content).toBe(paste) + expect(read.prompt[1].blob.id).toBe(restored.prompt[1].blob.id) + // The renamed ids are what later encodes publish, so saves of the still-live references (the + // composer keeps the original image id) upload nothing and keep one stable image id. + const puts = counter + for (const cursor of [1, 2, 3]) { + await store.setDocument("doc", { + prompt: [ + { type: "text", content: paste }, + { type: "image", blob: image }, + ], + cursor, + }) + expect(JSON.parse(memory.documents.get("doc")!).prompt[1].blob.id).toBe(restored.prompt[1].blob.id) + } + expect(counter).toBe(puts) + }) + + test("chunk boundaries never split a surrogate pair", async () => { + const memory = memoryDriver() + const store = createDraftStore(memory.driver) + const text = "x".repeat(draftTextChunk - 1) + "😀tail" + await store.setDocument("doc", { prompt: [{ type: "text", content: text }] }) + const stored = JSON.parse(memory.documents.get("doc")!) + const bytes = await Promise.all(stored.prompt[0].content.blob.ids.map((id: string) => memory.blobs.get(id)!.text())) + expect(bytes.join("")).toBe(text) + expect(bytes[0]!.length).toBe(draftTextChunk + 1) + const fresh = createDraftStore(memory.driver) + expect(JSON.parse((await fresh.getItem("doc"))!).prompt[0].content).toBe(text) + }) + + test("a failed chunk upload is retried on the next save instead of being reused", async () => { + const memory = memoryDriver() + let failNext = true + const putBlob = memory.driver.putBlob + memory.driver.putBlob = async (blob) => { + if (failNext) { + failNext = false + throw new Error("offline") + } + return putBlob(blob) + } + const store = createDraftStore(memory.driver) + await expect(store.setDocument("doc", { prompt: [{ type: "text", content: paste }] })).rejects.toThrow("offline") + await store.setDocument("doc", { prompt: [{ type: "text", content: `${paste}!` }] }) + const fresh = createDraftStore(memory.driver) + expect(JSON.parse((await fresh.getItem("doc"))!).prompt[0].content).toBe(`${paste}!`) + }) + + test("setItem still accepts a serialized document", async () => { + const memory = memoryDriver() + const store = createDraftStore(memory.driver) + await store.setItem("doc", JSON.stringify({ prompt: [{ type: "text", content: large }] })) + expect(JSON.parse(memory.documents.get("doc")!).prompt[0].content.blob.ids).toHaveLength(1) + }) +}) diff --git a/packages/app/src/runtime/persistence/drafts.ts b/packages/app/src/runtime/persistence/drafts.ts index 544f21ddd3d..d45949a17a2 100644 --- a/packages/app/src/runtime/persistence/drafts.ts +++ b/packages/app/src/runtime/persistence/drafts.ts @@ -5,20 +5,43 @@ export type BlobReference = { id: string; url: string } type Driver = { get(key: string): Promise - set(key: string, value: string): Promise + /** + * Store the document and report every blob id it references that the store does not hold. A + * strict write is refused (nothing stored) when any are missing, so a document is never visible + * with dangling references while its blobs are being restored. + */ + set(key: string, value: string, strict: boolean): Promise remove(key: string): Promise putBlob(blob: Blob): Promise getBlob(id: string): Promise } -export type DraftStore = AsyncStorage & { putBlob(blob: Blob): Promise } +export type DraftStore = AsyncStorage & { + putBlob(blob: Blob): Promise + /** Persist an already-encoded document without re-parsing its serialized form. */ + setDocument(key: string, document: unknown): Promise +} + +// Strings at least this long leave the document as fixed-size content-addressed chunks. Typing +// after a large paste changes only the final chunk, so a save uploads one chunk, not the paste. +export const draftTextThreshold = 16 * 1024 +export const draftTextChunk = 64 * 1024 +const textCacheLimit = 64 + const urls = new Map() +// The object URL already pins the Blob for the page's lifetime; keeping the Blob itself lets a +// collected image be uploaded again without fetching the URL. +const held = new Map() +// Image ids that were restored under a different id (a store without WebCrypto assigns fresh +// ones); live references still carry the original. +const aliases = new Map() function blobUrl(id: string, blob: Blob) { const existing = urls.get(id) if (existing) return existing const url = URL.createObjectURL(blob) urls.set(id, url) + held.set(id, blob) return url } @@ -56,25 +79,82 @@ export function createDraftStore(driver: Driver): DraftStore { const id = await driver.putBlob(blob) return { id, url: blobUrl(id, blob) } } - const encode = async (value: unknown): Promise => { - if (Array.isArray(value)) return Promise.all(value.map(encode)) + // Keyed by chunk content so unchanged chunks are never hashed or sent again while the draft is + // edited. Bounded because each entry pins up to draftTextChunk characters. A hit is safe even if + // the store has since collected the blob: the write reports it missing and it is uploaded again. + const chunkIds = new Map>() + const chunks = new Map() + const remember = (cache: Map, key: string, value: V) => { + cache.set(key, value) + if (cache.size > textCacheLimit) cache.delete(cache.keys().next().value!) + return value + } + const upload = (chunk: string) => { + const id = driver.putBlob(new Blob([chunk])).then( + (id) => { + remember(chunks, id, chunk) + return id + }, + (error: unknown) => { + // A failed upload must not be reused as the answer for this content on later saves. + if (chunkIds.get(chunk) === id) chunkIds.delete(chunk) + throw error + }, + ) + return remember(chunkIds, chunk, id) + } + const externalize = (text: string) => Promise.all(split(text).map((chunk) => chunkIds.get(chunk) ?? upload(chunk))) + const loadChunk = async (id: string) => { + const cached = chunks.get(id) + if (cached !== undefined) return cached + const blob = await driver.getBlob(id) + // A missing chunk loses that text but keeps the rest of the document decodable. + return remember(chunks, id, blob ? await blob.text() : "") + } + // `sources` collects, for every blob id the encoded document references, a way to produce its + // bytes again: the chunk text itself, or the image Blob (or object URL) the reference carries. + type Sources = Map Promise; chunk?: string }> + const encode = async (value: unknown, sources: Sources): Promise => { + if (typeof value === "string" && value.length >= draftTextThreshold) { + const pieces = split(value) + const ids = await externalize(value) + ids.forEach((id, index) => + sources.set(id, { blob: async () => new Blob([pieces[index]!]), chunk: pieces[index] }), + ) + return { blob: { kind: "text", ids } } + } + if (Array.isArray(value)) return Promise.all(value.map((entry) => encode(entry, sources))) if (!value || typeof value !== "object") return value const item = value as Record if (item.type === "image" && typeof item.dataUrl === "string") { const blob = await fetch(item.dataUrl).then((response) => response.blob()) const { dataUrl: _, ...rest } = item - return { ...rest, blob: { id: await driver.putBlob(blob) } } + const id = await driver.putBlob(blob) + sources.set(id, { blob: async () => blob }) + return { ...rest, blob: { id } } } if ("blob" in item && item.blob && typeof item.blob === "object") { const blob = item.blob as Record + if (blob.kind === "text") return item if (typeof blob.id === "string" && blob.id.startsWith("data:")) { const data = await fetch(blob.id).then((response) => response.blob()) - return { ...item, blob: { id: await driver.putBlob(data) } } + const id = await driver.putBlob(data) + sources.set(id, { blob: async () => data }) + return { ...item, blob: { id } } + } + if (typeof blob.id === "string") { + // A live reference keeps the id it was created with; publish the id its bytes now live under. + const id = aliases.get(blob.id) ?? blob.id + const kept = held.get(id) + const url = typeof blob.url === "string" ? blob.url : urls.get(id) + if (kept) sources.set(id, { blob: async () => kept }) + else if (url) sources.set(id, { blob: () => fetch(url).then((response) => response.blob()) }) + return { ...item, blob: { id } } } return { ...item, blob: { id: blob.id } } } return Object.fromEntries( - await Promise.all(Object.entries(item).map(async ([key, entry]) => [key, await encode(entry)])), + await Promise.all(Object.entries(item).map(async ([key, entry]) => [key, await encode(entry, sources)])), ) } const decode = async (value: unknown): Promise => { @@ -83,6 +163,9 @@ export function createDraftStore(driver: Driver): DraftStore { const item = value as Record if (item.blob && typeof item.blob === "object") { const ref = item.blob as Record + if (ref.kind === "text" && Array.isArray(ref.ids)) { + return (await Promise.all(ref.ids.map((id) => loadChunk(String(id))))).join("") + } if (typeof ref.id === "string") { const url = await loadBlobUrl(ref.id) if (url) return { ...item, blob: { id: ref.id, url } } @@ -92,6 +175,65 @@ export function createDraftStore(driver: Driver): DraftStore { await Promise.all(Object.entries(item).map(async ([key, entry]) => [key, await decode(entry)])), ) } + // Upload the bytes behind `ids` again and return the ids they were stored under. Ids are + // usually content hashes and come back unchanged, but a store without WebCrypto assigns fresh + // ones, so callers must rename references rather than assume. + const restore = async (ids: readonly string[], sources: Sources) => { + const renamed = new Map() + await Promise.all( + ids.map(async (id) => { + const source = sources.get(id) + if (!source) return + const blob = await source.blob() + const next = await driver.putBlob(blob) + renamed.set(id, next) + if (source.chunk !== undefined) { + remember(chunkIds, source.chunk, Promise.resolve(next)) + remember(chunks, next, source.chunk) + return + } + held.set(next, blob) + blobUrl(next, blob) + if (next === id) return + // Later encodes of the still-live reference resolve straight to the new id. Re-point any + // earlier alias chain so lookups stay one step. + for (const [from, to] of aliases) if (to === id) aliases.set(from, next) + aliases.set(id, next) + }), + ) + return renamed + } + const rename = (value: unknown, renamed: Map): unknown => { + if (Array.isArray(value)) return value.map((entry) => rename(entry, renamed)) + if (!value || typeof value !== "object") return value + const item = value as Record + if (item.blob && typeof item.blob === "object") { + const ref = item.blob as Record + if (Array.isArray(ref.ids)) + return { ...item, blob: { ...ref, ids: ref.ids.map((id) => renamed.get(String(id)) ?? id) } } + if (typeof ref.id === "string") return { ...item, blob: { ...ref, id: renamed.get(ref.id) ?? ref.id } } + } + return Object.fromEntries(Object.entries(item).map(([key, entry]) => [key, rename(entry, renamed)])) + } + const setDocument = async (key: string, document: unknown) => { + const version = (versions.get(key) ?? 0) + 1 + versions.set(key, version) + const sources: Sources = new Map() + const encoded = await encode(document, sources) + if (versions.get(key) !== version) return + // The store refuses the write while any referenced blob is missing, so the previous document + // stays visible until the bytes are back. Covers a blob collected while a cache, another tab, + // or the composer's history still held its id. + const missing = await driver.set(key, JSON.stringify(encoded), true) + if (missing.length === 0) return + const renamed = await restore(missing, sources) + if (versions.get(key) !== version) return + const unrestored = missing.filter((id) => !renamed.has(id)) + if (unrestored.length) + console.error(`[persistence] draft ${key} references blobs with no bytes to restore`, unrestored) + // Anything still missing has no bytes anywhere; the owning codec drops such references on read. + await driver.set(key, JSON.stringify(rename(encoded, renamed)), false) + } return { getItem: async (key) => { const value = await driver.get(key) @@ -101,12 +243,8 @@ export function createDraftStore(driver: Driver): DraftStore { if (Option.isNone(parsed)) return value return JSON.stringify(await decode(parsed.value)) }, - setItem: async (key, value) => { - const version = (versions.get(key) ?? 0) + 1 - versions.set(key, version) - const encoded = JSON.stringify(await encode(JSON.parse(value))) - if (versions.get(key) === version) await driver.set(key, encoded) - }, + setItem: (key, value) => setDocument(key, JSON.parse(value)), + setDocument, removeItem: async (key) => { versions.set(key, (versions.get(key) ?? 0) + 1) await driver.remove(key) @@ -115,6 +253,20 @@ export function createDraftStore(driver: Driver): DraftStore { } } +// Fixed-size pieces, except that a piece never ends between the two halves of a surrogate pair: +// each piece becomes its own Blob, and an unpaired surrogate would be encoded as U+FFFD. +function split(text: string) { + const pieces: string[] = [] + for (let start = 0; start < text.length; ) { + const end = Math.min(start + draftTextChunk, text.length) + const code = text.charCodeAt(end - 1) + const stop = end < text.length && code >= 0xd800 && code <= 0xdbff ? end + 1 : end + pieces.push(text.slice(start, stop)) + start = stop + } + return pieces +} + export function createBrowserDraftStore(): DraftStore { const request = indexedDB.open("opencode-drafts", 1) request.addEventListener("upgradeneeded", () => { @@ -127,11 +279,7 @@ export function createBrowserDraftStore(): DraftStore { const transaction = database.transaction(["documents", "blobs"], "readwrite") const documents = transaction.objectStore("documents").getAll() documents.addEventListener("success", () => { - const used = new Set() - JSON.parse(`[${documents.result.join(",")}]`, (_key, item) => { - if (item?.blob && typeof item.blob.id === "string") used.add(item.blob.id) - return item - }) + const used = referenced(`[${documents.result.join(",")}]`) const store = transaction.objectStore("blobs") const blobs = store.openKeyCursor() blobs.addEventListener("success", () => { @@ -164,7 +312,32 @@ export function createBrowserDraftStore(): DraftStore { } return createDraftStore({ get: async (key) => ((await get("documents", key)) as string | undefined) ?? null, - set: (key, value) => write("documents", key, value), + set: async (key, value, strict) => { + // One readwrite transaction over both stores: IndexedDB serialises overlapping readwrite + // transactions in creation order, so a later save or removal cannot commit between the + // reference check and this write. The put is issued from the last lookup's callback so the + // transaction is never left without a pending request. + const ids = [...referenced(value)] + const transaction = (await db).transaction(["blobs", "documents"], "readwrite") + const missing: string[] = [] + const publish = () => { + if (!strict || missing.length === 0) transaction.objectStore("documents").put(value, key) + } + let remaining = ids.length + if (remaining === 0) publish() + for (const id of ids) { + const lookup = transaction.objectStore("blobs").getKey(id) + lookup.addEventListener("success", () => { + if (lookup.result === undefined) missing.push(id) + if (--remaining === 0) publish() + }) + } + return new Promise((resolve, reject) => { + transaction.addEventListener("complete", () => resolve(missing)) + transaction.addEventListener("error", () => reject(transaction.error)) + transaction.addEventListener("abort", () => reject(transaction.error)) + }) + }, remove: (key) => write("documents", key), putBlob: async (blob) => { const id = await blobID(blob) @@ -175,6 +348,17 @@ export function createBrowserDraftStore(): DraftStore { }) } +// Every blob id a serialized document (or array of documents) references. +function referenced(json: string) { + const ids = new Set() + JSON.parse(json, (_key, item) => { + if (item?.blob && typeof item.blob.id === "string") ids.add(item.blob.id) + if (item?.blob && Array.isArray(item.blob.ids)) item.blob.ids.forEach((id: unknown) => ids.add(String(id))) + return item + }) + return ids +} + export async function blobDataUrl(blob: BlobReference, mime: string) { const data = await fetch(blob.url).then((response) => response.blob()) return new Promise((resolve, reject) => { diff --git a/packages/app/src/runtime/persistence/persist.ts b/packages/app/src/runtime/persistence/persist.ts index 7dfce845983..119fbbefe6b 100644 --- a/packages/app/src/runtime/persistence/persist.ts +++ b/packages/app/src/runtime/persistence/persist.ts @@ -32,6 +32,8 @@ export function persistStore(input: { deserialize: (raw: string) => T sync?: PersistenceSyncAPI delay?: number + /** Replaces `storage.setItem` for stores whose storage can take the value itself. */ + write?: (value: T, serialized: string) => void }) { const delay = input.delay ?? persistSaveDelay let dirty = false @@ -57,6 +59,7 @@ export function persistStore(input: { } last = next input.sync?.[1](input.name, next) + if (input.write) return input.write(input.store, next) void input.storage.setItem(input.name, next) } diff --git a/packages/app/src/runtime/persistence/storage.ts b/packages/app/src/runtime/persistence/storage.ts index 195b2fee68a..e23489426b5 100644 --- a/packages/app/src/runtime/persistence/storage.ts +++ b/packages/app/src/runtime/persistence/storage.ts @@ -484,6 +484,7 @@ export function persisted>( const initialized = Persistence.withInitial(schema, initial) const json = Schema.fromJsonString(initialized) const decode = Schema.decodeUnknownOption(json) + const encode = Schema.encodeSync(initialized) const serialize = Schema.encodeSync(json) const normalize = (raw: string) => { const value = decode(raw) @@ -492,10 +493,12 @@ export function persisted>( const store = createStore(Schema.decodeUnknownSync(Schema.toType(initialized))(initial)) const isDesktop = platform.platform === "desktop" && !!platform.storage const draft = config.draft ? platform.draftStore : undefined + const prefix = `${config.storage ?? "default"}:` + // The newest serialized draft, replayed into storage if a slow load finishes after an edit. + let draftLatest: string | undefined const currentStorage = (() => { if (draft) { - const prefix = `${config.storage ?? "default"}:` return { getItem: (key: string) => draft.getItem(prefix + key), setItem: (key: string, value: string) => draft.setItem(prefix + key, value), @@ -557,7 +560,6 @@ export function persisted>( ] .filter((source): source is { storage: SyncStorage | AsyncStorage; key?: string } => !!source?.storage) .map((source) => ({ ...source, storage: toAsyncStorage(source.storage) })) - let draftLatest: string | undefined const api: AsyncStorage = { getItem: async (key) => { @@ -602,6 +604,17 @@ export function persisted>( serialize, deserialize: Schema.decodeUnknownSync(json), sync: channel ? messageSync(channel) : undefined, + // Drafts take the encoded document itself so large text is externalized without the store + // re-parsing the serialized form on every save. + write: draft + ? (value, serialized) => { + draftLatest = serialized + // A failed chunk upload is retried by the next save; see drafts.ts. + void draft + .setDocument(prefix + config.key, encode(value)) + .catch((error: unknown) => console.error(`[persistence] draft write failed for ${config.key}`, error)) + } + : undefined, }) const state = store[0] const setState = persist.setStore diff --git a/packages/app/test-browser/draft-history-cache.test.ts b/packages/app/test-browser/draft-history-cache.test.ts index 3a29415dce4..61edf87c0e6 100644 --- a/packages/app/test-browser/draft-history-cache.test.ts +++ b/packages/app/test-browser/draft-history-cache.test.ts @@ -9,7 +9,10 @@ function fixture(id: string, getBlob: () => Promise) { ]) const store = createDraftStore({ get: async (key) => documents.get(key) ?? null, - set: async (key, value) => void documents.set(key, value), + set: async (key, value) => { + documents.set(key, value) + return [] + }, remove: async (key) => void documents.delete(key), putBlob: async () => id, getBlob, @@ -113,7 +116,7 @@ test("keeps different blob IDs independent", async () => { const reads: string[] = [] const store = createDraftStore({ get: async () => JSON.stringify(["history-cache-first", "history-cache-second"].map((id) => ({ blob: { id } }))), - set: async () => {}, + set: async () => [], remove: async () => {}, putBlob: async () => "unused", getBlob: async (id) => { diff --git a/packages/app/test-browser/prompt-persistence.test.ts b/packages/app/test-browser/prompt-persistence.test.ts index 09ab3d8c584..7f3e70da679 100644 --- a/packages/app/test-browser/prompt-persistence.test.ts +++ b/packages/app/test-browser/prompt-persistence.test.ts @@ -40,7 +40,7 @@ describe("prompt persistence", () => { async (raw) => { const store = createDraftStore({ get: async () => raw, - set: async () => undefined, + set: async () => [], remove: async () => undefined, putBlob: async () => "unused", getBlob: async () => null, @@ -68,7 +68,10 @@ describe("prompt persistence", () => { const blobs = new Map() const store = createDraftStore({ get: async (key) => documents.get(key) ?? null, - set: async (key, value) => void documents.set(key, value), + set: async (key, value) => { + documents.set(key, value) + return [] + }, remove: async (key) => void documents.delete(key), putBlob: async (blob) => { blobs.set("composer-image", blob) @@ -170,7 +173,10 @@ describe("prompt persistence", () => { const documents = new Map() const store = createDraftStore({ get: async (key) => documents.get(key) ?? null, - set: async (key, value) => void documents.set(key, value), + set: async (key, value) => { + documents.set(key, value) + return [] + }, remove: async (key) => void documents.delete(key), putBlob: async () => "blob", getBlob: async () => null, @@ -201,7 +207,10 @@ describe("prompt persistence", () => { const documents = new Map() const store = createDraftStore({ get: async (key) => documents.get(key) ?? null, - set: async (key, value) => void documents.set(key, value), + set: async (key, value) => { + documents.set(key, value) + return [] + }, remove: async (key) => void documents.delete(key), putBlob: async () => "blob", getBlob: async () => null, @@ -233,7 +242,10 @@ test("moves image data URLs into blobs and hydrates object URLs", async () => { const blobs = new Map() const store = createDraftStore({ get: async (key) => documents.get(key) ?? null, - set: async (key, value) => void documents.set(key, value), + set: async (key, value) => { + documents.set(key, value) + return [] + }, remove: async (key) => void documents.delete(key), putBlob: async (blob) => { const id = String(blob.size) @@ -255,7 +267,10 @@ test("does not let delayed blob migration overwrite a newer draft", async () => const migration = Promise.withResolvers() const store = createDraftStore({ get: async () => null, - set: async (key, value) => void documents.set(key, value), + set: async (key, value) => { + documents.set(key, value) + return [] + }, remove: async () => undefined, putBlob: async () => { await migration.promise diff --git a/packages/desktop/src/main/ipc-handlers/storage.ts b/packages/desktop/src/main/ipc-handlers/storage.ts index 62018d45d29..f5d2c294776 100644 --- a/packages/desktop/src/main/ipc-handlers/storage.ts +++ b/packages/desktop/src/main/ipc-handlers/storage.ts @@ -26,8 +26,8 @@ export const storageHandlers = StorageRpcs.toLayer( }), StorageClear: ({ name }) => Effect.sync(() => storage.state.clear(name)), DraftsGet: ({ key }) => Effect.sync(() => storage.drafts.get(key)), - DraftsSet: ({ key, value }) => Effect.sync(() => storage.drafts.set(key, value)), - DraftsDelete: ({ key }) => Effect.sync(() => storage.drafts.set(key, null)), + DraftsSet: ({ key, value, strict }) => Effect.sync(() => storage.drafts.set(key, value, strict)), + DraftsDelete: ({ key }) => Effect.sync(() => void storage.drafts.set(key, null)), DraftsPutBlob: ({ data }) => Effect.sync(() => storage.drafts.putBlob(data)), DraftsGetBlob: ({ id }) => Effect.sync(() => storage.drafts.getBlob(id)), }) diff --git a/packages/desktop/src/main/storage/drafts.test.ts b/packages/desktop/src/main/storage/drafts.test.ts index d39706490a4..da61d73624b 100644 --- a/packages/desktop/src/main/storage/drafts.test.ts +++ b/packages/desktop/src/main/storage/drafts.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test" import { sql } from "drizzle-orm" import { openDatabase } from "./database" -import { createDraftStore } from "./drafts" +import { blobGrace, createDraftStore } from "./drafts" describe("draft store", () => { test("queues documents and reads them back before and after flush", () => { @@ -29,4 +29,102 @@ describe("draft store", () => { expect(second.getBlob(used)).toEqual(new Uint8Array([1, 2, 3])) expect(second.getBlob(unused)).toBeNull() }) + + test("collects a retired chunk only once it is unreferenced and past the grace period", () => { + const database = openDatabase(":memory:") + let clock = 0 + const drafts = createDraftStore(database.db, { delay: 1_000, now: () => clock }) + const text = (...ids: string[]) => + JSON.stringify({ prompt: [{ type: "text", content: { blob: { kind: "text", ids } } }] }) + const a = drafts.putBlob(new TextEncoder().encode("chunk a")) + const b = drafts.putBlob(new TextEncoder().encode("chunk b")) + drafts.set("doc", text(a, b)) + drafts.flush() + const c = drafts.putBlob(new TextEncoder().encode("chunk c")) + drafts.set("doc", text(a, c)) + drafts.flush() + // Retired but recently touched: survives a due collection. + clock = 120_000 + drafts.putBlob(new TextEncoder().encode("chunk d")) + drafts.set("other", "{}") + drafts.flush() + expect(drafts.getBlob(b)).not.toBeNull() + // Past the grace period and still unreferenced: collected. Referenced chunks stay. + clock = 120_000 + blobGrace + 1 + drafts.putBlob(new TextEncoder().encode("chunk e")) + drafts.set("other", "{}") + drafts.flush() + expect(drafts.getBlob(a)).not.toBeNull() + expect(drafts.getBlob(c)).not.toBeNull() + expect(drafts.getBlob(b)).toBeNull() + }) + + test("a document that republishes a cached chunk id refreshes the chunk without an upload", () => { + const database = openDatabase(":memory:") + let clock = 0 + const drafts = createDraftStore(database.db, { delay: 1_000, now: () => clock }) + const text = (...ids: string[]) => + JSON.stringify({ prompt: [{ type: "text", content: { blob: { kind: "text", ids } } }] }) + const a = drafts.putBlob(new TextEncoder().encode("A")) + drafts.set("doc", text(a)) + drafts.flush() + // Edit away from A; A becomes unreferenced. + const b = drafts.putBlob(new TextEncoder().encode("A!")) + drafts.set("doc", text(b)) + drafts.flush() + // Long after, undo republishes A from the renderer cache with no upload. The write touches A. + clock = blobGrace - 1 + drafts.set("doc", text(a)) + drafts.flush() + clock = blobGrace + collectInterval + drafts.putBlob(new TextEncoder().encode("unrelated")) + drafts.set("other", "{}") + drafts.flush() + expect(drafts.getBlob(a)).not.toBeNull() + expect(drafts.getBlob(b)).toBeNull() + }) + + test("set reports referenced blobs the store does not hold so the renderer can upload them again", () => { + const database = openDatabase(":memory:") + const drafts = createDraftStore(database.db, { delay: 1_000 }) + const kept = drafts.putBlob(new TextEncoder().encode("kept")) + const image = drafts.putBlob(new Uint8Array([9])) + const document = JSON.stringify({ + prompt: [ + { type: "text", content: { blob: { kind: "text", ids: [kept, "gone-chunk"] } } }, + { type: "image", blob: { id: image } }, + { type: "image", blob: { id: "gone-image" } }, + ], + }) + // A strict write with missing blobs is refused: the previous document remains. + drafts.set("doc", JSON.stringify({ previous: true })) + expect(drafts.set("doc", document, true).sort()).toEqual(["gone-chunk", "gone-image"]) + expect(drafts.get("doc")).toBe(JSON.stringify({ previous: true })) + // A non-strict write stores it anyway and still reports what is missing. + expect(drafts.set("doc", document, false).sort()).toEqual(["gone-chunk", "gone-image"]) + expect(drafts.get("doc")).toBe(document) + expect(drafts.set("plain", "not json", true)).toEqual([]) + expect(drafts.set("doc", null, true)).toEqual([]) + expect(drafts.set("doc", document, true)).not.toContain(kept) + }) + + test("an uploaded attachment survives a due collection before its document is written", () => { + const database = openDatabase(":memory:") + let clock = 0 + const drafts = createDraftStore(database.db, { delay: 1_000, now: () => clock }) + drafts.putBlob(new TextEncoder().encode("old")) + drafts.set("other", JSON.stringify({ n: 1 })) + drafts.flush() + clock = collectInterval + 1 + // The renderer uploads first and saves the referencing document up to a second later. + const image = drafts.putBlob(new Uint8Array([1, 2, 3])) + drafts.set("other", JSON.stringify({ n: 2 })) + drafts.flush() + expect(drafts.getBlob(image)).not.toBeNull() + drafts.set("doc", JSON.stringify({ prompt: [{ type: "image", blob: { id: image } }] })) + drafts.flush() + expect(drafts.getBlob(image)).not.toBeNull() + }) }) + +const collectInterval = 60_000 diff --git a/packages/desktop/src/main/storage/drafts.ts b/packages/desktop/src/main/storage/drafts.ts index 726d29b7425..a97b1fc6bb7 100644 --- a/packages/desktop/src/main/storage/drafts.ts +++ b/packages/desktop/src/main/storage/drafts.ts @@ -6,8 +6,34 @@ import { createWriteBehind } from "./write-behind" export type DraftStore = ReturnType -export function createDraftStore(db: Database, input: { delay?: number; onError?: (error: unknown) => void } = {}) { - collectBlobs(db) +// Editing a large paste retires one text chunk per save, so orphans accumulate while the app runs. +const collectInterval = 60_000 +// A blob stays collectable-proof for this long after its last upload or document reference. The +// renderer reuses a cached chunk id without uploading for far less than this (see +// draftChunkCacheTtl), so a reference it publishes always points at a retained blob. +export const blobGrace = 15 * 60_000 + +// Every blob id a document references: image parts `{ blob: { id } }` and text chunk lists +// `{ blob: { kind: "text", ids: [...] } }`. SQLite walks the JSON; nothing parses drafts in JS. +const referenced = (value: unknown) => sql` + SELECT json_extract(node.value, '$.id') AS id + FROM json_tree(${value}) AS node + WHERE node.key = 'blob' AND node.type = 'object' AND json_type(node.value, '$.id') = 'text' + UNION + SELECT chunk.value AS id + FROM json_tree(${value}) AS node, json_each(node.value, '$.ids') AS chunk + WHERE node.key = 'blob' AND node.type = 'object' AND json_type(node.value, '$.ids') = 'array' +` + +export function createDraftStore( + db: Database, + input: { delay?: number; onError?: (error: unknown) => void; now?: () => number } = {}, +) { + const now = input.now ?? Date.now + // Nothing outside this process can hold a blob id at startup, so no grace applies. + collectBlobs(db, Infinity) + let collected = now() + let orphans = false const byKey = eq(document.key, sql.placeholder("key")) const read = db.select({ value: document.value }).from(document).where(byKey).prepare() const remove = db.delete(document).where(byKey).prepare() @@ -19,13 +45,27 @@ export function createDraftStore(db: Database, input: { delay?: number; onError? const writer = createWriteBehind({ delay: input.delay ?? 500, onError: input.onError, - write: (batch) => + write: (batch) => { + const at = now() db.transaction(() => { for (const [key, value] of batch) { - if (value === null) remove.run({ key }) - else upsert.run({ key, value }) + if (value === null) { + remove.run({ key }) + continue + } + upsert.run({ key, value }) + // Referencing a blob keeps it alive; done here so a reference the renderer republished + // from its cache is refreshed even though no upload happened. + if (json(value)) + db.run(sql`UPDATE ${blobs} SET touched_at = ${at} WHERE ${blobs.id} IN (${referenced(value)})`) } - }), + }) + // Only a document rewrite can orphan a blob, so collect right after one when due. + if (!orphans || at - collected < collectInterval) return + collectBlobs(db, at - blobGrace) + collected = at + orphans = false + }, }) return { @@ -33,13 +73,29 @@ export function createDraftStore(db: Database, input: { delay?: number; onError? if (writer.has(key)) return writer.get(key) ?? null return read.get({ key })?.value ?? null }, - set: (key: string, value: string | null) => writer.set(key, value), + // Returns the referenced blob ids this store does not hold so the renderer can upload them + // again. A strict write is refused while any are missing, so the previously stored document + // stays visible instead of one with dangling references. + set(key: string, value: string | null, strict = false) { + const missing = + value === null || !json(value) + ? [] + : db + .all<{ + id: string + }>(sql`SELECT ref.id FROM (${referenced(value)}) AS ref WHERE ref.id NOT IN (SELECT ${blobs.id} FROM ${blobs})`) + .map((row) => row.id) + if (!strict || missing.length === 0) writer.set(key, value) + return missing + }, putBlob(data: Uint8Array) { const id = createHash("sha256").update(data).digest("hex") + const touched_at = now() db.insert(blobs) - .values({ id, data: Buffer.from(data) }) - .onConflictDoNothing() + .values({ id, data: Buffer.from(data), touched_at }) + .onConflictDoUpdate({ target: blobs.id, set: { touched_at } }) .run() + orphans = true return id }, getBlob(id: string): Uint8Array | null { @@ -50,14 +106,25 @@ export function createDraftStore(db: Database, input: { delay?: number; onError? } } -// Blobs are content-addressed and shared; drop the ones no document references anymore. SQLite -// walks the JSON itself, so startup does not parse every draft and history entry in JavaScript. -function collectBlobs(db: Database) { +function json(value: string) { + return value.startsWith("{") || value.startsWith("[") +} + +// Drop blobs no stored document references and nothing has touched since `before`. +function collectBlobs(db: Database, before: number) { db.run(sql` - DELETE FROM ${blobs} WHERE ${blobs.id} NOT IN ( - SELECT json_extract(node.value, '$.id') - FROM ${document}, json_tree(${document.value}) AS node - WHERE json_valid(${document.value}) AND node.key = 'blob' AND node.type = 'object' - ) + DELETE FROM ${blobs} + WHERE ${blobs.touched_at} < ${before === Infinity ? Number.MAX_SAFE_INTEGER : before} + AND ${blobs.id} NOT IN ( + SELECT json_extract(node.value, '$.id') + FROM ${document}, json_tree(${document.value}) AS node + WHERE json_valid(${document.value}) AND node.key = 'blob' AND node.type = 'object' + AND json_type(node.value, '$.id') = 'text' + UNION + SELECT chunk.value + FROM ${document}, json_tree(${document.value}) AS node, json_each(node.value, '$.ids') AS chunk + WHERE json_valid(${document.value}) AND node.key = 'blob' AND node.type = 'object' + AND json_type(node.value, '$.ids') = 'array' + ) `) } diff --git a/packages/desktop/src/main/storage/migration.gen.ts b/packages/desktop/src/main/storage/migration.gen.ts index ac6c51756ad..f5800a9f5d8 100644 --- a/packages/desktop/src/main/storage/migration.gen.ts +++ b/packages/desktop/src/main/storage/migration.gen.ts @@ -14,4 +14,8 @@ export const migrations = [ "CREATE TABLE `state` (\n\t`name` text NOT NULL,\n\t`key` text NOT NULL,\n\t`value` text NOT NULL,\n\t`updated_at` integer NOT NULL,\n\tCONSTRAINT `state_pk` PRIMARY KEY(`name`, `key`)\n);", ], }, + { + id: "20260907031611_blob-touched", + statements: ["ALTER TABLE `blob` ADD `touched_at` integer DEFAULT 0 NOT NULL;"], + }, ] diff --git a/packages/desktop/src/main/storage/migration/20260907031611_blob-touched/migration.sql b/packages/desktop/src/main/storage/migration/20260907031611_blob-touched/migration.sql new file mode 100644 index 00000000000..88158d1e717 --- /dev/null +++ b/packages/desktop/src/main/storage/migration/20260907031611_blob-touched/migration.sql @@ -0,0 +1 @@ +ALTER TABLE `blob` ADD `touched_at` integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/packages/desktop/src/main/storage/migration/20260907031611_blob-touched/snapshot.json b/packages/desktop/src/main/storage/migration/20260907031611_blob-touched/snapshot.json new file mode 100644 index 00000000000..b0b59540dd9 --- /dev/null +++ b/packages/desktop/src/main/storage/migration/20260907031611_blob-touched/snapshot.json @@ -0,0 +1,141 @@ +{ + "version": "7", + "dialect": "sqlite", + "id": "53c65132-8703-42d6-8464-64356145dfb4", + "prevIds": [ + "5a2f8b7e-2765-4a0b-8877-a74ee2f29b20" + ], + "ddl": [ + { + "name": "blob", + "entityType": "tables" + }, + { + "name": "document", + "entityType": "tables" + }, + { + "name": "state", + "entityType": "tables" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "id", + "entityType": "columns", + "table": "blob" + }, + { + "type": "blob", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "data", + "entityType": "columns", + "table": "blob" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": "0", + "generated": null, + "name": "touched_at", + "entityType": "columns", + "table": "blob" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "key", + "entityType": "columns", + "table": "document" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "value", + "entityType": "columns", + "table": "document" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "name", + "entityType": "columns", + "table": "state" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "key", + "entityType": "columns", + "table": "state" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "value", + "entityType": "columns", + "table": "state" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "updated_at", + "entityType": "columns", + "table": "state" + }, + { + "columns": [ + "name", + "key" + ], + "nameExplicit": false, + "name": "state_pk", + "entityType": "pks", + "table": "state" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "blob_pk", + "table": "blob", + "entityType": "pks" + }, + { + "columns": [ + "key" + ], + "nameExplicit": false, + "name": "document_pk", + "table": "document", + "entityType": "pks" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/packages/desktop/src/main/storage/schema.ts b/packages/desktop/src/main/storage/schema.ts index 2051fdb6161..1fb0042d0a9 100644 --- a/packages/desktop/src/main/storage/schema.ts +++ b/packages/desktop/src/main/storage/schema.ts @@ -6,10 +6,12 @@ export const document = sqliteTable("document", { value: text().notNull(), }) -// Images referenced from documents by content hash. +// Images and text chunks referenced from documents by content hash. `touched_at` is the last time +// a blob was uploaded or a written document referenced it; collection leaves recent blobs alone. export const blobs = sqliteTable("blob", { id: text().primaryKey(), data: blob({ mode: "buffer" }).notNull(), + touched_at: integer().notNull().default(0), }) // Everything the renderer persists through `platform.storage(name)`. `name` is the storage diff --git a/packages/desktop/src/renderer/api-types.ts b/packages/desktop/src/renderer/api-types.ts index 7573646ec7e..1079dde3364 100644 --- a/packages/desktop/src/renderer/api-types.ts +++ b/packages/desktop/src/renderer/api-types.ts @@ -39,7 +39,7 @@ export type ElectronAPI = { cb: (name: string, insert: Record, remove: string[], revision: number) => void, ): () => void draftGet(key: string): Promise - draftSet(key: string, value: string): Promise + draftSet(key: string, value: string, strict: boolean): Promise draftDelete(key: string): Promise draftBlobPut(data: ArrayBuffer): Promise draftBlobGet(id: string): Promise diff --git a/packages/desktop/src/renderer/api.ts b/packages/desktop/src/renderer/api.ts index 770ed28e2f4..5f1c99ae5e0 100644 --- a/packages/desktop/src/renderer/api.ts +++ b/packages/desktop/src/renderer/api.ts @@ -81,7 +81,7 @@ export const api: ElectronAPI = { onStoreChanged: (cb) => listen("StorageChanged", (event) => cb(event.name, mutable(event.insert), mutable(event.remove), event.revision)), draftGet: (key) => invoke("DraftsGet", { key }), - draftSet: (key, value) => invoke("DraftsSet", { key, value }), + draftSet: (key, value, strict) => invoke("DraftsSet", { key, value, strict }).then(mutable), draftDelete: (key) => invoke("DraftsDelete", { key }), draftBlobPut: (data) => invoke("DraftsPutBlob", { data: new Uint8Array(data) }), draftBlobGet: (id) => invoke("DraftsGetBlob", { id }).then((data) => (data ? toArrayBuffer(data) : null)), diff --git a/packages/desktop/src/shared/ipc-rpc/storage.ts b/packages/desktop/src/shared/ipc-rpc/storage.ts index 12c02d8cd29..dc1b7554d37 100644 --- a/packages/desktop/src/shared/ipc-rpc/storage.ts +++ b/packages/desktop/src/shared/ipc-rpc/storage.ts @@ -19,7 +19,8 @@ export const DraftsGet = Rpc.make("DraftsGet", { success: Schema.NullOr(Schema.String), }) export const DraftsSet = Rpc.make("DraftsSet", { - payload: { key: Schema.String, value: Schema.String }, + payload: { key: Schema.String, value: Schema.String, strict: Schema.Boolean }, + success: Schema.Array(Schema.String), }) export const DraftsDelete = Rpc.make("DraftsDelete", { payload: { key: Schema.String } }) export const DraftsPutBlob = Rpc.make("DraftsPutBlob", {