mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-17 05:26:47 +03:00
perf(app): externalize large draft text into content-addressed chunks (#47706)
This commit is contained in:
committed by
GitHub
Unverified
parent
7ef9cb68d1
commit
5d971e5272
@@ -108,3 +108,28 @@ Storage-key relocation (`previousKey`, workspace aliases, draft storage moves)
|
||||
remains separate from schema migration. Draft blob externalization and hydration
|
||||
also remain in the storage adapter: composer codecs receive hydrated references,
|
||||
not raw ID-only blob documents.
|
||||
|
||||
## Large draft content
|
||||
|
||||
Draft documents never carry large text inline. Any string of `draftTextThreshold`
|
||||
characters or more is split into `draftTextChunk`-sized content-addressed blobs and
|
||||
stored as `{ blob: { kind: "text", ids: [...] } }`; reads join the chunks again. A
|
||||
content-keyed cache means unchanged chunks are not hashed or sent on later saves, so
|
||||
typing after a large paste uploads one chunk per save rather than the paste. Chunk
|
||||
boundaries never split a surrogate pair, and a failed upload is evicted so the next save
|
||||
retries it.
|
||||
|
||||
Blob collection is made safe by validation on write, not by timing. A strict document write
|
||||
is refused while it references blob ids the store does not hold, so the previous document
|
||||
stays visible; the renderer uploads the missing bytes again from the chunk text or the image
|
||||
`Blob` it still holds, renames the references to the ids the uploads returned (content hashes
|
||||
normally, fresh ids on a store without WebCrypto), and publishes. This covers a chunk another
|
||||
tab collected, and an image the composer kept in its history long after its blob was
|
||||
collected. The desktop host additionally refreshes `touched_at` for every blob a written
|
||||
document references and collects only blobs unreferenced and untouched for `blobGrace`, so
|
||||
repairs stay rare.
|
||||
`persisted()` hands the draft store the encoded document (`setDocument`) rather than
|
||||
a serialized string, so the store does not re-parse the full document to externalize
|
||||
it. Both blob collectors (desktop SQL, browser IndexedDB) keep chunk ids alive. This
|
||||
follows VS Code's rule that editor content lives in per-resource backups, not in the
|
||||
state database.
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { createDraftStore, draftTextChunk, draftTextThreshold } from "./drafts"
|
||||
|
||||
function memoryDriver() {
|
||||
const documents = new Map<string, string>()
|
||||
const blobs = new Map<string, Blob>()
|
||||
let puts = 0
|
||||
return {
|
||||
documents,
|
||||
blobs,
|
||||
puts: () => puts,
|
||||
driver: {
|
||||
get: async (key: string) => documents.get(key) ?? null,
|
||||
// Like the real stores: report referenced blobs that are not held; a strict write with any
|
||||
// missing is refused.
|
||||
set: async (key: string, value: string, strict: boolean) => {
|
||||
const ids = new Set<string>()
|
||||
JSON.parse(value, (_key, item) => {
|
||||
if (item?.blob && typeof item.blob.id === "string") ids.add(item.blob.id)
|
||||
if (item?.blob && Array.isArray(item.blob.ids)) item.blob.ids.forEach((id: unknown) => ids.add(String(id)))
|
||||
return item
|
||||
})
|
||||
const missing = [...ids].filter((id) => !blobs.has(id))
|
||||
if (!strict || missing.length === 0) documents.set(key, value)
|
||||
return missing
|
||||
},
|
||||
remove: async (key: string) => void documents.delete(key),
|
||||
putBlob: async (blob: Blob) => {
|
||||
puts++
|
||||
const id = `blob-${await blob.text().then((text) => Bun.hash(text).toString(16))}`
|
||||
blobs.set(id, blob)
|
||||
return id
|
||||
},
|
||||
getBlob: async (id: string) => blobs.get(id) ?? null,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const large = "x".repeat(draftTextThreshold)
|
||||
const paste = Array.from({ length: 3 * draftTextChunk }, (_, i) => String.fromCharCode(97 + (i % 26))).join("")
|
||||
|
||||
describe("draft store text externalization", () => {
|
||||
test("large strings become chunk lists and small ones stay inline", async () => {
|
||||
const memory = memoryDriver()
|
||||
const store = createDraftStore(memory.driver)
|
||||
await store.setDocument("doc", {
|
||||
prompt: [
|
||||
{ type: "text", content: paste },
|
||||
{ type: "text", content: "hi" },
|
||||
],
|
||||
})
|
||||
const stored = JSON.parse(memory.documents.get("doc")!)
|
||||
expect(stored.prompt[0].content.blob.kind).toBe("text")
|
||||
expect(stored.prompt[0].content.blob.ids).toHaveLength(3)
|
||||
expect(stored.prompt[1].content).toBe("hi")
|
||||
expect(memory.documents.get("doc")!.length).toBeLessThan(400)
|
||||
const chunks = await Promise.all(
|
||||
stored.prompt[0].content.blob.ids.map((id: string) => memory.blobs.get(id)!.text()),
|
||||
)
|
||||
expect(chunks.join("")).toBe(paste)
|
||||
})
|
||||
|
||||
test("appending to a large string re-uploads only the final chunk", async () => {
|
||||
const memory = memoryDriver()
|
||||
const store = createDraftStore(memory.driver)
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: paste }] })
|
||||
expect(memory.puts()).toBe(3)
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: `${paste}!` }] })
|
||||
expect(memory.puts()).toBe(4)
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: `${paste}!` }], cursor: 1 })
|
||||
expect(memory.puts()).toBe(4)
|
||||
})
|
||||
|
||||
test("reads join the chunks again and reuse cached content", async () => {
|
||||
const memory = memoryDriver()
|
||||
const store = createDraftStore(memory.driver)
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: paste }] })
|
||||
const fresh = createDraftStore(memory.driver)
|
||||
expect(JSON.parse((await fresh.getItem("doc"))!)).toEqual({ prompt: [{ type: "text", content: paste }] })
|
||||
memory.blobs.clear()
|
||||
expect(JSON.parse((await fresh.getItem("doc"))!)).toEqual({ prompt: [{ type: "text", content: paste }] })
|
||||
})
|
||||
|
||||
test("a missing chunk decodes to empty text instead of failing the document", async () => {
|
||||
const memory = memoryDriver()
|
||||
memory.documents.set(
|
||||
"doc",
|
||||
JSON.stringify({ prompt: [{ type: "text", content: { blob: { kind: "text", ids: ["gone"] } } }] }),
|
||||
)
|
||||
const store = createDraftStore(memory.driver)
|
||||
expect(JSON.parse((await store.getItem("doc"))!)).toEqual({ prompt: [{ type: "text", content: "" }] })
|
||||
})
|
||||
|
||||
test("a cached chunk id the store no longer holds is uploaded again on the next save", async () => {
|
||||
const memory = memoryDriver()
|
||||
const store = createDraftStore(memory.driver)
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: large }] })
|
||||
const [id] = JSON.parse(memory.documents.get("doc")!).prompt[0].content.blob.ids
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: `${large}!` }] })
|
||||
// Another tab collected the chunk for `large` while this tab still caches its id.
|
||||
memory.blobs.clear()
|
||||
// Undo republishes the cached id; the write reports it missing and the chunk is uploaded again.
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: large }] })
|
||||
expect(memory.puts()).toBe(3)
|
||||
const fresh = createDraftStore(memory.driver)
|
||||
expect(JSON.parse((await fresh.getItem("doc"))!).prompt[0].content).toBe(large)
|
||||
expect(memory.blobs.has(id)).toBe(true)
|
||||
})
|
||||
|
||||
test("an image reference whose blob was collected is restored from its object url", async () => {
|
||||
const memory = memoryDriver()
|
||||
const store = createDraftStore(memory.driver)
|
||||
const image = await store.putBlob(new Blob([new Uint8Array([1, 2, 3])], { type: "image/png" }))
|
||||
// The composer kept this reference (for example in its history) while the store collected the bytes.
|
||||
memory.blobs.clear()
|
||||
await store.setDocument("doc", { prompt: [{ type: "image", blob: { id: image.id, url: image.url } }] })
|
||||
expect(memory.blobs.has(image.id)).toBe(true)
|
||||
expect(new Uint8Array(await memory.blobs.get(image.id)!.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3]))
|
||||
})
|
||||
|
||||
test("the previous document stays visible until missing blobs are restored", async () => {
|
||||
const memory = memoryDriver()
|
||||
const store = createDraftStore(memory.driver)
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: large }] })
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: `${large}!` }] })
|
||||
const before = memory.documents.get("doc")
|
||||
memory.blobs.clear()
|
||||
// Hold the repair upload: while it is pending, another reader must still see the old document.
|
||||
const gate = Promise.withResolvers<void>()
|
||||
const putBlob = memory.driver.putBlob
|
||||
memory.driver.putBlob = async (blob) => {
|
||||
await gate.promise
|
||||
return putBlob(blob)
|
||||
}
|
||||
const saving = store.setDocument("doc", { prompt: [{ type: "text", content: large }] })
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
expect(memory.documents.get("doc")).toBe(before)
|
||||
gate.resolve()
|
||||
await saving
|
||||
expect(JSON.parse(memory.documents.get("doc")!).prompt[0].content.blob.ids).toHaveLength(1)
|
||||
const fresh = createDraftStore(memory.driver)
|
||||
expect(JSON.parse((await fresh.getItem("doc"))!).prompt[0].content).toBe(large)
|
||||
})
|
||||
|
||||
test("references are renamed when a restored blob comes back under a different id", async () => {
|
||||
const memory = memoryDriver()
|
||||
// A store without WebCrypto assigns a fresh id to every upload.
|
||||
let counter = 0
|
||||
memory.driver.putBlob = async (blob) => {
|
||||
const id = `random-${counter++}`
|
||||
memory.blobs.set(id, blob)
|
||||
return id
|
||||
}
|
||||
const store = createDraftStore(memory.driver)
|
||||
const image = await store.putBlob(new Blob([new Uint8Array([7])], { type: "image/png" }))
|
||||
await store.setDocument("doc", {
|
||||
prompt: [
|
||||
{ type: "text", content: paste },
|
||||
{ type: "image", blob: image },
|
||||
],
|
||||
})
|
||||
const original = JSON.parse(memory.documents.get("doc")!)
|
||||
memory.blobs.clear()
|
||||
await store.setDocument("doc", {
|
||||
prompt: [
|
||||
{ type: "text", content: paste },
|
||||
{ type: "image", blob: image },
|
||||
],
|
||||
})
|
||||
const restored = JSON.parse(memory.documents.get("doc")!)
|
||||
expect(restored.prompt[0].content.blob.ids).not.toEqual(original.prompt[0].content.blob.ids)
|
||||
expect(restored.prompt[1].blob.id).not.toBe(original.prompt[1].blob.id)
|
||||
for (const id of [...restored.prompt[0].content.blob.ids, restored.prompt[1].blob.id])
|
||||
expect(memory.blobs.has(id)).toBe(true)
|
||||
const fresh = createDraftStore(memory.driver)
|
||||
const read = JSON.parse((await fresh.getItem("doc"))!)
|
||||
expect(read.prompt[0].content).toBe(paste)
|
||||
expect(read.prompt[1].blob.id).toBe(restored.prompt[1].blob.id)
|
||||
// The renamed ids are what later encodes publish, so saves of the still-live references (the
|
||||
// composer keeps the original image id) upload nothing and keep one stable image id.
|
||||
const puts = counter
|
||||
for (const cursor of [1, 2, 3]) {
|
||||
await store.setDocument("doc", {
|
||||
prompt: [
|
||||
{ type: "text", content: paste },
|
||||
{ type: "image", blob: image },
|
||||
],
|
||||
cursor,
|
||||
})
|
||||
expect(JSON.parse(memory.documents.get("doc")!).prompt[1].blob.id).toBe(restored.prompt[1].blob.id)
|
||||
}
|
||||
expect(counter).toBe(puts)
|
||||
})
|
||||
|
||||
test("chunk boundaries never split a surrogate pair", async () => {
|
||||
const memory = memoryDriver()
|
||||
const store = createDraftStore(memory.driver)
|
||||
const text = "x".repeat(draftTextChunk - 1) + "😀tail"
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: text }] })
|
||||
const stored = JSON.parse(memory.documents.get("doc")!)
|
||||
const bytes = await Promise.all(stored.prompt[0].content.blob.ids.map((id: string) => memory.blobs.get(id)!.text()))
|
||||
expect(bytes.join("")).toBe(text)
|
||||
expect(bytes[0]!.length).toBe(draftTextChunk + 1)
|
||||
const fresh = createDraftStore(memory.driver)
|
||||
expect(JSON.parse((await fresh.getItem("doc"))!).prompt[0].content).toBe(text)
|
||||
})
|
||||
|
||||
test("a failed chunk upload is retried on the next save instead of being reused", async () => {
|
||||
const memory = memoryDriver()
|
||||
let failNext = true
|
||||
const putBlob = memory.driver.putBlob
|
||||
memory.driver.putBlob = async (blob) => {
|
||||
if (failNext) {
|
||||
failNext = false
|
||||
throw new Error("offline")
|
||||
}
|
||||
return putBlob(blob)
|
||||
}
|
||||
const store = createDraftStore(memory.driver)
|
||||
await expect(store.setDocument("doc", { prompt: [{ type: "text", content: paste }] })).rejects.toThrow("offline")
|
||||
await store.setDocument("doc", { prompt: [{ type: "text", content: `${paste}!` }] })
|
||||
const fresh = createDraftStore(memory.driver)
|
||||
expect(JSON.parse((await fresh.getItem("doc"))!).prompt[0].content).toBe(`${paste}!`)
|
||||
})
|
||||
|
||||
test("setItem still accepts a serialized document", async () => {
|
||||
const memory = memoryDriver()
|
||||
const store = createDraftStore(memory.driver)
|
||||
await store.setItem("doc", JSON.stringify({ prompt: [{ type: "text", content: large }] }))
|
||||
expect(JSON.parse(memory.documents.get("doc")!).prompt[0].content.blob.ids).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
@@ -5,20 +5,43 @@ export type BlobReference = { id: string; url: string }
|
||||
|
||||
type Driver = {
|
||||
get(key: string): Promise<string | null>
|
||||
set(key: string, value: string): Promise<void>
|
||||
/**
|
||||
* Store the document and report every blob id it references that the store does not hold. A
|
||||
* strict write is refused (nothing stored) when any are missing, so a document is never visible
|
||||
* with dangling references while its blobs are being restored.
|
||||
*/
|
||||
set(key: string, value: string, strict: boolean): Promise<readonly string[]>
|
||||
remove(key: string): Promise<void>
|
||||
putBlob(blob: Blob): Promise<string>
|
||||
getBlob(id: string): Promise<Blob | null>
|
||||
}
|
||||
|
||||
export type DraftStore = AsyncStorage & { putBlob(blob: Blob): Promise<BlobReference> }
|
||||
export type DraftStore = AsyncStorage & {
|
||||
putBlob(blob: Blob): Promise<BlobReference>
|
||||
/** Persist an already-encoded document without re-parsing its serialized form. */
|
||||
setDocument(key: string, document: unknown): Promise<void>
|
||||
}
|
||||
|
||||
// Strings at least this long leave the document as fixed-size content-addressed chunks. Typing
|
||||
// after a large paste changes only the final chunk, so a save uploads one chunk, not the paste.
|
||||
export const draftTextThreshold = 16 * 1024
|
||||
export const draftTextChunk = 64 * 1024
|
||||
const textCacheLimit = 64
|
||||
|
||||
const urls = new Map<string, string>()
|
||||
// The object URL already pins the Blob for the page's lifetime; keeping the Blob itself lets a
|
||||
// collected image be uploaded again without fetching the URL.
|
||||
const held = new Map<string, Blob>()
|
||||
// Image ids that were restored under a different id (a store without WebCrypto assigns fresh
|
||||
// ones); live references still carry the original.
|
||||
const aliases = new Map<string, string>()
|
||||
|
||||
function blobUrl(id: string, blob: Blob) {
|
||||
const existing = urls.get(id)
|
||||
if (existing) return existing
|
||||
const url = URL.createObjectURL(blob)
|
||||
urls.set(id, url)
|
||||
held.set(id, blob)
|
||||
return url
|
||||
}
|
||||
|
||||
@@ -56,25 +79,82 @@ export function createDraftStore(driver: Driver): DraftStore {
|
||||
const id = await driver.putBlob(blob)
|
||||
return { id, url: blobUrl(id, blob) }
|
||||
}
|
||||
const encode = async (value: unknown): Promise<unknown> => {
|
||||
if (Array.isArray(value)) return Promise.all(value.map(encode))
|
||||
// Keyed by chunk content so unchanged chunks are never hashed or sent again while the draft is
|
||||
// edited. Bounded because each entry pins up to draftTextChunk characters. A hit is safe even if
|
||||
// the store has since collected the blob: the write reports it missing and it is uploaded again.
|
||||
const chunkIds = new Map<string, Promise<string>>()
|
||||
const chunks = new Map<string, string>()
|
||||
const remember = <V>(cache: Map<string, V>, key: string, value: V) => {
|
||||
cache.set(key, value)
|
||||
if (cache.size > textCacheLimit) cache.delete(cache.keys().next().value!)
|
||||
return value
|
||||
}
|
||||
const upload = (chunk: string) => {
|
||||
const id = driver.putBlob(new Blob([chunk])).then(
|
||||
(id) => {
|
||||
remember(chunks, id, chunk)
|
||||
return id
|
||||
},
|
||||
(error: unknown) => {
|
||||
// A failed upload must not be reused as the answer for this content on later saves.
|
||||
if (chunkIds.get(chunk) === id) chunkIds.delete(chunk)
|
||||
throw error
|
||||
},
|
||||
)
|
||||
return remember(chunkIds, chunk, id)
|
||||
}
|
||||
const externalize = (text: string) => Promise.all(split(text).map((chunk) => chunkIds.get(chunk) ?? upload(chunk)))
|
||||
const loadChunk = async (id: string) => {
|
||||
const cached = chunks.get(id)
|
||||
if (cached !== undefined) return cached
|
||||
const blob = await driver.getBlob(id)
|
||||
// A missing chunk loses that text but keeps the rest of the document decodable.
|
||||
return remember(chunks, id, blob ? await blob.text() : "")
|
||||
}
|
||||
// `sources` collects, for every blob id the encoded document references, a way to produce its
|
||||
// bytes again: the chunk text itself, or the image Blob (or object URL) the reference carries.
|
||||
type Sources = Map<string, { blob: () => Promise<Blob>; chunk?: string }>
|
||||
const encode = async (value: unknown, sources: Sources): Promise<unknown> => {
|
||||
if (typeof value === "string" && value.length >= draftTextThreshold) {
|
||||
const pieces = split(value)
|
||||
const ids = await externalize(value)
|
||||
ids.forEach((id, index) =>
|
||||
sources.set(id, { blob: async () => new Blob([pieces[index]!]), chunk: pieces[index] }),
|
||||
)
|
||||
return { blob: { kind: "text", ids } }
|
||||
}
|
||||
if (Array.isArray(value)) return Promise.all(value.map((entry) => encode(entry, sources)))
|
||||
if (!value || typeof value !== "object") return value
|
||||
const item = value as Record<string, unknown>
|
||||
if (item.type === "image" && typeof item.dataUrl === "string") {
|
||||
const blob = await fetch(item.dataUrl).then((response) => response.blob())
|
||||
const { dataUrl: _, ...rest } = item
|
||||
return { ...rest, blob: { id: await driver.putBlob(blob) } }
|
||||
const id = await driver.putBlob(blob)
|
||||
sources.set(id, { blob: async () => blob })
|
||||
return { ...rest, blob: { id } }
|
||||
}
|
||||
if ("blob" in item && item.blob && typeof item.blob === "object") {
|
||||
const blob = item.blob as Record<string, unknown>
|
||||
if (blob.kind === "text") return item
|
||||
if (typeof blob.id === "string" && blob.id.startsWith("data:")) {
|
||||
const data = await fetch(blob.id).then((response) => response.blob())
|
||||
return { ...item, blob: { id: await driver.putBlob(data) } }
|
||||
const id = await driver.putBlob(data)
|
||||
sources.set(id, { blob: async () => data })
|
||||
return { ...item, blob: { id } }
|
||||
}
|
||||
if (typeof blob.id === "string") {
|
||||
// A live reference keeps the id it was created with; publish the id its bytes now live under.
|
||||
const id = aliases.get(blob.id) ?? blob.id
|
||||
const kept = held.get(id)
|
||||
const url = typeof blob.url === "string" ? blob.url : urls.get(id)
|
||||
if (kept) sources.set(id, { blob: async () => kept })
|
||||
else if (url) sources.set(id, { blob: () => fetch(url).then((response) => response.blob()) })
|
||||
return { ...item, blob: { id } }
|
||||
}
|
||||
return { ...item, blob: { id: blob.id } }
|
||||
}
|
||||
return Object.fromEntries(
|
||||
await Promise.all(Object.entries(item).map(async ([key, entry]) => [key, await encode(entry)])),
|
||||
await Promise.all(Object.entries(item).map(async ([key, entry]) => [key, await encode(entry, sources)])),
|
||||
)
|
||||
}
|
||||
const decode = async (value: unknown): Promise<unknown> => {
|
||||
@@ -83,6 +163,9 @@ export function createDraftStore(driver: Driver): DraftStore {
|
||||
const item = value as Record<string, unknown>
|
||||
if (item.blob && typeof item.blob === "object") {
|
||||
const ref = item.blob as Record<string, unknown>
|
||||
if (ref.kind === "text" && Array.isArray(ref.ids)) {
|
||||
return (await Promise.all(ref.ids.map((id) => loadChunk(String(id))))).join("")
|
||||
}
|
||||
if (typeof ref.id === "string") {
|
||||
const url = await loadBlobUrl(ref.id)
|
||||
if (url) return { ...item, blob: { id: ref.id, url } }
|
||||
@@ -92,6 +175,65 @@ export function createDraftStore(driver: Driver): DraftStore {
|
||||
await Promise.all(Object.entries(item).map(async ([key, entry]) => [key, await decode(entry)])),
|
||||
)
|
||||
}
|
||||
// Upload the bytes behind `ids` again and return the ids they were stored under. Ids are
|
||||
// usually content hashes and come back unchanged, but a store without WebCrypto assigns fresh
|
||||
// ones, so callers must rename references rather than assume.
|
||||
const restore = async (ids: readonly string[], sources: Sources) => {
|
||||
const renamed = new Map<string, string>()
|
||||
await Promise.all(
|
||||
ids.map(async (id) => {
|
||||
const source = sources.get(id)
|
||||
if (!source) return
|
||||
const blob = await source.blob()
|
||||
const next = await driver.putBlob(blob)
|
||||
renamed.set(id, next)
|
||||
if (source.chunk !== undefined) {
|
||||
remember(chunkIds, source.chunk, Promise.resolve(next))
|
||||
remember(chunks, next, source.chunk)
|
||||
return
|
||||
}
|
||||
held.set(next, blob)
|
||||
blobUrl(next, blob)
|
||||
if (next === id) return
|
||||
// Later encodes of the still-live reference resolve straight to the new id. Re-point any
|
||||
// earlier alias chain so lookups stay one step.
|
||||
for (const [from, to] of aliases) if (to === id) aliases.set(from, next)
|
||||
aliases.set(id, next)
|
||||
}),
|
||||
)
|
||||
return renamed
|
||||
}
|
||||
const rename = (value: unknown, renamed: Map<string, string>): unknown => {
|
||||
if (Array.isArray(value)) return value.map((entry) => rename(entry, renamed))
|
||||
if (!value || typeof value !== "object") return value
|
||||
const item = value as Record<string, unknown>
|
||||
if (item.blob && typeof item.blob === "object") {
|
||||
const ref = item.blob as Record<string, unknown>
|
||||
if (Array.isArray(ref.ids))
|
||||
return { ...item, blob: { ...ref, ids: ref.ids.map((id) => renamed.get(String(id)) ?? id) } }
|
||||
if (typeof ref.id === "string") return { ...item, blob: { ...ref, id: renamed.get(ref.id) ?? ref.id } }
|
||||
}
|
||||
return Object.fromEntries(Object.entries(item).map(([key, entry]) => [key, rename(entry, renamed)]))
|
||||
}
|
||||
const setDocument = async (key: string, document: unknown) => {
|
||||
const version = (versions.get(key) ?? 0) + 1
|
||||
versions.set(key, version)
|
||||
const sources: Sources = new Map()
|
||||
const encoded = await encode(document, sources)
|
||||
if (versions.get(key) !== version) return
|
||||
// The store refuses the write while any referenced blob is missing, so the previous document
|
||||
// stays visible until the bytes are back. Covers a blob collected while a cache, another tab,
|
||||
// or the composer's history still held its id.
|
||||
const missing = await driver.set(key, JSON.stringify(encoded), true)
|
||||
if (missing.length === 0) return
|
||||
const renamed = await restore(missing, sources)
|
||||
if (versions.get(key) !== version) return
|
||||
const unrestored = missing.filter((id) => !renamed.has(id))
|
||||
if (unrestored.length)
|
||||
console.error(`[persistence] draft ${key} references blobs with no bytes to restore`, unrestored)
|
||||
// Anything still missing has no bytes anywhere; the owning codec drops such references on read.
|
||||
await driver.set(key, JSON.stringify(rename(encoded, renamed)), false)
|
||||
}
|
||||
return {
|
||||
getItem: async (key) => {
|
||||
const value = await driver.get(key)
|
||||
@@ -101,12 +243,8 @@ export function createDraftStore(driver: Driver): DraftStore {
|
||||
if (Option.isNone(parsed)) return value
|
||||
return JSON.stringify(await decode(parsed.value))
|
||||
},
|
||||
setItem: async (key, value) => {
|
||||
const version = (versions.get(key) ?? 0) + 1
|
||||
versions.set(key, version)
|
||||
const encoded = JSON.stringify(await encode(JSON.parse(value)))
|
||||
if (versions.get(key) === version) await driver.set(key, encoded)
|
||||
},
|
||||
setItem: (key, value) => setDocument(key, JSON.parse(value)),
|
||||
setDocument,
|
||||
removeItem: async (key) => {
|
||||
versions.set(key, (versions.get(key) ?? 0) + 1)
|
||||
await driver.remove(key)
|
||||
@@ -115,6 +253,20 @@ export function createDraftStore(driver: Driver): DraftStore {
|
||||
}
|
||||
}
|
||||
|
||||
// Fixed-size pieces, except that a piece never ends between the two halves of a surrogate pair:
|
||||
// each piece becomes its own Blob, and an unpaired surrogate would be encoded as U+FFFD.
|
||||
function split(text: string) {
|
||||
const pieces: string[] = []
|
||||
for (let start = 0; start < text.length; ) {
|
||||
const end = Math.min(start + draftTextChunk, text.length)
|
||||
const code = text.charCodeAt(end - 1)
|
||||
const stop = end < text.length && code >= 0xd800 && code <= 0xdbff ? end + 1 : end
|
||||
pieces.push(text.slice(start, stop))
|
||||
start = stop
|
||||
}
|
||||
return pieces
|
||||
}
|
||||
|
||||
export function createBrowserDraftStore(): DraftStore {
|
||||
const request = indexedDB.open("opencode-drafts", 1)
|
||||
request.addEventListener("upgradeneeded", () => {
|
||||
@@ -127,11 +279,7 @@ export function createBrowserDraftStore(): DraftStore {
|
||||
const transaction = database.transaction(["documents", "blobs"], "readwrite")
|
||||
const documents = transaction.objectStore("documents").getAll()
|
||||
documents.addEventListener("success", () => {
|
||||
const used = new Set<string>()
|
||||
JSON.parse(`[${documents.result.join(",")}]`, (_key, item) => {
|
||||
if (item?.blob && typeof item.blob.id === "string") used.add(item.blob.id)
|
||||
return item
|
||||
})
|
||||
const used = referenced(`[${documents.result.join(",")}]`)
|
||||
const store = transaction.objectStore("blobs")
|
||||
const blobs = store.openKeyCursor()
|
||||
blobs.addEventListener("success", () => {
|
||||
@@ -164,7 +312,32 @@ export function createBrowserDraftStore(): DraftStore {
|
||||
}
|
||||
return createDraftStore({
|
||||
get: async (key) => ((await get("documents", key)) as string | undefined) ?? null,
|
||||
set: (key, value) => write("documents", key, value),
|
||||
set: async (key, value, strict) => {
|
||||
// One readwrite transaction over both stores: IndexedDB serialises overlapping readwrite
|
||||
// transactions in creation order, so a later save or removal cannot commit between the
|
||||
// reference check and this write. The put is issued from the last lookup's callback so the
|
||||
// transaction is never left without a pending request.
|
||||
const ids = [...referenced(value)]
|
||||
const transaction = (await db).transaction(["blobs", "documents"], "readwrite")
|
||||
const missing: string[] = []
|
||||
const publish = () => {
|
||||
if (!strict || missing.length === 0) transaction.objectStore("documents").put(value, key)
|
||||
}
|
||||
let remaining = ids.length
|
||||
if (remaining === 0) publish()
|
||||
for (const id of ids) {
|
||||
const lookup = transaction.objectStore("blobs").getKey(id)
|
||||
lookup.addEventListener("success", () => {
|
||||
if (lookup.result === undefined) missing.push(id)
|
||||
if (--remaining === 0) publish()
|
||||
})
|
||||
}
|
||||
return new Promise<string[]>((resolve, reject) => {
|
||||
transaction.addEventListener("complete", () => resolve(missing))
|
||||
transaction.addEventListener("error", () => reject(transaction.error))
|
||||
transaction.addEventListener("abort", () => reject(transaction.error))
|
||||
})
|
||||
},
|
||||
remove: (key) => write("documents", key),
|
||||
putBlob: async (blob) => {
|
||||
const id = await blobID(blob)
|
||||
@@ -175,6 +348,17 @@ export function createBrowserDraftStore(): DraftStore {
|
||||
})
|
||||
}
|
||||
|
||||
// Every blob id a serialized document (or array of documents) references.
|
||||
function referenced(json: string) {
|
||||
const ids = new Set<string>()
|
||||
JSON.parse(json, (_key, item) => {
|
||||
if (item?.blob && typeof item.blob.id === "string") ids.add(item.blob.id)
|
||||
if (item?.blob && Array.isArray(item.blob.ids)) item.blob.ids.forEach((id: unknown) => ids.add(String(id)))
|
||||
return item
|
||||
})
|
||||
return ids
|
||||
}
|
||||
|
||||
export async function blobDataUrl(blob: BlobReference, mime: string) {
|
||||
const data = await fetch(blob.url).then((response) => response.blob())
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
|
||||
@@ -32,6 +32,8 @@ export function persistStore<T extends object>(input: {
|
||||
deserialize: (raw: string) => T
|
||||
sync?: PersistenceSyncAPI
|
||||
delay?: number
|
||||
/** Replaces `storage.setItem` for stores whose storage can take the value itself. */
|
||||
write?: (value: T, serialized: string) => void
|
||||
}) {
|
||||
const delay = input.delay ?? persistSaveDelay
|
||||
let dirty = false
|
||||
@@ -57,6 +59,7 @@ export function persistStore<T extends object>(input: {
|
||||
}
|
||||
last = next
|
||||
input.sync?.[1](input.name, next)
|
||||
if (input.write) return input.write(input.store, next)
|
||||
void input.storage.setItem(input.name, next)
|
||||
}
|
||||
|
||||
|
||||
@@ -484,6 +484,7 @@ export function persisted<S extends Schema.ConstraintCodec<object, unknown>>(
|
||||
const initialized = Persistence.withInitial(schema, initial)
|
||||
const json = Schema.fromJsonString(initialized)
|
||||
const decode = Schema.decodeUnknownOption(json)
|
||||
const encode = Schema.encodeSync(initialized)
|
||||
const serialize = Schema.encodeSync(json)
|
||||
const normalize = (raw: string) => {
|
||||
const value = decode(raw)
|
||||
@@ -492,10 +493,12 @@ export function persisted<S extends Schema.ConstraintCodec<object, unknown>>(
|
||||
const store = createStore<S["Type"]>(Schema.decodeUnknownSync(Schema.toType(initialized))(initial))
|
||||
const isDesktop = platform.platform === "desktop" && !!platform.storage
|
||||
const draft = config.draft ? platform.draftStore : undefined
|
||||
const prefix = `${config.storage ?? "default"}:`
|
||||
// The newest serialized draft, replayed into storage if a slow load finishes after an edit.
|
||||
let draftLatest: string | undefined
|
||||
|
||||
const currentStorage = (() => {
|
||||
if (draft) {
|
||||
const prefix = `${config.storage ?? "default"}:`
|
||||
return {
|
||||
getItem: (key: string) => draft.getItem(prefix + key),
|
||||
setItem: (key: string, value: string) => draft.setItem(prefix + key, value),
|
||||
@@ -557,7 +560,6 @@ export function persisted<S extends Schema.ConstraintCodec<object, unknown>>(
|
||||
]
|
||||
.filter((source): source is { storage: SyncStorage | AsyncStorage; key?: string } => !!source?.storage)
|
||||
.map((source) => ({ ...source, storage: toAsyncStorage(source.storage) }))
|
||||
let draftLatest: string | undefined
|
||||
|
||||
const api: AsyncStorage = {
|
||||
getItem: async (key) => {
|
||||
@@ -602,6 +604,17 @@ export function persisted<S extends Schema.ConstraintCodec<object, unknown>>(
|
||||
serialize,
|
||||
deserialize: Schema.decodeUnknownSync(json),
|
||||
sync: channel ? messageSync(channel) : undefined,
|
||||
// Drafts take the encoded document itself so large text is externalized without the store
|
||||
// re-parsing the serialized form on every save.
|
||||
write: draft
|
||||
? (value, serialized) => {
|
||||
draftLatest = serialized
|
||||
// A failed chunk upload is retried by the next save; see drafts.ts.
|
||||
void draft
|
||||
.setDocument(prefix + config.key, encode(value))
|
||||
.catch((error: unknown) => console.error(`[persistence] draft write failed for ${config.key}`, error))
|
||||
}
|
||||
: undefined,
|
||||
})
|
||||
const state = store[0]
|
||||
const setState = persist.setStore
|
||||
|
||||
@@ -9,7 +9,10 @@ function fixture(id: string, getBlob: () => Promise<Blob | null>) {
|
||||
])
|
||||
const store = createDraftStore({
|
||||
get: async (key) => documents.get(key) ?? null,
|
||||
set: async (key, value) => void documents.set(key, value),
|
||||
set: async (key, value) => {
|
||||
documents.set(key, value)
|
||||
return []
|
||||
},
|
||||
remove: async (key) => void documents.delete(key),
|
||||
putBlob: async () => id,
|
||||
getBlob,
|
||||
@@ -113,7 +116,7 @@ test("keeps different blob IDs independent", async () => {
|
||||
const reads: string[] = []
|
||||
const store = createDraftStore({
|
||||
get: async () => JSON.stringify(["history-cache-first", "history-cache-second"].map((id) => ({ blob: { id } }))),
|
||||
set: async () => {},
|
||||
set: async () => [],
|
||||
remove: async () => {},
|
||||
putBlob: async () => "unused",
|
||||
getBlob: async (id) => {
|
||||
|
||||
@@ -40,7 +40,7 @@ describe("prompt persistence", () => {
|
||||
async (raw) => {
|
||||
const store = createDraftStore({
|
||||
get: async () => raw,
|
||||
set: async () => undefined,
|
||||
set: async () => [],
|
||||
remove: async () => undefined,
|
||||
putBlob: async () => "unused",
|
||||
getBlob: async () => null,
|
||||
@@ -68,7 +68,10 @@ describe("prompt persistence", () => {
|
||||
const blobs = new Map<string, Blob>()
|
||||
const store = createDraftStore({
|
||||
get: async (key) => documents.get(key) ?? null,
|
||||
set: async (key, value) => void documents.set(key, value),
|
||||
set: async (key, value) => {
|
||||
documents.set(key, value)
|
||||
return []
|
||||
},
|
||||
remove: async (key) => void documents.delete(key),
|
||||
putBlob: async (blob) => {
|
||||
blobs.set("composer-image", blob)
|
||||
@@ -170,7 +173,10 @@ describe("prompt persistence", () => {
|
||||
const documents = new Map<string, string>()
|
||||
const store = createDraftStore({
|
||||
get: async (key) => documents.get(key) ?? null,
|
||||
set: async (key, value) => void documents.set(key, value),
|
||||
set: async (key, value) => {
|
||||
documents.set(key, value)
|
||||
return []
|
||||
},
|
||||
remove: async (key) => void documents.delete(key),
|
||||
putBlob: async () => "blob",
|
||||
getBlob: async () => null,
|
||||
@@ -201,7 +207,10 @@ describe("prompt persistence", () => {
|
||||
const documents = new Map<string, string>()
|
||||
const store = createDraftStore({
|
||||
get: async (key) => documents.get(key) ?? null,
|
||||
set: async (key, value) => void documents.set(key, value),
|
||||
set: async (key, value) => {
|
||||
documents.set(key, value)
|
||||
return []
|
||||
},
|
||||
remove: async (key) => void documents.delete(key),
|
||||
putBlob: async () => "blob",
|
||||
getBlob: async () => null,
|
||||
@@ -233,7 +242,10 @@ test("moves image data URLs into blobs and hydrates object URLs", async () => {
|
||||
const blobs = new Map<string, Blob>()
|
||||
const store = createDraftStore({
|
||||
get: async (key) => documents.get(key) ?? null,
|
||||
set: async (key, value) => void documents.set(key, value),
|
||||
set: async (key, value) => {
|
||||
documents.set(key, value)
|
||||
return []
|
||||
},
|
||||
remove: async (key) => void documents.delete(key),
|
||||
putBlob: async (blob) => {
|
||||
const id = String(blob.size)
|
||||
@@ -255,7 +267,10 @@ test("does not let delayed blob migration overwrite a newer draft", async () =>
|
||||
const migration = Promise.withResolvers<void>()
|
||||
const store = createDraftStore({
|
||||
get: async () => null,
|
||||
set: async (key, value) => void documents.set(key, value),
|
||||
set: async (key, value) => {
|
||||
documents.set(key, value)
|
||||
return []
|
||||
},
|
||||
remove: async () => undefined,
|
||||
putBlob: async () => {
|
||||
await migration.promise
|
||||
|
||||
@@ -26,8 +26,8 @@ export const storageHandlers = StorageRpcs.toLayer(
|
||||
}),
|
||||
StorageClear: ({ name }) => Effect.sync(() => storage.state.clear(name)),
|
||||
DraftsGet: ({ key }) => Effect.sync(() => storage.drafts.get(key)),
|
||||
DraftsSet: ({ key, value }) => Effect.sync(() => storage.drafts.set(key, value)),
|
||||
DraftsDelete: ({ key }) => Effect.sync(() => storage.drafts.set(key, null)),
|
||||
DraftsSet: ({ key, value, strict }) => Effect.sync(() => storage.drafts.set(key, value, strict)),
|
||||
DraftsDelete: ({ key }) => Effect.sync(() => void storage.drafts.set(key, null)),
|
||||
DraftsPutBlob: ({ data }) => Effect.sync(() => storage.drafts.putBlob(data)),
|
||||
DraftsGetBlob: ({ id }) => Effect.sync(() => storage.drafts.getBlob(id)),
|
||||
})
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { sql } from "drizzle-orm"
|
||||
import { openDatabase } from "./database"
|
||||
import { createDraftStore } from "./drafts"
|
||||
import { blobGrace, createDraftStore } from "./drafts"
|
||||
|
||||
describe("draft store", () => {
|
||||
test("queues documents and reads them back before and after flush", () => {
|
||||
@@ -29,4 +29,102 @@ describe("draft store", () => {
|
||||
expect(second.getBlob(used)).toEqual(new Uint8Array([1, 2, 3]))
|
||||
expect(second.getBlob(unused)).toBeNull()
|
||||
})
|
||||
|
||||
test("collects a retired chunk only once it is unreferenced and past the grace period", () => {
|
||||
const database = openDatabase(":memory:")
|
||||
let clock = 0
|
||||
const drafts = createDraftStore(database.db, { delay: 1_000, now: () => clock })
|
||||
const text = (...ids: string[]) =>
|
||||
JSON.stringify({ prompt: [{ type: "text", content: { blob: { kind: "text", ids } } }] })
|
||||
const a = drafts.putBlob(new TextEncoder().encode("chunk a"))
|
||||
const b = drafts.putBlob(new TextEncoder().encode("chunk b"))
|
||||
drafts.set("doc", text(a, b))
|
||||
drafts.flush()
|
||||
const c = drafts.putBlob(new TextEncoder().encode("chunk c"))
|
||||
drafts.set("doc", text(a, c))
|
||||
drafts.flush()
|
||||
// Retired but recently touched: survives a due collection.
|
||||
clock = 120_000
|
||||
drafts.putBlob(new TextEncoder().encode("chunk d"))
|
||||
drafts.set("other", "{}")
|
||||
drafts.flush()
|
||||
expect(drafts.getBlob(b)).not.toBeNull()
|
||||
// Past the grace period and still unreferenced: collected. Referenced chunks stay.
|
||||
clock = 120_000 + blobGrace + 1
|
||||
drafts.putBlob(new TextEncoder().encode("chunk e"))
|
||||
drafts.set("other", "{}")
|
||||
drafts.flush()
|
||||
expect(drafts.getBlob(a)).not.toBeNull()
|
||||
expect(drafts.getBlob(c)).not.toBeNull()
|
||||
expect(drafts.getBlob(b)).toBeNull()
|
||||
})
|
||||
|
||||
test("a document that republishes a cached chunk id refreshes the chunk without an upload", () => {
|
||||
const database = openDatabase(":memory:")
|
||||
let clock = 0
|
||||
const drafts = createDraftStore(database.db, { delay: 1_000, now: () => clock })
|
||||
const text = (...ids: string[]) =>
|
||||
JSON.stringify({ prompt: [{ type: "text", content: { blob: { kind: "text", ids } } }] })
|
||||
const a = drafts.putBlob(new TextEncoder().encode("A"))
|
||||
drafts.set("doc", text(a))
|
||||
drafts.flush()
|
||||
// Edit away from A; A becomes unreferenced.
|
||||
const b = drafts.putBlob(new TextEncoder().encode("A!"))
|
||||
drafts.set("doc", text(b))
|
||||
drafts.flush()
|
||||
// Long after, undo republishes A from the renderer cache with no upload. The write touches A.
|
||||
clock = blobGrace - 1
|
||||
drafts.set("doc", text(a))
|
||||
drafts.flush()
|
||||
clock = blobGrace + collectInterval
|
||||
drafts.putBlob(new TextEncoder().encode("unrelated"))
|
||||
drafts.set("other", "{}")
|
||||
drafts.flush()
|
||||
expect(drafts.getBlob(a)).not.toBeNull()
|
||||
expect(drafts.getBlob(b)).toBeNull()
|
||||
})
|
||||
|
||||
test("set reports referenced blobs the store does not hold so the renderer can upload them again", () => {
|
||||
const database = openDatabase(":memory:")
|
||||
const drafts = createDraftStore(database.db, { delay: 1_000 })
|
||||
const kept = drafts.putBlob(new TextEncoder().encode("kept"))
|
||||
const image = drafts.putBlob(new Uint8Array([9]))
|
||||
const document = JSON.stringify({
|
||||
prompt: [
|
||||
{ type: "text", content: { blob: { kind: "text", ids: [kept, "gone-chunk"] } } },
|
||||
{ type: "image", blob: { id: image } },
|
||||
{ type: "image", blob: { id: "gone-image" } },
|
||||
],
|
||||
})
|
||||
// A strict write with missing blobs is refused: the previous document remains.
|
||||
drafts.set("doc", JSON.stringify({ previous: true }))
|
||||
expect(drafts.set("doc", document, true).sort()).toEqual(["gone-chunk", "gone-image"])
|
||||
expect(drafts.get("doc")).toBe(JSON.stringify({ previous: true }))
|
||||
// A non-strict write stores it anyway and still reports what is missing.
|
||||
expect(drafts.set("doc", document, false).sort()).toEqual(["gone-chunk", "gone-image"])
|
||||
expect(drafts.get("doc")).toBe(document)
|
||||
expect(drafts.set("plain", "not json", true)).toEqual([])
|
||||
expect(drafts.set("doc", null, true)).toEqual([])
|
||||
expect(drafts.set("doc", document, true)).not.toContain(kept)
|
||||
})
|
||||
|
||||
test("an uploaded attachment survives a due collection before its document is written", () => {
|
||||
const database = openDatabase(":memory:")
|
||||
let clock = 0
|
||||
const drafts = createDraftStore(database.db, { delay: 1_000, now: () => clock })
|
||||
drafts.putBlob(new TextEncoder().encode("old"))
|
||||
drafts.set("other", JSON.stringify({ n: 1 }))
|
||||
drafts.flush()
|
||||
clock = collectInterval + 1
|
||||
// The renderer uploads first and saves the referencing document up to a second later.
|
||||
const image = drafts.putBlob(new Uint8Array([1, 2, 3]))
|
||||
drafts.set("other", JSON.stringify({ n: 2 }))
|
||||
drafts.flush()
|
||||
expect(drafts.getBlob(image)).not.toBeNull()
|
||||
drafts.set("doc", JSON.stringify({ prompt: [{ type: "image", blob: { id: image } }] }))
|
||||
drafts.flush()
|
||||
expect(drafts.getBlob(image)).not.toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
const collectInterval = 60_000
|
||||
|
||||
@@ -6,8 +6,34 @@ import { createWriteBehind } from "./write-behind"
|
||||
|
||||
export type DraftStore = ReturnType<typeof createDraftStore>
|
||||
|
||||
export function createDraftStore(db: Database, input: { delay?: number; onError?: (error: unknown) => void } = {}) {
|
||||
collectBlobs(db)
|
||||
// Editing a large paste retires one text chunk per save, so orphans accumulate while the app runs.
|
||||
const collectInterval = 60_000
|
||||
// A blob stays collectable-proof for this long after its last upload or document reference. The
|
||||
// renderer reuses a cached chunk id without uploading for far less than this (see
|
||||
// draftChunkCacheTtl), so a reference it publishes always points at a retained blob.
|
||||
export const blobGrace = 15 * 60_000
|
||||
|
||||
// Every blob id a document references: image parts `{ blob: { id } }` and text chunk lists
|
||||
// `{ blob: { kind: "text", ids: [...] } }`. SQLite walks the JSON; nothing parses drafts in JS.
|
||||
const referenced = (value: unknown) => sql`
|
||||
SELECT json_extract(node.value, '$.id') AS id
|
||||
FROM json_tree(${value}) AS node
|
||||
WHERE node.key = 'blob' AND node.type = 'object' AND json_type(node.value, '$.id') = 'text'
|
||||
UNION
|
||||
SELECT chunk.value AS id
|
||||
FROM json_tree(${value}) AS node, json_each(node.value, '$.ids') AS chunk
|
||||
WHERE node.key = 'blob' AND node.type = 'object' AND json_type(node.value, '$.ids') = 'array'
|
||||
`
|
||||
|
||||
export function createDraftStore(
|
||||
db: Database,
|
||||
input: { delay?: number; onError?: (error: unknown) => void; now?: () => number } = {},
|
||||
) {
|
||||
const now = input.now ?? Date.now
|
||||
// Nothing outside this process can hold a blob id at startup, so no grace applies.
|
||||
collectBlobs(db, Infinity)
|
||||
let collected = now()
|
||||
let orphans = false
|
||||
const byKey = eq(document.key, sql.placeholder("key"))
|
||||
const read = db.select({ value: document.value }).from(document).where(byKey).prepare()
|
||||
const remove = db.delete(document).where(byKey).prepare()
|
||||
@@ -19,13 +45,27 @@ export function createDraftStore(db: Database, input: { delay?: number; onError?
|
||||
const writer = createWriteBehind<string | null>({
|
||||
delay: input.delay ?? 500,
|
||||
onError: input.onError,
|
||||
write: (batch) =>
|
||||
write: (batch) => {
|
||||
const at = now()
|
||||
db.transaction(() => {
|
||||
for (const [key, value] of batch) {
|
||||
if (value === null) remove.run({ key })
|
||||
else upsert.run({ key, value })
|
||||
if (value === null) {
|
||||
remove.run({ key })
|
||||
continue
|
||||
}
|
||||
upsert.run({ key, value })
|
||||
// Referencing a blob keeps it alive; done here so a reference the renderer republished
|
||||
// from its cache is refreshed even though no upload happened.
|
||||
if (json(value))
|
||||
db.run(sql`UPDATE ${blobs} SET touched_at = ${at} WHERE ${blobs.id} IN (${referenced(value)})`)
|
||||
}
|
||||
}),
|
||||
})
|
||||
// Only a document rewrite can orphan a blob, so collect right after one when due.
|
||||
if (!orphans || at - collected < collectInterval) return
|
||||
collectBlobs(db, at - blobGrace)
|
||||
collected = at
|
||||
orphans = false
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
@@ -33,13 +73,29 @@ export function createDraftStore(db: Database, input: { delay?: number; onError?
|
||||
if (writer.has(key)) return writer.get(key) ?? null
|
||||
return read.get({ key })?.value ?? null
|
||||
},
|
||||
set: (key: string, value: string | null) => writer.set(key, value),
|
||||
// Returns the referenced blob ids this store does not hold so the renderer can upload them
|
||||
// again. A strict write is refused while any are missing, so the previously stored document
|
||||
// stays visible instead of one with dangling references.
|
||||
set(key: string, value: string | null, strict = false) {
|
||||
const missing =
|
||||
value === null || !json(value)
|
||||
? []
|
||||
: db
|
||||
.all<{
|
||||
id: string
|
||||
}>(sql`SELECT ref.id FROM (${referenced(value)}) AS ref WHERE ref.id NOT IN (SELECT ${blobs.id} FROM ${blobs})`)
|
||||
.map((row) => row.id)
|
||||
if (!strict || missing.length === 0) writer.set(key, value)
|
||||
return missing
|
||||
},
|
||||
putBlob(data: Uint8Array) {
|
||||
const id = createHash("sha256").update(data).digest("hex")
|
||||
const touched_at = now()
|
||||
db.insert(blobs)
|
||||
.values({ id, data: Buffer.from(data) })
|
||||
.onConflictDoNothing()
|
||||
.values({ id, data: Buffer.from(data), touched_at })
|
||||
.onConflictDoUpdate({ target: blobs.id, set: { touched_at } })
|
||||
.run()
|
||||
orphans = true
|
||||
return id
|
||||
},
|
||||
getBlob(id: string): Uint8Array | null {
|
||||
@@ -50,14 +106,25 @@ export function createDraftStore(db: Database, input: { delay?: number; onError?
|
||||
}
|
||||
}
|
||||
|
||||
// Blobs are content-addressed and shared; drop the ones no document references anymore. SQLite
|
||||
// walks the JSON itself, so startup does not parse every draft and history entry in JavaScript.
|
||||
function collectBlobs(db: Database) {
|
||||
function json(value: string) {
|
||||
return value.startsWith("{") || value.startsWith("[")
|
||||
}
|
||||
|
||||
// Drop blobs no stored document references and nothing has touched since `before`.
|
||||
function collectBlobs(db: Database, before: number) {
|
||||
db.run(sql`
|
||||
DELETE FROM ${blobs} WHERE ${blobs.id} NOT IN (
|
||||
SELECT json_extract(node.value, '$.id')
|
||||
FROM ${document}, json_tree(${document.value}) AS node
|
||||
WHERE json_valid(${document.value}) AND node.key = 'blob' AND node.type = 'object'
|
||||
)
|
||||
DELETE FROM ${blobs}
|
||||
WHERE ${blobs.touched_at} < ${before === Infinity ? Number.MAX_SAFE_INTEGER : before}
|
||||
AND ${blobs.id} NOT IN (
|
||||
SELECT json_extract(node.value, '$.id')
|
||||
FROM ${document}, json_tree(${document.value}) AS node
|
||||
WHERE json_valid(${document.value}) AND node.key = 'blob' AND node.type = 'object'
|
||||
AND json_type(node.value, '$.id') = 'text'
|
||||
UNION
|
||||
SELECT chunk.value
|
||||
FROM ${document}, json_tree(${document.value}) AS node, json_each(node.value, '$.ids') AS chunk
|
||||
WHERE json_valid(${document.value}) AND node.key = 'blob' AND node.type = 'object'
|
||||
AND json_type(node.value, '$.ids') = 'array'
|
||||
)
|
||||
`)
|
||||
}
|
||||
|
||||
@@ -14,4 +14,8 @@ export const migrations = [
|
||||
"CREATE TABLE `state` (\n\t`name` text NOT NULL,\n\t`key` text NOT NULL,\n\t`value` text NOT NULL,\n\t`updated_at` integer NOT NULL,\n\tCONSTRAINT `state_pk` PRIMARY KEY(`name`, `key`)\n);",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "20260907031611_blob-touched",
|
||||
statements: ["ALTER TABLE `blob` ADD `touched_at` integer DEFAULT 0 NOT NULL;"],
|
||||
},
|
||||
]
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE `blob` ADD `touched_at` integer DEFAULT 0 NOT NULL;
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
{
|
||||
"version": "7",
|
||||
"dialect": "sqlite",
|
||||
"id": "53c65132-8703-42d6-8464-64356145dfb4",
|
||||
"prevIds": [
|
||||
"5a2f8b7e-2765-4a0b-8877-a74ee2f29b20"
|
||||
],
|
||||
"ddl": [
|
||||
{
|
||||
"name": "blob",
|
||||
"entityType": "tables"
|
||||
},
|
||||
{
|
||||
"name": "document",
|
||||
"entityType": "tables"
|
||||
},
|
||||
{
|
||||
"name": "state",
|
||||
"entityType": "tables"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "id",
|
||||
"entityType": "columns",
|
||||
"table": "blob"
|
||||
},
|
||||
{
|
||||
"type": "blob",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "data",
|
||||
"entityType": "columns",
|
||||
"table": "blob"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "0",
|
||||
"generated": null,
|
||||
"name": "touched_at",
|
||||
"entityType": "columns",
|
||||
"table": "blob"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "key",
|
||||
"entityType": "columns",
|
||||
"table": "document"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "value",
|
||||
"entityType": "columns",
|
||||
"table": "document"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "name",
|
||||
"entityType": "columns",
|
||||
"table": "state"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "key",
|
||||
"entityType": "columns",
|
||||
"table": "state"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "value",
|
||||
"entityType": "columns",
|
||||
"table": "state"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "updated_at",
|
||||
"entityType": "columns",
|
||||
"table": "state"
|
||||
},
|
||||
{
|
||||
"columns": [
|
||||
"name",
|
||||
"key"
|
||||
],
|
||||
"nameExplicit": false,
|
||||
"name": "state_pk",
|
||||
"entityType": "pks",
|
||||
"table": "state"
|
||||
},
|
||||
{
|
||||
"columns": [
|
||||
"id"
|
||||
],
|
||||
"nameExplicit": false,
|
||||
"name": "blob_pk",
|
||||
"table": "blob",
|
||||
"entityType": "pks"
|
||||
},
|
||||
{
|
||||
"columns": [
|
||||
"key"
|
||||
],
|
||||
"nameExplicit": false,
|
||||
"name": "document_pk",
|
||||
"table": "document",
|
||||
"entityType": "pks"
|
||||
}
|
||||
],
|
||||
"renames": []
|
||||
}
|
||||
@@ -6,10 +6,12 @@ export const document = sqliteTable("document", {
|
||||
value: text().notNull(),
|
||||
})
|
||||
|
||||
// Images referenced from documents by content hash.
|
||||
// Images and text chunks referenced from documents by content hash. `touched_at` is the last time
|
||||
// a blob was uploaded or a written document referenced it; collection leaves recent blobs alone.
|
||||
export const blobs = sqliteTable("blob", {
|
||||
id: text().primaryKey(),
|
||||
data: blob({ mode: "buffer" }).notNull(),
|
||||
touched_at: integer().notNull().default(0),
|
||||
})
|
||||
|
||||
// Everything the renderer persists through `platform.storage(name)`. `name` is the storage
|
||||
|
||||
@@ -39,7 +39,7 @@ export type ElectronAPI = {
|
||||
cb: (name: string, insert: Record<string, string>, remove: string[], revision: number) => void,
|
||||
): () => void
|
||||
draftGet(key: string): Promise<string | null>
|
||||
draftSet(key: string, value: string): Promise<void>
|
||||
draftSet(key: string, value: string, strict: boolean): Promise<string[]>
|
||||
draftDelete(key: string): Promise<void>
|
||||
draftBlobPut(data: ArrayBuffer): Promise<string>
|
||||
draftBlobGet(id: string): Promise<ArrayBuffer | null>
|
||||
|
||||
@@ -81,7 +81,7 @@ export const api: ElectronAPI = {
|
||||
onStoreChanged: (cb) =>
|
||||
listen("StorageChanged", (event) => cb(event.name, mutable(event.insert), mutable(event.remove), event.revision)),
|
||||
draftGet: (key) => invoke("DraftsGet", { key }),
|
||||
draftSet: (key, value) => invoke("DraftsSet", { key, value }),
|
||||
draftSet: (key, value, strict) => invoke("DraftsSet", { key, value, strict }).then(mutable),
|
||||
draftDelete: (key) => invoke("DraftsDelete", { key }),
|
||||
draftBlobPut: (data) => invoke("DraftsPutBlob", { data: new Uint8Array(data) }),
|
||||
draftBlobGet: (id) => invoke("DraftsGetBlob", { id }).then((data) => (data ? toArrayBuffer(data) : null)),
|
||||
|
||||
@@ -19,7 +19,8 @@ export const DraftsGet = Rpc.make("DraftsGet", {
|
||||
success: Schema.NullOr(Schema.String),
|
||||
})
|
||||
export const DraftsSet = Rpc.make("DraftsSet", {
|
||||
payload: { key: Schema.String, value: Schema.String },
|
||||
payload: { key: Schema.String, value: Schema.String, strict: Schema.Boolean },
|
||||
success: Schema.Array(Schema.String),
|
||||
})
|
||||
export const DraftsDelete = Rpc.make("DraftsDelete", { payload: { key: Schema.String } })
|
||||
export const DraftsPutBlob = Rpc.make("DraftsPutBlob", {
|
||||
|
||||
Reference in New Issue
Block a user