From 70c69f06f9d7ae3103d923898d926d88f3c97f4d Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sun, 25 Oct 2026 08:28:46 -0700 Subject: [PATCH] feat(core): store session media as media: refs in a content-addressed blob table (#58656) --- packages/core/schema.json | 190 ++++++++++- packages/core/src/database/migration.gen.ts | 2 + .../database/migration/20261025123711_blob.ts | 36 ++ packages/core/src/database/schema.gen.ts | 24 ++ packages/core/src/session/blob.ts | 107 ++++++ packages/core/src/session/history.ts | 53 ++- packages/core/src/session/inbox.ts | 40 ++- packages/core/src/session/projector.ts | 56 ++-- packages/core/src/session/recent-text.ts | 3 +- packages/core/src/session/runner/llm.ts | 11 +- packages/core/src/session/sql.ts | 44 ++- packages/core/src/session/store.ts | 14 +- packages/core/src/session/stored.ts | 41 +++ packages/core/test/session-projector.test.ts | 312 +++++++++++++++++- 14 files changed, 861 insertions(+), 72 deletions(-) create mode 100644 packages/core/src/database/migration/20261025123711_blob.ts create mode 100644 packages/core/src/session/blob.ts create mode 100644 packages/core/src/session/stored.ts diff --git a/packages/core/schema.json b/packages/core/schema.json index 9efc0260580..58c349f7d57 100644 --- a/packages/core/schema.json +++ b/packages/core/schema.json @@ -1,8 +1,8 @@ { "version": "7", "dialect": "sqlite", - "id": "76109fbf-f324-4af0-934c-3ae695275099", - "prevIds": ["ff2efc22-36c2-4af5-8f24-2288090b0ba6"], + "id": "f1f14616-b6a1-483d-87b0-d5b7a9132580", + "prevIds": ["76109fbf-f324-4af0-934c-3ae695275099"], "ddl": [ { "name": "account_state", @@ -44,6 +44,14 @@ "name": "project", "entityType": "tables" }, + { + "name": "blob_ref", + "entityType": "tables" + }, + { + "name": "blob", + "entityType": "tables" + }, { "name": "instruction_blob", "entityType": "tables" @@ -740,6 +748,116 @@ "entityType": "columns", "table": "project" }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "session_id", + "entityType": "columns", + "table": "blob_ref" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "message_id", + "entityType": "columns", + "table": "blob_ref" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "hash", + "entityType": "columns", + "table": "blob_ref" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "seq", + "entityType": "columns", + "table": "blob_ref" + }, + { + "type": "text", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "hash", + "entityType": "columns", + "table": "blob" + }, + { + "type": "text", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "mime", + "entityType": "columns", + "table": "blob" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "bytes", + "entityType": "columns", + "table": "blob" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "width", + "entityType": "columns", + "table": "blob" + }, + { + "type": "integer", + "notNull": false, + "autoincrement": false, + "default": null, + "generated": null, + "name": "height", + "entityType": "columns", + "table": "blob" + }, + { + "type": "blob", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "data", + "entityType": "columns", + "table": "blob" + }, + { + "type": "integer", + "notNull": true, + "autoincrement": false, + "default": null, + "generated": null, + "name": "time_created", + "entityType": "columns", + "table": "blob" + }, { "type": "text", "notNull": false, @@ -1574,6 +1692,28 @@ "entityType": "fks", "table": "project_directory" }, + { + "columns": ["session_id"], + "tableTo": "session_v2", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_blob_ref_session_id_session_v2_id_fk", + "entityType": "fks", + "table": "blob_ref" + }, + { + "columns": ["hash"], + "tableTo": "blob", + "columnsTo": ["hash"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "nameExplicit": false, + "name": "fk_blob_ref_hash_blob_hash_fk", + "entityType": "fks", + "table": "blob_ref" + }, { "columns": ["session_id"], "tableTo": "session_v2", @@ -1665,6 +1805,13 @@ "entityType": "pks", "table": "project_directory" }, + { + "columns": ["session_id", "message_id", "hash"], + "nameExplicit": false, + "name": "blob_ref_pk", + "entityType": "pks", + "table": "blob_ref" + }, { "columns": ["session_id", "key"], "nameExplicit": false, @@ -1735,6 +1882,13 @@ "table": "project", "entityType": "pks" }, + { + "columns": ["hash"], + "nameExplicit": false, + "name": "blob_pk", + "table": "blob", + "entityType": "pks" + }, { "columns": ["hash"], "nameExplicit": false, @@ -1846,6 +2000,38 @@ "entityType": "indexes", "table": "permission" }, + { + "columns": [ + { + "value": "hash", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "blob_ref_hash_idx", + "entityType": "indexes", + "table": "blob_ref" + }, + { + "columns": [ + { + "value": "session_id", + "isExpression": false + }, + { + "value": "seq", + "isExpression": false + } + ], + "isUnique": false, + "where": null, + "origin": "manual", + "name": "blob_ref_session_seq_idx", + "entityType": "indexes", + "table": "blob_ref" + }, { "columns": [ { diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index d7c7f001d7e..53df88ae0d1 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -51,6 +51,7 @@ import m48 from "./migration/20261007190000_azure_cli_external_credential.js" import m49 from "./migration/20261010180456_project_time_archived.js" import m50 from "./migration/20261024002554_session_allowance.js" import m51 from "./migration/20261024175301_drop_session_allowance.js" +import m52 from "./migration/20261025123711_blob.js" export const migrations = [ m00, @@ -105,4 +106,5 @@ export const migrations = [ m49, m50, m51, + m52, ] satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20261025123711_blob.ts b/packages/core/src/database/migration/20261025123711_blob.ts new file mode 100644 index 00000000000..266407d603d --- /dev/null +++ b/packages/core/src/database/migration/20261025123711_blob.ts @@ -0,0 +1,36 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration.js" + +const migration: DatabaseMigration.Migration = { + id: "20261025123711_blob", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`blob_ref\` ( + \`session_id\` text NOT NULL, + \`message_id\` text NOT NULL, + \`hash\` text NOT NULL, + \`seq\` integer NOT NULL, + CONSTRAINT \`blob_ref_pk\` PRIMARY KEY(\`session_id\`, \`message_id\`, \`hash\`), + CONSTRAINT \`fk_blob_ref_session_id_session_v2_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session_v2\`(\`id\`) ON DELETE CASCADE, + CONSTRAINT \`fk_blob_ref_hash_blob_hash_fk\` FOREIGN KEY (\`hash\`) REFERENCES \`blob\`(\`hash\`) ON DELETE CASCADE + ); + `) + yield* tx.run(` + CREATE TABLE \`blob\` ( + \`hash\` text PRIMARY KEY, + \`mime\` text NOT NULL, + \`bytes\` integer NOT NULL, + \`width\` integer, + \`height\` integer, + \`data\` blob NOT NULL, + \`time_created\` integer NOT NULL + ); + `) + yield* tx.run(`CREATE INDEX \`blob_ref_hash_idx\` ON \`blob_ref\` (\`hash\`);`) + yield* tx.run(`CREATE INDEX \`blob_ref_session_seq_idx\` ON \`blob_ref\` (\`session_id\`,\`seq\`);`) + }) + }, +} + +export default migration diff --git a/packages/core/src/database/schema.gen.ts b/packages/core/src/database/schema.gen.ts index 3ce8c6fdab9..fd54c34b780 100644 --- a/packages/core/src/database/schema.gen.ts +++ b/packages/core/src/database/schema.gen.ts @@ -116,6 +116,28 @@ const schema: Omit = { \`commands\` text ); `) + yield* tx.run(` + CREATE TABLE \`blob_ref\` ( + \`session_id\` text NOT NULL, + \`message_id\` text NOT NULL, + \`hash\` text NOT NULL, + \`seq\` integer NOT NULL, + CONSTRAINT \`blob_ref_pk\` PRIMARY KEY(\`session_id\`, \`message_id\`, \`hash\`), + CONSTRAINT \`fk_blob_ref_session_id_session_v2_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session_v2\`(\`id\`) ON DELETE CASCADE, + CONSTRAINT \`fk_blob_ref_hash_blob_hash_fk\` FOREIGN KEY (\`hash\`) REFERENCES \`blob\`(\`hash\`) ON DELETE CASCADE + ); + `) + yield* tx.run(` + CREATE TABLE \`blob\` ( + \`hash\` text PRIMARY KEY, + \`mime\` text NOT NULL, + \`bytes\` integer NOT NULL, + \`width\` integer, + \`height\` integer, + \`data\` blob NOT NULL, + \`time_created\` integer NOT NULL + ); + `) yield* tx.run(` CREATE TABLE \`instruction_blob\` ( \`hash\` text PRIMARY KEY, @@ -245,6 +267,8 @@ const schema: Omit = { yield* tx.run( `CREATE UNIQUE INDEX \`permission_project_action_resource_idx\` ON \`permission\` (\`project_id\`,\`action\`,\`resource\`);`, ) + yield* tx.run(`CREATE INDEX \`blob_ref_hash_idx\` ON \`blob_ref\` (\`hash\`);`) + yield* tx.run(`CREATE INDEX \`blob_ref_session_seq_idx\` ON \`blob_ref\` (\`session_id\`,\`seq\`);`) yield* tx.run( `CREATE INDEX \`session_inbox_session_delivery_seq_idx\` ON \`session_inbox\` (\`session_id\`,\`delivery\`,\`enqueued_seq\`);`, ) diff --git a/packages/core/src/session/blob.ts b/packages/core/src/session/blob.ts new file mode 100644 index 00000000000..72b4bcc61da --- /dev/null +++ b/packages/core/src/session/blob.ts @@ -0,0 +1,107 @@ +export * as SessionBlob from "./blob.js" + +import { inArray } from "drizzle-orm" +import { Effect, Schema } from "effect" +import { map } from "effect/Array" +import type { FileAttachment } from "@opencode/schema/prompt" +import type { Tool } from "@opencode/schema/tool" +import type { Database } from "../database/database.js" +import { BlobTable, SessionInboxTable, SessionMessageTable } from "./sql.js" +import { SessionStored } from "./stored.js" + +type DatabaseService = Database.Interface["db"] +type MessageRow = typeof SessionMessageTable.$inferSelect +type InboxRow = typeof SessionInboxTable.$inferSelect +type Part = Extract["content"][number] +/** Base64 bytes by blob hash. */ +type Blobs = ReadonlyMap + +const isFileRef = Schema.is(SessionStored.FileRef) +const isToolFileRef = Schema.is(SessionStored.ToolFileRef) + +const toolItems = (part: Part): ReadonlyArray => + part.type === "tool" && (part.state.status === "completed" || part.state.status === "error") + ? (part.state.content ?? []) + : [] + +function collect(data: MessageRow["data"] | InboxRow["payload"], hashes: Set) { + if ("files" in data) + for (const file of data.files ?? []) if (isFileRef(file)) hashes.add(SessionStored.refHash(file.uri)) + if ("content" in data) + for (const part of data.content) + for (const item of toolItems(part)) if (isToolFileRef(item)) hashes.add(SessionStored.refHash(item.uri)) +} + +/** A ref whose blob is gone reads as this text, so clients and the model never see a bare ref. */ +const unavailable = (mime: string, name?: string) => + `[media unavailable: ${name === undefined ? mime : `${name} (${mime})`}]` + +function inlineFile(file: SessionStored.File, blobs: Blobs): FileAttachment { + if (!isFileRef(file)) return file + const { uri, ...rest } = file + const data = blobs.get(SessionStored.refHash(uri)) + if (data === undefined) + return { ...rest, data: Buffer.from(unavailable(file.mime, file.name)).toString("base64"), mime: "text/plain" } + return { ...rest, data } +} + +function inlineToolItem(item: Tool.Content, blobs: Blobs): Tool.Content { + if (!isToolFileRef(item)) return item + const data = blobs.get(SessionStored.refHash(item.uri)) + if (data === undefined) return { type: "text", text: unavailable(item.mime, item.name) } + return { ...item, uri: `data:${item.mime};base64,${data}` } +} + +function inlinePart(part: Part, blobs: Blobs): Part { + if (part.type !== "tool" || (part.state.status !== "completed" && part.state.status !== "error")) return part + const content = part.state.content + if (!content?.some(isToolFileRef)) return part + return { ...part, state: { ...part.state, content: map(content, (item) => inlineToolItem(item, blobs)) } } +} + +const load = Effect.fnUntraced(function* (db: DatabaseService, hashes: ReadonlySet) { + const all = [...hashes] + const blobs = new Map() + for (let offset = 0; offset < all.length; offset += 500) { + const found = yield* db + .select({ hash: BlobTable.hash, data: BlobTable.data }) + .from(BlobTable) + .where(inArray(BlobTable.hash, all.slice(offset, offset + 500))) + .all() + .pipe(Effect.orDie) + for (const blob of found) blobs.set(blob.hash, view(blob.data).toString("base64")) + } + return blobs +}) + +const view = (data: Uint8Array) => + Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength) + +/** Restores inline media for the refs in `session_message` rows. Rows without refs are returned untouched. */ +export const hydrate = Effect.fnUntraced(function* (db: DatabaseService, rows: ReadonlyArray) { + const hashes = new Set() + for (const row of rows) collect(row.data, hashes) + if (hashes.size === 0) return rows + const blobs = yield* load(db, hashes) + return rows.map((row): MessageRow => { + const data = row.data + if ("files" in data && data.files?.some(isFileRef)) + return { ...row, data: { ...data, files: data.files.map((file) => inlineFile(file, blobs)) } } + if ("content" in data && data.content.some((part) => toolItems(part).some(isToolFileRef))) + return { ...row, data: { ...data, content: data.content.map((part) => inlinePart(part, blobs)) } } + return row + }) +}) + +/** Restores inline media for the refs in `session_inbox` rows. Rows without refs are returned untouched. */ +export const hydrateInbox = Effect.fnUntraced(function* (db: DatabaseService, rows: ReadonlyArray) { + const hashes = new Set() + for (const row of rows) collect(row.payload, hashes) + if (hashes.size === 0) return rows + const blobs = yield* load(db, hashes) + return rows.map((row): InboxRow => { + const payload = row.payload + if (!("files" in payload) || !payload.files?.some(isFileRef)) return row + return { ...row, payload: { ...payload, files: payload.files.map((file) => inlineFile(file, blobs)) } } + }) +}) diff --git a/packages/core/src/session/history.ts b/packages/core/src/session/history.ts index ddaf3c5064e..ea7a69ea56d 100644 --- a/packages/core/src/session/history.ts +++ b/packages/core/src/session/history.ts @@ -9,13 +9,16 @@ import { SessionMessage } from "./message.js" import type { SessionSchema } from "./schema.js" import { Instructions } from "../instructions/index.js" import { InstructionState } from "./instruction-state.js" +import { SessionBlob } from "./blob.js" import { SessionRecentText } from "./recent-text.js" import { SessionMessageTable } from "./sql.js" +import { SessionStored } from "./stored.js" type DatabaseService = Database.Interface["db"] const decode = Schema.decodeUnknownEffect(SessionMessage.Info) const decodeSync = Schema.decodeUnknownSync(SessionMessage.Info) +const decodeStoredUser = Schema.decodeUnknownEffect(SessionStored.User) /** * Which completed compactions bound a history read. Local summaries always do. Native @@ -89,16 +92,28 @@ const rowInput = (row: typeof SessionMessageTable.$inferSelect) => ({ export const decodeMessageRowSync = (row: typeof SessionMessageTable.$inferSelect) => decodeSync(rowInput(row)) +const decodeError = (row: typeof SessionMessageTable.$inferSelect) => () => + new MessageDecodeError({ + sessionID: row.session_id, + messageID: row.id, + }) + export const decodeMessageRow = (row: typeof SessionMessageTable.$inferSelect) => - decode(rowInput(row)).pipe( - Effect.mapError( - () => - new MessageDecodeError({ - sessionID: row.session_id, - messageID: row.id, - }), - ), - ) + decode(rowInput(row)).pipe(Effect.mapError(decodeError(row))) + +/** Decodes a row without its blobs, so user files may stay refs. Only for readers that never use file bytes. */ +const decodeStoredRow = ( + row: typeof SessionMessageTable.$inferSelect, +): Effect.Effect => + row.type === "user" ? decodeStoredUser(rowInput(row)).pipe(Effect.mapError(decodeError(row))) : decodeMessageRow(row) + +/** Decodes rows after restoring media stored in `blob`. Reads that return media must use this. */ +export const decodeMessageRows = Effect.fnUntraced(function* ( + db: DatabaseService, + rows: ReadonlyArray, +) { + return yield* Effect.forEach(yield* SessionBlob.hydrate(db, rows), decodeMessageRow, { concurrency: 1 }) +}) const messageEntries = Effect.fnUntraced(function* ( db: DatabaseService, @@ -122,8 +137,10 @@ const messageEntries = Effect.fnUntraced(function* ( .orderBy(asc(SessionMessageTable.seq)) .all() .pipe(Effect.orDie) - const decoded = yield* Effect.forEach(rows, (row) => - decodeMessageRow(row).pipe(Effect.map((message) => ({ seq: row.seq, message }))), + const decoded = yield* Effect.forEach( + yield* SessionBlob.hydrate(db, rows), + (row) => decodeMessageRow(row).pipe(Effect.map((message) => ({ seq: row.seq, message }))), + { concurrency: 1 }, ) // A window that cannot replay leaves nothing to start from, so read from the compaction before it. const start = decoded[0] @@ -176,7 +193,8 @@ const withRecentText = Effect.fnUntraced(function* ( .orderBy(asc(SessionMessageTable.seq)) .all() .pipe(Effect.orDie) - const kept = yield* Effect.forEach(rows, decodeMessageRow) + // The recent text names attachments by mime and name only, so their bytes are never read. + const kept = yield* Effect.forEach(rows, decodeStoredRow, { concurrency: 1 }) const recent = kept.map(SessionRecentText.messageToText).filter(Boolean).join("\n\n") return [{ ...first, message: { ...compaction, metadata: { ...compaction.metadata, recent } } }, ...entries.slice(1)] }) @@ -235,21 +253,22 @@ export const preview = Effect.fn("SessionHistory.preview")(function* ( .pipe(Effect.catch((error) => (error instanceof Instructions.InitializationBlocked ? error : Effect.die(error)))) }) -/** Returns the session's first user message. */ +/** The id and text of the session's first user message. */ export const firstUserMessage = Effect.fn("SessionHistory.firstUserMessage")(function* ( db: DatabaseService, sessionID: SessionSchema.ID, ) { const row = yield* db - .select() + .select({ + id: SessionMessageTable.id, + text: sql`json_extract(${SessionMessageTable.data}, '$.text')`, + }) .from(SessionMessageTable) .where(and(eq(SessionMessageTable.session_id, sessionID), eq(SessionMessageTable.type, "user"))) .orderBy(asc(SessionMessageTable.seq)) .get() .pipe(Effect.orDie) - if (!row) return undefined - const message = yield* decodeMessageRow(row).pipe(Effect.orElseSucceed(() => undefined)) - return message?.type === "user" ? message : undefined + return row && Schema.is(Schema.String)(row.text) ? { id: row.id, text: row.text } : undefined }) export * as SessionHistory from "./history.js" diff --git a/packages/core/src/session/inbox.ts b/packages/core/src/session/inbox.ts index 31cd11f95d0..36eb12d34af 100644 --- a/packages/core/src/session/inbox.ts +++ b/packages/core/src/session/inbox.ts @@ -20,8 +20,10 @@ import { Database } from "../database/database.js" import { Bus } from "../bus.js" import { KeyedMutex } from "../effect/keyed-mutex.js" import { SessionEvent } from "./event.js" +import { SessionBlob } from "./blob.js" import { SessionMessage } from "./message.js" import { SessionSchema } from "./schema.js" +import { SessionStored } from "./stored.js" import { SessionInboxTable, SessionMessageTable } from "./sql.js" import { SessionHistory } from "./history.js" @@ -49,6 +51,7 @@ export { export type Promotable = "input" | "steer" const decodeUser = Schema.decodeUnknownSync(UserPayload) +const decodeStoredUser = Schema.decodeUnknownSync(SessionStored.UserPayload) const encodeUser = Schema.encodeSync(UserPayload) const decodeSynthetic = Schema.decodeUnknownSync(SyntheticPayload) const encodeSynthetic = Schema.encodeSync(SyntheticPayload) @@ -107,7 +110,9 @@ const fromRow = (row: typeof SessionInboxTable.$inferSelect): Info => { export const find = Effect.fn("SessionInbox.find")(function* (db: DatabaseService, id: SessionMessage.ID) { const row = yield* db.select().from(SessionInboxTable).where(eq(SessionInboxTable.id, id)).get().pipe(Effect.orDie) - return row === undefined ? undefined : fromRow(row) + if (row === undefined) return undefined + const [hydrated] = yield* SessionBlob.hydrateInbox(db, [row]) + return hydrated && fromRow(hydrated) }) const promotedFromMessage = Effect.fn("SessionInbox.promotedFromMessage")(function* ( @@ -125,7 +130,8 @@ const promotedFromMessage = Effect.fn("SessionInbox.promotedFromMessage")(functi if (row === undefined) return undefined if (row.session_id !== sessionID || (row.type !== "user" && row.type !== "synthetic")) return yield* new LifecycleConflict({ id }) - const message = SessionHistory.decodeMessageRowSync(row) + const [message] = yield* SessionHistory.decodeMessageRows(db, [row]).pipe(Effect.orDie) + if (!message) return undefined const base = { id, sessionID, time: { created: message.time.created }, delivery } if (message.type === "user") return User.make({ @@ -341,9 +347,21 @@ export const projectDelivered = Effect.fn("SessionInbox.projectDelivered")(funct .get() .pipe(Effect.orDie) if (!deleted) return yield* Effect.die(new LifecycleConflict({ id: input.id })) - return fromRow(deleted) + return delivered(deleted) }) +/** A consumed row with its payload as stored, so user file refs reach the message without their blobs. */ +export type Delivered = + | { readonly id: SessionMessage.ID; readonly type: "user"; readonly payload: SessionStored.UserPayload } + | { readonly id: SessionMessage.ID; readonly type: "synthetic"; readonly payload: SyntheticPayload } + | { readonly id: SessionMessage.ID; readonly type: "compaction" | "move" } + +const delivered = (row: typeof SessionInboxTable.$inferSelect): Delivered => { + if (row.type === "user") return { id: row.id, type: "user", payload: decodeStoredUser(row.payload) } + if (row.type === "synthetic") return { id: row.id, type: "synthetic", payload: decodeSynthetic(row.payload) } + return { id: row.id, type: row.type } +} + export const projectCancelled = Effect.fn("SessionInbox.projectCancelled")(function* ( db: DatabaseService, input: PendingRef, @@ -400,7 +418,7 @@ export const list = Effect.fn("SessionInbox.list")(function* (db: DatabaseServic .orderBy(asc(SessionInboxTable.enqueued_seq)) .all() .pipe(Effect.orDie) - return rows.map(fromRow) + return (yield* SessionBlob.hydrateInbox(db, rows)).map(fromRow) }) export const moveIDs = Effect.fn("SessionInbox.moveIDs")(function* (db: DatabaseService, sessionID: SessionSchema.ID) { @@ -419,7 +437,7 @@ export const nextPromotable = Effect.fn("SessionInbox.nextPromotable")(function* promotable: Promotable, ) { const steer = (yield* pendingSteers(db, sessionID))[0] - if (steer) return fromRow(steer) + if (steer) return promotion(steer) if (promotable !== "input") return undefined const queued = yield* db .select() @@ -429,9 +447,19 @@ export const nextPromotable = Effect.fn("SessionInbox.nextPromotable")(function* .limit(1) .get() .pipe(Effect.orDie) - return queued ? fromRow(queued) : undefined + return queued ? promotion(queued) : undefined }) +/** What a boundary decides on: prompts are promoted by id, so only a move's payload is read. */ +export type Promotion = + | { readonly id: SessionMessage.ID; readonly type: "user" | "synthetic" | "compaction"; readonly delivery: Delivery } + | { readonly id: SessionMessage.ID; readonly type: "move"; readonly delivery: Delivery; readonly payload: MovePayload } + +const promotion = (row: typeof SessionInboxTable.$inferSelect): Promotion => + row.type === "move" + ? { id: row.id, type: "move", delivery: row.delivery, payload: decodeMove(row.payload) } + : { id: row.id, type: row.type, delivery: row.delivery } + /** Which pending rows count: "input" means any item in either delivery mode. */ export type Scope = "input" | Delivery diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index f7602eafda7..e5dba0813ac 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -13,6 +13,7 @@ import { SessionMessageUpdater } from "./message-updater.js" import { SessionInbox } from "./inbox.js" import { InstructionState } from "./instruction-state.js" import { SessionInboxTable, SessionMessageTable, SessionTable } from "./sql.js" +import { SessionStored } from "./stored.js" import { InstructionEntry } from "./instruction-entry.js" import { Slug } from "../util/slug.js" import { FSUtil } from "@opencode/util/fs-util" @@ -32,6 +33,7 @@ type MessageEvent = Exclude< > const encodeMessage = Schema.encodeSync(SessionMessage.Info) +const encodeStoredUser = Schema.encodeSync(SessionStored.User) export class SessionAlreadyProjected extends Error {} @@ -398,9 +400,12 @@ function run(db: DatabaseService, event: MessageEvent) { }) } -function insertMessage(db: DatabaseService, event: SessionEvent.DurableEvent, message: SessionMessage.Info) { - const encoded = encodeMessage(message) - const { id: _, type, ...data } = encoded +function insertMessage( + db: DatabaseService, + event: SessionEvent.DurableEvent, + message: SessionMessage.Info | SessionStored.User, +) { + const { id: _, type, ...data } = message.type === "user" ? encodeStoredUser(message) : encodeMessage(message) return db .insert(SessionMessageTable) .values({ @@ -643,30 +648,27 @@ const layer = Layer.effectDiscard( id: event.data.inboxID, sessionID: event.data.sessionID, }) - if (input.type === "compaction" || input.type === "move") return - yield* insertMessage( - db, - event, - input.type === "user" - ? { - id: input.id, - type: "user", - metadata: input.payload.metadata, - text: input.payload.text, - files: input.payload.files, - agents: input.payload.agents, - skills: input.payload.skills, - time: { created: DateTime.makeUnsafe(event.created) }, - } - : { - id: input.id, - type: "synthetic", - text: input.payload.text, - description: input.payload.description, - metadata: input.payload.metadata, - time: { created: DateTime.makeUnsafe(event.created) }, - }, - ) + const time = { created: DateTime.makeUnsafe(event.created) } + if (input.type === "user") + yield* insertMessage(db, event, { + id: input.id, + type: "user", + metadata: input.payload.metadata, + text: input.payload.text, + files: input.payload.files, + agents: input.payload.agents, + skills: input.payload.skills, + time, + }) + if (input.type === "synthetic") + yield* insertMessage(db, event, { + id: input.id, + type: "synthetic", + text: input.payload.text, + description: input.payload.description, + metadata: input.payload.metadata, + time, + }) }), ) yield* bus.project(SessionEvent.InboxEnqueued, (event) => diff --git a/packages/core/src/session/recent-text.ts b/packages/core/src/session/recent-text.ts index 1f5c923249b..dd188a89b27 100644 --- a/packages/core/src/session/recent-text.ts +++ b/packages/core/src/session/recent-text.ts @@ -1,6 +1,7 @@ export * as SessionRecentText from "./recent-text.js" import type { SessionMessage } from "./message.js" +import type { SessionStored } from "./stored.js" const TOOL_OUTPUT_MAX_CHARS = 1_250 @@ -12,7 +13,7 @@ export const kept = (message: SessionMessage.CompactionCompleted) => typeof message.metadata?.recent === "string" ? message.metadata.recent : "" /** One message as the recent, verbatim part of a summary shows it. Empty for messages that part leaves out. */ -export const messageToText = (message: SessionMessage.Info): string => { +export const messageToText = (message: SessionMessage.Info | SessionStored.User): string => { switch (message.type) { // Earlier summaries and instruction updates are handled outside the recent part. case "compaction": diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 9406ab1bd5c..df3fdb09193 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -298,8 +298,11 @@ const layer = Layer.effect( }), OutputLimit: Effect.fn("SessionRunner.continueOutputLimit")(function* (outcome) { // The current response is included. User input or any other finish breaks the streak. - const rows = yield* db - .select() + const recent = yield* db + .select({ + type: SessionMessageTable.type, + finish: sql`json_extract(${SessionMessageTable.data}, '$.finish')`, + }) .from(SessionMessageTable) .where( and( @@ -311,10 +314,8 @@ const layer = Layer.effect( .limit(3) .all() .pipe(Effect.orDie) - const recent = yield* Effect.forEach(rows, SessionHistory.decodeMessageRow) const exhausted = - recent.length === 3 && - recent.every((message) => message.type === "assistant" && message.finish === "length") + recent.length === 3 && recent.every((row) => row.type === "assistant" && row.finish === "length") if (exhausted) return yield* new StepFailedError({ error: { diff --git a/packages/core/src/session/sql.ts b/packages/core/src/session/sql.ts index 88e9f925600..760326f8728 100644 --- a/packages/core/src/session/sql.ts +++ b/packages/core/src/session/sql.ts @@ -1,4 +1,4 @@ -import { sqliteTable, text, integer, index, primaryKey, real, uniqueIndex } from "drizzle-orm/sqlite-core" +import { blob, sqliteTable, text, integer, index, primaryKey, real, uniqueIndex } from "drizzle-orm/sqlite-core" import { sql } from "drizzle-orm" import { directoryColumn, pathColumn } from "../database/path.js" import { ProjectTable } from "../project/sql.js" @@ -9,6 +9,7 @@ import type { FileDiff } from "@opencode/schema/file-diff" import type { Permission } from "@opencode/schema/permission" import type { Project } from "@opencode/schema/project" import type { SessionSchema } from "./schema.js" +import type { SessionStored } from "./stored.js" import type { Workspace } from "@opencode/schema/workspace" import { Timestamps } from "../database/schema.sql.js" import type { Instruction } from "@opencode/schema/instruction" @@ -18,7 +19,10 @@ import type { RevertV1 } from "@opencode/schema/session-revert" import type { Schema } from "effect" type DistributiveOmit = T extends unknown ? Omit : never -type SessionMessageData = DistributiveOmit<(typeof SessionMessage.Info)["Encoded"], "type" | "id"> +type SessionMessageData = DistributiveOmit< + Exclude<(typeof SessionMessage.Info)["Encoded"], { readonly type: "user" }> | (typeof SessionStored.User)["Encoded"], + "type" | "id" +> export const SessionTable = sqliteTable( "session_v2", @@ -132,7 +136,9 @@ export const SessionInboxTable = sqliteTable( .notNull() .references(() => SessionTable.id, { onDelete: "cascade" }), type: text().$type().notNull(), - payload: text({ mode: "json" }).$type().notNull(), + payload: text({ mode: "json" }) + .$type() + .notNull(), delivery: text().$type().notNull(), enqueued_seq: integer().notNull(), time_created: integer() @@ -175,3 +181,35 @@ export const InstructionStateTable = sqliteTable("instruction_state", { initial_values: text({ mode: "json" }).notNull().$type(), current_values: text({ mode: "json" }).notNull().$type(), }) + +export const BlobTable = sqliteTable("blob", { + hash: text().primaryKey(), + mime: text().notNull(), + bytes: integer().notNull(), + width: integer(), + height: integer(), + data: blob({ mode: "buffer" }).notNull(), + time_created: integer() + .notNull() + .$default(() => Date.now()), +}) + +export const BlobRefTable = sqliteTable( + "blob_ref", + { + session_id: text() + .$type() + .notNull() + .references(() => SessionTable.id, { onDelete: "cascade" }), + message_id: text().$type().notNull(), + hash: text() + .notNull() + .references(() => BlobTable.hash, { onDelete: "cascade" }), + seq: integer().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.session_id, table.message_id, table.hash] }), + index("blob_ref_hash_idx").on(table.hash), + index("blob_ref_session_seq_idx").on(table.session_id, table.seq), + ], +) diff --git a/packages/core/src/session/store.ts b/packages/core/src/session/store.ts index c9698231bbc..00a7eeb0104 100644 --- a/packages/core/src/session/store.ts +++ b/packages/core/src/session/store.ts @@ -174,10 +174,7 @@ const layer = Layer.effect( const rows = yield* (input.limit === undefined ? query.all() : query.limit(input.limit).all()).pipe( Effect.orDie, ) - return yield* Effect.forEach( - direction === "previous" ? rows.toReversed() : rows, - SessionHistory.decodeMessageRow, - ) + return yield* SessionHistory.decodeMessageRows(db, direction === "previous" ? rows.toReversed() : rows) }), context: Effect.fn("SessionStore.context")((sessionID) => SessionHistory.load(db, sessionID, "latest")), message: Effect.fn("SessionStore.message")(function* (messageID) { @@ -187,12 +184,9 @@ const layer = Layer.effect( .where(eq(SessionMessageTable.id, messageID)) .get() .pipe(Effect.orDie) - return row - ? { - sessionID: row.session_id, - message: yield* SessionHistory.decodeMessageRow(row).pipe(Effect.orDie), - } - : undefined + if (!row) return undefined + const [message] = yield* SessionHistory.decodeMessageRows(db, [row]).pipe(Effect.orDie) + return message && { sessionID: row.session_id, message } }), listSuspended: Effect.fn("SessionStore.listSuspended")(function* () { return yield* db diff --git a/packages/core/src/session/stored.ts b/packages/core/src/session/stored.ts new file mode 100644 index 00000000000..717a28e2a7a --- /dev/null +++ b/packages/core/src/session/stored.ts @@ -0,0 +1,41 @@ +export * as SessionStored from "./stored.js" + +import { Schema } from "effect" +import { FileAttachment } from "@opencode/schema/prompt" +import { optional } from "@opencode/schema/schema" +import { UserPayload as InboxUserPayload } from "@opencode/schema/session-inbox" +import { Tool } from "@opencode/schema/tool" +import { SessionMessage } from "./message.js" + +/** + * Stored rows may reference a `blob` row as `media:` and the lowercase hex SHA-256 of its raw bytes. The blob + * row holds the bytes, size and dimensions; the ref names no server route. + */ +export const Ref = Schema.String.check(Schema.isPattern(/^media:[0-9a-f]{64}$/)) +export type Ref = typeof Ref.Type + +/** The `blob` hash a {@link Ref} names. */ +export const refHash = (ref: Ref) => ref.slice("media:".length) + +const { data: _, ...attachment } = FileAttachment.fields + +/** A user file whose bytes live in `blob`. */ +export interface FileRef extends Schema.Schema.Type {} +export const FileRef = Schema.Struct({ ...attachment, uri: Ref }) + +export const File = Schema.Union([FileAttachment, FileRef]) +export type File = typeof File.Type + +/** A tool result file whose bytes live in `blob`. */ +export interface ToolFileRef extends Schema.Schema.Type {} +export const ToolFileRef = Schema.Struct({ ...Tool.FileContent.fields, uri: Ref }) + +const files = Schema.Array(File).pipe(optional) + +/** A `session_inbox` user payload as stored, whose files may be refs. */ +export interface UserPayload extends Schema.Schema.Type {} +export const UserPayload = Schema.Struct({ ...InboxUserPayload.fields, files }) + +/** A `session_message` user message as stored, whose files may be refs. */ +export interface User extends Schema.Schema.Type {} +export const User = Schema.Struct({ ...SessionMessage.User.fields, files }) diff --git a/packages/core/test/session-projector.test.ts b/packages/core/test/session-projector.test.ts index 96f45aa7f51..dd17856b790 100644 --- a/packages/core/test/session-projector.test.ts +++ b/packages/core/test/session-projector.test.ts @@ -23,8 +23,16 @@ import { SessionExecution } from "@opencode/core/session/execution" import { fromRow } from "@opencode/core/session/info" import { SessionInbox } from "@opencode/core/session/inbox" import { SessionStore } from "@opencode/core/session/store" +import { SessionStored } from "@opencode/core/session/stored" +import { SessionHistory } from "@opencode/core/session/history" import { Shell } from "@opencode/schema/shell" -import { InstructionStateTable, SessionInboxTable, SessionMessageTable, SessionTable } from "@opencode/core/session/sql" +import { + InstructionStateTable, + BlobTable, + SessionInboxTable, + SessionMessageTable, + SessionTable, +} from "@opencode/core/session/sql" import { testEffect } from "./lib/effect" import { Snapshot } from "@opencode/core/snapshot" @@ -993,4 +1001,306 @@ describe("SessionProjector", () => { expect(yield* updated()).toEqual({ time_updated: 15 }) }), ) + + it.effect("hydrates media: refs on message and inbox reads", () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const inbox = yield* SessionInbox.Service + const store = yield* SessionStore.Service + + const rawBytes = Buffer.from("externalized-media-payload") + const base64 = rawBytes.toString("base64") + const hash = "a".repeat(64) + const uri = `media:${hash}` + // Only media: is a ref; a plain path that looks like a media URL is left alone. + const pathUri = `/api/media/${hash}` + + yield* db + .insert(BlobTable) + .values({ + hash, + mime: "application/pdf", + bytes: rawBytes.byteLength, + width: null, + height: null, + data: rawBytes, + }) + .run() + .pipe(Effect.orDie) + + const userID = SessionMessage.ID.make("msg_read_user") + const assistantID = SessionMessage.ID.make("msg_read_assistant") + const inboxID = SessionMessage.ID.make("msg_read_inbox") + + yield* db + .insert(SessionMessageTable) + .values([ + { + id: userID, + session_id: sessionID, + type: "user", + seq: 1, + time_created: 1, + data: { + text: "user attachment", + files: [SessionStored.FileRef.make({ uri, mime: "application/pdf", source: { type: "inline" }, name: "spec.pdf" })], + time: { created: 1 }, + }, + }, + { + id: assistantID, + session_id: sessionID, + type: "assistant", + seq: 2, + time_created: 2, + data: { + agent: build, + model, + content: [ + { + type: "tool", + id: "call_1", + name: "read", + executed: true, + state: { + status: "completed", + input: { path: "spec.pdf" }, + content: [ + SessionStored.ToolFileRef.make({ type: "file", uri, mime: "application/pdf", name: "spec.pdf" }), + { type: "file", uri: pathUri, mime: "application/pdf", name: "path.pdf" }, + ], + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + ], + time: { created: 2, completed: 2 }, + }, + }, + ]) + .run() + .pipe(Effect.orDie) + + yield* db + .insert(SessionInboxTable) + .values({ + id: inboxID, + session_id: sessionID, + type: "user", + payload: { + text: "queued attachment", + files: [SessionStored.FileRef.make({ uri, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" })], + }, + delivery: "queue", + enqueued_seq: 3, + time_created: 3, + }) + .run() + .pipe(Effect.orDie) + + const messages = yield* store.messages({ sessionID, order: "asc" }) + expect(messages).toHaveLength(2) + expect(messages[0]).toMatchObject({ + id: userID, + type: "user", + files: [{ data: base64, mime: "application/pdf", source: { type: "inline" }, name: "spec.pdf" }], + }) + expect(messages[1]).toMatchObject({ + id: assistantID, + type: "assistant", + content: [ + { + type: "tool", + state: { + status: "completed", + content: [ + { type: "file", uri: `data:application/pdf;base64,${base64}`, mime: "application/pdf", name: "spec.pdf" }, + { type: "file", uri: pathUri, mime: "application/pdf", name: "path.pdf" }, + ], + }, + }, + ], + }) + + expect((yield* store.message(userID))?.message).toEqual(messages[0]!) + expect(yield* store.context(sessionID)).toEqual(messages) + + const listedInbox = yield* inbox.list(sessionID) + expect(listedInbox).toHaveLength(1) + expect(listedInbox[0]).toMatchObject({ + id: inboxID, + type: "user", + payload: { + text: "queued attachment", + files: [{ data: base64, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" }], + }, + }) + + expect(yield* SessionHistory.firstUserMessage(db, sessionID)).toEqual({ id: userID, text: "user attachment" }) + expect(yield* SessionInbox.nextPromotable(db, sessionID, "input")).toEqual({ + id: inboxID, + type: "user", + delivery: "queue", + }) + + yield* bus.publish(SessionEvent.InboxDelivered, { sessionID, inboxID }) + expect((yield* store.message(inboxID))?.message).toMatchObject({ + id: inboxID, + type: "user", + files: [{ data: base64, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" }], + }) + // Delivery keeps the ref in the stored row instead of writing the bytes back inline. + const delivered = yield* db + .select({ data: SessionMessageTable.data }) + .from(SessionMessageTable) + .where(eq(SessionMessageTable.id, inboxID)) + .get() + .pipe(Effect.orDie) + expect(delivered?.data).toMatchObject({ + text: "queued attachment", + files: [{ uri, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" }], + }) + expect(JSON.stringify(delivered?.data)).not.toContain(base64) + }), + ) + + it.effect("replaces refs to missing blobs with an unavailable placeholder on read", () => + Effect.gen(function* () { + const db = yield* seedSession() + const inbox = yield* SessionInbox.Service + const store = yield* SessionStore.Service + const hash = "b".repeat(64) + const userID = SessionMessage.ID.make("msg_missing_user") + const assistantID = SessionMessage.ID.make("msg_missing_assistant") + const inboxID = SessionMessage.ID.make("msg_missing_inbox") + const placeholder = Buffer.from("[media unavailable: shot.png (image/png)]").toString("base64") + const ref = SessionStored.FileRef.make({ + uri: `media:${hash}`, + mime: "image/png", + source: { type: "inline" }, + name: "shot.png", + }) + + yield* db + .insert(SessionMessageTable) + .values([ + { + id: userID, + session_id: sessionID, + type: "user", + seq: 1, + time_created: 1, + data: { + text: "gone", + files: [ref], + time: { created: 1 }, + }, + }, + { + id: assistantID, + session_id: sessionID, + type: "assistant", + seq: 2, + time_created: 2, + data: { + agent: build, + model, + content: [ + { + type: "tool", + id: "call_1", + name: "read", + executed: true, + state: { + status: "completed", + input: { path: "shot.png" }, + content: [SessionStored.ToolFileRef.make({ type: "file", uri: ref.uri, mime: "image/png" })], + }, + time: { created: 2, ran: 2, completed: 2 }, + }, + ], + time: { created: 2, completed: 2 }, + }, + }, + ]) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionInboxTable) + .values({ + id: inboxID, + session_id: sessionID, + type: "user", + payload: { text: "queued", files: [ref] }, + delivery: "queue", + enqueued_seq: 3, + time_created: 3, + }) + .run() + .pipe(Effect.orDie) + + const messages = yield* store.messages({ sessionID, order: "asc" }) + expect(messages[0]).toMatchObject({ + files: [{ data: placeholder, mime: "text/plain", source: { type: "inline" }, name: "shot.png" }], + }) + expect(messages[1]).toMatchObject({ + content: [{ state: { content: [{ type: "text", text: "[media unavailable: image/png]" }] } }], + }) + expect(yield* store.context(sessionID)).toEqual(messages) + const listed = yield* inbox.list(sessionID) + expect(listed[0]).toMatchObject({ payload: { files: [{ data: placeholder, mime: "text/plain" }] } }) + expect(JSON.stringify([messages, listed])).not.toContain(hash) + }), + ) + + it.effect("names ref attachments in kept text without reading their blobs", () => + Effect.gen(function* () { + const db = yield* seedSession() + const store = yield* SessionStore.Service + const userID = SessionMessage.ID.make("msg_kept_user") + const compactionID = SessionMessage.ID.make("msg_kept_compaction") + const { id: _, type: __, ...compaction } = encodeMessage( + SessionMessage.CompactionCompleted.make({ + id: compactionID, + type: "compaction", + status: "completed", + reason: "manual", + summary: "summary", + metadata: { keepID: userID }, + time: { created }, + }), + ) + yield* db + .insert(SessionMessageTable) + .values([ + { + id: userID, + session_id: sessionID, + type: "user", + seq: 1, + time_created: 1, + data: { + text: "look", + files: [ + SessionStored.FileRef.make({ + uri: `media:${"c".repeat(64)}`, + mime: "image/png", + source: { type: "inline" }, + name: "shot.png", + }), + ], + time: { created: 1 }, + }, + }, + { id: compactionID, session_id: sessionID, type: "compaction", seq: 2, time_created: 2, data: compaction }, + ]) + .run() + .pipe(Effect.orDie) + + // The blob is missing, so a hydrated read would name a text/plain placeholder instead. + expect(yield* store.context(sessionID)).toMatchObject([ + { id: compactionID, metadata: { recent: "[User]: look\n[Attached image/png: shot.png]" } }, + ]) + }), + ) })