feat(core): store session media as media:<hash> refs in a content-addressed blob table (#58656)

This commit is contained in:
Kit Langton
2026-10-25 08:28:46 -07:00
committed by GitHub
Unverified
parent 9cc9e58c73
commit 70c69f06f9
14 changed files with 861 additions and 72 deletions
+188 -2
View File
@@ -1,8 +1,8 @@
{
"version": "7",
"dialect": "sqlite",
"id": "76109fbf-f324-4af0-934c-3ae695275099",
"prevIds": ["ff2efc22-36c2-4af5-8f24-2288090b0ba6"],
"id": "f1f14616-b6a1-483d-87b0-d5b7a9132580",
"prevIds": ["76109fbf-f324-4af0-934c-3ae695275099"],
"ddl": [
{
"name": "account_state",
@@ -44,6 +44,14 @@
"name": "project",
"entityType": "tables"
},
{
"name": "blob_ref",
"entityType": "tables"
},
{
"name": "blob",
"entityType": "tables"
},
{
"name": "instruction_blob",
"entityType": "tables"
@@ -740,6 +748,116 @@
"entityType": "columns",
"table": "project"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "session_id",
"entityType": "columns",
"table": "blob_ref"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "message_id",
"entityType": "columns",
"table": "blob_ref"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "hash",
"entityType": "columns",
"table": "blob_ref"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "seq",
"entityType": "columns",
"table": "blob_ref"
},
{
"type": "text",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "hash",
"entityType": "columns",
"table": "blob"
},
{
"type": "text",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "mime",
"entityType": "columns",
"table": "blob"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "bytes",
"entityType": "columns",
"table": "blob"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "width",
"entityType": "columns",
"table": "blob"
},
{
"type": "integer",
"notNull": false,
"autoincrement": false,
"default": null,
"generated": null,
"name": "height",
"entityType": "columns",
"table": "blob"
},
{
"type": "blob",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "data",
"entityType": "columns",
"table": "blob"
},
{
"type": "integer",
"notNull": true,
"autoincrement": false,
"default": null,
"generated": null,
"name": "time_created",
"entityType": "columns",
"table": "blob"
},
{
"type": "text",
"notNull": false,
@@ -1574,6 +1692,28 @@
"entityType": "fks",
"table": "project_directory"
},
{
"columns": ["session_id"],
"tableTo": "session_v2",
"columnsTo": ["id"],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_blob_ref_session_id_session_v2_id_fk",
"entityType": "fks",
"table": "blob_ref"
},
{
"columns": ["hash"],
"tableTo": "blob",
"columnsTo": ["hash"],
"onUpdate": "NO ACTION",
"onDelete": "CASCADE",
"nameExplicit": false,
"name": "fk_blob_ref_hash_blob_hash_fk",
"entityType": "fks",
"table": "blob_ref"
},
{
"columns": ["session_id"],
"tableTo": "session_v2",
@@ -1665,6 +1805,13 @@
"entityType": "pks",
"table": "project_directory"
},
{
"columns": ["session_id", "message_id", "hash"],
"nameExplicit": false,
"name": "blob_ref_pk",
"entityType": "pks",
"table": "blob_ref"
},
{
"columns": ["session_id", "key"],
"nameExplicit": false,
@@ -1735,6 +1882,13 @@
"table": "project",
"entityType": "pks"
},
{
"columns": ["hash"],
"nameExplicit": false,
"name": "blob_pk",
"table": "blob",
"entityType": "pks"
},
{
"columns": ["hash"],
"nameExplicit": false,
@@ -1846,6 +2000,38 @@
"entityType": "indexes",
"table": "permission"
},
{
"columns": [
{
"value": "hash",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "blob_ref_hash_idx",
"entityType": "indexes",
"table": "blob_ref"
},
{
"columns": [
{
"value": "session_id",
"isExpression": false
},
{
"value": "seq",
"isExpression": false
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "blob_ref_session_seq_idx",
"entityType": "indexes",
"table": "blob_ref"
},
{
"columns": [
{
+2
View File
@@ -51,6 +51,7 @@ import m48 from "./migration/20261007190000_azure_cli_external_credential.js"
import m49 from "./migration/20261010180456_project_time_archived.js"
import m50 from "./migration/20261024002554_session_allowance.js"
import m51 from "./migration/20261024175301_drop_session_allowance.js"
import m52 from "./migration/20261025123711_blob.js"
export const migrations = [
m00,
@@ -105,4 +106,5 @@ export const migrations = [
m49,
m50,
m51,
m52,
] satisfies DatabaseMigration.Migration[]
@@ -0,0 +1,36 @@
import { Effect } from "effect"
import type { DatabaseMigration } from "../migration.js"
const migration: DatabaseMigration.Migration = {
id: "20261025123711_blob",
up(tx) {
return Effect.gen(function* () {
yield* tx.run(`
CREATE TABLE \`blob_ref\` (
\`session_id\` text NOT NULL,
\`message_id\` text NOT NULL,
\`hash\` text NOT NULL,
\`seq\` integer NOT NULL,
CONSTRAINT \`blob_ref_pk\` PRIMARY KEY(\`session_id\`, \`message_id\`, \`hash\`),
CONSTRAINT \`fk_blob_ref_session_id_session_v2_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session_v2\`(\`id\`) ON DELETE CASCADE,
CONSTRAINT \`fk_blob_ref_hash_blob_hash_fk\` FOREIGN KEY (\`hash\`) REFERENCES \`blob\`(\`hash\`) ON DELETE CASCADE
);
`)
yield* tx.run(`
CREATE TABLE \`blob\` (
\`hash\` text PRIMARY KEY,
\`mime\` text NOT NULL,
\`bytes\` integer NOT NULL,
\`width\` integer,
\`height\` integer,
\`data\` blob NOT NULL,
\`time_created\` integer NOT NULL
);
`)
yield* tx.run(`CREATE INDEX \`blob_ref_hash_idx\` ON \`blob_ref\` (\`hash\`);`)
yield* tx.run(`CREATE INDEX \`blob_ref_session_seq_idx\` ON \`blob_ref\` (\`session_id\`,\`seq\`);`)
})
},
}
export default migration
+24
View File
@@ -116,6 +116,28 @@ const schema: Omit<DatabaseMigration.Migration, "id"> = {
\`commands\` text
);
`)
yield* tx.run(`
CREATE TABLE \`blob_ref\` (
\`session_id\` text NOT NULL,
\`message_id\` text NOT NULL,
\`hash\` text NOT NULL,
\`seq\` integer NOT NULL,
CONSTRAINT \`blob_ref_pk\` PRIMARY KEY(\`session_id\`, \`message_id\`, \`hash\`),
CONSTRAINT \`fk_blob_ref_session_id_session_v2_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session_v2\`(\`id\`) ON DELETE CASCADE,
CONSTRAINT \`fk_blob_ref_hash_blob_hash_fk\` FOREIGN KEY (\`hash\`) REFERENCES \`blob\`(\`hash\`) ON DELETE CASCADE
);
`)
yield* tx.run(`
CREATE TABLE \`blob\` (
\`hash\` text PRIMARY KEY,
\`mime\` text NOT NULL,
\`bytes\` integer NOT NULL,
\`width\` integer,
\`height\` integer,
\`data\` blob NOT NULL,
\`time_created\` integer NOT NULL
);
`)
yield* tx.run(`
CREATE TABLE \`instruction_blob\` (
\`hash\` text PRIMARY KEY,
@@ -245,6 +267,8 @@ const schema: Omit<DatabaseMigration.Migration, "id"> = {
yield* tx.run(
`CREATE UNIQUE INDEX \`permission_project_action_resource_idx\` ON \`permission\` (\`project_id\`,\`action\`,\`resource\`);`,
)
yield* tx.run(`CREATE INDEX \`blob_ref_hash_idx\` ON \`blob_ref\` (\`hash\`);`)
yield* tx.run(`CREATE INDEX \`blob_ref_session_seq_idx\` ON \`blob_ref\` (\`session_id\`,\`seq\`);`)
yield* tx.run(
`CREATE INDEX \`session_inbox_session_delivery_seq_idx\` ON \`session_inbox\` (\`session_id\`,\`delivery\`,\`enqueued_seq\`);`,
)
+107
View File
@@ -0,0 +1,107 @@
export * as SessionBlob from "./blob.js"
import { inArray } from "drizzle-orm"
import { Effect, Schema } from "effect"
import { map } from "effect/Array"
import type { FileAttachment } from "@opencode/schema/prompt"
import type { Tool } from "@opencode/schema/tool"
import type { Database } from "../database/database.js"
import { BlobTable, SessionInboxTable, SessionMessageTable } from "./sql.js"
import { SessionStored } from "./stored.js"
type DatabaseService = Database.Interface["db"]
type MessageRow = typeof SessionMessageTable.$inferSelect
type InboxRow = typeof SessionInboxTable.$inferSelect
type Part = Extract<MessageRow["data"], { readonly content: unknown }>["content"][number]
/** Base64 bytes by blob hash. */
type Blobs = ReadonlyMap<string, string>
const isFileRef = Schema.is(SessionStored.FileRef)
const isToolFileRef = Schema.is(SessionStored.ToolFileRef)
const toolItems = (part: Part): ReadonlyArray<Tool.Content> =>
part.type === "tool" && (part.state.status === "completed" || part.state.status === "error")
? (part.state.content ?? [])
: []
function collect(data: MessageRow["data"] | InboxRow["payload"], hashes: Set<string>) {
if ("files" in data)
for (const file of data.files ?? []) if (isFileRef(file)) hashes.add(SessionStored.refHash(file.uri))
if ("content" in data)
for (const part of data.content)
for (const item of toolItems(part)) if (isToolFileRef(item)) hashes.add(SessionStored.refHash(item.uri))
}
/** A ref whose blob is gone reads as this text, so clients and the model never see a bare ref. */
const unavailable = (mime: string, name?: string) =>
`[media unavailable: ${name === undefined ? mime : `${name} (${mime})`}]`
function inlineFile(file: SessionStored.File, blobs: Blobs): FileAttachment {
if (!isFileRef(file)) return file
const { uri, ...rest } = file
const data = blobs.get(SessionStored.refHash(uri))
if (data === undefined)
return { ...rest, data: Buffer.from(unavailable(file.mime, file.name)).toString("base64"), mime: "text/plain" }
return { ...rest, data }
}
function inlineToolItem(item: Tool.Content, blobs: Blobs): Tool.Content {
if (!isToolFileRef(item)) return item
const data = blobs.get(SessionStored.refHash(item.uri))
if (data === undefined) return { type: "text", text: unavailable(item.mime, item.name) }
return { ...item, uri: `data:${item.mime};base64,${data}` }
}
function inlinePart(part: Part, blobs: Blobs): Part {
if (part.type !== "tool" || (part.state.status !== "completed" && part.state.status !== "error")) return part
const content = part.state.content
if (!content?.some(isToolFileRef)) return part
return { ...part, state: { ...part.state, content: map(content, (item) => inlineToolItem(item, blobs)) } }
}
const load = Effect.fnUntraced(function* (db: DatabaseService, hashes: ReadonlySet<string>) {
const all = [...hashes]
const blobs = new Map<string, string>()
for (let offset = 0; offset < all.length; offset += 500) {
const found = yield* db
.select({ hash: BlobTable.hash, data: BlobTable.data })
.from(BlobTable)
.where(inArray(BlobTable.hash, all.slice(offset, offset + 500)))
.all()
.pipe(Effect.orDie)
for (const blob of found) blobs.set(blob.hash, view(blob.data).toString("base64"))
}
return blobs
})
const view = (data: Uint8Array) =>
Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength)
/** Restores inline media for the refs in `session_message` rows. Rows without refs are returned untouched. */
export const hydrate = Effect.fnUntraced(function* (db: DatabaseService, rows: ReadonlyArray<MessageRow>) {
const hashes = new Set<string>()
for (const row of rows) collect(row.data, hashes)
if (hashes.size === 0) return rows
const blobs = yield* load(db, hashes)
return rows.map((row): MessageRow => {
const data = row.data
if ("files" in data && data.files?.some(isFileRef))
return { ...row, data: { ...data, files: data.files.map((file) => inlineFile(file, blobs)) } }
if ("content" in data && data.content.some((part) => toolItems(part).some(isToolFileRef)))
return { ...row, data: { ...data, content: data.content.map((part) => inlinePart(part, blobs)) } }
return row
})
})
/** Restores inline media for the refs in `session_inbox` rows. Rows without refs are returned untouched. */
export const hydrateInbox = Effect.fnUntraced(function* (db: DatabaseService, rows: ReadonlyArray<InboxRow>) {
const hashes = new Set<string>()
for (const row of rows) collect(row.payload, hashes)
if (hashes.size === 0) return rows
const blobs = yield* load(db, hashes)
return rows.map((row): InboxRow => {
const payload = row.payload
if (!("files" in payload) || !payload.files?.some(isFileRef)) return row
return { ...row, payload: { ...payload, files: payload.files.map((file) => inlineFile(file, blobs)) } }
})
})
+36 -17
View File
@@ -9,13 +9,16 @@ import { SessionMessage } from "./message.js"
import type { SessionSchema } from "./schema.js"
import { Instructions } from "../instructions/index.js"
import { InstructionState } from "./instruction-state.js"
import { SessionBlob } from "./blob.js"
import { SessionRecentText } from "./recent-text.js"
import { SessionMessageTable } from "./sql.js"
import { SessionStored } from "./stored.js"
type DatabaseService = Database.Interface["db"]
const decode = Schema.decodeUnknownEffect(SessionMessage.Info)
const decodeSync = Schema.decodeUnknownSync(SessionMessage.Info)
const decodeStoredUser = Schema.decodeUnknownEffect(SessionStored.User)
/**
* Which completed compactions bound a history read. Local summaries always do. Native
@@ -89,16 +92,28 @@ const rowInput = (row: typeof SessionMessageTable.$inferSelect) => ({
export const decodeMessageRowSync = (row: typeof SessionMessageTable.$inferSelect) => decodeSync(rowInput(row))
const decodeError = (row: typeof SessionMessageTable.$inferSelect) => () =>
new MessageDecodeError({
sessionID: row.session_id,
messageID: row.id,
})
export const decodeMessageRow = (row: typeof SessionMessageTable.$inferSelect) =>
decode(rowInput(row)).pipe(
Effect.mapError(
() =>
new MessageDecodeError({
sessionID: row.session_id,
messageID: row.id,
}),
),
)
decode(rowInput(row)).pipe(Effect.mapError(decodeError(row)))
/** Decodes a row without its blobs, so user files may stay refs. Only for readers that never use file bytes. */
const decodeStoredRow = (
row: typeof SessionMessageTable.$inferSelect,
): Effect.Effect<SessionMessage.Info | SessionStored.User, MessageDecodeError> =>
row.type === "user" ? decodeStoredUser(rowInput(row)).pipe(Effect.mapError(decodeError(row))) : decodeMessageRow(row)
/** Decodes rows after restoring media stored in `blob`. Reads that return media must use this. */
export const decodeMessageRows = Effect.fnUntraced(function* (
db: DatabaseService,
rows: ReadonlyArray<typeof SessionMessageTable.$inferSelect>,
) {
return yield* Effect.forEach(yield* SessionBlob.hydrate(db, rows), decodeMessageRow, { concurrency: 1 })
})
const messageEntries = Effect.fnUntraced(function* (
db: DatabaseService,
@@ -122,8 +137,10 @@ const messageEntries = Effect.fnUntraced(function* (
.orderBy(asc(SessionMessageTable.seq))
.all()
.pipe(Effect.orDie)
const decoded = yield* Effect.forEach(rows, (row) =>
decodeMessageRow(row).pipe(Effect.map((message) => ({ seq: row.seq, message }))),
const decoded = yield* Effect.forEach(
yield* SessionBlob.hydrate(db, rows),
(row) => decodeMessageRow(row).pipe(Effect.map((message) => ({ seq: row.seq, message }))),
{ concurrency: 1 },
)
// A window that cannot replay leaves nothing to start from, so read from the compaction before it.
const start = decoded[0]
@@ -176,7 +193,8 @@ const withRecentText = Effect.fnUntraced(function* (
.orderBy(asc(SessionMessageTable.seq))
.all()
.pipe(Effect.orDie)
const kept = yield* Effect.forEach(rows, decodeMessageRow)
// The recent text names attachments by mime and name only, so their bytes are never read.
const kept = yield* Effect.forEach(rows, decodeStoredRow, { concurrency: 1 })
const recent = kept.map(SessionRecentText.messageToText).filter(Boolean).join("\n\n")
return [{ ...first, message: { ...compaction, metadata: { ...compaction.metadata, recent } } }, ...entries.slice(1)]
})
@@ -235,21 +253,22 @@ export const preview = Effect.fn("SessionHistory.preview")(function* (
.pipe(Effect.catch((error) => (error instanceof Instructions.InitializationBlocked ? error : Effect.die(error))))
})
/** Returns the session's first user message. */
/** The id and text of the session's first user message. */
export const firstUserMessage = Effect.fn("SessionHistory.firstUserMessage")(function* (
db: DatabaseService,
sessionID: SessionSchema.ID,
) {
const row = yield* db
.select()
.select({
id: SessionMessageTable.id,
text: sql<unknown>`json_extract(${SessionMessageTable.data}, '$.text')`,
})
.from(SessionMessageTable)
.where(and(eq(SessionMessageTable.session_id, sessionID), eq(SessionMessageTable.type, "user")))
.orderBy(asc(SessionMessageTable.seq))
.get()
.pipe(Effect.orDie)
if (!row) return undefined
const message = yield* decodeMessageRow(row).pipe(Effect.orElseSucceed(() => undefined))
return message?.type === "user" ? message : undefined
return row && Schema.is(Schema.String)(row.text) ? { id: row.id, text: row.text } : undefined
})
export * as SessionHistory from "./history.js"
+34 -6
View File
@@ -20,8 +20,10 @@ import { Database } from "../database/database.js"
import { Bus } from "../bus.js"
import { KeyedMutex } from "../effect/keyed-mutex.js"
import { SessionEvent } from "./event.js"
import { SessionBlob } from "./blob.js"
import { SessionMessage } from "./message.js"
import { SessionSchema } from "./schema.js"
import { SessionStored } from "./stored.js"
import { SessionInboxTable, SessionMessageTable } from "./sql.js"
import { SessionHistory } from "./history.js"
@@ -49,6 +51,7 @@ export {
export type Promotable = "input" | "steer"
const decodeUser = Schema.decodeUnknownSync(UserPayload)
const decodeStoredUser = Schema.decodeUnknownSync(SessionStored.UserPayload)
const encodeUser = Schema.encodeSync(UserPayload)
const decodeSynthetic = Schema.decodeUnknownSync(SyntheticPayload)
const encodeSynthetic = Schema.encodeSync(SyntheticPayload)
@@ -107,7 +110,9 @@ const fromRow = (row: typeof SessionInboxTable.$inferSelect): Info => {
export const find = Effect.fn("SessionInbox.find")(function* (db: DatabaseService, id: SessionMessage.ID) {
const row = yield* db.select().from(SessionInboxTable).where(eq(SessionInboxTable.id, id)).get().pipe(Effect.orDie)
return row === undefined ? undefined : fromRow(row)
if (row === undefined) return undefined
const [hydrated] = yield* SessionBlob.hydrateInbox(db, [row])
return hydrated && fromRow(hydrated)
})
const promotedFromMessage = Effect.fn("SessionInbox.promotedFromMessage")(function* (
@@ -125,7 +130,8 @@ const promotedFromMessage = Effect.fn("SessionInbox.promotedFromMessage")(functi
if (row === undefined) return undefined
if (row.session_id !== sessionID || (row.type !== "user" && row.type !== "synthetic"))
return yield* new LifecycleConflict({ id })
const message = SessionHistory.decodeMessageRowSync(row)
const [message] = yield* SessionHistory.decodeMessageRows(db, [row]).pipe(Effect.orDie)
if (!message) return undefined
const base = { id, sessionID, time: { created: message.time.created }, delivery }
if (message.type === "user")
return User.make({
@@ -341,9 +347,21 @@ export const projectDelivered = Effect.fn("SessionInbox.projectDelivered")(funct
.get()
.pipe(Effect.orDie)
if (!deleted) return yield* Effect.die(new LifecycleConflict({ id: input.id }))
return fromRow(deleted)
return delivered(deleted)
})
/** A consumed row with its payload as stored, so user file refs reach the message without their blobs. */
export type Delivered =
| { readonly id: SessionMessage.ID; readonly type: "user"; readonly payload: SessionStored.UserPayload }
| { readonly id: SessionMessage.ID; readonly type: "synthetic"; readonly payload: SyntheticPayload }
| { readonly id: SessionMessage.ID; readonly type: "compaction" | "move" }
const delivered = (row: typeof SessionInboxTable.$inferSelect): Delivered => {
if (row.type === "user") return { id: row.id, type: "user", payload: decodeStoredUser(row.payload) }
if (row.type === "synthetic") return { id: row.id, type: "synthetic", payload: decodeSynthetic(row.payload) }
return { id: row.id, type: row.type }
}
export const projectCancelled = Effect.fn("SessionInbox.projectCancelled")(function* (
db: DatabaseService,
input: PendingRef,
@@ -400,7 +418,7 @@ export const list = Effect.fn("SessionInbox.list")(function* (db: DatabaseServic
.orderBy(asc(SessionInboxTable.enqueued_seq))
.all()
.pipe(Effect.orDie)
return rows.map(fromRow)
return (yield* SessionBlob.hydrateInbox(db, rows)).map(fromRow)
})
export const moveIDs = Effect.fn("SessionInbox.moveIDs")(function* (db: DatabaseService, sessionID: SessionSchema.ID) {
@@ -419,7 +437,7 @@ export const nextPromotable = Effect.fn("SessionInbox.nextPromotable")(function*
promotable: Promotable,
) {
const steer = (yield* pendingSteers(db, sessionID))[0]
if (steer) return fromRow(steer)
if (steer) return promotion(steer)
if (promotable !== "input") return undefined
const queued = yield* db
.select()
@@ -429,9 +447,19 @@ export const nextPromotable = Effect.fn("SessionInbox.nextPromotable")(function*
.limit(1)
.get()
.pipe(Effect.orDie)
return queued ? fromRow(queued) : undefined
return queued ? promotion(queued) : undefined
})
/** What a boundary decides on: prompts are promoted by id, so only a move's payload is read. */
export type Promotion =
| { readonly id: SessionMessage.ID; readonly type: "user" | "synthetic" | "compaction"; readonly delivery: Delivery }
| { readonly id: SessionMessage.ID; readonly type: "move"; readonly delivery: Delivery; readonly payload: MovePayload }
const promotion = (row: typeof SessionInboxTable.$inferSelect): Promotion =>
row.type === "move"
? { id: row.id, type: "move", delivery: row.delivery, payload: decodeMove(row.payload) }
: { id: row.id, type: row.type, delivery: row.delivery }
/** Which pending rows count: "input" means any item in either delivery mode. */
export type Scope = "input" | Delivery
+29 -27
View File
@@ -13,6 +13,7 @@ import { SessionMessageUpdater } from "./message-updater.js"
import { SessionInbox } from "./inbox.js"
import { InstructionState } from "./instruction-state.js"
import { SessionInboxTable, SessionMessageTable, SessionTable } from "./sql.js"
import { SessionStored } from "./stored.js"
import { InstructionEntry } from "./instruction-entry.js"
import { Slug } from "../util/slug.js"
import { FSUtil } from "@opencode/util/fs-util"
@@ -32,6 +33,7 @@ type MessageEvent = Exclude<
>
const encodeMessage = Schema.encodeSync(SessionMessage.Info)
const encodeStoredUser = Schema.encodeSync(SessionStored.User)
export class SessionAlreadyProjected extends Error {}
@@ -398,9 +400,12 @@ function run(db: DatabaseService, event: MessageEvent) {
})
}
function insertMessage(db: DatabaseService, event: SessionEvent.DurableEvent, message: SessionMessage.Info) {
const encoded = encodeMessage(message)
const { id: _, type, ...data } = encoded
function insertMessage(
db: DatabaseService,
event: SessionEvent.DurableEvent,
message: SessionMessage.Info | SessionStored.User,
) {
const { id: _, type, ...data } = message.type === "user" ? encodeStoredUser(message) : encodeMessage(message)
return db
.insert(SessionMessageTable)
.values({
@@ -643,30 +648,27 @@ const layer = Layer.effectDiscard(
id: event.data.inboxID,
sessionID: event.data.sessionID,
})
if (input.type === "compaction" || input.type === "move") return
yield* insertMessage(
db,
event,
input.type === "user"
? {
id: input.id,
type: "user",
metadata: input.payload.metadata,
text: input.payload.text,
files: input.payload.files,
agents: input.payload.agents,
skills: input.payload.skills,
time: { created: DateTime.makeUnsafe(event.created) },
}
: {
id: input.id,
type: "synthetic",
text: input.payload.text,
description: input.payload.description,
metadata: input.payload.metadata,
time: { created: DateTime.makeUnsafe(event.created) },
},
)
const time = { created: DateTime.makeUnsafe(event.created) }
if (input.type === "user")
yield* insertMessage(db, event, {
id: input.id,
type: "user",
metadata: input.payload.metadata,
text: input.payload.text,
files: input.payload.files,
agents: input.payload.agents,
skills: input.payload.skills,
time,
})
if (input.type === "synthetic")
yield* insertMessage(db, event, {
id: input.id,
type: "synthetic",
text: input.payload.text,
description: input.payload.description,
metadata: input.payload.metadata,
time,
})
}),
)
yield* bus.project(SessionEvent.InboxEnqueued, (event) =>
+2 -1
View File
@@ -1,6 +1,7 @@
export * as SessionRecentText from "./recent-text.js"
import type { SessionMessage } from "./message.js"
import type { SessionStored } from "./stored.js"
const TOOL_OUTPUT_MAX_CHARS = 1_250
@@ -12,7 +13,7 @@ export const kept = (message: SessionMessage.CompactionCompleted) =>
typeof message.metadata?.recent === "string" ? message.metadata.recent : ""
/** One message as the recent, verbatim part of a summary shows it. Empty for messages that part leaves out. */
export const messageToText = (message: SessionMessage.Info): string => {
export const messageToText = (message: SessionMessage.Info | SessionStored.User): string => {
switch (message.type) {
// Earlier summaries and instruction updates are handled outside the recent part.
case "compaction":
+6 -5
View File
@@ -298,8 +298,11 @@ const layer = Layer.effect(
}),
OutputLimit: Effect.fn("SessionRunner.continueOutputLimit")(function* (outcome) {
// The current response is included. User input or any other finish breaks the streak.
const rows = yield* db
.select()
const recent = yield* db
.select({
type: SessionMessageTable.type,
finish: sql<string | null>`json_extract(${SessionMessageTable.data}, '$.finish')`,
})
.from(SessionMessageTable)
.where(
and(
@@ -311,10 +314,8 @@ const layer = Layer.effect(
.limit(3)
.all()
.pipe(Effect.orDie)
const recent = yield* Effect.forEach(rows, SessionHistory.decodeMessageRow)
const exhausted =
recent.length === 3 &&
recent.every((message) => message.type === "assistant" && message.finish === "length")
recent.length === 3 && recent.every((row) => row.type === "assistant" && row.finish === "length")
if (exhausted)
return yield* new StepFailedError({
error: {
+41 -3
View File
@@ -1,4 +1,4 @@
import { sqliteTable, text, integer, index, primaryKey, real, uniqueIndex } from "drizzle-orm/sqlite-core"
import { blob, sqliteTable, text, integer, index, primaryKey, real, uniqueIndex } from "drizzle-orm/sqlite-core"
import { sql } from "drizzle-orm"
import { directoryColumn, pathColumn } from "../database/path.js"
import { ProjectTable } from "../project/sql.js"
@@ -9,6 +9,7 @@ import type { FileDiff } from "@opencode/schema/file-diff"
import type { Permission } from "@opencode/schema/permission"
import type { Project } from "@opencode/schema/project"
import type { SessionSchema } from "./schema.js"
import type { SessionStored } from "./stored.js"
import type { Workspace } from "@opencode/schema/workspace"
import { Timestamps } from "../database/schema.sql.js"
import type { Instruction } from "@opencode/schema/instruction"
@@ -18,7 +19,10 @@ import type { RevertV1 } from "@opencode/schema/session-revert"
import type { Schema } from "effect"
type DistributiveOmit<T, K extends PropertyKey> = T extends unknown ? Omit<T, K> : never
type SessionMessageData = DistributiveOmit<(typeof SessionMessage.Info)["Encoded"], "type" | "id">
type SessionMessageData = DistributiveOmit<
Exclude<(typeof SessionMessage.Info)["Encoded"], { readonly type: "user" }> | (typeof SessionStored.User)["Encoded"],
"type" | "id"
>
export const SessionTable = sqliteTable(
"session_v2",
@@ -132,7 +136,9 @@ export const SessionInboxTable = sqliteTable(
.notNull()
.references(() => SessionTable.id, { onDelete: "cascade" }),
type: text().$type<SessionInbox.Info["type"]>().notNull(),
payload: text({ mode: "json" }).$type<UserPayload | SyntheticPayload | CompactionPayload | MovePayload>().notNull(),
payload: text({ mode: "json" })
.$type<SessionStored.UserPayload | SyntheticPayload | CompactionPayload | MovePayload>()
.notNull(),
delivery: text().$type<SessionInbox.Delivery>().notNull(),
enqueued_seq: integer().notNull(),
time_created: integer()
@@ -175,3 +181,35 @@ export const InstructionStateTable = sqliteTable("instruction_state", {
initial_values: text({ mode: "json" }).notNull().$type<Instruction.Values>(),
current_values: text({ mode: "json" }).notNull().$type<Instruction.Values>(),
})
export const BlobTable = sqliteTable("blob", {
hash: text().primaryKey(),
mime: text().notNull(),
bytes: integer().notNull(),
width: integer(),
height: integer(),
data: blob({ mode: "buffer" }).notNull(),
time_created: integer()
.notNull()
.$default(() => Date.now()),
})
export const BlobRefTable = sqliteTable(
"blob_ref",
{
session_id: text()
.$type<SessionSchema.ID>()
.notNull()
.references(() => SessionTable.id, { onDelete: "cascade" }),
message_id: text().$type<SessionMessage.ID>().notNull(),
hash: text()
.notNull()
.references(() => BlobTable.hash, { onDelete: "cascade" }),
seq: integer().notNull(),
},
(table) => [
primaryKey({ columns: [table.session_id, table.message_id, table.hash] }),
index("blob_ref_hash_idx").on(table.hash),
index("blob_ref_session_seq_idx").on(table.session_id, table.seq),
],
)
+4 -10
View File
@@ -174,10 +174,7 @@ const layer = Layer.effect(
const rows = yield* (input.limit === undefined ? query.all() : query.limit(input.limit).all()).pipe(
Effect.orDie,
)
return yield* Effect.forEach(
direction === "previous" ? rows.toReversed() : rows,
SessionHistory.decodeMessageRow,
)
return yield* SessionHistory.decodeMessageRows(db, direction === "previous" ? rows.toReversed() : rows)
}),
context: Effect.fn("SessionStore.context")((sessionID) => SessionHistory.load(db, sessionID, "latest")),
message: Effect.fn("SessionStore.message")(function* (messageID) {
@@ -187,12 +184,9 @@ const layer = Layer.effect(
.where(eq(SessionMessageTable.id, messageID))
.get()
.pipe(Effect.orDie)
return row
? {
sessionID: row.session_id,
message: yield* SessionHistory.decodeMessageRow(row).pipe(Effect.orDie),
}
: undefined
if (!row) return undefined
const [message] = yield* SessionHistory.decodeMessageRows(db, [row]).pipe(Effect.orDie)
return message && { sessionID: row.session_id, message }
}),
listSuspended: Effect.fn("SessionStore.listSuspended")(function* () {
return yield* db
+41
View File
@@ -0,0 +1,41 @@
export * as SessionStored from "./stored.js"
import { Schema } from "effect"
import { FileAttachment } from "@opencode/schema/prompt"
import { optional } from "@opencode/schema/schema"
import { UserPayload as InboxUserPayload } from "@opencode/schema/session-inbox"
import { Tool } from "@opencode/schema/tool"
import { SessionMessage } from "./message.js"
/**
* Stored rows may reference a `blob` row as `media:` and the lowercase hex SHA-256 of its raw bytes. The blob
* row holds the bytes, size and dimensions; the ref names no server route.
*/
export const Ref = Schema.String.check(Schema.isPattern(/^media:[0-9a-f]{64}$/))
export type Ref = typeof Ref.Type
/** The `blob` hash a {@link Ref} names. */
export const refHash = (ref: Ref) => ref.slice("media:".length)
const { data: _, ...attachment } = FileAttachment.fields
/** A user file whose bytes live in `blob`. */
export interface FileRef extends Schema.Schema.Type<typeof FileRef> {}
export const FileRef = Schema.Struct({ ...attachment, uri: Ref })
export const File = Schema.Union([FileAttachment, FileRef])
export type File = typeof File.Type
/** A tool result file whose bytes live in `blob`. */
export interface ToolFileRef extends Schema.Schema.Type<typeof ToolFileRef> {}
export const ToolFileRef = Schema.Struct({ ...Tool.FileContent.fields, uri: Ref })
const files = Schema.Array(File).pipe(optional)
/** A `session_inbox` user payload as stored, whose files may be refs. */
export interface UserPayload extends Schema.Schema.Type<typeof UserPayload> {}
export const UserPayload = Schema.Struct({ ...InboxUserPayload.fields, files })
/** A `session_message` user message as stored, whose files may be refs. */
export interface User extends Schema.Schema.Type<typeof User> {}
export const User = Schema.Struct({ ...SessionMessage.User.fields, files })
+311 -1
View File
@@ -23,8 +23,16 @@ import { SessionExecution } from "@opencode/core/session/execution"
import { fromRow } from "@opencode/core/session/info"
import { SessionInbox } from "@opencode/core/session/inbox"
import { SessionStore } from "@opencode/core/session/store"
import { SessionStored } from "@opencode/core/session/stored"
import { SessionHistory } from "@opencode/core/session/history"
import { Shell } from "@opencode/schema/shell"
import { InstructionStateTable, SessionInboxTable, SessionMessageTable, SessionTable } from "@opencode/core/session/sql"
import {
InstructionStateTable,
BlobTable,
SessionInboxTable,
SessionMessageTable,
SessionTable,
} from "@opencode/core/session/sql"
import { testEffect } from "./lib/effect"
import { Snapshot } from "@opencode/core/snapshot"
@@ -993,4 +1001,306 @@ describe("SessionProjector", () => {
expect(yield* updated()).toEqual({ time_updated: 15 })
}),
)
it.effect("hydrates media:<hash> refs on message and inbox reads", () =>
Effect.gen(function* () {
const db = yield* seedSession()
const bus = yield* Bus.Service
const inbox = yield* SessionInbox.Service
const store = yield* SessionStore.Service
const rawBytes = Buffer.from("externalized-media-payload")
const base64 = rawBytes.toString("base64")
const hash = "a".repeat(64)
const uri = `media:${hash}`
// Only media:<hash> is a ref; a plain path that looks like a media URL is left alone.
const pathUri = `/api/media/${hash}`
yield* db
.insert(BlobTable)
.values({
hash,
mime: "application/pdf",
bytes: rawBytes.byteLength,
width: null,
height: null,
data: rawBytes,
})
.run()
.pipe(Effect.orDie)
const userID = SessionMessage.ID.make("msg_read_user")
const assistantID = SessionMessage.ID.make("msg_read_assistant")
const inboxID = SessionMessage.ID.make("msg_read_inbox")
yield* db
.insert(SessionMessageTable)
.values([
{
id: userID,
session_id: sessionID,
type: "user",
seq: 1,
time_created: 1,
data: {
text: "user attachment",
files: [SessionStored.FileRef.make({ uri, mime: "application/pdf", source: { type: "inline" }, name: "spec.pdf" })],
time: { created: 1 },
},
},
{
id: assistantID,
session_id: sessionID,
type: "assistant",
seq: 2,
time_created: 2,
data: {
agent: build,
model,
content: [
{
type: "tool",
id: "call_1",
name: "read",
executed: true,
state: {
status: "completed",
input: { path: "spec.pdf" },
content: [
SessionStored.ToolFileRef.make({ type: "file", uri, mime: "application/pdf", name: "spec.pdf" }),
{ type: "file", uri: pathUri, mime: "application/pdf", name: "path.pdf" },
],
},
time: { created: 2, ran: 2, completed: 2 },
},
],
time: { created: 2, completed: 2 },
},
},
])
.run()
.pipe(Effect.orDie)
yield* db
.insert(SessionInboxTable)
.values({
id: inboxID,
session_id: sessionID,
type: "user",
payload: {
text: "queued attachment",
files: [SessionStored.FileRef.make({ uri, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" })],
},
delivery: "queue",
enqueued_seq: 3,
time_created: 3,
})
.run()
.pipe(Effect.orDie)
const messages = yield* store.messages({ sessionID, order: "asc" })
expect(messages).toHaveLength(2)
expect(messages[0]).toMatchObject({
id: userID,
type: "user",
files: [{ data: base64, mime: "application/pdf", source: { type: "inline" }, name: "spec.pdf" }],
})
expect(messages[1]).toMatchObject({
id: assistantID,
type: "assistant",
content: [
{
type: "tool",
state: {
status: "completed",
content: [
{ type: "file", uri: `data:application/pdf;base64,${base64}`, mime: "application/pdf", name: "spec.pdf" },
{ type: "file", uri: pathUri, mime: "application/pdf", name: "path.pdf" },
],
},
},
],
})
expect((yield* store.message(userID))?.message).toEqual(messages[0]!)
expect(yield* store.context(sessionID)).toEqual(messages)
const listedInbox = yield* inbox.list(sessionID)
expect(listedInbox).toHaveLength(1)
expect(listedInbox[0]).toMatchObject({
id: inboxID,
type: "user",
payload: {
text: "queued attachment",
files: [{ data: base64, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" }],
},
})
expect(yield* SessionHistory.firstUserMessage(db, sessionID)).toEqual({ id: userID, text: "user attachment" })
expect(yield* SessionInbox.nextPromotable(db, sessionID, "input")).toEqual({
id: inboxID,
type: "user",
delivery: "queue",
})
yield* bus.publish(SessionEvent.InboxDelivered, { sessionID, inboxID })
expect((yield* store.message(inboxID))?.message).toMatchObject({
id: inboxID,
type: "user",
files: [{ data: base64, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" }],
})
// Delivery keeps the ref in the stored row instead of writing the bytes back inline.
const delivered = yield* db
.select({ data: SessionMessageTable.data })
.from(SessionMessageTable)
.where(eq(SessionMessageTable.id, inboxID))
.get()
.pipe(Effect.orDie)
expect(delivered?.data).toMatchObject({
text: "queued attachment",
files: [{ uri, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" }],
})
expect(JSON.stringify(delivered?.data)).not.toContain(base64)
}),
)
it.effect("replaces refs to missing blobs with an unavailable placeholder on read", () =>
Effect.gen(function* () {
const db = yield* seedSession()
const inbox = yield* SessionInbox.Service
const store = yield* SessionStore.Service
const hash = "b".repeat(64)
const userID = SessionMessage.ID.make("msg_missing_user")
const assistantID = SessionMessage.ID.make("msg_missing_assistant")
const inboxID = SessionMessage.ID.make("msg_missing_inbox")
const placeholder = Buffer.from("[media unavailable: shot.png (image/png)]").toString("base64")
const ref = SessionStored.FileRef.make({
uri: `media:${hash}`,
mime: "image/png",
source: { type: "inline" },
name: "shot.png",
})
yield* db
.insert(SessionMessageTable)
.values([
{
id: userID,
session_id: sessionID,
type: "user",
seq: 1,
time_created: 1,
data: {
text: "gone",
files: [ref],
time: { created: 1 },
},
},
{
id: assistantID,
session_id: sessionID,
type: "assistant",
seq: 2,
time_created: 2,
data: {
agent: build,
model,
content: [
{
type: "tool",
id: "call_1",
name: "read",
executed: true,
state: {
status: "completed",
input: { path: "shot.png" },
content: [SessionStored.ToolFileRef.make({ type: "file", uri: ref.uri, mime: "image/png" })],
},
time: { created: 2, ran: 2, completed: 2 },
},
],
time: { created: 2, completed: 2 },
},
},
])
.run()
.pipe(Effect.orDie)
yield* db
.insert(SessionInboxTable)
.values({
id: inboxID,
session_id: sessionID,
type: "user",
payload: { text: "queued", files: [ref] },
delivery: "queue",
enqueued_seq: 3,
time_created: 3,
})
.run()
.pipe(Effect.orDie)
const messages = yield* store.messages({ sessionID, order: "asc" })
expect(messages[0]).toMatchObject({
files: [{ data: placeholder, mime: "text/plain", source: { type: "inline" }, name: "shot.png" }],
})
expect(messages[1]).toMatchObject({
content: [{ state: { content: [{ type: "text", text: "[media unavailable: image/png]" }] } }],
})
expect(yield* store.context(sessionID)).toEqual(messages)
const listed = yield* inbox.list(sessionID)
expect(listed[0]).toMatchObject({ payload: { files: [{ data: placeholder, mime: "text/plain" }] } })
expect(JSON.stringify([messages, listed])).not.toContain(hash)
}),
)
it.effect("names ref attachments in kept text without reading their blobs", () =>
Effect.gen(function* () {
const db = yield* seedSession()
const store = yield* SessionStore.Service
const userID = SessionMessage.ID.make("msg_kept_user")
const compactionID = SessionMessage.ID.make("msg_kept_compaction")
const { id: _, type: __, ...compaction } = encodeMessage(
SessionMessage.CompactionCompleted.make({
id: compactionID,
type: "compaction",
status: "completed",
reason: "manual",
summary: "summary",
metadata: { keepID: userID },
time: { created },
}),
)
yield* db
.insert(SessionMessageTable)
.values([
{
id: userID,
session_id: sessionID,
type: "user",
seq: 1,
time_created: 1,
data: {
text: "look",
files: [
SessionStored.FileRef.make({
uri: `media:${"c".repeat(64)}`,
mime: "image/png",
source: { type: "inline" },
name: "shot.png",
}),
],
time: { created: 1 },
},
},
{ id: compactionID, session_id: sessionID, type: "compaction", seq: 2, time_created: 2, data: compaction },
])
.run()
.pipe(Effect.orDie)
// The blob is missing, so a hydrated read would name a text/plain placeholder instead.
expect(yield* store.context(sessionID)).toMatchObject([
{ id: compactionID, metadata: { recent: "[User]: look\n[Attached image/png: shot.png]" } },
])
}),
)
})