mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-20 18:57:17 +03:00
feat(core): store session media as media:<hash> refs in a content-addressed blob table (#58656)
This commit is contained in:
committed by
GitHub
Unverified
parent
9cc9e58c73
commit
70c69f06f9
+188
-2
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"version": "7",
|
||||
"dialect": "sqlite",
|
||||
"id": "76109fbf-f324-4af0-934c-3ae695275099",
|
||||
"prevIds": ["ff2efc22-36c2-4af5-8f24-2288090b0ba6"],
|
||||
"id": "f1f14616-b6a1-483d-87b0-d5b7a9132580",
|
||||
"prevIds": ["76109fbf-f324-4af0-934c-3ae695275099"],
|
||||
"ddl": [
|
||||
{
|
||||
"name": "account_state",
|
||||
@@ -44,6 +44,14 @@
|
||||
"name": "project",
|
||||
"entityType": "tables"
|
||||
},
|
||||
{
|
||||
"name": "blob_ref",
|
||||
"entityType": "tables"
|
||||
},
|
||||
{
|
||||
"name": "blob",
|
||||
"entityType": "tables"
|
||||
},
|
||||
{
|
||||
"name": "instruction_blob",
|
||||
"entityType": "tables"
|
||||
@@ -740,6 +748,116 @@
|
||||
"entityType": "columns",
|
||||
"table": "project"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "session_id",
|
||||
"entityType": "columns",
|
||||
"table": "blob_ref"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "message_id",
|
||||
"entityType": "columns",
|
||||
"table": "blob_ref"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "hash",
|
||||
"entityType": "columns",
|
||||
"table": "blob_ref"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "seq",
|
||||
"entityType": "columns",
|
||||
"table": "blob_ref"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "hash",
|
||||
"entityType": "columns",
|
||||
"table": "blob"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "mime",
|
||||
"entityType": "columns",
|
||||
"table": "blob"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "bytes",
|
||||
"entityType": "columns",
|
||||
"table": "blob"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "width",
|
||||
"entityType": "columns",
|
||||
"table": "blob"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "height",
|
||||
"entityType": "columns",
|
||||
"table": "blob"
|
||||
},
|
||||
{
|
||||
"type": "blob",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "data",
|
||||
"entityType": "columns",
|
||||
"table": "blob"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": null,
|
||||
"generated": null,
|
||||
"name": "time_created",
|
||||
"entityType": "columns",
|
||||
"table": "blob"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"notNull": false,
|
||||
@@ -1574,6 +1692,28 @@
|
||||
"entityType": "fks",
|
||||
"table": "project_directory"
|
||||
},
|
||||
{
|
||||
"columns": ["session_id"],
|
||||
"tableTo": "session_v2",
|
||||
"columnsTo": ["id"],
|
||||
"onUpdate": "NO ACTION",
|
||||
"onDelete": "CASCADE",
|
||||
"nameExplicit": false,
|
||||
"name": "fk_blob_ref_session_id_session_v2_id_fk",
|
||||
"entityType": "fks",
|
||||
"table": "blob_ref"
|
||||
},
|
||||
{
|
||||
"columns": ["hash"],
|
||||
"tableTo": "blob",
|
||||
"columnsTo": ["hash"],
|
||||
"onUpdate": "NO ACTION",
|
||||
"onDelete": "CASCADE",
|
||||
"nameExplicit": false,
|
||||
"name": "fk_blob_ref_hash_blob_hash_fk",
|
||||
"entityType": "fks",
|
||||
"table": "blob_ref"
|
||||
},
|
||||
{
|
||||
"columns": ["session_id"],
|
||||
"tableTo": "session_v2",
|
||||
@@ -1665,6 +1805,13 @@
|
||||
"entityType": "pks",
|
||||
"table": "project_directory"
|
||||
},
|
||||
{
|
||||
"columns": ["session_id", "message_id", "hash"],
|
||||
"nameExplicit": false,
|
||||
"name": "blob_ref_pk",
|
||||
"entityType": "pks",
|
||||
"table": "blob_ref"
|
||||
},
|
||||
{
|
||||
"columns": ["session_id", "key"],
|
||||
"nameExplicit": false,
|
||||
@@ -1735,6 +1882,13 @@
|
||||
"table": "project",
|
||||
"entityType": "pks"
|
||||
},
|
||||
{
|
||||
"columns": ["hash"],
|
||||
"nameExplicit": false,
|
||||
"name": "blob_pk",
|
||||
"table": "blob",
|
||||
"entityType": "pks"
|
||||
},
|
||||
{
|
||||
"columns": ["hash"],
|
||||
"nameExplicit": false,
|
||||
@@ -1846,6 +2000,38 @@
|
||||
"entityType": "indexes",
|
||||
"table": "permission"
|
||||
},
|
||||
{
|
||||
"columns": [
|
||||
{
|
||||
"value": "hash",
|
||||
"isExpression": false
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": null,
|
||||
"origin": "manual",
|
||||
"name": "blob_ref_hash_idx",
|
||||
"entityType": "indexes",
|
||||
"table": "blob_ref"
|
||||
},
|
||||
{
|
||||
"columns": [
|
||||
{
|
||||
"value": "session_id",
|
||||
"isExpression": false
|
||||
},
|
||||
{
|
||||
"value": "seq",
|
||||
"isExpression": false
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": null,
|
||||
"origin": "manual",
|
||||
"name": "blob_ref_session_seq_idx",
|
||||
"entityType": "indexes",
|
||||
"table": "blob_ref"
|
||||
},
|
||||
{
|
||||
"columns": [
|
||||
{
|
||||
|
||||
+2
@@ -51,6 +51,7 @@ import m48 from "./migration/20261007190000_azure_cli_external_credential.js"
|
||||
import m49 from "./migration/20261010180456_project_time_archived.js"
|
||||
import m50 from "./migration/20261024002554_session_allowance.js"
|
||||
import m51 from "./migration/20261024175301_drop_session_allowance.js"
|
||||
import m52 from "./migration/20261025123711_blob.js"
|
||||
|
||||
export const migrations = [
|
||||
m00,
|
||||
@@ -105,4 +106,5 @@ export const migrations = [
|
||||
m49,
|
||||
m50,
|
||||
m51,
|
||||
m52,
|
||||
] satisfies DatabaseMigration.Migration[]
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import { Effect } from "effect"
|
||||
import type { DatabaseMigration } from "../migration.js"
|
||||
|
||||
const migration: DatabaseMigration.Migration = {
|
||||
id: "20261025123711_blob",
|
||||
up(tx) {
|
||||
return Effect.gen(function* () {
|
||||
yield* tx.run(`
|
||||
CREATE TABLE \`blob_ref\` (
|
||||
\`session_id\` text NOT NULL,
|
||||
\`message_id\` text NOT NULL,
|
||||
\`hash\` text NOT NULL,
|
||||
\`seq\` integer NOT NULL,
|
||||
CONSTRAINT \`blob_ref_pk\` PRIMARY KEY(\`session_id\`, \`message_id\`, \`hash\`),
|
||||
CONSTRAINT \`fk_blob_ref_session_id_session_v2_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session_v2\`(\`id\`) ON DELETE CASCADE,
|
||||
CONSTRAINT \`fk_blob_ref_hash_blob_hash_fk\` FOREIGN KEY (\`hash\`) REFERENCES \`blob\`(\`hash\`) ON DELETE CASCADE
|
||||
);
|
||||
`)
|
||||
yield* tx.run(`
|
||||
CREATE TABLE \`blob\` (
|
||||
\`hash\` text PRIMARY KEY,
|
||||
\`mime\` text NOT NULL,
|
||||
\`bytes\` integer NOT NULL,
|
||||
\`width\` integer,
|
||||
\`height\` integer,
|
||||
\`data\` blob NOT NULL,
|
||||
\`time_created\` integer NOT NULL
|
||||
);
|
||||
`)
|
||||
yield* tx.run(`CREATE INDEX \`blob_ref_hash_idx\` ON \`blob_ref\` (\`hash\`);`)
|
||||
yield* tx.run(`CREATE INDEX \`blob_ref_session_seq_idx\` ON \`blob_ref\` (\`session_id\`,\`seq\`);`)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
export default migration
|
||||
Generated
+24
@@ -116,6 +116,28 @@ const schema: Omit<DatabaseMigration.Migration, "id"> = {
|
||||
\`commands\` text
|
||||
);
|
||||
`)
|
||||
yield* tx.run(`
|
||||
CREATE TABLE \`blob_ref\` (
|
||||
\`session_id\` text NOT NULL,
|
||||
\`message_id\` text NOT NULL,
|
||||
\`hash\` text NOT NULL,
|
||||
\`seq\` integer NOT NULL,
|
||||
CONSTRAINT \`blob_ref_pk\` PRIMARY KEY(\`session_id\`, \`message_id\`, \`hash\`),
|
||||
CONSTRAINT \`fk_blob_ref_session_id_session_v2_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session_v2\`(\`id\`) ON DELETE CASCADE,
|
||||
CONSTRAINT \`fk_blob_ref_hash_blob_hash_fk\` FOREIGN KEY (\`hash\`) REFERENCES \`blob\`(\`hash\`) ON DELETE CASCADE
|
||||
);
|
||||
`)
|
||||
yield* tx.run(`
|
||||
CREATE TABLE \`blob\` (
|
||||
\`hash\` text PRIMARY KEY,
|
||||
\`mime\` text NOT NULL,
|
||||
\`bytes\` integer NOT NULL,
|
||||
\`width\` integer,
|
||||
\`height\` integer,
|
||||
\`data\` blob NOT NULL,
|
||||
\`time_created\` integer NOT NULL
|
||||
);
|
||||
`)
|
||||
yield* tx.run(`
|
||||
CREATE TABLE \`instruction_blob\` (
|
||||
\`hash\` text PRIMARY KEY,
|
||||
@@ -245,6 +267,8 @@ const schema: Omit<DatabaseMigration.Migration, "id"> = {
|
||||
yield* tx.run(
|
||||
`CREATE UNIQUE INDEX \`permission_project_action_resource_idx\` ON \`permission\` (\`project_id\`,\`action\`,\`resource\`);`,
|
||||
)
|
||||
yield* tx.run(`CREATE INDEX \`blob_ref_hash_idx\` ON \`blob_ref\` (\`hash\`);`)
|
||||
yield* tx.run(`CREATE INDEX \`blob_ref_session_seq_idx\` ON \`blob_ref\` (\`session_id\`,\`seq\`);`)
|
||||
yield* tx.run(
|
||||
`CREATE INDEX \`session_inbox_session_delivery_seq_idx\` ON \`session_inbox\` (\`session_id\`,\`delivery\`,\`enqueued_seq\`);`,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
export * as SessionBlob from "./blob.js"
|
||||
|
||||
import { inArray } from "drizzle-orm"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { map } from "effect/Array"
|
||||
import type { FileAttachment } from "@opencode/schema/prompt"
|
||||
import type { Tool } from "@opencode/schema/tool"
|
||||
import type { Database } from "../database/database.js"
|
||||
import { BlobTable, SessionInboxTable, SessionMessageTable } from "./sql.js"
|
||||
import { SessionStored } from "./stored.js"
|
||||
|
||||
type DatabaseService = Database.Interface["db"]
|
||||
type MessageRow = typeof SessionMessageTable.$inferSelect
|
||||
type InboxRow = typeof SessionInboxTable.$inferSelect
|
||||
type Part = Extract<MessageRow["data"], { readonly content: unknown }>["content"][number]
|
||||
/** Base64 bytes by blob hash. */
|
||||
type Blobs = ReadonlyMap<string, string>
|
||||
|
||||
const isFileRef = Schema.is(SessionStored.FileRef)
|
||||
const isToolFileRef = Schema.is(SessionStored.ToolFileRef)
|
||||
|
||||
const toolItems = (part: Part): ReadonlyArray<Tool.Content> =>
|
||||
part.type === "tool" && (part.state.status === "completed" || part.state.status === "error")
|
||||
? (part.state.content ?? [])
|
||||
: []
|
||||
|
||||
function collect(data: MessageRow["data"] | InboxRow["payload"], hashes: Set<string>) {
|
||||
if ("files" in data)
|
||||
for (const file of data.files ?? []) if (isFileRef(file)) hashes.add(SessionStored.refHash(file.uri))
|
||||
if ("content" in data)
|
||||
for (const part of data.content)
|
||||
for (const item of toolItems(part)) if (isToolFileRef(item)) hashes.add(SessionStored.refHash(item.uri))
|
||||
}
|
||||
|
||||
/** A ref whose blob is gone reads as this text, so clients and the model never see a bare ref. */
|
||||
const unavailable = (mime: string, name?: string) =>
|
||||
`[media unavailable: ${name === undefined ? mime : `${name} (${mime})`}]`
|
||||
|
||||
function inlineFile(file: SessionStored.File, blobs: Blobs): FileAttachment {
|
||||
if (!isFileRef(file)) return file
|
||||
const { uri, ...rest } = file
|
||||
const data = blobs.get(SessionStored.refHash(uri))
|
||||
if (data === undefined)
|
||||
return { ...rest, data: Buffer.from(unavailable(file.mime, file.name)).toString("base64"), mime: "text/plain" }
|
||||
return { ...rest, data }
|
||||
}
|
||||
|
||||
function inlineToolItem(item: Tool.Content, blobs: Blobs): Tool.Content {
|
||||
if (!isToolFileRef(item)) return item
|
||||
const data = blobs.get(SessionStored.refHash(item.uri))
|
||||
if (data === undefined) return { type: "text", text: unavailable(item.mime, item.name) }
|
||||
return { ...item, uri: `data:${item.mime};base64,${data}` }
|
||||
}
|
||||
|
||||
function inlinePart(part: Part, blobs: Blobs): Part {
|
||||
if (part.type !== "tool" || (part.state.status !== "completed" && part.state.status !== "error")) return part
|
||||
const content = part.state.content
|
||||
if (!content?.some(isToolFileRef)) return part
|
||||
return { ...part, state: { ...part.state, content: map(content, (item) => inlineToolItem(item, blobs)) } }
|
||||
}
|
||||
|
||||
const load = Effect.fnUntraced(function* (db: DatabaseService, hashes: ReadonlySet<string>) {
|
||||
const all = [...hashes]
|
||||
const blobs = new Map<string, string>()
|
||||
for (let offset = 0; offset < all.length; offset += 500) {
|
||||
const found = yield* db
|
||||
.select({ hash: BlobTable.hash, data: BlobTable.data })
|
||||
.from(BlobTable)
|
||||
.where(inArray(BlobTable.hash, all.slice(offset, offset + 500)))
|
||||
.all()
|
||||
.pipe(Effect.orDie)
|
||||
for (const blob of found) blobs.set(blob.hash, view(blob.data).toString("base64"))
|
||||
}
|
||||
return blobs
|
||||
})
|
||||
|
||||
const view = (data: Uint8Array) =>
|
||||
Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength)
|
||||
|
||||
/** Restores inline media for the refs in `session_message` rows. Rows without refs are returned untouched. */
|
||||
export const hydrate = Effect.fnUntraced(function* (db: DatabaseService, rows: ReadonlyArray<MessageRow>) {
|
||||
const hashes = new Set<string>()
|
||||
for (const row of rows) collect(row.data, hashes)
|
||||
if (hashes.size === 0) return rows
|
||||
const blobs = yield* load(db, hashes)
|
||||
return rows.map((row): MessageRow => {
|
||||
const data = row.data
|
||||
if ("files" in data && data.files?.some(isFileRef))
|
||||
return { ...row, data: { ...data, files: data.files.map((file) => inlineFile(file, blobs)) } }
|
||||
if ("content" in data && data.content.some((part) => toolItems(part).some(isToolFileRef)))
|
||||
return { ...row, data: { ...data, content: data.content.map((part) => inlinePart(part, blobs)) } }
|
||||
return row
|
||||
})
|
||||
})
|
||||
|
||||
/** Restores inline media for the refs in `session_inbox` rows. Rows without refs are returned untouched. */
|
||||
export const hydrateInbox = Effect.fnUntraced(function* (db: DatabaseService, rows: ReadonlyArray<InboxRow>) {
|
||||
const hashes = new Set<string>()
|
||||
for (const row of rows) collect(row.payload, hashes)
|
||||
if (hashes.size === 0) return rows
|
||||
const blobs = yield* load(db, hashes)
|
||||
return rows.map((row): InboxRow => {
|
||||
const payload = row.payload
|
||||
if (!("files" in payload) || !payload.files?.some(isFileRef)) return row
|
||||
return { ...row, payload: { ...payload, files: payload.files.map((file) => inlineFile(file, blobs)) } }
|
||||
})
|
||||
})
|
||||
@@ -9,13 +9,16 @@ import { SessionMessage } from "./message.js"
|
||||
import type { SessionSchema } from "./schema.js"
|
||||
import { Instructions } from "../instructions/index.js"
|
||||
import { InstructionState } from "./instruction-state.js"
|
||||
import { SessionBlob } from "./blob.js"
|
||||
import { SessionRecentText } from "./recent-text.js"
|
||||
import { SessionMessageTable } from "./sql.js"
|
||||
import { SessionStored } from "./stored.js"
|
||||
|
||||
type DatabaseService = Database.Interface["db"]
|
||||
|
||||
const decode = Schema.decodeUnknownEffect(SessionMessage.Info)
|
||||
const decodeSync = Schema.decodeUnknownSync(SessionMessage.Info)
|
||||
const decodeStoredUser = Schema.decodeUnknownEffect(SessionStored.User)
|
||||
|
||||
/**
|
||||
* Which completed compactions bound a history read. Local summaries always do. Native
|
||||
@@ -89,16 +92,28 @@ const rowInput = (row: typeof SessionMessageTable.$inferSelect) => ({
|
||||
|
||||
export const decodeMessageRowSync = (row: typeof SessionMessageTable.$inferSelect) => decodeSync(rowInput(row))
|
||||
|
||||
const decodeError = (row: typeof SessionMessageTable.$inferSelect) => () =>
|
||||
new MessageDecodeError({
|
||||
sessionID: row.session_id,
|
||||
messageID: row.id,
|
||||
})
|
||||
|
||||
export const decodeMessageRow = (row: typeof SessionMessageTable.$inferSelect) =>
|
||||
decode(rowInput(row)).pipe(
|
||||
Effect.mapError(
|
||||
() =>
|
||||
new MessageDecodeError({
|
||||
sessionID: row.session_id,
|
||||
messageID: row.id,
|
||||
}),
|
||||
),
|
||||
)
|
||||
decode(rowInput(row)).pipe(Effect.mapError(decodeError(row)))
|
||||
|
||||
/** Decodes a row without its blobs, so user files may stay refs. Only for readers that never use file bytes. */
|
||||
const decodeStoredRow = (
|
||||
row: typeof SessionMessageTable.$inferSelect,
|
||||
): Effect.Effect<SessionMessage.Info | SessionStored.User, MessageDecodeError> =>
|
||||
row.type === "user" ? decodeStoredUser(rowInput(row)).pipe(Effect.mapError(decodeError(row))) : decodeMessageRow(row)
|
||||
|
||||
/** Decodes rows after restoring media stored in `blob`. Reads that return media must use this. */
|
||||
export const decodeMessageRows = Effect.fnUntraced(function* (
|
||||
db: DatabaseService,
|
||||
rows: ReadonlyArray<typeof SessionMessageTable.$inferSelect>,
|
||||
) {
|
||||
return yield* Effect.forEach(yield* SessionBlob.hydrate(db, rows), decodeMessageRow, { concurrency: 1 })
|
||||
})
|
||||
|
||||
const messageEntries = Effect.fnUntraced(function* (
|
||||
db: DatabaseService,
|
||||
@@ -122,8 +137,10 @@ const messageEntries = Effect.fnUntraced(function* (
|
||||
.orderBy(asc(SessionMessageTable.seq))
|
||||
.all()
|
||||
.pipe(Effect.orDie)
|
||||
const decoded = yield* Effect.forEach(rows, (row) =>
|
||||
decodeMessageRow(row).pipe(Effect.map((message) => ({ seq: row.seq, message }))),
|
||||
const decoded = yield* Effect.forEach(
|
||||
yield* SessionBlob.hydrate(db, rows),
|
||||
(row) => decodeMessageRow(row).pipe(Effect.map((message) => ({ seq: row.seq, message }))),
|
||||
{ concurrency: 1 },
|
||||
)
|
||||
// A window that cannot replay leaves nothing to start from, so read from the compaction before it.
|
||||
const start = decoded[0]
|
||||
@@ -176,7 +193,8 @@ const withRecentText = Effect.fnUntraced(function* (
|
||||
.orderBy(asc(SessionMessageTable.seq))
|
||||
.all()
|
||||
.pipe(Effect.orDie)
|
||||
const kept = yield* Effect.forEach(rows, decodeMessageRow)
|
||||
// The recent text names attachments by mime and name only, so their bytes are never read.
|
||||
const kept = yield* Effect.forEach(rows, decodeStoredRow, { concurrency: 1 })
|
||||
const recent = kept.map(SessionRecentText.messageToText).filter(Boolean).join("\n\n")
|
||||
return [{ ...first, message: { ...compaction, metadata: { ...compaction.metadata, recent } } }, ...entries.slice(1)]
|
||||
})
|
||||
@@ -235,21 +253,22 @@ export const preview = Effect.fn("SessionHistory.preview")(function* (
|
||||
.pipe(Effect.catch((error) => (error instanceof Instructions.InitializationBlocked ? error : Effect.die(error))))
|
||||
})
|
||||
|
||||
/** Returns the session's first user message. */
|
||||
/** The id and text of the session's first user message. */
|
||||
export const firstUserMessage = Effect.fn("SessionHistory.firstUserMessage")(function* (
|
||||
db: DatabaseService,
|
||||
sessionID: SessionSchema.ID,
|
||||
) {
|
||||
const row = yield* db
|
||||
.select()
|
||||
.select({
|
||||
id: SessionMessageTable.id,
|
||||
text: sql<unknown>`json_extract(${SessionMessageTable.data}, '$.text')`,
|
||||
})
|
||||
.from(SessionMessageTable)
|
||||
.where(and(eq(SessionMessageTable.session_id, sessionID), eq(SessionMessageTable.type, "user")))
|
||||
.orderBy(asc(SessionMessageTable.seq))
|
||||
.get()
|
||||
.pipe(Effect.orDie)
|
||||
if (!row) return undefined
|
||||
const message = yield* decodeMessageRow(row).pipe(Effect.orElseSucceed(() => undefined))
|
||||
return message?.type === "user" ? message : undefined
|
||||
return row && Schema.is(Schema.String)(row.text) ? { id: row.id, text: row.text } : undefined
|
||||
})
|
||||
|
||||
export * as SessionHistory from "./history.js"
|
||||
|
||||
@@ -20,8 +20,10 @@ import { Database } from "../database/database.js"
|
||||
import { Bus } from "../bus.js"
|
||||
import { KeyedMutex } from "../effect/keyed-mutex.js"
|
||||
import { SessionEvent } from "./event.js"
|
||||
import { SessionBlob } from "./blob.js"
|
||||
import { SessionMessage } from "./message.js"
|
||||
import { SessionSchema } from "./schema.js"
|
||||
import { SessionStored } from "./stored.js"
|
||||
import { SessionInboxTable, SessionMessageTable } from "./sql.js"
|
||||
import { SessionHistory } from "./history.js"
|
||||
|
||||
@@ -49,6 +51,7 @@ export {
|
||||
export type Promotable = "input" | "steer"
|
||||
|
||||
const decodeUser = Schema.decodeUnknownSync(UserPayload)
|
||||
const decodeStoredUser = Schema.decodeUnknownSync(SessionStored.UserPayload)
|
||||
const encodeUser = Schema.encodeSync(UserPayload)
|
||||
const decodeSynthetic = Schema.decodeUnknownSync(SyntheticPayload)
|
||||
const encodeSynthetic = Schema.encodeSync(SyntheticPayload)
|
||||
@@ -107,7 +110,9 @@ const fromRow = (row: typeof SessionInboxTable.$inferSelect): Info => {
|
||||
|
||||
export const find = Effect.fn("SessionInbox.find")(function* (db: DatabaseService, id: SessionMessage.ID) {
|
||||
const row = yield* db.select().from(SessionInboxTable).where(eq(SessionInboxTable.id, id)).get().pipe(Effect.orDie)
|
||||
return row === undefined ? undefined : fromRow(row)
|
||||
if (row === undefined) return undefined
|
||||
const [hydrated] = yield* SessionBlob.hydrateInbox(db, [row])
|
||||
return hydrated && fromRow(hydrated)
|
||||
})
|
||||
|
||||
const promotedFromMessage = Effect.fn("SessionInbox.promotedFromMessage")(function* (
|
||||
@@ -125,7 +130,8 @@ const promotedFromMessage = Effect.fn("SessionInbox.promotedFromMessage")(functi
|
||||
if (row === undefined) return undefined
|
||||
if (row.session_id !== sessionID || (row.type !== "user" && row.type !== "synthetic"))
|
||||
return yield* new LifecycleConflict({ id })
|
||||
const message = SessionHistory.decodeMessageRowSync(row)
|
||||
const [message] = yield* SessionHistory.decodeMessageRows(db, [row]).pipe(Effect.orDie)
|
||||
if (!message) return undefined
|
||||
const base = { id, sessionID, time: { created: message.time.created }, delivery }
|
||||
if (message.type === "user")
|
||||
return User.make({
|
||||
@@ -341,9 +347,21 @@ export const projectDelivered = Effect.fn("SessionInbox.projectDelivered")(funct
|
||||
.get()
|
||||
.pipe(Effect.orDie)
|
||||
if (!deleted) return yield* Effect.die(new LifecycleConflict({ id: input.id }))
|
||||
return fromRow(deleted)
|
||||
return delivered(deleted)
|
||||
})
|
||||
|
||||
/** A consumed row with its payload as stored, so user file refs reach the message without their blobs. */
|
||||
export type Delivered =
|
||||
| { readonly id: SessionMessage.ID; readonly type: "user"; readonly payload: SessionStored.UserPayload }
|
||||
| { readonly id: SessionMessage.ID; readonly type: "synthetic"; readonly payload: SyntheticPayload }
|
||||
| { readonly id: SessionMessage.ID; readonly type: "compaction" | "move" }
|
||||
|
||||
const delivered = (row: typeof SessionInboxTable.$inferSelect): Delivered => {
|
||||
if (row.type === "user") return { id: row.id, type: "user", payload: decodeStoredUser(row.payload) }
|
||||
if (row.type === "synthetic") return { id: row.id, type: "synthetic", payload: decodeSynthetic(row.payload) }
|
||||
return { id: row.id, type: row.type }
|
||||
}
|
||||
|
||||
export const projectCancelled = Effect.fn("SessionInbox.projectCancelled")(function* (
|
||||
db: DatabaseService,
|
||||
input: PendingRef,
|
||||
@@ -400,7 +418,7 @@ export const list = Effect.fn("SessionInbox.list")(function* (db: DatabaseServic
|
||||
.orderBy(asc(SessionInboxTable.enqueued_seq))
|
||||
.all()
|
||||
.pipe(Effect.orDie)
|
||||
return rows.map(fromRow)
|
||||
return (yield* SessionBlob.hydrateInbox(db, rows)).map(fromRow)
|
||||
})
|
||||
|
||||
export const moveIDs = Effect.fn("SessionInbox.moveIDs")(function* (db: DatabaseService, sessionID: SessionSchema.ID) {
|
||||
@@ -419,7 +437,7 @@ export const nextPromotable = Effect.fn("SessionInbox.nextPromotable")(function*
|
||||
promotable: Promotable,
|
||||
) {
|
||||
const steer = (yield* pendingSteers(db, sessionID))[0]
|
||||
if (steer) return fromRow(steer)
|
||||
if (steer) return promotion(steer)
|
||||
if (promotable !== "input") return undefined
|
||||
const queued = yield* db
|
||||
.select()
|
||||
@@ -429,9 +447,19 @@ export const nextPromotable = Effect.fn("SessionInbox.nextPromotable")(function*
|
||||
.limit(1)
|
||||
.get()
|
||||
.pipe(Effect.orDie)
|
||||
return queued ? fromRow(queued) : undefined
|
||||
return queued ? promotion(queued) : undefined
|
||||
})
|
||||
|
||||
/** What a boundary decides on: prompts are promoted by id, so only a move's payload is read. */
|
||||
export type Promotion =
|
||||
| { readonly id: SessionMessage.ID; readonly type: "user" | "synthetic" | "compaction"; readonly delivery: Delivery }
|
||||
| { readonly id: SessionMessage.ID; readonly type: "move"; readonly delivery: Delivery; readonly payload: MovePayload }
|
||||
|
||||
const promotion = (row: typeof SessionInboxTable.$inferSelect): Promotion =>
|
||||
row.type === "move"
|
||||
? { id: row.id, type: "move", delivery: row.delivery, payload: decodeMove(row.payload) }
|
||||
: { id: row.id, type: row.type, delivery: row.delivery }
|
||||
|
||||
/** Which pending rows count: "input" means any item in either delivery mode. */
|
||||
export type Scope = "input" | Delivery
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import { SessionMessageUpdater } from "./message-updater.js"
|
||||
import { SessionInbox } from "./inbox.js"
|
||||
import { InstructionState } from "./instruction-state.js"
|
||||
import { SessionInboxTable, SessionMessageTable, SessionTable } from "./sql.js"
|
||||
import { SessionStored } from "./stored.js"
|
||||
import { InstructionEntry } from "./instruction-entry.js"
|
||||
import { Slug } from "../util/slug.js"
|
||||
import { FSUtil } from "@opencode/util/fs-util"
|
||||
@@ -32,6 +33,7 @@ type MessageEvent = Exclude<
|
||||
>
|
||||
|
||||
const encodeMessage = Schema.encodeSync(SessionMessage.Info)
|
||||
const encodeStoredUser = Schema.encodeSync(SessionStored.User)
|
||||
|
||||
export class SessionAlreadyProjected extends Error {}
|
||||
|
||||
@@ -398,9 +400,12 @@ function run(db: DatabaseService, event: MessageEvent) {
|
||||
})
|
||||
}
|
||||
|
||||
function insertMessage(db: DatabaseService, event: SessionEvent.DurableEvent, message: SessionMessage.Info) {
|
||||
const encoded = encodeMessage(message)
|
||||
const { id: _, type, ...data } = encoded
|
||||
function insertMessage(
|
||||
db: DatabaseService,
|
||||
event: SessionEvent.DurableEvent,
|
||||
message: SessionMessage.Info | SessionStored.User,
|
||||
) {
|
||||
const { id: _, type, ...data } = message.type === "user" ? encodeStoredUser(message) : encodeMessage(message)
|
||||
return db
|
||||
.insert(SessionMessageTable)
|
||||
.values({
|
||||
@@ -643,30 +648,27 @@ const layer = Layer.effectDiscard(
|
||||
id: event.data.inboxID,
|
||||
sessionID: event.data.sessionID,
|
||||
})
|
||||
if (input.type === "compaction" || input.type === "move") return
|
||||
yield* insertMessage(
|
||||
db,
|
||||
event,
|
||||
input.type === "user"
|
||||
? {
|
||||
id: input.id,
|
||||
type: "user",
|
||||
metadata: input.payload.metadata,
|
||||
text: input.payload.text,
|
||||
files: input.payload.files,
|
||||
agents: input.payload.agents,
|
||||
skills: input.payload.skills,
|
||||
time: { created: DateTime.makeUnsafe(event.created) },
|
||||
}
|
||||
: {
|
||||
id: input.id,
|
||||
type: "synthetic",
|
||||
text: input.payload.text,
|
||||
description: input.payload.description,
|
||||
metadata: input.payload.metadata,
|
||||
time: { created: DateTime.makeUnsafe(event.created) },
|
||||
},
|
||||
)
|
||||
const time = { created: DateTime.makeUnsafe(event.created) }
|
||||
if (input.type === "user")
|
||||
yield* insertMessage(db, event, {
|
||||
id: input.id,
|
||||
type: "user",
|
||||
metadata: input.payload.metadata,
|
||||
text: input.payload.text,
|
||||
files: input.payload.files,
|
||||
agents: input.payload.agents,
|
||||
skills: input.payload.skills,
|
||||
time,
|
||||
})
|
||||
if (input.type === "synthetic")
|
||||
yield* insertMessage(db, event, {
|
||||
id: input.id,
|
||||
type: "synthetic",
|
||||
text: input.payload.text,
|
||||
description: input.payload.description,
|
||||
metadata: input.payload.metadata,
|
||||
time,
|
||||
})
|
||||
}),
|
||||
)
|
||||
yield* bus.project(SessionEvent.InboxEnqueued, (event) =>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
export * as SessionRecentText from "./recent-text.js"
|
||||
|
||||
import type { SessionMessage } from "./message.js"
|
||||
import type { SessionStored } from "./stored.js"
|
||||
|
||||
const TOOL_OUTPUT_MAX_CHARS = 1_250
|
||||
|
||||
@@ -12,7 +13,7 @@ export const kept = (message: SessionMessage.CompactionCompleted) =>
|
||||
typeof message.metadata?.recent === "string" ? message.metadata.recent : ""
|
||||
|
||||
/** One message as the recent, verbatim part of a summary shows it. Empty for messages that part leaves out. */
|
||||
export const messageToText = (message: SessionMessage.Info): string => {
|
||||
export const messageToText = (message: SessionMessage.Info | SessionStored.User): string => {
|
||||
switch (message.type) {
|
||||
// Earlier summaries and instruction updates are handled outside the recent part.
|
||||
case "compaction":
|
||||
|
||||
@@ -298,8 +298,11 @@ const layer = Layer.effect(
|
||||
}),
|
||||
OutputLimit: Effect.fn("SessionRunner.continueOutputLimit")(function* (outcome) {
|
||||
// The current response is included. User input or any other finish breaks the streak.
|
||||
const rows = yield* db
|
||||
.select()
|
||||
const recent = yield* db
|
||||
.select({
|
||||
type: SessionMessageTable.type,
|
||||
finish: sql<string | null>`json_extract(${SessionMessageTable.data}, '$.finish')`,
|
||||
})
|
||||
.from(SessionMessageTable)
|
||||
.where(
|
||||
and(
|
||||
@@ -311,10 +314,8 @@ const layer = Layer.effect(
|
||||
.limit(3)
|
||||
.all()
|
||||
.pipe(Effect.orDie)
|
||||
const recent = yield* Effect.forEach(rows, SessionHistory.decodeMessageRow)
|
||||
const exhausted =
|
||||
recent.length === 3 &&
|
||||
recent.every((message) => message.type === "assistant" && message.finish === "length")
|
||||
recent.length === 3 && recent.every((row) => row.type === "assistant" && row.finish === "length")
|
||||
if (exhausted)
|
||||
return yield* new StepFailedError({
|
||||
error: {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { sqliteTable, text, integer, index, primaryKey, real, uniqueIndex } from "drizzle-orm/sqlite-core"
|
||||
import { blob, sqliteTable, text, integer, index, primaryKey, real, uniqueIndex } from "drizzle-orm/sqlite-core"
|
||||
import { sql } from "drizzle-orm"
|
||||
import { directoryColumn, pathColumn } from "../database/path.js"
|
||||
import { ProjectTable } from "../project/sql.js"
|
||||
@@ -9,6 +9,7 @@ import type { FileDiff } from "@opencode/schema/file-diff"
|
||||
import type { Permission } from "@opencode/schema/permission"
|
||||
import type { Project } from "@opencode/schema/project"
|
||||
import type { SessionSchema } from "./schema.js"
|
||||
import type { SessionStored } from "./stored.js"
|
||||
import type { Workspace } from "@opencode/schema/workspace"
|
||||
import { Timestamps } from "../database/schema.sql.js"
|
||||
import type { Instruction } from "@opencode/schema/instruction"
|
||||
@@ -18,7 +19,10 @@ import type { RevertV1 } from "@opencode/schema/session-revert"
|
||||
import type { Schema } from "effect"
|
||||
|
||||
type DistributiveOmit<T, K extends PropertyKey> = T extends unknown ? Omit<T, K> : never
|
||||
type SessionMessageData = DistributiveOmit<(typeof SessionMessage.Info)["Encoded"], "type" | "id">
|
||||
type SessionMessageData = DistributiveOmit<
|
||||
Exclude<(typeof SessionMessage.Info)["Encoded"], { readonly type: "user" }> | (typeof SessionStored.User)["Encoded"],
|
||||
"type" | "id"
|
||||
>
|
||||
|
||||
export const SessionTable = sqliteTable(
|
||||
"session_v2",
|
||||
@@ -132,7 +136,9 @@ export const SessionInboxTable = sqliteTable(
|
||||
.notNull()
|
||||
.references(() => SessionTable.id, { onDelete: "cascade" }),
|
||||
type: text().$type<SessionInbox.Info["type"]>().notNull(),
|
||||
payload: text({ mode: "json" }).$type<UserPayload | SyntheticPayload | CompactionPayload | MovePayload>().notNull(),
|
||||
payload: text({ mode: "json" })
|
||||
.$type<SessionStored.UserPayload | SyntheticPayload | CompactionPayload | MovePayload>()
|
||||
.notNull(),
|
||||
delivery: text().$type<SessionInbox.Delivery>().notNull(),
|
||||
enqueued_seq: integer().notNull(),
|
||||
time_created: integer()
|
||||
@@ -175,3 +181,35 @@ export const InstructionStateTable = sqliteTable("instruction_state", {
|
||||
initial_values: text({ mode: "json" }).notNull().$type<Instruction.Values>(),
|
||||
current_values: text({ mode: "json" }).notNull().$type<Instruction.Values>(),
|
||||
})
|
||||
|
||||
export const BlobTable = sqliteTable("blob", {
|
||||
hash: text().primaryKey(),
|
||||
mime: text().notNull(),
|
||||
bytes: integer().notNull(),
|
||||
width: integer(),
|
||||
height: integer(),
|
||||
data: blob({ mode: "buffer" }).notNull(),
|
||||
time_created: integer()
|
||||
.notNull()
|
||||
.$default(() => Date.now()),
|
||||
})
|
||||
|
||||
export const BlobRefTable = sqliteTable(
|
||||
"blob_ref",
|
||||
{
|
||||
session_id: text()
|
||||
.$type<SessionSchema.ID>()
|
||||
.notNull()
|
||||
.references(() => SessionTable.id, { onDelete: "cascade" }),
|
||||
message_id: text().$type<SessionMessage.ID>().notNull(),
|
||||
hash: text()
|
||||
.notNull()
|
||||
.references(() => BlobTable.hash, { onDelete: "cascade" }),
|
||||
seq: integer().notNull(),
|
||||
},
|
||||
(table) => [
|
||||
primaryKey({ columns: [table.session_id, table.message_id, table.hash] }),
|
||||
index("blob_ref_hash_idx").on(table.hash),
|
||||
index("blob_ref_session_seq_idx").on(table.session_id, table.seq),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -174,10 +174,7 @@ const layer = Layer.effect(
|
||||
const rows = yield* (input.limit === undefined ? query.all() : query.limit(input.limit).all()).pipe(
|
||||
Effect.orDie,
|
||||
)
|
||||
return yield* Effect.forEach(
|
||||
direction === "previous" ? rows.toReversed() : rows,
|
||||
SessionHistory.decodeMessageRow,
|
||||
)
|
||||
return yield* SessionHistory.decodeMessageRows(db, direction === "previous" ? rows.toReversed() : rows)
|
||||
}),
|
||||
context: Effect.fn("SessionStore.context")((sessionID) => SessionHistory.load(db, sessionID, "latest")),
|
||||
message: Effect.fn("SessionStore.message")(function* (messageID) {
|
||||
@@ -187,12 +184,9 @@ const layer = Layer.effect(
|
||||
.where(eq(SessionMessageTable.id, messageID))
|
||||
.get()
|
||||
.pipe(Effect.orDie)
|
||||
return row
|
||||
? {
|
||||
sessionID: row.session_id,
|
||||
message: yield* SessionHistory.decodeMessageRow(row).pipe(Effect.orDie),
|
||||
}
|
||||
: undefined
|
||||
if (!row) return undefined
|
||||
const [message] = yield* SessionHistory.decodeMessageRows(db, [row]).pipe(Effect.orDie)
|
||||
return message && { sessionID: row.session_id, message }
|
||||
}),
|
||||
listSuspended: Effect.fn("SessionStore.listSuspended")(function* () {
|
||||
return yield* db
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
export * as SessionStored from "./stored.js"
|
||||
|
||||
import { Schema } from "effect"
|
||||
import { FileAttachment } from "@opencode/schema/prompt"
|
||||
import { optional } from "@opencode/schema/schema"
|
||||
import { UserPayload as InboxUserPayload } from "@opencode/schema/session-inbox"
|
||||
import { Tool } from "@opencode/schema/tool"
|
||||
import { SessionMessage } from "./message.js"
|
||||
|
||||
/**
|
||||
* Stored rows may reference a `blob` row as `media:` and the lowercase hex SHA-256 of its raw bytes. The blob
|
||||
* row holds the bytes, size and dimensions; the ref names no server route.
|
||||
*/
|
||||
export const Ref = Schema.String.check(Schema.isPattern(/^media:[0-9a-f]{64}$/))
|
||||
export type Ref = typeof Ref.Type
|
||||
|
||||
/** The `blob` hash a {@link Ref} names. */
|
||||
export const refHash = (ref: Ref) => ref.slice("media:".length)
|
||||
|
||||
const { data: _, ...attachment } = FileAttachment.fields
|
||||
|
||||
/** A user file whose bytes live in `blob`. */
|
||||
export interface FileRef extends Schema.Schema.Type<typeof FileRef> {}
|
||||
export const FileRef = Schema.Struct({ ...attachment, uri: Ref })
|
||||
|
||||
export const File = Schema.Union([FileAttachment, FileRef])
|
||||
export type File = typeof File.Type
|
||||
|
||||
/** A tool result file whose bytes live in `blob`. */
|
||||
export interface ToolFileRef extends Schema.Schema.Type<typeof ToolFileRef> {}
|
||||
export const ToolFileRef = Schema.Struct({ ...Tool.FileContent.fields, uri: Ref })
|
||||
|
||||
const files = Schema.Array(File).pipe(optional)
|
||||
|
||||
/** A `session_inbox` user payload as stored, whose files may be refs. */
|
||||
export interface UserPayload extends Schema.Schema.Type<typeof UserPayload> {}
|
||||
export const UserPayload = Schema.Struct({ ...InboxUserPayload.fields, files })
|
||||
|
||||
/** A `session_message` user message as stored, whose files may be refs. */
|
||||
export interface User extends Schema.Schema.Type<typeof User> {}
|
||||
export const User = Schema.Struct({ ...SessionMessage.User.fields, files })
|
||||
@@ -23,8 +23,16 @@ import { SessionExecution } from "@opencode/core/session/execution"
|
||||
import { fromRow } from "@opencode/core/session/info"
|
||||
import { SessionInbox } from "@opencode/core/session/inbox"
|
||||
import { SessionStore } from "@opencode/core/session/store"
|
||||
import { SessionStored } from "@opencode/core/session/stored"
|
||||
import { SessionHistory } from "@opencode/core/session/history"
|
||||
import { Shell } from "@opencode/schema/shell"
|
||||
import { InstructionStateTable, SessionInboxTable, SessionMessageTable, SessionTable } from "@opencode/core/session/sql"
|
||||
import {
|
||||
InstructionStateTable,
|
||||
BlobTable,
|
||||
SessionInboxTable,
|
||||
SessionMessageTable,
|
||||
SessionTable,
|
||||
} from "@opencode/core/session/sql"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { Snapshot } from "@opencode/core/snapshot"
|
||||
|
||||
@@ -993,4 +1001,306 @@ describe("SessionProjector", () => {
|
||||
expect(yield* updated()).toEqual({ time_updated: 15 })
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("hydrates media:<hash> refs on message and inbox reads", () =>
|
||||
Effect.gen(function* () {
|
||||
const db = yield* seedSession()
|
||||
const bus = yield* Bus.Service
|
||||
const inbox = yield* SessionInbox.Service
|
||||
const store = yield* SessionStore.Service
|
||||
|
||||
const rawBytes = Buffer.from("externalized-media-payload")
|
||||
const base64 = rawBytes.toString("base64")
|
||||
const hash = "a".repeat(64)
|
||||
const uri = `media:${hash}`
|
||||
// Only media:<hash> is a ref; a plain path that looks like a media URL is left alone.
|
||||
const pathUri = `/api/media/${hash}`
|
||||
|
||||
yield* db
|
||||
.insert(BlobTable)
|
||||
.values({
|
||||
hash,
|
||||
mime: "application/pdf",
|
||||
bytes: rawBytes.byteLength,
|
||||
width: null,
|
||||
height: null,
|
||||
data: rawBytes,
|
||||
})
|
||||
.run()
|
||||
.pipe(Effect.orDie)
|
||||
|
||||
const userID = SessionMessage.ID.make("msg_read_user")
|
||||
const assistantID = SessionMessage.ID.make("msg_read_assistant")
|
||||
const inboxID = SessionMessage.ID.make("msg_read_inbox")
|
||||
|
||||
yield* db
|
||||
.insert(SessionMessageTable)
|
||||
.values([
|
||||
{
|
||||
id: userID,
|
||||
session_id: sessionID,
|
||||
type: "user",
|
||||
seq: 1,
|
||||
time_created: 1,
|
||||
data: {
|
||||
text: "user attachment",
|
||||
files: [SessionStored.FileRef.make({ uri, mime: "application/pdf", source: { type: "inline" }, name: "spec.pdf" })],
|
||||
time: { created: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
id: assistantID,
|
||||
session_id: sessionID,
|
||||
type: "assistant",
|
||||
seq: 2,
|
||||
time_created: 2,
|
||||
data: {
|
||||
agent: build,
|
||||
model,
|
||||
content: [
|
||||
{
|
||||
type: "tool",
|
||||
id: "call_1",
|
||||
name: "read",
|
||||
executed: true,
|
||||
state: {
|
||||
status: "completed",
|
||||
input: { path: "spec.pdf" },
|
||||
content: [
|
||||
SessionStored.ToolFileRef.make({ type: "file", uri, mime: "application/pdf", name: "spec.pdf" }),
|
||||
{ type: "file", uri: pathUri, mime: "application/pdf", name: "path.pdf" },
|
||||
],
|
||||
},
|
||||
time: { created: 2, ran: 2, completed: 2 },
|
||||
},
|
||||
],
|
||||
time: { created: 2, completed: 2 },
|
||||
},
|
||||
},
|
||||
])
|
||||
.run()
|
||||
.pipe(Effect.orDie)
|
||||
|
||||
yield* db
|
||||
.insert(SessionInboxTable)
|
||||
.values({
|
||||
id: inboxID,
|
||||
session_id: sessionID,
|
||||
type: "user",
|
||||
payload: {
|
||||
text: "queued attachment",
|
||||
files: [SessionStored.FileRef.make({ uri, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" })],
|
||||
},
|
||||
delivery: "queue",
|
||||
enqueued_seq: 3,
|
||||
time_created: 3,
|
||||
})
|
||||
.run()
|
||||
.pipe(Effect.orDie)
|
||||
|
||||
const messages = yield* store.messages({ sessionID, order: "asc" })
|
||||
expect(messages).toHaveLength(2)
|
||||
expect(messages[0]).toMatchObject({
|
||||
id: userID,
|
||||
type: "user",
|
||||
files: [{ data: base64, mime: "application/pdf", source: { type: "inline" }, name: "spec.pdf" }],
|
||||
})
|
||||
expect(messages[1]).toMatchObject({
|
||||
id: assistantID,
|
||||
type: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "tool",
|
||||
state: {
|
||||
status: "completed",
|
||||
content: [
|
||||
{ type: "file", uri: `data:application/pdf;base64,${base64}`, mime: "application/pdf", name: "spec.pdf" },
|
||||
{ type: "file", uri: pathUri, mime: "application/pdf", name: "path.pdf" },
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
expect((yield* store.message(userID))?.message).toEqual(messages[0]!)
|
||||
expect(yield* store.context(sessionID)).toEqual(messages)
|
||||
|
||||
const listedInbox = yield* inbox.list(sessionID)
|
||||
expect(listedInbox).toHaveLength(1)
|
||||
expect(listedInbox[0]).toMatchObject({
|
||||
id: inboxID,
|
||||
type: "user",
|
||||
payload: {
|
||||
text: "queued attachment",
|
||||
files: [{ data: base64, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" }],
|
||||
},
|
||||
})
|
||||
|
||||
expect(yield* SessionHistory.firstUserMessage(db, sessionID)).toEqual({ id: userID, text: "user attachment" })
|
||||
expect(yield* SessionInbox.nextPromotable(db, sessionID, "input")).toEqual({
|
||||
id: inboxID,
|
||||
type: "user",
|
||||
delivery: "queue",
|
||||
})
|
||||
|
||||
yield* bus.publish(SessionEvent.InboxDelivered, { sessionID, inboxID })
|
||||
expect((yield* store.message(inboxID))?.message).toMatchObject({
|
||||
id: inboxID,
|
||||
type: "user",
|
||||
files: [{ data: base64, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" }],
|
||||
})
|
||||
// Delivery keeps the ref in the stored row instead of writing the bytes back inline.
|
||||
const delivered = yield* db
|
||||
.select({ data: SessionMessageTable.data })
|
||||
.from(SessionMessageTable)
|
||||
.where(eq(SessionMessageTable.id, inboxID))
|
||||
.get()
|
||||
.pipe(Effect.orDie)
|
||||
expect(delivered?.data).toMatchObject({
|
||||
text: "queued attachment",
|
||||
files: [{ uri, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" }],
|
||||
})
|
||||
expect(JSON.stringify(delivered?.data)).not.toContain(base64)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("replaces refs to missing blobs with an unavailable placeholder on read", () =>
|
||||
Effect.gen(function* () {
|
||||
const db = yield* seedSession()
|
||||
const inbox = yield* SessionInbox.Service
|
||||
const store = yield* SessionStore.Service
|
||||
const hash = "b".repeat(64)
|
||||
const userID = SessionMessage.ID.make("msg_missing_user")
|
||||
const assistantID = SessionMessage.ID.make("msg_missing_assistant")
|
||||
const inboxID = SessionMessage.ID.make("msg_missing_inbox")
|
||||
const placeholder = Buffer.from("[media unavailable: shot.png (image/png)]").toString("base64")
|
||||
const ref = SessionStored.FileRef.make({
|
||||
uri: `media:${hash}`,
|
||||
mime: "image/png",
|
||||
source: { type: "inline" },
|
||||
name: "shot.png",
|
||||
})
|
||||
|
||||
yield* db
|
||||
.insert(SessionMessageTable)
|
||||
.values([
|
||||
{
|
||||
id: userID,
|
||||
session_id: sessionID,
|
||||
type: "user",
|
||||
seq: 1,
|
||||
time_created: 1,
|
||||
data: {
|
||||
text: "gone",
|
||||
files: [ref],
|
||||
time: { created: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
id: assistantID,
|
||||
session_id: sessionID,
|
||||
type: "assistant",
|
||||
seq: 2,
|
||||
time_created: 2,
|
||||
data: {
|
||||
agent: build,
|
||||
model,
|
||||
content: [
|
||||
{
|
||||
type: "tool",
|
||||
id: "call_1",
|
||||
name: "read",
|
||||
executed: true,
|
||||
state: {
|
||||
status: "completed",
|
||||
input: { path: "shot.png" },
|
||||
content: [SessionStored.ToolFileRef.make({ type: "file", uri: ref.uri, mime: "image/png" })],
|
||||
},
|
||||
time: { created: 2, ran: 2, completed: 2 },
|
||||
},
|
||||
],
|
||||
time: { created: 2, completed: 2 },
|
||||
},
|
||||
},
|
||||
])
|
||||
.run()
|
||||
.pipe(Effect.orDie)
|
||||
yield* db
|
||||
.insert(SessionInboxTable)
|
||||
.values({
|
||||
id: inboxID,
|
||||
session_id: sessionID,
|
||||
type: "user",
|
||||
payload: { text: "queued", files: [ref] },
|
||||
delivery: "queue",
|
||||
enqueued_seq: 3,
|
||||
time_created: 3,
|
||||
})
|
||||
.run()
|
||||
.pipe(Effect.orDie)
|
||||
|
||||
const messages = yield* store.messages({ sessionID, order: "asc" })
|
||||
expect(messages[0]).toMatchObject({
|
||||
files: [{ data: placeholder, mime: "text/plain", source: { type: "inline" }, name: "shot.png" }],
|
||||
})
|
||||
expect(messages[1]).toMatchObject({
|
||||
content: [{ state: { content: [{ type: "text", text: "[media unavailable: image/png]" }] } }],
|
||||
})
|
||||
expect(yield* store.context(sessionID)).toEqual(messages)
|
||||
const listed = yield* inbox.list(sessionID)
|
||||
expect(listed[0]).toMatchObject({ payload: { files: [{ data: placeholder, mime: "text/plain" }] } })
|
||||
expect(JSON.stringify([messages, listed])).not.toContain(hash)
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("names ref attachments in kept text without reading their blobs", () =>
|
||||
Effect.gen(function* () {
|
||||
const db = yield* seedSession()
|
||||
const store = yield* SessionStore.Service
|
||||
const userID = SessionMessage.ID.make("msg_kept_user")
|
||||
const compactionID = SessionMessage.ID.make("msg_kept_compaction")
|
||||
const { id: _, type: __, ...compaction } = encodeMessage(
|
||||
SessionMessage.CompactionCompleted.make({
|
||||
id: compactionID,
|
||||
type: "compaction",
|
||||
status: "completed",
|
||||
reason: "manual",
|
||||
summary: "summary",
|
||||
metadata: { keepID: userID },
|
||||
time: { created },
|
||||
}),
|
||||
)
|
||||
yield* db
|
||||
.insert(SessionMessageTable)
|
||||
.values([
|
||||
{
|
||||
id: userID,
|
||||
session_id: sessionID,
|
||||
type: "user",
|
||||
seq: 1,
|
||||
time_created: 1,
|
||||
data: {
|
||||
text: "look",
|
||||
files: [
|
||||
SessionStored.FileRef.make({
|
||||
uri: `media:${"c".repeat(64)}`,
|
||||
mime: "image/png",
|
||||
source: { type: "inline" },
|
||||
name: "shot.png",
|
||||
}),
|
||||
],
|
||||
time: { created: 1 },
|
||||
},
|
||||
},
|
||||
{ id: compactionID, session_id: sessionID, type: "compaction", seq: 2, time_created: 2, data: compaction },
|
||||
])
|
||||
.run()
|
||||
.pipe(Effect.orDie)
|
||||
|
||||
// The blob is missing, so a hydrated read would name a text/plain placeholder instead.
|
||||
expect(yield* store.context(sessionID)).toMatchObject([
|
||||
{ id: compactionID, metadata: { recent: "[User]: look\n[Attached image/png: shot.png]" } },
|
||||
])
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user