diff --git a/.opencode/opencode.jsonc b/.opencode/opencode.jsonc index 4506cf49fef..3a8acaa600b 100644 --- a/.opencode/opencode.jsonc +++ b/.opencode/opencode.jsonc @@ -16,4 +16,7 @@ "tools": { "github-pr-search": false, }, + "experimental": { + "part_rows": true, + }, } diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index 058ec6b110f..98cba302a73 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -2546,6 +2546,7 @@ export type ConfigEntry = todo?: boolean background_compaction?: boolean media_blob?: boolean + part_rows?: boolean image_generate?: boolean | { model?: string | { providerID: string; model: string; variant?: string } } schedules?: boolean artifacts?: boolean diff --git a/packages/core/src/config/normalize.ts b/packages/core/src/config/normalize.ts index 1ef08471294..86966dfec34 100644 --- a/packages/core/src/config/normalize.ts +++ b/packages/core/src/config/normalize.ts @@ -495,6 +495,15 @@ function normalizeExperimental( ) if (value !== undefined) result.media_blob = value } + if (own(experimental, "part_rows")) { + const value = decodeEncoded( + ConfigExperimental.Info.fields.part_rows, + experimental.part_rows, + ["experimental", "part_rows"], + diagnostics, + ) + if (value !== undefined) result.part_rows = value + } if (own(experimental, "image_generate")) { const value = decodeEncoded( ConfigExperimental.Info.fields.image_generate, diff --git a/packages/core/src/database/batch.ts b/packages/core/src/database/batch.ts new file mode 100644 index 00000000000..4c18888607f --- /dev/null +++ b/packages/core/src/database/batch.ts @@ -0,0 +1,21 @@ +export * as DatabaseBatch from "./batch.js" + +import { getTableColumns, type Table } from "drizzle-orm" + +/** Durable Object SQLite rejects a statement that binds more than 100 parameters, so every runtime stays under it. */ +export const MaxBoundParameters = 100 + +/** + * Splits `items` into batches whose statement binds at most {@link MaxBoundParameters}: `perItem` parameters for each + * item (a multi-row insert binds one per column) plus `fixed` for the rest of the statement. + */ +export const batches = (items: ReadonlyArray, perItem: number, fixed = 0): Array> => { + const size = Math.max(1, Math.floor((MaxBoundParameters - fixed) / perItem)) + return Array.from({ length: Math.ceil(items.length / size) }, (_, index) => + items.slice(index * size, (index + 1) * size), + ) +} + +/** {@link batches} for a multi-row insert into `table`, which binds one parameter per column of each row. */ +export const insertBatches = (table: Table, rows: ReadonlyArray, fixed = 0) => + batches(rows, Object.keys(getTableColumns(table)).length, fixed) diff --git a/packages/core/src/database/database.ts b/packages/core/src/database/database.ts index 93724052fc0..b18521b3084 100644 --- a/packages/core/src/database/database.ts +++ b/packages/core/src/database/database.ts @@ -35,8 +35,10 @@ const databaseLayer = (lock: Effect.Effect) => Service, Effect.gen(function* () { const db = yield* makeDatabase + // An injected Durable Object client may run under a non-workerd module condition in tests. + const durableObject = EffectDrizzleSqlite.managesTransactionsNatively(db.$client) - if (supportsTuningPragmas) { + if (supportsTuningPragmas && !durableObject) { yield* db.run("PRAGMA journal_mode = WAL") yield* db.run("PRAGMA synchronous = NORMAL") yield* db.run("PRAGMA busy_timeout = 5000") @@ -44,7 +46,7 @@ const databaseLayer = (lock: Effect.Effect) => yield* db.run("PRAGMA wal_checkpoint(PASSIVE)") } // Durable Object SQLite always enforces foreign keys and rejects the pragma. - if (supportsForeignKeyToggle) yield* db.run("PRAGMA foreign_keys = ON") + if (supportsForeignKeyToggle && !durableObject) yield* db.run("PRAGMA foreign_keys = ON") const semaphore = yield* lock yield* semaphore.withPermit(DatabaseMigration.apply(db)) diff --git a/packages/core/src/database/migration.ts b/packages/core/src/database/migration.ts index fb0fd72a523..8de838ff6d3 100644 --- a/packages/core/src/database/migration.ts +++ b/packages/core/src/database/migration.ts @@ -2,13 +2,14 @@ export * as DatabaseMigration from "./migration.js" import { sql } from "drizzle-orm" import { Effect } from "effect" +import type { SqlClient } from "effect/sql" import { supportsForeignKeyToggle } from "#sqlite" -import type { EffectDrizzleSqlite } from "./drizzle.js" +import { EffectDrizzleSqlite } from "./drizzle.js" import { migrations } from "./migration.gen.js" import schema from "./schema.gen.js" import { Global } from "@opencode/util/global" -type Database = EffectDrizzleSqlite.EffectSQLiteDatabase +type Database = EffectDrizzleSqlite.EffectSQLiteDatabase & { readonly $client?: SqlClient.SqlClient } type Transaction = Parameters[0]>[0] export type Migration = { @@ -122,10 +123,13 @@ export function applyOnly(db: Database, input: Migration[]) { : Effect.gen(function* () { // Durable Object SQLite rejects the foreign_keys toggle; the closest // allowlisted relaxation is deferring enforcement to transaction commit. - const relaxForeignKeys = supportsForeignKeyToggle + const toggleForeignKeys = + supportsForeignKeyToggle && + !(db.$client !== undefined && EffectDrizzleSqlite.managesTransactionsNatively(db.$client)) + const relaxForeignKeys = toggleForeignKeys ? db.run(sql`PRAGMA foreign_keys = OFF`) : db.run(sql`PRAGMA defer_foreign_keys = ON`) - const restoreForeignKeys = supportsForeignKeyToggle + const restoreForeignKeys = toggleForeignKeys ? Effect.asVoid(db.run(sql`PRAGMA foreign_keys = ON`)) : Effect.void yield* relaxForeignKeys diff --git a/packages/core/src/session/blob.ts b/packages/core/src/session/blob.ts index de7e0b6b51f..a264d41bc42 100644 --- a/packages/core/src/session/blob.ts +++ b/packages/core/src/session/blob.ts @@ -1,12 +1,13 @@ export * as SessionBlob from "./blob.js" import { createHash } from "node:crypto" -import { and, eq, exists, gte, inArray, notExists, notInArray, type SQL, sql } from "drizzle-orm" +import { and, eq, exists, gte, inArray, notExists, type SQL, sql } from "drizzle-orm" import { Cause, Context, Effect, Option, Schema } from "effect" import { map } from "effect/Array" import type { Attachment, FileAttachment } from "@opencode/schema/prompt" import type { Tool } from "@opencode/schema/tool" import { Config } from "../config.js" +import { DatabaseBatch } from "../database/batch.js" import type { Database } from "../database/database.js" import type { Instance } from "../instance/service.js" import type { SessionMessage } from "./message.js" @@ -29,12 +30,18 @@ const toolItems = (part: Part): ReadonlyArray => ? (part.state.content ?? []) : [] -function collect(data: MessageRow["data"] | InboxRow["payload"], hashes: Set) { +/** A stored row, one `session_part` row, or the `session_part` rows of one assistant message. */ +type Refs = MessageRow["data"] | InboxRow["payload"] | Part | { readonly content: ReadonlyArray } + +function collect(data: Refs, hashes: Set) { if ("files" in data) for (const file of data.files ?? []) if (isFileRef(file)) hashes.add(SessionStored.refHash(file.uri)) - if ("content" in data) + // A corrupt row's non-array content is left for decoding to reject. + if ("content" in data && Array.isArray(data.content)) for (const part of data.content) for (const item of toolItems(part)) if (isToolFileRef(item)) hashes.add(SessionStored.refHash(item.uri)) + if ("type" in data && data.type === "tool") + for (const item of toolItems(data)) if (isToolFileRef(item)) hashes.add(SessionStored.refHash(item.uri)) } /** A ref whose blob is gone reads as this text, so clients and the model never see a bare ref. */ @@ -97,13 +104,12 @@ function rewriteInbox(row: InboxRow, file: (file: SessionStored.File) => Session } const load = Effect.fnUntraced(function* (db: DatabaseService, hashes: ReadonlySet) { - const all = [...hashes] const blobs = new Map() - for (let offset = 0; offset < all.length; offset += 500) { + for (const batch of DatabaseBatch.batches([...hashes], 1)) { const found = yield* db .select({ hash: BlobTable.hash, data: BlobTable.data }) .from(BlobTable) - .where(inArray(BlobTable.hash, all.slice(offset, offset + 500))) + .where(inArray(BlobTable.hash, batch)) .all() .pipe(Effect.orDie) for (const blob of found) blobs.set(blob.hash, view(blob.data).toString("base64")) @@ -166,14 +172,13 @@ export const keepInboxRefs = Effect.fnUntraced(function* (db: DatabaseService, r const loadMeta = Effect.fnUntraced(function* (db: DatabaseService, hashes: ReadonlySet) { const meta = new Map() - const all = [...hashes] - for (let offset = 0; offset < all.length; offset += 500) { + for (const batch of DatabaseBatch.batches([...hashes], 1)) { const found = yield* db .select({ hash: BlobTable.hash, bytes: BlobTable.bytes, width: BlobTable.width, height: BlobTable.height }) .from(BlobTable) .where( and( - inArray(BlobTable.hash, all.slice(offset, offset + 500)), + inArray(BlobTable.hash, batch), exists( db .select({ one: sql`1` }) @@ -313,13 +318,8 @@ export const save = Effect.fnUntraced(function* ( db: DatabaseService, blobs: ReadonlyArray, ) { - if (blobs.length === 0) return - yield* db - .insert(BlobTable) - .values([...blobs]) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie) + for (const batch of DatabaseBatch.insertBatches(BlobTable, blobs)) + yield* db.insert(BlobTable).values(batch).onConflictDoNothing().run().pipe(Effect.orDie) }) /** {@link externalize} for a user message or pending inbox user payload. */ @@ -351,7 +351,17 @@ export const externalize = Effect.fnUntraced(function* ( return { ...message, content } }) -const hashesOf = (data: MessageRow["data"] | InboxRow["payload"]) => { +/** {@link externalize} for one assistant part. Returns the same part when nothing moved. */ +export const externalizePart = Effect.fnUntraced(function* (db: DatabaseService, part: AssistantPart) { + if (!(yield* enabled)) return part + const pending: Pending = new Map() + const stored = toolPartRefs(pending, part) + if (pending.size === 0) return part + yield* save(db, [...pending.values()]) + return stored +}) + +const hashesOf = (data: Refs) => { const hashes = new Set() collect(data, hashes) return hashes @@ -368,17 +378,17 @@ function storedPartRefs(pending: Pending, part: Part): Part { * Moves a stored row's inline media into blobs whatever the flag says, for offline backfills. Returns the row to * store (the same object when nothing moved), the blobs to insert, and every hash the row then names. */ -export function split( - data: T, -): { data: T; moved: boolean; blobs: Array; hashes: Set } -export function split(data: MessageRow["data"] | InboxRow["payload"]) { +export function split(data: T) { const pending: Pending = new Map() - const stored = + const stored = ( "files" in data && data.files?.some((file) => !isFileRef(file)) ? { ...data, files: data.files.map((file) => fileRef(pending, file)) } : "content" in data ? { ...data, content: data.content.map((part) => storedPartRefs(pending, part)) } - : data + : "type" in data && data.type === "tool" + ? storedPartRefs(pending, data) + : data + ) as T const result = pending.size > 0 ? stored : data return { data: result, moved: pending.size > 0, blobs: [...pending.values()], hashes: hashesOf(result) } } @@ -395,11 +405,7 @@ export const externalizeRow = Effect.fnUntraced(function* (db: DatabaseService, }) /** Records that `owner` references each blob its stored row names, so release keeps them until it lets go. */ -export const recordRefs = Effect.fnUntraced(function* ( - db: DatabaseService, - owner: Owner, - data: MessageRow["data"] | InboxRow["payload"], -) { +export const recordRefs = Effect.fnUntraced(function* (db: DatabaseService, owner: Owner, data: Refs) { yield* recordHashes(db, owner, hashesOf(data)) }) @@ -412,34 +418,50 @@ export const recordHashes = Effect.fnUntraced(function* ( owner: Owner, hashes: ReadonlySet, ) { - if (hashes.size === 0) return // A ref whose blob is already gone keeps nothing alive; reads render it as unavailable. - const found = yield* db - .select({ hash: BlobTable.hash }) - .from(BlobTable) - .where(inArray(BlobTable.hash, [...hashes])) - .all() - .pipe(Effect.orDie) - if (found.length === 0) return - yield* db - .insert(BlobRefTable) - .values( - found.map(({ hash }) => ({ session_id: owner.sessionID, message_id: owner.messageID, hash, seq: owner.seq })), + const found: Array<{ hash: string }> = [] + for (const batch of DatabaseBatch.batches([...hashes], 1)) + found.push( + ...(yield* db + .select({ hash: BlobTable.hash }) + .from(BlobTable) + .where(inArray(BlobTable.hash, batch)) + .all() + .pipe(Effect.orDie)), ) - .onConflictDoUpdate({ - target: [BlobRefTable.session_id, BlobRefTable.message_id, BlobRefTable.hash], - set: { seq: owner.seq }, - }) - .run() - .pipe(Effect.orDie) + const refs = found.map(({ hash }) => ({ + session_id: owner.sessionID, + message_id: owner.messageID, + hash, + seq: owner.seq, + })) + // The conflict update binds `seq` once more. + for (const batch of DatabaseBatch.insertBatches(BlobRefTable, refs, 1)) + yield* db + .insert(BlobRefTable) + .values(batch) + .onConflictDoUpdate({ + target: [BlobRefTable.session_id, BlobRefTable.message_id, BlobRefTable.hash], + set: { seq: owner.seq }, + }) + .run() + .pipe(Effect.orDie) }) /** Records the refs a rewritten message names and releases the ones it no longer does. */ -export const syncRefs = Effect.fnUntraced(function* (db: DatabaseService, owner: Owner, data: MessageRow["data"]) { +export const syncRefs = Effect.fnUntraced(function* (db: DatabaseService, owner: Owner, data: Refs) { yield* recordRefs(db, owner, data) - const present = [...hashesOf(data)] + const present = hashesOf(data) const own = [eq(BlobRefTable.session_id, owner.sessionID), eq(BlobRefTable.message_id, owner.messageID)] as const - yield* present.length === 0 ? release(db, ...own) : release(db, ...own, notInArray(BlobRefTable.hash, present)) + if (present.size === 0) return yield* release(db, ...own) + const stale = (yield* db + .select({ hash: BlobRefTable.hash }) + .from(BlobRefTable) + .where(and(...own)) + .all() + .pipe(Effect.orDie)).flatMap((ref) => (present.has(ref.hash) ? [] : [ref.hash])) + for (const batch of DatabaseBatch.batches(stale, 1, own.length)) + yield* release(db, ...own, inArray(BlobRefTable.hash, batch)) }) /** Deletes the matching refs, then every blob among theirs that no ref names anymore. */ @@ -450,13 +472,12 @@ const release = Effect.fnUntraced(function* (db: DatabaseService, ...where: read .returning({ hash: BlobRefTable.hash }) .all() .pipe(Effect.orDie) - const hashes = [...new Set(released.map((ref) => ref.hash))] - for (let offset = 0; offset < hashes.length; offset += 500) + for (const batch of DatabaseBatch.batches([...new Set(released.map((ref) => ref.hash))], 1)) yield* db .delete(BlobTable) .where( and( - inArray(BlobTable.hash, hashes.slice(offset, offset + 500)), + inArray(BlobTable.hash, batch), notExists( db .select({ one: sql`1` }) diff --git a/packages/core/src/session/history.ts b/packages/core/src/session/history.ts index 5a4220e0d34..b25e5365085 100644 --- a/packages/core/src/session/history.ts +++ b/packages/core/src/session/history.ts @@ -4,6 +4,7 @@ import { Effect, Option, Schema } from "effect" import type { Model } from "@opencode/schema/model" import { SessionLegacy } from "@opencode/schema/session-legacy" import { Database } from "../database/database.js" +import { DatabaseBatch } from "../database/batch.js" import { MessageDecodeError } from "./error.js" import { SessionMessage } from "./message.js" import type { SessionSchema } from "./schema.js" @@ -20,7 +21,8 @@ const decode = Schema.decodeUnknownEffect(SessionMessage.Info) const decodeSync = Schema.decodeUnknownSync(SessionMessage.Info) const decodeStoredUser = Schema.decodeUnknownEffect(SessionStored.User) const decodeMedia = Schema.decodeUnknownEffect(SessionMessage.MediaInfo) -const PartQueryBatchSize = 500 +const decodePartSync = Schema.decodeUnknownSync(SessionMessage.AssistantContent) +export const encodePart = Schema.encodeSync(SessionMessage.AssistantContent) /** * Which completed compactions bound a history read. Local summaries always do. Native @@ -87,7 +89,9 @@ const latestCompaction = Effect.fnUntraced(function* ( /** * Puts each assistant row's `session_part` rows into its `data.content`, in ordinal order. Rows without parts keep - * their inline content, so legacy rows read unchanged. Runs before blob hydration so media in parts restores too. + * their inline content, so legacy rows read unchanged. A row an older build wrote inline content to after its parts + * were split reads as the parts followed by that content, the order the projector moves them in. Runs before blob + * hydration so media in parts restores too. */ export const withParts = Effect.fnUntraced(function* ( db: DatabaseService, @@ -96,11 +100,11 @@ export const withParts = Effect.fnUntraced(function* ( const assistantIDs = rows.flatMap((row) => (row.type === "assistant" ? [row.id] : [])) if (assistantIDs.length === 0) return rows const partsByMessage = new Map>() - for (let index = 0; index < assistantIDs.length; index += PartQueryBatchSize) { + for (const ids of DatabaseBatch.batches(assistantIDs, 1)) { const partRows = yield* db .select() .from(SessionPartTable) - .where(inArray(SessionPartTable.message_id, assistantIDs.slice(index, index + PartQueryBatchSize))) + .where(inArray(SessionPartTable.message_id, ids)) .orderBy(asc(SessionPartTable.message_id), asc(SessionPartTable.ordinal)) .all() .pipe(Effect.orDie) @@ -113,7 +117,9 @@ export const withParts = Effect.fnUntraced(function* ( if (partsByMessage.size === 0) return rows return rows.map((row) => { const parts = partsByMessage.get(row.id) - return parts && "content" in row.data ? { ...row, data: { ...row.data, content: parts } } : row + if (!parts) return row + const inline = "content" in row.data ? row.data.content : undefined + return { ...row, data: { ...row.data, content: Array.isArray(inline) ? [...parts, ...inline] : parts } } }) }) @@ -124,6 +130,9 @@ const rowInput = (row: typeof SessionMessageTable.$inferSelect) => ({ type: row.type, }) +export const decodePartRowSync = (row: typeof SessionPartTable.$inferSelect) => + decodePartSync(SessionLegacy.part({ ...row.data, type: row.type })) + export const decodeMessageRowSync = (row: typeof SessionMessageTable.$inferSelect) => decodeSync(rowInput(row)) const decodeError = (row: typeof SessionMessageTable.$inferSelect) => () => @@ -151,6 +160,15 @@ export const decodeMessageRows = Effect.fnUntraced(function* ( }) }) +/** {@link decodeMessageRows} for one row. */ +export const loadMessageRow = Effect.fnUntraced(function* ( + db: DatabaseService, + row: typeof SessionMessageTable.$inferSelect, +) { + const [hydrated = row] = yield* SessionBlob.hydrate(db, yield* withParts(db, [row])) + return yield* decodeMessageRow(hydrated) +}) + /** Decodes rows for clients that fetch media themselves, so files stored in `blob` stay `media:` refs. */ export const decodeMessageRowsWithRefs = Effect.fnUntraced(function* ( db: DatabaseService, diff --git a/packages/core/src/session/message-updater.ts b/packages/core/src/session/message-updater.ts index 24d9ce35280..6297fe679e4 100644 --- a/packages/core/src/session/message-updater.ts +++ b/packages/core/src/session/message-updater.ts @@ -12,6 +12,23 @@ export interface Adapter { readonly getShell: (shellID: SessionMessage.Shell["shellID"]) => Effect.Effect readonly getCompaction: () => Effect.Effect readonly updateAssistant: (assistant: SessionMessage.Assistant) => Effect.Effect + readonly appendAssistantPart: ( + messageID: SessionMessage.ID, + part: SessionMessage.AssistantContent, + ) => Effect.Effect + readonly updateAssistantPart: ( + messageID: SessionMessage.ID, + match: { + readonly type: T + readonly callID?: string + readonly incompleteReasoning?: boolean + }, + recipe: (draft: WritableDraft>) => void, + ) => Effect.Effect + readonly replaceAssistantParts: ( + messageID: SessionMessage.ID, + parts: ReadonlyArray, + ) => Effect.Effect readonly updateShell: (shell: SessionMessage.Shell) => Effect.Effect readonly updateCompaction: (compaction: SessionMessage.Compaction) => Effect.Effect readonly appendMessage: (message: SessionMessage.Info) => Effect.Effect @@ -31,20 +48,8 @@ const projectTerminalSnapshot = (draft: DraftAssistant, event: SessionEvent.Step } export function update(adapter: Adapter, event: SessionEvent.DurableEvent) { - type DraftTool = WritableDraft - type DraftText = WritableDraft - type DraftReasoning = WritableDraft const created = DateTime.makeUnsafe(event.created) - const latestTool = (assistant: DraftAssistant, id: string) => - assistant.content.findLast((item): item is DraftTool => item.type === "tool" && item.id === id) - - const latestText = (assistant: DraftAssistant) => - assistant.content.findLast((item): item is DraftText => item.type === "text") - - const latestReasoning = (assistant: DraftAssistant) => - assistant.content.findLast((item): item is DraftReasoning => item.type === "reasoning" && !item.time?.completed) - const updateOwnedAssistant = (messageID: SessionMessage.ID, recipe: (draft: DraftAssistant) => void) => Effect.gen(function* () { const assistant = yield* adapter.getAssistant(messageID) @@ -86,9 +91,7 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) { "session.archived": () => Effect.void, "session.unarchived": () => Effect.void, "session.message.content.updated": (event) => - updateOwnedAssistant(event.data.messageID, (draft) => { - draft.content = castDraft(decodeAssistantContent(event.data.content)) - }), + adapter.replaceAssistantParts(event.data.messageID, decodeAssistantContent(event.data.content)), "session.usage.recorded": () => Effect.void, "session.agent.selected": (event) => Effect.gen(function* () { @@ -289,44 +292,43 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) { }) }, "session.text.started": (event) => { - return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { - draft.content.push(castDraft(SessionMessage.AssistantText.make({ type: "text", text: "" }))) - }) + return adapter.appendAssistantPart( + event.data.assistantMessageID, + SessionMessage.AssistantText.make({ type: "text", text: "" }), + ) }, "session.text.ended": (event) => { - return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { - const match = latestText(draft) - if (match) { - match.text = event.data.text - match.native = castDraft(event.data.native) - } + return adapter.updateAssistantPart(event.data.assistantMessageID, { type: "text" }, (match) => { + match.text = event.data.text + match.native = castDraft(event.data.native) }) }, "session.tool.input.started": (event) => { - return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { - draft.content.push( - castDraft( - SessionMessage.AssistantTool.make({ - type: "tool", - id: event.data.id, - name: event.data.name, - time: { created }, - state: SessionMessage.ToolStateStreaming.make({ status: "streaming", input: "" }), - }), - ), - ) - }) + return adapter.appendAssistantPart( + event.data.assistantMessageID, + SessionMessage.AssistantTool.make({ + type: "tool", + id: event.data.id, + name: event.data.name, + time: { created }, + state: SessionMessage.ToolStateStreaming.make({ status: "streaming", input: "" }), + }), + ) }, "session.tool.input.ended": (event) => { - return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { - const match = latestTool(draft, event.data.id) - if (match && match.state.status === "streaming") match.state.input = event.data.text - }) + return adapter.updateAssistantPart( + event.data.assistantMessageID, + { type: "tool", callID: event.data.id }, + (match) => { + if (match.state.status === "streaming") match.state.input = event.data.text + }, + ) }, "session.tool.called": (event) => { - return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { - const match = latestTool(draft, event.data.id) - if (match) { + return adapter.updateAssistantPart( + event.data.assistantMessageID, + { type: "tool", callID: event.data.id }, + (match) => { match.executed = event.data.executed match.native = event.data.native match.time.ran = created @@ -337,71 +339,75 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) { metadata: {}, }), ) - } - }) + }, + ) }, // Terminal tool events are self-contained; projection is a direct copy and // never reaches into ephemeral progress history. "session.tool.success": (event) => { - return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { - const match = latestTool(draft, event.data.id) - if (match && match.state.status === "running") { - match.executed = event.data.executed || match.executed === true - if (event.data.native) match.native = { ...match.native, ...event.data.native } - match.time.completed = created - match.state = castDraft( - SessionMessage.ToolStateCompleted.make({ - status: "completed", - input: match.state.input, - content: event.data.content, - ...(event.data.metadata === undefined ? {} : { metadata: event.data.metadata }), - }), - ) - } - }) + return adapter.updateAssistantPart( + event.data.assistantMessageID, + { type: "tool", callID: event.data.id }, + (match) => { + if (match.state.status === "running") { + match.executed = event.data.executed || match.executed === true + if (event.data.native) match.native = { ...match.native, ...event.data.native } + match.time.completed = created + match.state = castDraft( + SessionMessage.ToolStateCompleted.make({ + status: "completed", + input: match.state.input, + content: event.data.content, + ...(event.data.metadata === undefined ? {} : { metadata: event.data.metadata }), + }), + ) + } + }, + ) }, "session.tool.failed": (event) => { - return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { - const match = latestTool(draft, event.data.id) - if (match && (match.state.status === "streaming" || match.state.status === "running")) { - match.executed = event.data.executed || match.executed === true - if (event.data.native) match.native = { ...match.native, ...event.data.native } - match.time.completed = created - match.state = castDraft( - SessionMessage.ToolStateError.make({ - status: "error", - error: event.data.error, - input: typeof match.state.input === "string" ? {} : match.state.input, - ...(event.data.content === undefined ? {} : { content: event.data.content }), - ...(event.data.metadata === undefined ? {} : { metadata: event.data.metadata }), - }), - ) - } - }) + return adapter.updateAssistantPart( + event.data.assistantMessageID, + { type: "tool", callID: event.data.id }, + (match) => { + if (match.state.status === "streaming" || match.state.status === "running") { + match.executed = event.data.executed || match.executed === true + if (event.data.native) match.native = { ...match.native, ...event.data.native } + match.time.completed = created + match.state = castDraft( + SessionMessage.ToolStateError.make({ + status: "error", + error: event.data.error, + input: typeof match.state.input === "string" ? {} : match.state.input, + ...(event.data.content === undefined ? {} : { content: event.data.content }), + ...(event.data.metadata === undefined ? {} : { metadata: event.data.metadata }), + }), + ) + } + }, + ) }, "session.reasoning.started": (event) => { - return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { - draft.content.push( - castDraft( - SessionMessage.AssistantReasoning.make({ - type: "reasoning", - text: "", - native: event.data.native, - time: { created }, - }), - ), - ) - }) + return adapter.appendAssistantPart( + event.data.assistantMessageID, + SessionMessage.AssistantReasoning.make({ + type: "reasoning", + text: "", + native: event.data.native, + time: { created }, + }), + ) }, "session.reasoning.ended": (event) => { - return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { - const match = latestReasoning(draft) - if (match) { + return adapter.updateAssistantPart( + event.data.assistantMessageID, + { type: "reasoning", incompleteReasoning: true }, + (match) => { match.text = event.data.text match.time = { created: match.time?.created ?? created, completed: created } if (event.data.native !== undefined) match.native = event.data.native - } - }) + }, + ) }, "session.retry.scheduled": (event) => { return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index e8bdc4d0d6e..35f038b00a6 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -1,9 +1,12 @@ export * as SessionProjector from "./projector.js" +import { castDraft, produce, type WritableDraft } from "immer" import { and, asc, desc, eq, gt, gte, inArray, isNull, lt, lte, or, sql } from "drizzle-orm" -import { DateTime, Effect, Layer, Schema, Stream } from "effect" +import { DateTime, Effect, Layer, Option, Schema, Stream } from "effect" import path from "path" +import { Config } from "../config.js" import { Database } from "../database/database.js" +import { DatabaseBatch } from "../database/batch.js" import { Bus } from "../bus.js" import { makeGlobalNode } from "@opencode/util/effect/app-node" import { Model } from "@opencode/schema/model" @@ -46,6 +49,13 @@ type Usage = { const ForkBatchSize = 500 +const insertParts = (db: DatabaseService, rows: ReadonlyArray) => + Effect.forEach( + DatabaseBatch.insertBatches(SessionPartTable, rows), + (batch) => db.insert(SessionPartTable).values(batch).run().pipe(Effect.orDie), + { discard: true }, + ) + const forkTitle = (value?: string) => { if (value === undefined) return const match = value.match(/^(.+) \(fork #(\d+)\)$/) @@ -190,7 +200,6 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* ( const media = yield* SessionBlob.ForkMedia let cursor = -1 while (copiedSeq !== undefined) { - // A parent message whose parts live in session_part is copied with them inline, so the fork reads the same. const rows = yield* db .select() .from(SessionMessageTable) @@ -208,7 +217,7 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* ( .orderBy(asc(SessionMessageTable.seq)) .limit(ForkBatchSize) .all() - .pipe(Effect.orDie, Effect.flatMap((rows) => SessionHistory.withParts(db, rows))) + .pipe(Effect.orDie) if (rows.length === 0) break const values = yield* Effect.forEach(rows, (row) => @@ -220,6 +229,7 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* ( seq: row.seq, time_created: row.time_created, time_updated: row.time_updated, + // Split parents' envelopes carry no content, so only unsplit parents' inline media moves here. data: media ? yield* SessionBlob.externalizeRow(db, row.data) : row.data, } // A summary references the first message it kept, which the fork copied under a new ID. @@ -238,7 +248,11 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* ( } }), ) - yield* db.insert(SessionMessageTable).values(values).run().pipe(Effect.orDie) + yield* Effect.forEach( + DatabaseBatch.insertBatches(SessionMessageTable, values), + (batch) => db.insert(SessionMessageTable).values(batch).run().pipe(Effect.orDie), + { discard: true }, + ) yield* Effect.forEach( values, (value) => @@ -250,6 +264,48 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* ( { discard: true }, ) + const assistantRows = rows.filter((row) => row.type === "assistant") + if (assistantRows.length > 0) { + const idMap = new Map(assistantRows.map((row) => [row.id, forkedID(row.seq)])) + const storedParts = yield* Effect.forEach(DatabaseBatch.batches([...idMap.keys()], 1, 1), (ids) => + db + .select() + .from(SessionPartTable) + .where(and(eq(SessionPartTable.session_id, event.data.parentID), inArray(SessionPartTable.message_id, ids))) + .orderBy(asc(SessionPartTable.message_id), asc(SessionPartTable.ordinal)) + .all() + .pipe(Effect.orDie), + ).pipe(Effect.map((batches) => batches.flat())) + const partValues: Array = yield* Effect.forEach(storedParts, (part) => + Effect.gen(function* () { + const data = media ? SessionBlob.split(part.data) : undefined + if (data?.moved) yield* SessionBlob.save(db, data.blobs) + return { + message_id: idMap.get(part.message_id)!, + session_id: event.data.sessionID, + ordinal: part.ordinal, + type: part.type, + call_id: part.call_id, + time_created: part.time_created, + time_updated: part.time_updated, + data: data?.data ?? part.data, + } + }), + ) + yield* insertParts(db, partValues) + // Split parents' envelopes carry no content, so the parts' refs are recorded for each forked message here. + const partsByMessage = Map.groupBy(partValues, (part) => part.message_id) + yield* Effect.forEach( + assistantRows, + (row) => { + const messageID = idMap.get(row.id)! + const content = (partsByMessage.get(messageID) ?? []).map((part) => part.data) + return SessionBlob.recordRefs(db, { sessionID: event.data.sessionID, messageID, seq: row.seq }, { content }) + }, + { discard: true }, + ) + } + cursor = rows.at(-1)!.seq } if (copiedSeq !== undefined) yield* Bus.reserveSequence(db, event.data.sessionID, copiedSeq) @@ -257,86 +313,32 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* ( yield* InstructionState.initialize(db, event.data.sessionID, event.durable.seq, event.data.instructions) }) +/** Reads `experimental.part_rows` from the Config in scope on every call; without one, it is off. */ +const partRowsEnabled = Effect.gen(function* () { + const config = yield* Effect.serviceOption(Config.Service) + if (Option.isNone(config)) return false + return Config.latest(yield* config.value.entries(), "experimental")?.part_rows === true +}) + +type MessageRow = typeof SessionMessageTable.$inferSelect +type PartRow = typeof SessionPartTable.$inferSelect +type Writes = Pick< + SessionMessageUpdater.Adapter, + | "getCurrentAssistant" + | "getAssistant" + | "updateAssistant" + | "appendAssistantPart" + | "updateAssistantPart" + | "replaceAssistantParts" + | "updateShell" + | "updateCompaction" +> + function run(db: DatabaseService, event: MessageEvent) { return Effect.gen(function* () { - const partRows = (messageID: SessionMessage.ID) => - db - .select() - .from(SessionPartTable) - .where(eq(SessionPartTable.message_id, messageID)) - .orderBy(asc(SessionPartTable.ordinal)) - .all() - .pipe(Effect.orDie) - // Writes `content` over a message's existing part rows, touching only the rows that changed. - const writeParts = Effect.fnUntraced(function* ( - messageID: SessionMessage.ID, - existing: ReadonlyArray, - content: ReadonlyArray, - ) { - let next = (existing.at(-1)?.ordinal ?? -1) + 1 - for (const [index, part] of content.entries()) { - const current = existing[index] - const columns = { type: part.type, call_id: part.type === "tool" ? part.id : null, data: part } - if (current === undefined) - yield* db - .insert(SessionPartTable) - .values({ - ...columns, - message_id: messageID, - session_id: event.data.sessionID, - ordinal: next++, - time_created: event.created, - time_updated: event.created, - }) - .run() - .pipe(Effect.orDie) - else if (JSON.stringify(current.data) !== JSON.stringify(part)) - yield* db - .update(SessionPartTable) - .set({ ...columns, time_updated: event.created }) - .where(and(eq(SessionPartTable.message_id, messageID), eq(SessionPartTable.ordinal, current.ordinal))) - .run() - .pipe(Effect.orDie) - } - const stale = existing.slice(content.length).map((part) => part.ordinal) - if (stale.length > 0) - yield* db - .delete(SessionPartTable) - .where(and(eq(SessionPartTable.message_id, messageID), inArray(SessionPartTable.ordinal, stale))) - .run() - .pipe(Effect.orDie) - }) - const decodeAssistantRow = Effect.fnUntraced(function* (row: typeof SessionMessageTable.$inferSelect) { - const [withParts = row] = yield* SessionHistory.withParts(db, [row]) - return SessionHistory.decodeMessageRowSync(withParts) - }) - const updateMessage = Effect.fnUntraced(function* (message: SessionMessage.Info) { - const stored = yield* SessionBlob.externalize(db, message) - const { id: _, type, ...data } = encodeStored(stored) - // A newer release may have moved this message's parts to session_part. Reads prefer those rows, so updates - // go there too and the row keeps an empty envelope; messages without part rows are written inline as before. - const parts = message.type === "assistant" && "content" in data ? yield* partRows(message.id) : [] - const updated = yield* db - .update(SessionMessageTable) - .set({ - type, - time_created: DateTime.toEpochMillis(message.time.created), - data: parts.length > 0 && "content" in data ? { ...data, content: [] } : data, - }) - .where(and(eq(SessionMessageTable.id, message.id), eq(SessionMessageTable.session_id, event.data.sessionID))) - .returning({ seq: SessionMessageTable.seq }) - .get() - .pipe(Effect.orDie) - if (!updated) return - if (parts.length > 0 && "content" in data) yield* writeParts(message.id, parts, data.content) - const owner = { sessionID: event.data.sessionID, messageID: message.id, seq: updated.seq } - // Most updates only carry existing refs forward; a content rewrite may also drop some. `externalize` returns - // the same message when it extracted nothing, so streaming updates skip the ref tables entirely. - if (event.type === SessionEvent.MessageContentUpdated.type) yield* SessionBlob.syncRefs(db, owner, data) - else if (stored !== message) yield* SessionBlob.recordRefs(db, owner, data) - }) - const appendMessage = (message: SessionMessage.Info) => insertMessage(db, event, message) + const writes = (yield* partRowsEnabled) ? partRowWrites(db, event) : messageWrites(db, event) const adapter: SessionMessageUpdater.Adapter = { + ...writes, getAgent() { return db .select({ agent: SessionTable.agent }) @@ -386,43 +388,6 @@ function run(db: DatabaseService, event: MessageEvent) { ), ) }, - getCurrentAssistant() { - return Effect.gen(function* () { - // A newer step supersedes stale incomplete rows; never resume an older assistant projection. - const row = yield* db - .select() - .from(SessionMessageTable) - .where( - and(eq(SessionMessageTable.session_id, event.data.sessionID), eq(SessionMessageTable.type, "assistant")), - ) - .orderBy(desc(SessionMessageTable.seq)) - .limit(1) - .get() - .pipe(Effect.orDie) - if (!row || ("completed" in row.data.time && row.data.time.completed)) return - const message = yield* decodeAssistantRow(row) - return message.type === "assistant" && !message.time.completed ? message : undefined - }) - }, - getAssistant(messageID) { - return Effect.gen(function* () { - const row = yield* db - .select() - .from(SessionMessageTable) - .where( - and( - eq(SessionMessageTable.id, messageID), - eq(SessionMessageTable.session_id, event.data.sessionID), - eq(SessionMessageTable.type, "assistant"), - ), - ) - .get() - .pipe(Effect.orDie) - if (!row) return - const message = yield* decodeAssistantRow(row) - return message.type === "assistant" ? message : undefined - }) - }, getShell(shellID) { return Effect.gen(function* () { const row = yield* db @@ -465,15 +430,368 @@ function run(db: DatabaseService, event: MessageEvent) { return message.type === "compaction" ? message : undefined }) }, - updateAssistant: updateMessage, - updateShell: updateMessage, - updateCompaction: updateMessage, - appendMessage, + appendMessage: (message) => insertMessage(db, event, message), } yield* SessionMessageUpdater.update(adapter, event) }) } +/** Assistant reads shared by both write paths; `decode` decides whether part rows are folded into `content`. */ +function assistantReads( + db: DatabaseService, + event: MessageEvent, + decode: (row: MessageRow) => Effect.Effect, +): Pick { + return { + getCurrentAssistant() { + return Effect.gen(function* () { + // A newer step supersedes stale incomplete rows; never resume an older assistant projection. + const row = yield* db + .select() + .from(SessionMessageTable) + .where( + and(eq(SessionMessageTable.session_id, event.data.sessionID), eq(SessionMessageTable.type, "assistant")), + ) + .orderBy(desc(SessionMessageTable.seq)) + .limit(1) + .get() + .pipe(Effect.orDie) + if (!row || ("completed" in row.data.time && row.data.time.completed)) return + const message = yield* decode(row) + return message.type === "assistant" && !message.time.completed ? message : undefined + }) + }, + getAssistant(messageID) { + return Effect.gen(function* () { + const row = yield* db + .select() + .from(SessionMessageTable) + .where( + and( + eq(SessionMessageTable.id, messageID), + eq(SessionMessageTable.session_id, event.data.sessionID), + eq(SessionMessageTable.type, "assistant"), + ), + ) + .get() + .pipe(Effect.orDie) + if (!row) return + const message = yield* decode(row) + return message.type === "assistant" ? message : undefined + }) + }, + } +} + +/** + * The write path with `experimental.part_rows` off, unchanged from before part rows were written: every assistant + * event rewrites the whole message. A message whose parts already live in session_part (written with the flag on, + * or by a newer release) is written into those rows, and its row keeps an empty envelope. + */ +function messageWrites(db: DatabaseService, event: MessageEvent): Writes { + const partRows = (messageID: SessionMessage.ID) => + db + .select() + .from(SessionPartTable) + .where(eq(SessionPartTable.message_id, messageID)) + .orderBy(asc(SessionPartTable.ordinal)) + .all() + .pipe(Effect.orDie) + // Writes `content` over a message's existing part rows, touching only the rows that changed. + const writeParts = Effect.fnUntraced(function* ( + messageID: SessionMessage.ID, + existing: ReadonlyArray, + content: ReadonlyArray, + ) { + let next = (existing.at(-1)?.ordinal ?? -1) + 1 + for (const [index, part] of content.entries()) { + const current = existing[index] + const columns = { type: part.type, call_id: part.type === "tool" ? part.id : null, data: part } + if (current === undefined) + yield* db + .insert(SessionPartTable) + .values({ + ...columns, + message_id: messageID, + session_id: event.data.sessionID, + ordinal: next++, + time_created: event.created, + time_updated: event.created, + }) + .run() + .pipe(Effect.orDie) + else if (JSON.stringify(current.data) !== JSON.stringify(part)) + yield* db + .update(SessionPartTable) + .set({ ...columns, time_updated: event.created }) + .where(and(eq(SessionPartTable.message_id, messageID), eq(SessionPartTable.ordinal, current.ordinal))) + .run() + .pipe(Effect.orDie) + } + const stale = existing.slice(content.length).map((part) => part.ordinal) + if (stale.length > 0) + yield* db + .delete(SessionPartTable) + .where(and(eq(SessionPartTable.message_id, messageID), inArray(SessionPartTable.ordinal, stale))) + .run() + .pipe(Effect.orDie) + }) + const updateMessage = Effect.fnUntraced(function* (message: SessionMessage.Info) { + const stored = yield* SessionBlob.externalize(db, message) + const { id: _, type, ...data } = encodeStored(stored) + const parts = message.type === "assistant" && "content" in data ? yield* partRows(message.id) : [] + const updated = yield* db + .update(SessionMessageTable) + .set({ + type, + time_created: DateTime.toEpochMillis(message.time.created), + data: parts.length > 0 && "content" in data ? { ...data, content: [] } : data, + }) + .where(and(eq(SessionMessageTable.id, message.id), eq(SessionMessageTable.session_id, event.data.sessionID))) + .returning({ seq: SessionMessageTable.seq }) + .get() + .pipe(Effect.orDie) + if (!updated) return + if (parts.length > 0 && "content" in data) yield* writeParts(message.id, parts, data.content) + const owner = { sessionID: event.data.sessionID, messageID: message.id, seq: updated.seq } + // Most updates only carry existing refs forward; a content rewrite may also drop some. `externalize` returns + // the same message when it extracted nothing, so streaming updates skip the ref tables entirely. + if (event.type === SessionEvent.MessageContentUpdated.type) yield* SessionBlob.syncRefs(db, owner, data) + else if (stored !== message) yield* SessionBlob.recordRefs(db, owner, data) + }) + const reads = assistantReads(db, event, (row) => + SessionHistory.withParts(db, [row]).pipe( + Effect.map(([withParts = row]) => SessionHistory.decodeMessageRowSync(withParts)), + ), + ) + const editAssistant = ( + messageID: SessionMessage.ID, + recipe: (draft: WritableDraft) => void, + ) => + Effect.gen(function* () { + const assistant = yield* reads.getAssistant(messageID) + if (!assistant) return + yield* updateMessage(produce(assistant, recipe)) + }) + return { + ...reads, + updateAssistant: updateMessage, + updateShell: updateMessage, + updateCompaction: updateMessage, + appendAssistantPart: (messageID, part) => + editAssistant(messageID, (draft) => { + draft.content.push(castDraft(part)) + }), + updateAssistantPart: (messageID, match, recipe) => + editAssistant(messageID, (draft) => { + const found = draft.content.findLast( + ( + item, + ): item is WritableDraft> => + item.type === match.type && + (match.callID === undefined || (item.type === "tool" && item.id === match.callID)) && + (!match.incompleteReasoning || (item.type === "reasoning" && !item.time?.completed)), + ) + if (found) recipe(found) + }), + replaceAssistantParts: (messageID, parts) => + editAssistant(messageID, (draft) => { + draft.content = castDraft([...parts]) + }), + } +} + +/** The write path with `experimental.part_rows` on: each assistant part is its own session_part row. */ +function partRowWrites(db: DatabaseService, event: MessageEvent): Writes { + /** + * The only place a legacy inline assistant row moves to session_part, called right before its parts are written. + * `carry: false` drops the inline parts instead of copying them, for a write that replaces them all. Returns the + * owner the message's part refs belong to. + */ + const prepareAssistantParts = Effect.fnUntraced(function* (messageID: SessionMessage.ID, carry: boolean) { + const row = yield* db + .select({ + id: SessionMessageTable.id, + seq: SessionMessageTable.seq, + inlineCount: sql`json_array_length(${SessionMessageTable.data}, '$.content')`, + }) + .from(SessionMessageTable) + .where( + and( + eq(SessionMessageTable.id, messageID), + eq(SessionMessageTable.session_id, event.data.sessionID), + eq(SessionMessageTable.type, "assistant"), + ), + ) + .get() + .pipe(Effect.orDie) + if (!row) return + const full = + row.inlineCount > 0 + ? yield* db + .select() + .from(SessionMessageTable) + .where(eq(SessionMessageTable.id, messageID)) + .get() + .pipe(Effect.orDie) + : undefined + const message = full ? SessionHistory.decodeMessageRowSync(full) : undefined + if (full && message?.type === "assistant") { + if (carry) { + // An older build that wrote inline content while part rows existed leaves both shapes on one message. + // Keep the rows and append the inline parts after them, so a downgrade and re-upgrade loses nothing. + const last = yield* db + .select({ ordinal: sql`max(${SessionPartTable.ordinal})` }) + .from(SessionPartTable) + .where(eq(SessionPartTable.message_id, messageID)) + .get() + .pipe(Effect.orDie) + const first = (last?.ordinal ?? -1) + 1 + const partRows = message.content.map((part, index) => ({ + message_id: messageID, + session_id: event.data.sessionID, + ordinal: first + index, + type: part.type, + call_id: part.type === "tool" ? part.id : null, + time_created: full.time_created, + time_updated: event.created, + data: SessionHistory.encodePart(part), + })) + yield* insertParts(db, partRows) + } + const { id: _, type: __, ...data } = encodeMessage({ ...message, content: [] }) + yield* db + .update(SessionMessageTable) + .set({ data }) + .where(and(eq(SessionMessageTable.id, messageID), eq(SessionMessageTable.session_id, event.data.sessionID))) + .run() + .pipe(Effect.orDie) + } + return { sessionID: event.data.sessionID, messageID, seq: row.seq } satisfies SessionBlob.Owner + }) + // Assistant content is written only by the part methods below, which also record its blob refs. Envelope updates + // keep whatever content the row already holds: inline for a legacy row that was never split, [] once it was. + const updateMessage = (message: SessionMessage.Info) => { + const { + id: _, + type: __, + ...data + } = encodeMessage(message.type === "assistant" ? { ...message, content: [] } : message) + return db + .update(SessionMessageTable) + .set({ + ...(event.type === SessionEvent.Step.Started.type + ? { time_created: DateTime.toEpochMillis(message.time.created) } + : {}), + data: + message.type === "assistant" + ? sql`json_set(${JSON.stringify(data)}, '$.content', json(coalesce(${SessionMessageTable.data} -> '$.content', '[]')))` + : data, + }) + .where(and(eq(SessionMessageTable.id, message.id), eq(SessionMessageTable.session_id, event.data.sessionID))) + .run() + .pipe(Effect.orDie) + } + const reads = assistantReads(db, event, (row) => Effect.sync(() => SessionHistory.decodeMessageRowSync(row))) + return { + ...reads, + appendAssistantPart(messageID, part) { + return Effect.gen(function* () { + const owner = yield* prepareAssistantParts(messageID, true) + if (!owner) return + const data = SessionHistory.encodePart(yield* SessionBlob.externalizePart(db, part)) + const next = yield* db + .select({ ordinal: sql`coalesce(max(${SessionPartTable.ordinal}) + 1, 0)` }) + .from(SessionPartTable) + .where(eq(SessionPartTable.message_id, messageID)) + .get() + .pipe(Effect.orDie) + yield* db + .insert(SessionPartTable) + .values({ + message_id: messageID, + session_id: event.data.sessionID, + ordinal: next?.ordinal ?? 0, + type: part.type, + call_id: part.type === "tool" ? part.id : null, + time_created: event.created, + time_updated: event.created, + data, + }) + .run() + .pipe(Effect.orDie) + yield* SessionBlob.recordRefs(db, owner, { content: [data] }) + }) + }, + updateAssistantPart(messageID, match, recipe) { + return Effect.gen(function* () { + const owner = yield* prepareAssistantParts(messageID, true) + if (!owner) return + const row = yield* db + .select() + .from(SessionPartTable) + .where( + and( + eq(SessionPartTable.message_id, messageID), + eq(SessionPartTable.session_id, event.data.sessionID), + eq(SessionPartTable.type, match.type), + match.callID !== undefined ? eq(SessionPartTable.call_id, match.callID) : undefined, + match.incompleteReasoning + ? sql`json_extract(${SessionPartTable.data}, '$.time.completed') is null` + : undefined, + ), + ) + .orderBy(desc(SessionPartTable.ordinal)) + .limit(1) + .get() + .pipe(Effect.orDie) + if (!row) return + + const current = SessionHistory.decodePartRowSync(row) as Extract< + SessionMessage.AssistantContent, + { readonly type: typeof match.type } + > + const updated = produce(current, recipe) + if (updated === current) return + // `externalizePart` returns the same part when it moved nothing, so streaming updates skip the ref tables. + const stored = yield* SessionBlob.externalizePart(db, updated) + const data = SessionHistory.encodePart(stored) + yield* db + .update(SessionPartTable) + .set({ data, time_updated: event.created }) + .where(and(eq(SessionPartTable.message_id, messageID), eq(SessionPartTable.ordinal, row.ordinal))) + .run() + .pipe(Effect.orDie) + if (stored !== updated) yield* SessionBlob.recordRefs(db, owner, { content: [data] }) + }) + }, + replaceAssistantParts(messageID, parts) { + return Effect.gen(function* () { + const owner = yield* prepareAssistantParts(messageID, false) + if (!owner) return + yield* db.delete(SessionPartTable).where(eq(SessionPartTable.message_id, messageID)).run().pipe(Effect.orDie) + const stored = yield* Effect.forEach(parts, (part) => SessionBlob.externalizePart(db, part)) + const partRows = stored.map((part, ordinal) => ({ + message_id: messageID, + session_id: event.data.sessionID, + ordinal, + type: part.type, + call_id: part.type === "tool" ? part.id : null, + time_created: event.created, + time_updated: event.created, + data: SessionHistory.encodePart(part), + })) + yield* insertParts(db, partRows) + // A content rewrite may drop media, so refs the new parts no longer name are released. + yield* SessionBlob.syncRefs(db, owner, { content: partRows.map((part) => part.data) }) + }) + }, + updateAssistant: updateMessage, + updateShell: updateMessage, + updateCompaction: updateMessage, + } +} + const encodeStored = (message: SessionMessage.Info | SessionStored.User) => message.type === "user" ? encodeStoredUser(message) : encodeMessage(message) @@ -482,7 +800,11 @@ const insertMessage = Effect.fnUntraced(function* ( event: SessionEvent.DurableEvent, message: SessionMessage.Info | SessionStored.User, ) { - const { id: _, type, ...data } = encodeStored(yield* SessionBlob.externalize(db, message)) + const stored = yield* SessionBlob.externalize(db, message) + const { id: _, type, ...data } = encodeStored(stored) + const timeCreated = DateTime.toEpochMillis(message.time.created) + // Assistant content goes to session_part rows; the message row keeps an empty envelope. + const parts = stored.type === "assistant" ? stored.content : [] yield* db .insert(SessionMessageTable) .values({ @@ -490,11 +812,22 @@ const insertMessage = Effect.fnUntraced(function* ( session_id: event.data.sessionID, type, seq: event.durable.seq, - time_created: DateTime.toEpochMillis(message.time.created), - data, + time_created: timeCreated, + data: "content" in data && parts.length > 0 ? { ...data, content: [] } : data, }) .run() .pipe(Effect.orDie) + const partRows = parts.map((part, ordinal) => ({ + message_id: message.id, + session_id: event.data.sessionID, + ordinal, + type: part.type, + call_id: part.type === "tool" ? part.id : null, + time_created: timeCreated, + time_updated: event.created, + data: SessionHistory.encodePart(part), + })) + yield* insertParts(db, partRows) yield* SessionBlob.recordRefs( db, { sessionID: event.data.sessionID, messageID: message.id, seq: event.durable.seq }, diff --git a/packages/core/src/session/stats.ts b/packages/core/src/session/stats.ts index 741367377f4..e0d9eef84a2 100644 --- a/packages/core/src/session/stats.ts +++ b/packages/core/src/session/stats.ts @@ -11,6 +11,7 @@ import { SessionEvent } from "@opencode/schema/session-event" import { ToolMode } from "@opencode/schema/session-stats" import { TokenUsage } from "@opencode/schema/token-usage" import { Database } from "../database/database.js" +import { DatabaseBatch } from "../database/batch.js" import { EventTable } from "../event/sql.js" import { SessionMessageTable, SessionPartTable, SessionTable } from "./sql.js" @@ -212,7 +213,7 @@ export const get = Effect.fn("SessionStats.get")(function* (input: Input = {}) { AND message.time_created >= ${range.from} AND message.time_created < ${range.to} AND (session.fork_session_id IS NULL OR message.time_created >= session.time_created) - AND NOT EXISTS (SELECT 1 FROM ${SessionPartTable} AS part WHERE part.message_id = message.id) + AND json_array_length(message.data, '$.content') > 0 AND json_extract(content.value, '$.type') = 'tool' ${project} ) @@ -273,7 +274,7 @@ export const get = Effect.fn("SessionStats.get")(function* (input: Input = {}) { AND message.time_created >= ${range.from} AND message.time_created < ${range.to} AND (session.fork_session_id IS NULL OR message.time_created >= session.time_created) - AND NOT EXISTS (SELECT 1 FROM ${SessionPartTable} AS part WHERE part.message_id = message.id) + AND json_array_length(message.data, '$.content') > 0 AND json_extract(content.value, '$.type') = 'tool' ${project} `, @@ -312,7 +313,11 @@ export const get = Effect.fn("SessionStats.get")(function* (input: Input = {}) { .all() .pipe(Effect.orDie) const events = (yield* Effect.forEach( - batches(ids.map((row) => row.id)), + DatabaseBatch.batches( + ids.map((row) => row.id), + 1, + 3, + ), (batch) => db .select({ data: EventTable.data }) @@ -384,10 +389,6 @@ function windows(from: number, to: number) { })) } -function batches(ids: string[]) { - return Array.from({ length: Math.ceil(ids.length / 500) }, (_, index) => ids.slice(index * 500, (index + 1) * 500)) -} - function rowTokens(row: MessageRow): Tokens { return { input: row.input ?? 0, diff --git a/packages/core/src/session/transfer.ts b/packages/core/src/session/transfer.ts index f123a20497e..5d6c8f05126 100644 --- a/packages/core/src/session/transfer.ts +++ b/packages/core/src/session/transfer.ts @@ -11,6 +11,7 @@ import { makeGlobalNode } from "@opencode/util/effect/app-node" import { App } from "../app.js" import { Bus } from "../bus.js" import { Database } from "../database/database.js" +import { DatabaseBatch } from "../database/batch.js" import { Instance } from "../instance/service.js" import { Location } from "../location.js" import { Project } from "../project.js" @@ -129,7 +130,8 @@ const layer = Layer.effect( const stored = media ? messages.map((message, i) => ({ ...message, data: split[i]!.data })) : messages - yield* db.insert(SessionMessageTable).values(stored).run().pipe(Effect.orDie) + for (const batch of DatabaseBatch.insertBatches(SessionMessageTable, stored)) + yield* db.insert(SessionMessageTable).values(batch).run().pipe(Effect.orDie) if (media) yield* Effect.forEach( stored, diff --git a/packages/core/test/config/normalization.test.ts b/packages/core/test/config/normalization.test.ts index 239ab67165c..884bb5feb4b 100644 --- a/packages/core/test/config/normalization.test.ts +++ b/packages/core/test/config/normalization.test.ts @@ -553,6 +553,7 @@ describe("ConfigNormalize", () => { todo: true, background_compaction: true, media_blob: true, + part_rows: true, image_generate: { model: "openai/gpt-image-2" }, subagent_depth: 0, location_idle_timeout: "2 hours", @@ -565,6 +566,7 @@ describe("ConfigNormalize", () => { todo: true, background_compaction: true, media_blob: true, + part_rows: true, image_generate: { model: { providerID: "openai", model: "gpt-image-2" } }, subagent_depth: 0, location_idle_timeout: "7200000 millis", diff --git a/packages/core/test/fixture/durable-object-storage.ts b/packages/core/test/fixture/durable-object-storage.ts index a961d71c4cc..a0b491a1bd1 100644 --- a/packages/core/test/fixture/durable-object-storage.ts +++ b/packages/core/test/fixture/durable-object-storage.ts @@ -6,6 +6,8 @@ import type { DurableObjectStorage } from "@opencode/core/database/sqlite.worker // Cloudflare runtime dependencies. export const makeDurableObjectStorage = (): DurableObjectStorage => { const native = new Database(":memory:") + // Durable Object SQLite compiles with SQLITE_DEFAULT_FOREIGN_KEYS=1 and rejects toggling it via PRAGMA. + native.run("PRAGMA foreign_keys = ON") const toSqlStorageValue = (value: unknown) => { if (!(value instanceof Uint8Array)) return value as ArrayBuffer | string | number | null const buffer = new ArrayBuffer(value.byteLength) @@ -15,6 +17,9 @@ export const makeDurableObjectStorage = (): DurableObjectStorage => { return { sql: { exec(query: string, ...bindings: Array) { + // Durable Object SQLite rejects a statement with more than 100 bound parameters. + if (bindings.length > 100) throw new Error(`too many SQL variables: ${bindings.length}`) + if (/^\s*pragma\s+(?:\w+\.)?foreign_keys\b/i.test(query)) throw new Error("not authorized") const statement = native.query(query) const rows = (statement.values(...(bindings as never[])) ?? []).map((row) => row.map(toSqlStorageValue)) const columnNames = statement.columnNames diff --git a/packages/core/test/session-create.test.ts b/packages/core/test/session-create.test.ts index 0caa7731b1d..ad0508e9eab 100644 --- a/packages/core/test/session-create.test.ts +++ b/packages/core/test/session-create.test.ts @@ -42,6 +42,8 @@ import { offlineModels } from "./fixture/models" import { promptLocationNode } from "./fixture/prompt-location" import { globalProjectNode } from "./lib/project" import { tmpdirScoped } from "./fixture/tmpdir" +import { makeDurableObjectStorage } from "./fixture/durable-object-storage" +import { sqliteLayer } from "@opencode/core/database/sqlite.workerd" const it = testEffect( AppNodeBuilder.build( @@ -62,6 +64,29 @@ const it = testEffect( ], ), ) +// Durable Object SQLite, which rejects statements that bind more than 100 parameters. +const durableIt = testEffect( + AppNodeBuilder.build( + LayerNode.group([ + Database.node, + Bus.node, + SessionProjector.node, + SessionStore.node, + Session.node, + SessionTransfer.node, + InstructionEntry.node, + ]), + [ + Bus.node.replace(Bus.configured({ persist: true })), + Project.node.replace(globalProjectNode), + LocationServiceMap.node.replace(promptLocationNode), + SessionExecution.node.replace(SessionExecution.noopLayer), + Database.node.replace( + Database.configuredClient(Layer.suspend(() => sqliteLayer({ storage: makeDurableObjectStorage() }))), + ), + ], + ), +) const liveIt = testEffect( AppNodeBuilder.build( LayerNode.group([Database.node, Bus.node, Project.node, SessionProjector.node, SessionStore.node, Session.node]), @@ -358,9 +383,7 @@ describe("Session.create", () => { const created = yield* session.create({ location: { directory: AbsolutePath.make("/project/") } }) expect(created.location.directory).toBe(location.directory) - expect((yield* session.list({ directory: location.directory })).data.map((item) => item.id)).toEqual([ - created.id, - ]) + expect((yield* session.list({ directory: location.directory })).data.map((item) => item.id)).toEqual([created.id]) }), ) @@ -852,9 +875,10 @@ describe("Session.create", () => { const session = yield* Session.Service const parent = yield* session.create({ location }) - expect( - yield* session.fork({ sessionID: parent.id }).pipe(Effect.flip), - ).toMatchObject({ _tag: "Session.ForkEmptyError", sessionID: parent.id }) + expect(yield* session.fork({ sessionID: parent.id }).pipe(Effect.flip)).toMatchObject({ + _tag: "Session.ForkEmptyError", + sessionID: parent.id, + }) }), ) @@ -969,7 +993,12 @@ describe("Session.create", () => { const input = { id, location } const created = yield* session.create(input) - yield* db.update(SessionTable).set({ agent: Agent.defaultID }).where(eq(SessionTable.id, id)).run().pipe(Effect.orDie) + yield* db + .update(SessionTable) + .set({ agent: Agent.defaultID }) + .where(eq(SessionTable.id, id)) + .run() + .pipe(Effect.orDie) expect(yield* session.create(input)).toMatchObject({ id: created.id, agent: "build" }) }), @@ -1564,4 +1593,28 @@ describe("SessionTransfer", () => { expect(imported.outcome).toBe("succeeded") }), ) + + durableIt.effect("imports a long transcript in statements under 100 bound parameters", () => + Effect.gen(function* () { + const session = yield* Session.Service + const transfer = yield* SessionTransfer.Service + const template = yield* session.create({ location }) + const sessionID = Session.ID.create() + const messages = Array.from({ length: 30 }, (_, index) => + SessionMessage.Assistant.make({ + id: SessionMessage.ID.create(), + type: "assistant", + agent: Agent.defaultID, + model: { id: Model.ID.make("model"), providerID: Provider.ID.make("provider") }, + content: Array.from({ length: 30 }, (_, ordinal) => ({ type: "text" as const, text: `${index}:${ordinal}` })), + time: { created: DateTime.makeUnsafe(index), completed: DateTime.makeUnsafe(index) }, + }), + ) + + yield* transfer.import({ data: { info: { ...template, id: sessionID }, messages }, location }) + expect(yield* session.messages({ sessionID, order: "asc" })).toMatchObject( + messages.map((message) => ({ id: message.id, content: message.content })), + ) + }), + ) }) diff --git a/packages/core/test/session-projector.test.ts b/packages/core/test/session-projector.test.ts index ab50f233d93..a818c954f54 100644 --- a/packages/core/test/session-projector.test.ts +++ b/packages/core/test/session-projector.test.ts @@ -1,8 +1,9 @@ import { describe, expect } from "bun:test" -import { DateTime, Effect, Fiber, Option, Schema, Stream } from "effect" +import { DateTime, Effect, Fiber, Layer, Option, Schema, Stream } from "effect" import { TestClock } from "effect/testing" -import { asc, eq, sql } from "drizzle-orm" +import { and, asc, eq, sql } from "drizzle-orm" import { Database } from "@opencode/core/database/database" +import { sqliteLayer } from "@opencode/core/database/sqlite.workerd" import { Agent } from "@opencode/core/agent" import { LayerNode } from "@opencode/util/effect/layer-node" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" @@ -40,6 +41,7 @@ import { SessionTable, } from "@opencode/core/session/sql" import { testEffect } from "./lib/effect" +import { makeDurableObjectStorage } from "./fixture/durable-object-storage" import { Snapshot } from "@opencode/core/snapshot" const it = testEffect( @@ -48,18 +50,31 @@ const it = testEffect( [Bus.node.replace(Bus.configured({ persist: true }))], ), ) +// Durable Object SQLite, which rejects statements that bind more than 100 parameters. +const durableIt = testEffect( + AppNodeBuilder.build( + LayerNode.group([Database.node, Bus.node, SessionProjector.node, SessionInbox.node, SessionStore.node]), + [ + Bus.node.replace(Bus.configured({ persist: true })), + Database.node.replace( + Database.configuredClient(Layer.suspend(() => sqliteLayer({ storage: makeDurableObjectStorage() }))), + ), + ], + ), +) const sessionsLayer = AppNodeBuilder.build(Session.node, [SessionExecution.node.replace(SessionExecution.noopLayer)]) const sessionID = Session.ID.make("ses_projector_test") const created = DateTime.makeUnsafe(0) const model = { id: Model.ID.make("model"), providerID: Provider.ID.make("provider") } const previousModel = { ...model, variant: Model.VariantID.make("medium") } const encodeMessage = Schema.encodeSync(SessionMessage.Info) -const mediaBlob = Config.testLayer([ - new Document({ - type: "document", - info: new Info({ experimental: new ConfigExperimental.Info({ media_blob: true }) }), - }), -]) +const experimental = (flags: ConstructorParameters[0]) => + Config.testLayer([ + new Document({ type: "document", info: new Info({ experimental: new ConfigExperimental.Info(flags) }) }), + ]) +const mediaBlob = experimental({ media_blob: true }) +const partRows = experimental({ part_rows: true }) +const mediaBlobPartRows = experimental({ media_blob: true, part_rows: true }) const png = Buffer.from( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64", @@ -811,26 +826,38 @@ describe("SessionProjector", () => { expect(JSON.parse(replayed?.data ?? "{}")).toMatchObject({ native: { step: "replayed" } }) expect(JSON.parse(replayed?.data ?? "{}")).not.toHaveProperty("providerState") - // Projecting onto the legacy row wrote it back in the current shape. - const rewritten = JSON.parse( - (yield* db.get<{ data: string }>(sql`SELECT data FROM session_message WHERE id = ${assistantID}`))?.data ?? - "{}", - ) + // An envelope update rewrites the legacy row in the current shape but leaves its inline parts where they are. + const messageData = () => + db + .get<{ data: string }>(sql`SELECT data FROM session_message WHERE id = ${assistantID}`) + .pipe(Effect.map((row) => JSON.parse(row?.data ?? "{}"))) + const rewritten = yield* messageData() expect(rewritten).not.toHaveProperty("providerState") - expect(rewritten).toMatchObject({ - native: { step: "replayed" }, - content: [ - { type: "text", native: { signature: "text" } }, - { type: "reasoning", native: { signature: "reasoning" } }, - { type: "tool", native: { call: "kept", result: "kept" } }, - ], - }) - expect(rewritten.content.map((part: Record) => Object.keys(part).toSorted())).toEqual([ + expect(rewritten).toMatchObject({ native: { step: "replayed" } }) + expect(rewritten.content).toHaveLength(3) + + // The first part write splits the legacy parts into session_part in the current shape. + yield* bus.publish(SessionEvent.Text.Started, { sessionID, assistantMessageID: assistantID, ordinal: 3 }) + expect(yield* messageData()).toMatchObject({ native: { step: "replayed" }, content: [] }) + const partRows = yield* db + .select({ data: SessionPartTable.data }) + .from(SessionPartTable) + .where(eq(SessionPartTable.message_id, assistantID)) + .orderBy(asc(SessionPartTable.ordinal)) + .all() + .pipe(Effect.orDie) + const rewrittenParts = partRows.map((row) => row.data).slice(0, 3) + expect(rewrittenParts).toMatchObject([ + { type: "text", native: { signature: "text" } }, + { type: "reasoning", native: { signature: "reasoning" } }, + { type: "tool", native: { call: "kept", result: "kept" } }, + ]) + expect(rewrittenParts.map((part) => Object.keys(part).toSorted())).toEqual([ ["native", "text", "type"], ["native", "text", "type"], ["executed", "id", "name", "native", "state", "time", "type"], ]) - }), + }).pipe(Effect.provide(partRows)), ) it.effect("merges tool result native data into the call's native data", () => @@ -876,13 +903,14 @@ describe("SessionProjector", () => { .where(eq(SessionMessageTable.id, assistantMessageID)) .get() .pipe(Effect.orDie) - expect(row?.data).toMatchObject({ + expect(row?.data).toMatchObject({ content: [] }) + expect(yield* SessionHistory.loadMessageRow(db, row!)).toMatchObject({ content: [ { id: "succeeded", native: { call: "succeeded", result: "succeeded" }, state: { status: "completed" } }, { id: "failed", native: { call: "failed" }, state: { status: "error" } }, ], }) - }), + }).pipe(Effect.provide(partRows)), ) it.effect("projects ended and failed step terminal state", () => @@ -984,9 +1012,7 @@ describe("SessionProjector", () => { .orderBy(asc(SessionMessageTable.id)) .all() .pipe(Effect.orDie) - const messages = rows.map((row) => - Schema.decodeUnknownSync(SessionMessage.Info)({ ...row.data, id: row.id, type: row.type }), - ) + const messages = yield* SessionHistory.decodeMessageRows(db, rows).pipe(Effect.orDie) expect(messages).toEqual([ SessionMessage.Assistant.make({ id: SessionMessage.ID.make("msg_assistant_completed"), @@ -1055,6 +1081,43 @@ describe("SessionProjector", () => { }), ) + it.effect("leaves a settled inline assistant message untouched when a new step starts", () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const settledID = SessionMessage.ID.make("msg_settled_inline") + const seeded = assistantRow(settledID, 0, { created, completed: created }) + yield* db + .insert(SessionMessageTable) + .values({ ...seeded, data: { ...seeded.data, content: [{ type: "text", text: "old answer" }] } }) + .run() + .pipe(Effect.orDie) + yield* Bus.reserveSequence(db, sessionID, 0) + const settledRow = () => + db.select().from(SessionMessageTable).where(eq(SessionMessageTable.id, settledID)).get().pipe(Effect.orDie) + const before = yield* settledRow() + + yield* bus.publish(SessionEvent.Step.Started, { + sessionID, + assistantMessageID: SessionMessage.ID.make("msg_next_step"), + agent: build, + model, + started: 1, + }) + yield* bus.publish(SessionEvent.Execution.Succeeded, { sessionID }) + + expect(yield* settledRow()).toEqual(before) + expect( + yield* db + .select() + .from(SessionPartTable) + .where(eq(SessionPartTable.message_id, settledID)) + .all() + .pipe(Effect.orDie), + ).toEqual([]) + }), + ) + it.effect("hydrates media: refs on message and inbox reads", () => Effect.gen(function* () { const db = yield* seedSession() @@ -1097,7 +1160,14 @@ describe("SessionProjector", () => { time_created: 1, data: { text: "user attachment", - files: [SessionStored.FileRef.make({ uri, mime: "application/pdf", source: { type: "inline" }, name: "spec.pdf" })], + files: [ + SessionStored.FileRef.make({ + uri, + mime: "application/pdf", + source: { type: "inline" }, + name: "spec.pdf", + }), + ], time: { created: 1 }, }, }, @@ -1142,7 +1212,14 @@ describe("SessionProjector", () => { type: "user", payload: { text: "queued attachment", - files: [SessionStored.FileRef.make({ uri, mime: "application/pdf", source: { type: "inline" }, name: "queued.pdf" })], + files: [ + SessionStored.FileRef.make({ + uri, + mime: "application/pdf", + source: { type: "inline" }, + name: "queued.pdf", + }), + ], }, delivery: "queue", enqueued_seq: 3, @@ -1167,7 +1244,12 @@ describe("SessionProjector", () => { state: { status: "completed", content: [ - { type: "file", uri: `data:application/pdf;base64,${base64}`, mime: "application/pdf", name: "spec.pdf" }, + { + type: "file", + uri: `data:application/pdf;base64,${base64}`, + mime: "application/pdf", + name: "spec.pdf", + }, { type: "file", uri: pathUri, mime: "application/pdf", name: "path.pdf" }, ], }, @@ -1312,7 +1394,11 @@ describe("SessionProjector", () => { const store = yield* SessionStore.Service const userID = SessionMessage.ID.make("msg_kept_user") const compactionID = SessionMessage.ID.make("msg_kept_compaction") - const { id: _, type: __, ...compaction } = encodeMessage( + const { + id: _, + type: __, + ...compaction + } = encodeMessage( SessionMessage.CompactionCompleted.make({ id: compactionID, type: "compaction", @@ -1434,12 +1520,12 @@ describe("SessionProjector", () => { }) const tool = yield* db .select() - .from(SessionMessageTable) - .where(eq(SessionMessageTable.id, assistantMessageID)) + .from(SessionPartTable) + .where(and(eq(SessionPartTable.message_id, assistantMessageID), eq(SessionPartTable.call_id, "shot"))) .get() .pipe(Effect.orDie) expect(tool?.data).toMatchObject({ - content: [{ state: { status: "completed", content: [{ type: "file", uri, mime: "image/png" }] } }], + state: { status: "completed", content: [{ type: "file", uri, mime: "image/png" }] }, }) expect((yield* store.message(assistantMessageID))?.message).toMatchObject({ content: [ @@ -1473,7 +1559,7 @@ describe("SessionProjector", () => { { message: assistantMessageID, hash, seq: 0 }, { message: id, hash, seq: delivered.durable?.seq }, ]) - }).pipe(Effect.provide(mediaBlob)), + }).pipe(Effect.provide(mediaBlobPartRows)), ) it.effect("deletes a cancelled inbox row's blob refs and blobs", () => @@ -1562,6 +1648,52 @@ describe("SessionProjector", () => { }).pipe(Effect.provide(mediaBlob)), ) + it.effect("keeps a tool part's blob while a fork references it", () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const assistantMessageID = SessionMessage.ID.make("msg_media_tool_forked") + const childID = Session.ID.make("ses_media_tool_fork") + yield* bus.publish(SessionEvent.Step.Started, { sessionID, assistantMessageID, agent: build, model, started: 1 }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { sessionID, assistantMessageID, id: "shot", name: "shot" }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID, + assistantMessageID, + id: "shot", + input: {}, + executed: true, + }) + yield* bus.publish(SessionEvent.Tool.Success, { + sessionID, + assistantMessageID, + id: "shot", + content: [{ type: "file", uri: `data:image/png;base64,${png.toString("base64")}`, mime: "image/png" }], + executed: true, + }) + yield* bus.publish(SessionEvent.Step.Ended, { + sessionID, + assistantMessageID, + finish: "stop", + cost: Money.USD.make(0), + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + }) + yield* bus.publish(SessionEvent.Forked, { + sessionID: childID, + parentID: sessionID, + boundary: { type: "through", messageID: assistantMessageID }, + }) + + yield* bus.publish(SessionEvent.Deleted, { sessionID }) + expect( + yield* db.select({ session: BlobRefTable.session_id }).from(BlobRefTable).all().pipe(Effect.orDie), + ).toEqual([{ session: childID }]) + expect(yield* db.select().from(BlobTable).all().pipe(Effect.orDie)).toHaveLength(1) + + yield* bus.publish(SessionEvent.Deleted, { sessionID: childID }) + expect(yield* db.select().from(BlobTable).all().pipe(Effect.orDie)).toEqual([]) + }).pipe(Effect.provide(mediaBlob)), + ) + it.effect("releases a tool's blob when a content update drops its ref", () => Effect.gen(function* () { const db = yield* seedSession() @@ -1592,6 +1724,270 @@ describe("SessionProjector", () => { }).pipe(Effect.provide(mediaBlob)), ) + it.effect( + "stores interleaved assistant parts in session_part by ordinal across out-of-order tool completions and forks", + () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const assistantMessageID = SessionMessage.ID.make("msg_interleaved") + + yield* bus.publish(SessionEvent.Step.Started, { + sessionID, + assistantMessageID, + agent: build, + model, + started: 1, + }) + yield* bus.publish(SessionEvent.Reasoning.Started, { sessionID, assistantMessageID, ordinal: 0 }) + yield* bus.publish(SessionEvent.Reasoning.Ended, { + sessionID, + assistantMessageID, + ordinal: 0, + text: "plan first", + }) + yield* bus.publish(SessionEvent.Text.Started, { sessionID, assistantMessageID, ordinal: 1 }) + yield* bus.publish(SessionEvent.Text.Ended, { + sessionID, + assistantMessageID, + ordinal: 1, + text: "running two tools", + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { + sessionID, + assistantMessageID, + id: "call_first", + name: "read", + }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID, + assistantMessageID, + id: "call_first", + input: { file: "a.ts" }, + executed: true, + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { + sessionID, + assistantMessageID, + id: "call_second", + name: "read", + }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID, + assistantMessageID, + id: "call_second", + input: { file: "b.ts" }, + executed: true, + }) + // Complete the second tool before the first tool + yield* bus.publish(SessionEvent.Tool.Success, { + sessionID, + assistantMessageID, + id: "call_second", + content: [{ type: "text", text: "b contents" }], + executed: true, + }) + yield* bus.publish(SessionEvent.Tool.Success, { + sessionID, + assistantMessageID, + id: "call_first", + content: [{ type: "text", text: "a contents" }], + executed: true, + }) + yield* bus.publish(SessionEvent.Text.Started, { sessionID, assistantMessageID, ordinal: 4 }) + yield* bus.publish(SessionEvent.Text.Ended, { + sessionID, + assistantMessageID, + ordinal: 4, + text: "all done", + }) + yield* bus.publish(SessionEvent.Step.Ended, { + sessionID, + assistantMessageID, + finish: "stop", + cost: Money.USD.make(0), + tokens: { input: 10, output: 5, reasoning: 2, cache: { read: 0, write: 0 } }, + }) + + const envelope = yield* db + .select() + .from(SessionMessageTable) + .where(eq(SessionMessageTable.id, assistantMessageID)) + .get() + .pipe(Effect.orDie) + expect(envelope?.data).toMatchObject({ content: [] }) + + const parts = yield* db + .select() + .from(SessionPartTable) + .where(eq(SessionPartTable.message_id, assistantMessageID)) + .orderBy(asc(SessionPartTable.ordinal)) + .all() + .pipe(Effect.orDie) + expect(parts.map((row) => ({ ordinal: row.ordinal, type: row.type, callID: row.call_id }))).toEqual([ + { ordinal: 0, type: "reasoning", callID: null }, + { ordinal: 1, type: "text", callID: null }, + { ordinal: 2, type: "tool", callID: "call_first" }, + { ordinal: 3, type: "tool", callID: "call_second" }, + { ordinal: 4, type: "text", callID: null }, + ]) + + const forkedSessionID = Session.ID.make("ses_projected_fork") + yield* bus.publish(SessionEvent.Forked, { + sessionID: forkedSessionID, + parentID: sessionID, + boundary: { type: "through", messageID: assistantMessageID }, + }) + const forkedMessage = yield* db + .select() + .from(SessionMessageTable) + .where(eq(SessionMessageTable.session_id, forkedSessionID)) + .get() + .pipe(Effect.orDie) + const loadedFork = yield* SessionHistory.loadMessageRow(db, forkedMessage!).pipe(Effect.orDie) + expect(loadedFork).toMatchObject({ + type: "assistant", + content: [ + { type: "reasoning", text: "plan first" }, + { type: "text", text: "running two tools" }, + { type: "tool", id: "call_first", state: { status: "completed" } }, + { type: "tool", id: "call_second", state: { status: "completed" } }, + { type: "text", text: "all done" }, + ], + }) + }).pipe(Effect.provide(partRows)), + ) + + it.effect("appends inline content after existing session_part rows when splitting an inline assistant row", () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const conflictID = SessionMessage.ID.make("msg_conflict_replace") + + // Simulate a downgrade: part rows (ordinals 0..2) exist, then an older build wrote inline content + yield* db + .insert(SessionMessageTable) + .values({ + id: conflictID, + session_id: sessionID, + seq: 1, + type: "assistant", + time_created: 10, + time_updated: 20, + data: { + agent: build, + model, + content: [ + { type: "reasoning", text: "fresh inline reasoning", time: { created: 10, completed: 12 } }, + { type: "text", text: "fresh inline text" }, + ], + time: { created: 10 }, + }, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionPartTable) + .values([ + { + message_id: conflictID, + session_id: sessionID, + ordinal: 0, + type: "text", + call_id: null, + time_created: 1, + time_updated: 1, + data: { type: "text", text: "stale part 0" }, + }, + { + message_id: conflictID, + session_id: sessionID, + ordinal: 1, + type: "text", + call_id: null, + time_created: 2, + time_updated: 2, + data: { type: "text", text: "stale part 1" }, + }, + { + message_id: conflictID, + session_id: sessionID, + ordinal: 2, + type: "text", + call_id: null, + time_created: 3, + time_updated: 3, + data: { type: "text", text: "stale trailing part 2" }, + }, + ]) + .run() + .pipe(Effect.orDie) + + yield* bus.publish(SessionEvent.Text.Started, { + sessionID, + assistantMessageID: conflictID, + ordinal: 5, + }) + yield* bus.publish(SessionEvent.Text.Ended, { + sessionID, + assistantMessageID: conflictID, + ordinal: 5, + text: "appended after split", + }) + + const messageRow = yield* db + .select() + .from(SessionMessageTable) + .where(eq(SessionMessageTable.id, conflictID)) + .get() + .pipe(Effect.orDie) + expect(messageRow?.data).toMatchObject({ content: [] }) + + const partRows = yield* db + .select() + .from(SessionPartTable) + .where(eq(SessionPartTable.message_id, conflictID)) + .orderBy(asc(SessionPartTable.ordinal)) + .all() + .pipe(Effect.orDie) + expect(partRows.map((row) => ({ ordinal: row.ordinal, data: row.data }))).toMatchObject([ + { ordinal: 0, data: { type: "text", text: "stale part 0" } }, + { ordinal: 1, data: { type: "text", text: "stale part 1" } }, + { ordinal: 2, data: { type: "text", text: "stale trailing part 2" } }, + { ordinal: 3, data: { type: "reasoning", text: "fresh inline reasoning" } }, + { ordinal: 4, data: { type: "text", text: "fresh inline text" } }, + { ordinal: 5, data: { type: "text", text: "appended after split" } }, + ]) + }), + ) + + it.effect("splits inline media out of a session_part tool row for offline blob backfills", () => + Effect.sync(() => { + const hash = SessionBlob.hashBytes(png) + const split = SessionBlob.split({ + type: "tool", + id: "call_shot", + name: "shot", + state: { + status: "completed", + input: {}, + content: [{ type: "file", uri: `data:image/png;base64,${png.toString("base64")}`, mime: "image/png" }], + time: { created: 1, completed: 2 }, + }, + }) + expect(split.moved).toBe(true) + expect(split.hashes).toEqual(new Set([hash])) + expect(split.blobs).toEqual([{ hash, mime: "image/png", bytes: png.byteLength, data: png }]) + expect(split.data).toMatchObject({ + type: "tool", + state: { + status: "completed", + content: [{ type: "file", uri: SessionStored.ref(hash), mime: "image/png" }], + }, + }) + }), + ) + // A newer release wrote this in-progress message's parts as session_part rows, then the user rolled back. it.effect("keeps projecting into session_part rows a newer release wrote for an in-progress message", () => Effect.gen(function* () { @@ -1649,7 +2045,11 @@ describe("SessionProjector", () => { expect((yield* store.message(assistantMessageID))?.message).toMatchObject({ content: [ { type: "text", text: "before rollback" }, - { type: "tool", id: "call_a", state: { status: "completed", content: [{ type: "text", text: "export {}" }] } }, + { + type: "tool", + id: "call_a", + state: { status: "completed", content: [{ type: "text", text: "export {}" }] }, + }, { type: "text", text: "after rollback" }, { type: "tool", id: "call_b", state: { status: "streaming" } }, ], @@ -1720,6 +2120,55 @@ describe("SessionProjector", () => { }), ) + it.effect("keeps session_part rows when an older build left inline content on the same message", () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const messageID = SessionMessage.ID.make("msg_downgraded") + const row = assistantRow(messageID, 0) + yield* db + .insert(SessionMessageTable) + .values({ + ...row, + data: { ...row.data, content: [{ type: "text", text: "after downgrade" }] }, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionPartTable) + .values( + ["first", "second"].map((text, ordinal) => ({ + message_id: messageID, + session_id: sessionID, + ordinal, + type: "text" as const, + call_id: null, + time_created: 0, + time_updated: 0, + data: { type: "text" as const, text }, + })), + ) + .run() + .pipe(Effect.orDie) + + yield* bus.publish(SessionEvent.Text.Started, { sessionID, assistantMessageID: messageID, ordinal: 3 }) + + const parts = yield* db + .select({ ordinal: SessionPartTable.ordinal, data: SessionPartTable.data }) + .from(SessionPartTable) + .where(eq(SessionPartTable.message_id, messageID)) + .orderBy(asc(SessionPartTable.ordinal)) + .all() + .pipe(Effect.orDie) + expect(parts.map((part) => [part.ordinal, part.data.type === "text" ? part.data.text : undefined])).toEqual([ + [0, "first"], + [1, "second"], + [2, "after downgrade"], + [3, ""], + ]) + }), + ) + it.effect("stores a fork's copies of inline parent media as blobs and leaves the parent's rows inline", () => Effect.gen(function* () { const db = yield* seedSession() @@ -1740,13 +2189,21 @@ describe("SessionProjector", () => { const hash = SessionBlob.hashBytes(png) const uri = SessionStored.ref(hash) const childRows = yield* messageRows(db, childID) - expect(childRows.map((row) => row.data)).toMatchObject([ + const childFull = yield* SessionHistory.withParts(db, childRows) + expect(childFull.map((row) => row.data)).toMatchObject([ { files: [{ uri, mime: "image/png", name: "shot.png" }] }, { content: [{ type: "tool", state: { content: [{ type: "file", uri, mime: "image/png" }] } }] }, ]) - expect(JSON.stringify(childRows)).not.toContain(png.toString("base64")) + expect(JSON.stringify(childFull)).not.toContain(png.toString("base64")) expect(yield* messageRows(db, sessionID)).toEqual(parentRows) - expect(yield* db.select().from(SessionPartTable).all().pipe(Effect.orDie)).toEqual(parentParts) + expect( + yield* db + .select() + .from(SessionPartTable) + .where(eq(SessionPartTable.session_id, sessionID)) + .all() + .pipe(Effect.orDie), + ).toEqual(parentParts) expect(yield* db.select({ hash: BlobTable.hash }).from(BlobTable).all().pipe(Effect.orDie)).toEqual([{ hash }]) expect( yield* db @@ -1782,7 +2239,7 @@ describe("SessionProjector", () => { }) const childRows = yield* messageRows(db, childID) - expect(childRows.map((row) => row.data)).toEqual( + expect((yield* SessionHistory.withParts(db, childRows)).map((row) => row.data)).toEqual( (yield* SessionHistory.withParts(db, parentRows)).map((row) => row.data), ) expect(yield* db.select().from(BlobTable).all().pipe(Effect.orDie)).toEqual([]) @@ -1793,6 +2250,457 @@ describe("SessionProjector", () => { // The caller's own config doesn't decide: Session.fork sets ForkMedia from the parent's location. }).pipe(Effect.provide(mediaBlob)), ) + + it.effect("reads a split message an older build added inline content to as its parts then that content", () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const store = yield* SessionStore.Service + const messageID = SessionMessage.ID.make("msg_hybrid_read") + yield* db + .insert(SessionMessageTable) + .values( + hybridRow(messageID, [ + { type: "text", text: "inline after" }, + { + type: "tool", + id: "call_inline", + name: "read", + executed: true, + state: { status: "running", input: { path: "a.ts" }, metadata: {} }, + time: { created: 1, ran: 1 }, + }, + ]), + ) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionPartTable) + .values(textPart(messageID, 0, "split first")) + .run() + .pipe(Effect.orDie) + + const content = [ + { type: "text", text: "split first" }, + { type: "text", text: "inline after" }, + { type: "tool", id: "call_inline", state: { status: "running" } }, + ] + expect((yield* store.message(messageID))?.message).toMatchObject({ content }) + // The runner settles stale tool calls from context, so an inline one on a split message must show up there. + expect(yield* store.context(sessionID)).toMatchObject([{ id: messageID, content }]) + + yield* bus.publish(SessionEvent.Tool.Failed, { + sessionID, + assistantMessageID: messageID, + id: "call_inline", + error: { type: "aborted", message: "Tool execution interrupted: read" }, + executed: true, + }) + expect((yield* store.message(messageID))?.message).toMatchObject({ + content: [content[0], content[1], { type: "tool", id: "call_inline", state: { status: "error" } }], + }) + expect((yield* messageRows(db, sessionID))[0]?.data).toMatchObject({ content: [] }) + }), + ) + + it.effect("forks a split message an older build added inline media to", () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const store = yield* SessionStore.Service + const messageID = SessionMessage.ID.make("msg_hybrid_fork") + const childID = Session.ID.make("ses_hybrid_fork") + yield* db + .insert(SessionMessageTable) + .values(hybridRow(messageID, [inlineShot], { created, completed: created })) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionPartTable) + .values(textPart(messageID, 0, "split first")) + .run() + .pipe(Effect.orDie) + const parent = yield* store.messages({ sessionID, order: "asc" }) + expect(parent).toMatchObject([ + { + content: [ + { type: "text", text: "split first" }, + { type: "tool", id: "call_inline_shot" }, + ], + }, + ]) + + yield* bus.publish(SessionEvent.Forked, { + sessionID: childID, + parentID: sessionID, + boundary: { type: "through", messageID }, + }) + + const childRows = yield* messageRows(db, childID) + expect(JSON.stringify(yield* SessionHistory.withParts(db, childRows))).not.toContain(png.toString("base64")) + expect( + yield* db + .select({ session: BlobRefTable.session_id, message: BlobRefTable.message_id, hash: BlobRefTable.hash }) + .from(BlobRefTable) + .all() + .pipe(Effect.orDie), + ).toEqual([{ session: childID, message: childRows[0]!.id, hash: SessionBlob.hashBytes(png) }]) + yield* bus.publish(SessionEvent.Deleted, { sessionID }) + expect(withoutIDs(yield* store.messages({ sessionID: childID, order: "asc" }))).toEqual(withoutIDs(parent)) + }).pipe(Effect.provideService(SessionBlob.ForkMedia, true)), + ) + + it.effect("deletes a reverted message's session_part rows and the blobs only they referenced", () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const assistantMessageID = SessionMessage.ID.make("msg_media_tool_reverted") + yield* bus.publish(SessionEvent.Step.Started, { sessionID, assistantMessageID, agent: build, model, started: 1 }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { sessionID, assistantMessageID, id: "shot", name: "shot" }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID, + assistantMessageID, + id: "shot", + input: {}, + executed: true, + }) + yield* bus.publish(SessionEvent.Tool.Success, { + sessionID, + assistantMessageID, + id: "shot", + content: [{ type: "file", uri: `data:image/png;base64,${png.toString("base64")}`, mime: "image/png" }], + executed: true, + }) + expect(yield* db.select().from(SessionPartTable).all().pipe(Effect.orDie)).toHaveLength(1) + expect(yield* db.select().from(BlobTable).all().pipe(Effect.orDie)).toHaveLength(1) + + yield* bus.publish(SessionEvent.RevertEvent.Committed, { sessionID, to: assistantMessageID }) + expect(yield* db.select().from(SessionPartTable).all().pipe(Effect.orDie)).toEqual([]) + expect(yield* db.select().from(BlobRefTable).all().pipe(Effect.orDie)).toEqual([]) + expect(yield* db.select().from(BlobTable).all().pipe(Effect.orDie)).toEqual([]) + }).pipe(Effect.provide(mediaBlobPartRows)), + ) + + it.effect("moves a legacy inline media message's rewritten content to session_part with its media as a blob", () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const store = yield* SessionStore.Service + const messageID = SessionMessage.ID.make("msg_legacy_media_rewritten") + const hash = SessionBlob.hashBytes(png) + yield* db + .insert(SessionMessageTable) + .values(hybridRow(messageID, [inlineShot], { created, completed: created })) + .run() + .pipe(Effect.orDie) + + yield* bus.publish(SessionEvent.MessageContentUpdated, { + sessionID, + messageID, + content: [{ type: "text", text: "rewritten" }, inlineShot], + }) + + expect((yield* messageRows(db, sessionID))[0]?.data).toMatchObject({ content: [] }) + expect( + yield* db + .select({ data: SessionPartTable.data }) + .from(SessionPartTable) + .orderBy(asc(SessionPartTable.ordinal)) + .all() + .pipe(Effect.orDie), + ).toMatchObject([ + { data: { type: "text", text: "rewritten" } }, + { data: { type: "tool", state: { content: [{ type: "file", uri: SessionStored.ref(hash) }] } } }, + ]) + expect(yield* db.select({ hash: BlobTable.hash }).from(BlobTable).all().pipe(Effect.orDie)).toEqual([{ hash }]) + expect( + yield* db + .select({ message: BlobRefTable.message_id, hash: BlobRefTable.hash }) + .from(BlobRefTable) + .all() + .pipe(Effect.orDie), + ).toEqual([{ message: messageID, hash }]) + expect((yield* store.message(messageID))?.message).toMatchObject({ + content: [ + { type: "text", text: "rewritten" }, + { + type: "tool", + id: "call_inline_shot", + state: { content: [{ type: "file", uri: inlineShot.state.content[0].uri }] }, + }, + ], + }) + }).pipe(Effect.provide(mediaBlobPartRows)), + ) + + durableIt.effect("forks and reads a long session in statements under 100 bound parameters", () => + Effect.gen(function* () { + const db = yield* seedSession() + const bus = yield* Bus.Service + const store = yield* SessionStore.Service + const childID = Session.ID.make("ses_long_fork") + const ids = Array.from({ length: 120 }, (_, seq) => + SessionMessage.ID.make(`msg_long_${String(seq).padStart(3, "0")}`), + ) + yield* Effect.forEach( + ids, + (id, seq) => + db + .insert(SessionMessageTable) + .values(assistantRow(id, seq, { created, completed: created })) + .run() + .pipe(Effect.orDie), + { discard: true }, + ) + const texts = Array.from({ length: 30 }, (_, ordinal) => `part ${ordinal}`) + yield* Effect.forEach( + texts, + (text, ordinal) => + db + .insert(SessionPartTable) + .values(textPart(ids[0]!, ordinal, text)) + .run() + .pipe(Effect.orDie), + { discard: true }, + ) + const parent = yield* store.messages({ sessionID, order: "asc" }) + expect(parent).toHaveLength(ids.length) + expect(parent[0]).toMatchObject({ content: texts.map((text) => ({ type: "text", text })) }) + + yield* bus.publish(SessionEvent.Forked, { + sessionID: childID, + parentID: sessionID, + boundary: { type: "through", messageID: ids.at(-1)! }, + }) + expect(withoutIDs(yield* store.messages({ sessionID: childID, order: "asc" }))).toEqual(withoutIDs(parent)) + yield* bus.publish(SessionEvent.Deleted, { sessionID }) + expect( + yield* db + .select() + .from(SessionPartTable) + .where(eq(SessionPartTable.session_id, sessionID)) + .all() + .pipe(Effect.orDie), + ).toEqual([]) + expect(withoutIDs(yield* store.messages({ sessionID: childID, order: "asc" }))).toEqual(withoutIDs(parent)) + }).pipe(Effect.provideService(SessionBlob.ForkMedia, true)), + ) + + it.effect( + "hydrates parts when envelope content is omitted or non-array, and fails to decode non-array content without parts", + () => + Effect.gen(function* () { + const db = yield* seedSession() + const store = yield* SessionStore.Service + const omittedID = SessionMessage.ID.make("msg_parts_omitted_content") + const nullID = SessionMessage.ID.make("msg_parts_null_content") + const corruptID = SessionMessage.ID.make("msg_no_parts_null_content") + yield* db + .insert(SessionMessageTable) + .values([ + assistantRow(omittedID, 0, { created, completed: created }), + assistantRow(nullID, 1, { created, completed: created }), + assistantRow(corruptID, 2, { created, completed: created }), + ]) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionPartTable) + .values([textPart(omittedID, 0, "from part"), textPart(nullID, 0, "from part")]) + .run() + .pipe(Effect.orDie) + + yield* db.run(sql`update session_message set data = json_remove(data, '$.content') where id = ${omittedID}`) + expect((yield* store.message(omittedID))?.message).toMatchObject({ + content: [{ type: "text", text: "from part" }], + }) + + // With part rows, a non-array `content` counts as no inline parts. + yield* db.run( + sql`update session_message set data = json_set(data, '$.content', json('null')) where id in (${nullID}, ${corruptID})`, + ) + expect((yield* store.message(nullID))?.message).toMatchObject({ + agent: build, + content: [{ type: "text", text: "from part" }], + }) + + // Without part rows there is nothing to read instead, so the row fails to decode. + expect(yield* store.messages({ sessionID }).pipe(Effect.flip)).toMatchObject({ + _tag: "Session.MessageDecodeError", + sessionID, + messageID: corruptID, + }) + }), + ) + + it.effect("writes whole messages inline with experimental.part_rows off", () => + Effect.gen(function* () { + const db = yield* seedSession() + const store = yield* SessionStore.Service + yield* projectTwoSteps(sessionID, () => false) + + expect(yield* db.select().from(SessionPartTable).all().pipe(Effect.orDie)).toEqual([]) + const messages = yield* store.messages({ sessionID, order: "asc" }) + const assistants = messages.filter((message) => message.type === "assistant") + expect(assistants.map((message) => message.content.length)).toEqual([3, 1]) + // Each row holds the whole encoded message, as before part rows existed. + const rows = yield* messageRows(db, sessionID) + for (const message of assistants) { + const { id: _, type: __, ...data } = encodeMessage(message) + expect(rows.find((row) => row.id === message.id)?.data).toEqual(data) + } + }), + ) + + it.effect("reads the same reply when experimental.part_rows is switched on, off and on again mid-session", () => + Effect.gen(function* () { + const db = yield* seedSession() + const store = yield* SessionStore.Service + const off = Session.ID.make("ses_part_rows_off") + const on = Session.ID.make("ses_part_rows_on") + yield* db + .insert(SessionTable) + .values( + [off, on].map((id) => ({ + id, + project_id: Project.ID.global, + slug: id, + directory: "/project", + title: "test", + version: "test", + })), + ) + .run() + .pipe(Effect.orDie) + + yield* projectTwoSteps(off, () => false) + yield* projectTwoSteps(on, () => true) + // On for the first step's start, off for its tool call, on for its end; then off for the second step's text start. + yield* projectTwoSteps(sessionID, (index) => ![4, 5, 6, 7, 10].includes(index)) + + const read = (session: Session.ID) => + store.messages({ sessionID: session, order: "asc" }).pipe(Effect.map(withoutIDs)) + const mixed = yield* read(sessionID) + expect(mixed).toEqual(yield* read(off)) + expect(mixed).toEqual(yield* read(on)) + expect(mixed.flatMap((message) => (message.type === "assistant" ? [message.content.length] : []))).toEqual([3, 1]) + + const parts = yield* db + .select({ message: SessionPartTable.message_id, ordinal: SessionPartTable.ordinal }) + .from(SessionPartTable) + .where(eq(SessionPartTable.session_id, sessionID)) + .orderBy(asc(SessionPartTable.message_id), asc(SessionPartTable.ordinal)) + .all() + .pipe(Effect.orDie) + expect( + Object.values(Object.groupBy(parts, (part) => part.message)).map((rows) => rows?.map((row) => row.ordinal)), + ).toEqual([[0, 1, 2], [0]]) + for (const row of yield* messageRows(db, sessionID)) + if (row.type === "assistant") expect(row.data).toMatchObject({ content: [] }) + }), + ) +}) + +/** Two assistant steps, the first with reasoning, text and a tool call. Event `index` is projected with part rows on when `partRowsAt(index)`. */ +const projectTwoSteps = Effect.fnUntraced(function* (session: Session.ID, partRowsAt: (index: number) => boolean) { + const bus = yield* Bus.Service + const first = SessionMessage.ID.make(`msg_${session}_first`) + const second = SessionMessage.ID.make(`msg_${session}_second`) + const usage = { cost: Money.USD.make(0), tokens: { input: 1, output: 1, reasoning: 0, cache: { read: 0, write: 0 } } } + const events: ReadonlyArray> = [ + bus.publish(SessionEvent.Step.Started, { + sessionID: session, + assistantMessageID: first, + agent: build, + model, + started: 1, + }), + bus.publish(SessionEvent.Reasoning.Started, { sessionID: session, assistantMessageID: first, ordinal: 0 }), + bus.publish(SessionEvent.Reasoning.Ended, { + sessionID: session, + assistantMessageID: first, + ordinal: 0, + text: "plan", + }), + bus.publish(SessionEvent.Text.Started, { sessionID: session, assistantMessageID: first, ordinal: 1 }), + bus.publish(SessionEvent.Text.Ended, { + sessionID: session, + assistantMessageID: first, + ordinal: 1, + text: "reading", + }), + bus.publish(SessionEvent.Tool.Input.Started, { + sessionID: session, + assistantMessageID: first, + id: "call", + name: "read", + }), + bus.publish(SessionEvent.Tool.Called, { + sessionID: session, + assistantMessageID: first, + id: "call", + input: { file: "a.ts" }, + executed: true, + }), + bus.publish(SessionEvent.Tool.Success, { + sessionID: session, + assistantMessageID: first, + id: "call", + content: [{ type: "text", text: "a contents" }], + executed: true, + }), + bus.publish(SessionEvent.Step.Ended, { + sessionID: session, + assistantMessageID: first, + finish: "tool-calls", + ...usage, + }), + bus.publish(SessionEvent.Step.Started, { + sessionID: session, + assistantMessageID: second, + agent: build, + model, + started: 2, + }), + bus.publish(SessionEvent.Text.Started, { sessionID: session, assistantMessageID: second, ordinal: 0 }), + bus.publish(SessionEvent.Text.Ended, { sessionID: session, assistantMessageID: second, ordinal: 0, text: "done" }), + bus.publish(SessionEvent.Step.Ended, { sessionID: session, assistantMessageID: second, finish: "stop", ...usage }), + ] + for (const [index, event] of events.entries()) yield* partRowsAt(index) ? event.pipe(Effect.provide(partRows)) : event +}) + +const inlineShot = { + type: "tool", + id: "call_inline_shot", + name: "screenshot", + executed: true, + state: { + status: "completed", + input: {}, + content: [{ type: "file", uri: `data:image/png;base64,${png.toString("base64")}`, mime: "image/png" }], + }, + time: { created: 2, ran: 2, completed: 2 }, +} as const + +/** An assistant row with inline content, the shape an older build writes even when session_part rows exist. */ +const hybridRow = ( + messageID: SessionMessage.ID, + content: ReadonlyArray, + time?: { created: DateTime.Utc; completed?: DateTime.Utc }, +) => { + const row = assistantRow(messageID, 0, time) + return { ...row, data: { ...row.data, content } } +} + +const textPart = (messageID: SessionMessage.ID, ordinal: number, text: string) => ({ + message_id: messageID, + session_id: sessionID, + ordinal, + type: "text" as const, + call_id: null, + time_created: 0, + time_updated: 0, + data: { type: "text" as const, text }, }) const messageRows = (db: Database.Interface["db"], session: Session.ID) => diff --git a/packages/core/test/session-stats.test.ts b/packages/core/test/session-stats.test.ts index c71d1fe86e8..7e828d4ac93 100644 --- a/packages/core/test/session-stats.test.ts +++ b/packages/core/test/session-stats.test.ts @@ -9,14 +9,18 @@ import { SessionEvent } from "@opencode/schema/session-event" import { SessionMessage } from "@opencode/schema/session-message" import { AbsolutePath } from "@opencode/core/schema" import { Bus } from "@opencode/core/bus" +import { DatabaseBatch } from "@opencode/core/database/batch" import { Database } from "@opencode/core/database/database" +import { sqliteLayer } from "@opencode/core/database/sqlite.workerd" import { ProjectTable } from "@opencode/core/project/sql" -import { SessionMessageTable, SessionTable } from "@opencode/core/session/sql" +import { SessionMessageTable, SessionPartTable, SessionTable } from "@opencode/core/session/sql" +import { SessionHistory } from "@opencode/core/session/history" import { SessionStats } from "@opencode/core/session/stats" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { LayerNode } from "@opencode/util/effect/layer-node" -import { DateTime, Effect, Schema } from "effect" +import { DateTime, Effect, Layer, Schema } from "effect" import { TestClock } from "effect/testing" +import { makeDurableObjectStorage } from "./fixture/durable-object-storage" import { testEffect } from "./lib/effect" const it = testEffect( @@ -24,6 +28,14 @@ const it = testEffect( Bus.node.replace(Bus.configured({ persist: true })), ]), ) +const durableIt = testEffect( + AppNodeBuilder.build(LayerNode.group([Database.node, Bus.node]), [ + Bus.node.replace(Bus.configured({ persist: true })), + Database.node.replace( + Database.configuredClient(Layer.suspend(() => sqliteLayer({ storage: makeDurableObjectStorage() }))), + ), + ]), +) const projectID = Project.ID.make("stats-project") const otherProjectID = Project.ID.make("stats-other-project") const sessionID = Session.ID.make("ses_stats_root") @@ -246,6 +258,145 @@ describe("SessionStats", () => { expect(future._tag).toBe("SessionStats.InvalidRangeError") }), ) + + it.effect("counts tool calls in both session_part rows and inline content an older build left on one message", () => + Effect.gen(function* () { + const db = (yield* Database.Service).db + yield* db + .insert(ProjectTable) + .values({ id: projectID, worktree: AbsolutePath.make("/stats"), name: "stats", sandboxes: [] }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ id: sessionID, project_id: projectID, slug: "root", directory: "/stats", version: "test" }) + .run() + .pipe(Effect.orDie) + const created = Date.UTC(2026, 0, 2, 10) + const time = { created: DateTime.makeUnsafe(created), completed: DateTime.makeUnsafe(created + 1_000) } + const row = messageRow( + sessionID, + 1, + assistant("msg_stats_hybrid", created, [ + SessionMessage.AssistantTool.make({ + type: "tool", + id: "call_inline", + name: "edit", + state: SessionMessage.ToolStateError.make({ + status: "error", + input: {}, + error: { type: "tool", message: "failed" }, + }), + time, + }), + ]), + ) + yield* db.insert(SessionMessageTable).values(row).run().pipe(Effect.orDie) + yield* db + .insert(SessionPartTable) + .values({ + message_id: row.id, + session_id: sessionID, + ordinal: 0, + type: "tool", + call_id: "call_part", + time_created: created, + time_updated: created, + data: SessionHistory.encodePart( + SessionMessage.AssistantTool.make({ + type: "tool", + id: "call_part", + name: "read", + state: SessionMessage.ToolStateCompleted.make({ + status: "completed", + input: {}, + content: [{ type: "text", text: "ok" }], + }), + time, + }), + ), + }) + .run() + .pipe(Effect.orDie) + + const range = { from: Date.UTC(2026, 0, 1), to: Date.UTC(2026, 1, 1), timezone: "UTC" } + expect((yield* SessionStats.get(range)).tools).toEqual({ + mode: "summary", + totals: { calls: 2, succeeded: 1, failed: 1, unfinished: 0 }, + }) + const detail = (yield* SessionStats.get({ ...range, tools: "detail" })).tools + if (detail.mode !== "detail") throw new Error("Expected detailed tool statistics") + expect(detail.usage.map((tool) => tool.name).toSorted()).toEqual(["edit", "read"]) + }), + ) + + durableIt.effect("aggregates compaction usage across more than 100 sessions under the 100-parameter limit", () => + Effect.gen(function* () { + const db = (yield* Database.Service).db + const bus = yield* Bus.Service + yield* db + .insert(ProjectTable) + .values({ id: projectID, worktree: AbsolutePath.make("/stats"), name: "stats", sandboxes: [] }) + .run() + .pipe(Effect.orDie) + + const ids = Array.from({ length: 120 }, (_, i) => Session.ID.make(`ses_stats_${String(i).padStart(3, "0")}`)) + const created = Date.UTC(2026, 0, 2, 10) + yield* Effect.forEach( + DatabaseBatch.insertBatches( + SessionTable, + ids.map((id, i) => ({ + id, + project_id: projectID, + slug: `session-${i}`, + directory: "/stats", + version: "test", + })), + ), + (batch) => db.insert(SessionTable).values(batch).run().pipe(Effect.orDie), + { discard: true }, + ) + yield* Effect.forEach( + DatabaseBatch.insertBatches( + SessionMessageTable, + ids.map((id, i) => messageRow(id, 1, assistant(`msg_stats_${i}`, created, []))), + ), + (batch) => db.insert(SessionMessageTable).values(batch).run().pipe(Effect.orDie), + { discard: true }, + ) + + yield* TestClock.setTime(created + 1_000) + yield* bus.publish(SessionEvent.UsageRecorded, { + sessionID: ids[0]!, + source: "compaction", + cost: Money.USD.make(0.25), + tokens: { input: 2, output: 1, reasoning: 0, cache: { read: 0, write: 0 } }, + }) + yield* bus.publish(SessionEvent.UsageRecorded, { + sessionID: ids.at(-1)!, + source: "compaction", + cost: Money.USD.make(0.75), + tokens: { input: 3, output: 4, reasoning: 0, cache: { read: 0, write: 0 } }, + }) + + const stats = yield* SessionStats.get({ + from: Date.UTC(2026, 0, 1), + to: Date.UTC(2026, 1, 1), + projectID, + timezone: "UTC", + tools: "none", + }) + expect(stats.sessions).toBe(120) + expect(stats.steps).toBe(120) + expect(stats.tokens).toEqual({ + input: 120 * 10 + 5, + output: 120 * 5 + 5, + reasoning: 120 * 2, + cache: { read: 120 * 4, write: 120 }, + }) + expect(stats.cost).toBe(Money.USD.make(120 * 1.5 + 1)) + }), + ) }) function assistant( diff --git a/packages/core/test/session-store.test.ts b/packages/core/test/session-store.test.ts index c30369fb935..a1bb439f29f 100644 --- a/packages/core/test/session-store.test.ts +++ b/packages/core/test/session-store.test.ts @@ -2,14 +2,17 @@ import { describe, expect } from "bun:test" import { asc, eq } from "drizzle-orm" import { Effect } from "effect" import { Bus } from "@opencode/core/bus" +import { Config } from "@opencode/core/config" import { Database } from "@opencode/core/database/database" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { ProjectTable } from "@opencode/core/project/sql" -import { SessionHistory } from "@opencode/core/session/history" import { SessionProjector } from "@opencode/core/session/projector" +import { SessionHistory } from "@opencode/core/session/history" import { SessionMessageTable, SessionPartTable } from "@opencode/core/session/sql" import { SessionStore } from "@opencode/core/session/store" import { Agent } from "@opencode/schema/agent" +import { Document, Info } from "@opencode/schema/config" +import { ConfigExperimental } from "@opencode/schema/config/experimental" import { Event } from "@opencode/schema/event" import { Model } from "@opencode/schema/model" import { Project } from "@opencode/schema/project" @@ -26,6 +29,12 @@ const it = testEffect( Bus.node.replace(Bus.configured({ persist: true })), ]), ) +const partRows = Config.testLayer([ + new Document({ + type: "document", + info: new Info({ experimental: new ConfigExperimental.Info({ part_rows: true }) }), + }), +]) const seedSessions = (rows: { id: string; updated: number }[]) => Effect.gen(function* () { @@ -194,107 +203,142 @@ describe("SessionStore", () => { }), ) - it.effect("hydrates assistant parts from session_part when present and falls back to inline content", () => + it.effect("hydrates and settles assistant messages across session_part rows and legacy inline content rows", () => Effect.gen(function* () { - const sessionID = Session.ID.make("ses_part_read_fallback") - yield* seedSessions([{ id: sessionID, updated: 0 }]) + const sessionID = Session.ID.make("ses_dual_read") + const bus = yield* seedSessions([{ id: sessionID, updated: 0 }]) const database = yield* Database.Service const store = yield* SessionStore.Service const agent = Agent.ID.make("build") const model = { providerID: Provider.ID.make("anthropic"), id: Model.ID.make("claude-sonnet-4-5") } - const inlineOnlyID = SessionMessage.ID.make("msg_inline_only") - const splitID = SessionMessage.ID.make("msg_split_parts") + const legacyUntouchedID = SessionMessage.ID.make("msg_legacy_untouched") + const legacyMutatedID = SessionMessage.ID.make("msg_legacy_mutated") + const splitID = SessionMessage.ID.make("msg_split") - // 1. Assistant message with inline content only (no session_part rows) -> falls back to inline content + // 1. Untouched legacy row with inline content and no session_part rows yield* database.db .insert(SessionMessageTable) .values({ - id: inlineOnlyID, + id: legacyUntouchedID, session_id: sessionID, - seq: 10, + seq: -2, type: "assistant", time_created: 10, time_updated: 10, data: { agent, model, - content: [{ type: "text", text: "inline fallback content" }], + content: [{ type: "text", text: "legacy inline only" }], time: { created: 10, completed: 11 }, }, }) .run() - // 2. Assistant message with session_part rows -> hydrates parts in ordinal order + // 2. Legacy row with inline content that gets mutated via projector and split into session_part yield* database.db .insert(SessionMessageTable) .values({ - id: splitID, + id: legacyMutatedID, session_id: sessionID, - seq: 20, + seq: -1, type: "assistant", time_created: 20, time_updated: 20, data: { agent, model, - content: [], - time: { created: 20, completed: 25 }, + content: [{ type: "reasoning", text: "legacy reasoning" }], + time: { created: 20 }, }, }) .run() - yield* database.db - .insert(SessionPartTable) - .values([ - { - message_id: splitID, - session_id: sessionID, - ordinal: 0, - type: "reasoning", - call_id: null, - time_created: 20, - time_updated: 21, - data: { type: "reasoning", text: "first reasoning", time: { created: 20, completed: 21 } }, - }, - { - message_id: splitID, - session_id: sessionID, - ordinal: 1, - type: "tool", - call_id: "call_read", - time_created: 21, - time_updated: 23, - data: { - type: "tool", - id: "call_read", - name: "read", - executed: true, - state: { - status: "completed", - input: { path: "src/index.ts" }, - content: [{ type: "text", text: "export {}" }], - }, - time: { created: 21, ran: 22, completed: 23 }, - }, - }, - { - message_id: splitID, - session_id: sessionID, - ordinal: 2, - type: "text", - call_id: null, - time_created: 23, - time_updated: 25, - data: { type: "text", text: "hydrated from session_part" }, - }, - ]) - .run() + yield* bus.publish(SessionEvent.Text.Started, { + sessionID, + assistantMessageID: legacyMutatedID, + ordinal: 1, + }) + yield* bus.publish(SessionEvent.Text.Ended, { + sessionID, + assistantMessageID: legacyMutatedID, + ordinal: 1, + text: "appended into session_part", + }) - expect(yield* store.message(inlineOnlyID)).toMatchObject({ + // 3. Fresh step projected directly into session_part + yield* bus.publish(SessionEvent.Step.Started, { + sessionID, + assistantMessageID: splitID, + agent, + model, + started: 30, + }) + yield* bus.publish(SessionEvent.Reasoning.Started, { sessionID, assistantMessageID: splitID, ordinal: 0 }) + yield* bus.publish(SessionEvent.Reasoning.Ended, { + sessionID, + assistantMessageID: splitID, + ordinal: 0, + text: "step reasoning", + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { + sessionID, + assistantMessageID: splitID, + id: "call_1", + name: "bash", + }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID, + assistantMessageID: splitID, + id: "call_1", + input: { command: "ls" }, + executed: true, + }) + yield* bus.publish(SessionEvent.Tool.Success, { + sessionID, + assistantMessageID: splitID, + id: "call_1", + content: [{ type: "text", text: "ok" }], + executed: true, + }) + yield* bus.publish(SessionEvent.Text.Started, { sessionID, assistantMessageID: splitID, ordinal: 2 }) + yield* bus.publish(SessionEvent.Text.Ended, { + sessionID, + assistantMessageID: splitID, + ordinal: 2, + text: "step answer", + }) + + // Verify raw storage split on the mutated legacy row + const mutatedRaw = yield* database.db + .select() + .from(SessionMessageTable) + .where(eq(SessionMessageTable.id, legacyMutatedID)) + .get() + expect(mutatedRaw?.data).toMatchObject({ content: [] }) + const mutatedParts = yield* database.db + .select() + .from(SessionPartTable) + .where(eq(SessionPartTable.message_id, legacyMutatedID)) + .orderBy(asc(SessionPartTable.ordinal)) + .all() + expect(mutatedParts).toHaveLength(2) + + // Settle reads (single-message lookup) + expect(yield* store.message(legacyUntouchedID)).toMatchObject({ sessionID, message: { - id: inlineOnlyID, - content: [{ type: "text", text: "inline fallback content" }], + id: legacyUntouchedID, + content: [{ type: "text", text: "legacy inline only" }], + }, + }) + expect(yield* store.message(legacyMutatedID)).toMatchObject({ + sessionID, + message: { + id: legacyMutatedID, + content: [ + { type: "reasoning", text: "legacy reasoning" }, + { type: "text", text: "appended into session_part" }, + ], }, }) expect(yield* store.message(splitID)).toMatchObject({ @@ -302,26 +346,33 @@ describe("SessionStore", () => { message: { id: splitID, content: [ - { type: "reasoning", text: "first reasoning" }, - { type: "tool", id: "call_read", state: { status: "completed" } }, - { type: "text", text: "hydrated from session_part" }, + { type: "reasoning", text: "step reasoning" }, + { type: "tool", id: "call_1", state: { status: "completed" } }, + { type: "text", text: "step answer" }, ], }, }) + // Hydrate reads (paged store.messages and full SessionHistory.get) const hydrated = yield* store.messages({ sessionID, order: "asc" }) expect(hydrated).toMatchObject([ - { id: inlineOnlyID, content: [{ type: "text", text: "inline fallback content" }] }, + { id: legacyUntouchedID, content: [{ type: "text", text: "legacy inline only" }] }, + { + id: legacyMutatedID, + content: [ + { type: "reasoning", text: "legacy reasoning" }, + { type: "text", text: "appended into session_part" }, + ], + }, { id: splitID, content: [ - { type: "reasoning", text: "first reasoning" }, - { type: "tool", id: "call_read", state: { status: "completed" } }, - { type: "text", text: "hydrated from session_part" }, + { type: "reasoning", text: "step reasoning" }, + { type: "tool", id: "call_1", state: { status: "completed" } }, + { type: "text", text: "step answer" }, ], }, ]) - const rows = yield* database.db .select() .from(SessionMessageTable) @@ -329,6 +380,6 @@ describe("SessionStore", () => { .orderBy(asc(SessionMessageTable.seq)) .all() expect(yield* SessionHistory.decodeMessageRows(database.db, rows)).toEqual(hydrated) - }), + }).pipe(Effect.provide(partRows)), ) }) diff --git a/packages/core/test/sqlite-workerd.test.ts b/packages/core/test/sqlite-workerd.test.ts index ae5ef260114..07fd5ef7f5b 100644 --- a/packages/core/test/sqlite-workerd.test.ts +++ b/packages/core/test/sqlite-workerd.test.ts @@ -93,5 +93,19 @@ describe("sqlite.workerd", () => { .map((row) => row.name) expect(names).toContain("migration") expect(names).toContain("session_v2") + expect(() => storage.sql.exec("PRAGMA foreign_keys = OFF")).toThrow("not authorized") + + await run( + storage, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + yield* sql`CREATE TABLE _parent (id TEXT PRIMARY KEY)` + yield* sql`CREATE TABLE _child (parent_id TEXT NOT NULL REFERENCES _parent(id) ON DELETE CASCADE)` + yield* sql`INSERT INTO _parent (id) VALUES (${"p1"})` + yield* sql`INSERT INTO _child (parent_id) VALUES (${"p1"})` + yield* sql`DELETE FROM _parent WHERE id = ${"p1"}` + expect(yield* sql`SELECT * FROM _child`).toEqual([]) + }), + ) }) }) diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index 7eb93847a7a..5ed182b44ed 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -16664,6 +16664,9 @@ "media_blob": { "type": "boolean" }, + "part_rows": { + "type": "boolean" + }, "image_generate": { "anyOf": [ { diff --git a/packages/schema/src/config/experimental.ts b/packages/schema/src/config/experimental.ts index 123e24fd3c2..4f3eae37b8d 100644 --- a/packages/schema/src/config/experimental.ts +++ b/packages/schema/src/config/experimental.ts @@ -20,6 +20,10 @@ export class Info extends Schema.Class("ConfigExperimental.Info")({ description: "Store new session media once in a deduplicated blob table instead of inline in every message row. Defaults to false.", }), + part_rows: Schema.Boolean.pipe(optional).annotate({ + description: + "Write each assistant part as its own session_part row instead of rewriting the whole message on every event. Defaults to false.", + }), image_generate: Schema.Union([ Schema.Boolean, Schema.Struct({