perf(core): write each assistant part as its own row instead of rewriting the message (#58916)

This commit is contained in:
Kit Langton
2026-11-04 14:16:44 -08:00
committed by GitHub
Unverified
parent 88cc1e71af
commit 6000b7c51b
21 changed files with 2027 additions and 415 deletions
+3
View File
@@ -16,4 +16,7 @@
"tools": {
"github-pr-search": false,
},
"experimental": {
"part_rows": true,
},
}
+1
View File
@@ -2546,6 +2546,7 @@ export type ConfigEntry =
todo?: boolean
background_compaction?: boolean
media_blob?: boolean
part_rows?: boolean
image_generate?: boolean | { model?: string | { providerID: string; model: string; variant?: string } }
schedules?: boolean
artifacts?: boolean
+9
View File
@@ -495,6 +495,15 @@ function normalizeExperimental(
)
if (value !== undefined) result.media_blob = value
}
if (own(experimental, "part_rows")) {
const value = decodeEncoded(
ConfigExperimental.Info.fields.part_rows,
experimental.part_rows,
["experimental", "part_rows"],
diagnostics,
)
if (value !== undefined) result.part_rows = value
}
if (own(experimental, "image_generate")) {
const value = decodeEncoded(
ConfigExperimental.Info.fields.image_generate,
+21
View File
@@ -0,0 +1,21 @@
export * as DatabaseBatch from "./batch.js"
import { getTableColumns, type Table } from "drizzle-orm"
/** Durable Object SQLite rejects a statement that binds more than 100 parameters, so every runtime stays under it. */
export const MaxBoundParameters = 100
/**
* Splits `items` into batches whose statement binds at most {@link MaxBoundParameters}: `perItem` parameters for each
* item (a multi-row insert binds one per column) plus `fixed` for the rest of the statement.
*/
export const batches = <T>(items: ReadonlyArray<T>, perItem: number, fixed = 0): Array<Array<T>> => {
const size = Math.max(1, Math.floor((MaxBoundParameters - fixed) / perItem))
return Array.from({ length: Math.ceil(items.length / size) }, (_, index) =>
items.slice(index * size, (index + 1) * size),
)
}
/** {@link batches} for a multi-row insert into `table`, which binds one parameter per column of each row. */
export const insertBatches = <T>(table: Table, rows: ReadonlyArray<T>, fixed = 0) =>
batches(rows, Object.keys(getTableColumns(table)).length, fixed)
+4 -2
View File
@@ -35,8 +35,10 @@ const databaseLayer = (lock: Effect.Effect<Semaphore.Semaphore>) =>
Service,
Effect.gen(function* () {
const db = yield* makeDatabase
// An injected Durable Object client may run under a non-workerd module condition in tests.
const durableObject = EffectDrizzleSqlite.managesTransactionsNatively(db.$client)
if (supportsTuningPragmas) {
if (supportsTuningPragmas && !durableObject) {
yield* db.run("PRAGMA journal_mode = WAL")
yield* db.run("PRAGMA synchronous = NORMAL")
yield* db.run("PRAGMA busy_timeout = 5000")
@@ -44,7 +46,7 @@ const databaseLayer = (lock: Effect.Effect<Semaphore.Semaphore>) =>
yield* db.run("PRAGMA wal_checkpoint(PASSIVE)")
}
// Durable Object SQLite always enforces foreign keys and rejects the pragma.
if (supportsForeignKeyToggle) yield* db.run("PRAGMA foreign_keys = ON")
if (supportsForeignKeyToggle && !durableObject) yield* db.run("PRAGMA foreign_keys = ON")
const semaphore = yield* lock
yield* semaphore.withPermit(DatabaseMigration.apply(db))
+8 -4
View File
@@ -2,13 +2,14 @@ export * as DatabaseMigration from "./migration.js"
import { sql } from "drizzle-orm"
import { Effect } from "effect"
import type { SqlClient } from "effect/sql"
import { supportsForeignKeyToggle } from "#sqlite"
import type { EffectDrizzleSqlite } from "./drizzle.js"
import { EffectDrizzleSqlite } from "./drizzle.js"
import { migrations } from "./migration.gen.js"
import schema from "./schema.gen.js"
import { Global } from "@opencode/util/global"
type Database = EffectDrizzleSqlite.EffectSQLiteDatabase
type Database = EffectDrizzleSqlite.EffectSQLiteDatabase & { readonly $client?: SqlClient.SqlClient }
type Transaction = Parameters<Parameters<Database["transaction"]>[0]>[0]
export type Migration = {
@@ -122,10 +123,13 @@ export function applyOnly(db: Database, input: Migration[]) {
: Effect.gen(function* () {
// Durable Object SQLite rejects the foreign_keys toggle; the closest
// allowlisted relaxation is deferring enforcement to transaction commit.
const relaxForeignKeys = supportsForeignKeyToggle
const toggleForeignKeys =
supportsForeignKeyToggle &&
!(db.$client !== undefined && EffectDrizzleSqlite.managesTransactionsNatively(db.$client))
const relaxForeignKeys = toggleForeignKeys
? db.run(sql`PRAGMA foreign_keys = OFF`)
: db.run(sql`PRAGMA defer_foreign_keys = ON`)
const restoreForeignKeys = supportsForeignKeyToggle
const restoreForeignKeys = toggleForeignKeys
? Effect.asVoid(db.run(sql`PRAGMA foreign_keys = ON`))
: Effect.void
yield* relaxForeignKeys
+73 -52
View File
@@ -1,12 +1,13 @@
export * as SessionBlob from "./blob.js"
import { createHash } from "node:crypto"
import { and, eq, exists, gte, inArray, notExists, notInArray, type SQL, sql } from "drizzle-orm"
import { and, eq, exists, gte, inArray, notExists, type SQL, sql } from "drizzle-orm"
import { Cause, Context, Effect, Option, Schema } from "effect"
import { map } from "effect/Array"
import type { Attachment, FileAttachment } from "@opencode/schema/prompt"
import type { Tool } from "@opencode/schema/tool"
import { Config } from "../config.js"
import { DatabaseBatch } from "../database/batch.js"
import type { Database } from "../database/database.js"
import type { Instance } from "../instance/service.js"
import type { SessionMessage } from "./message.js"
@@ -29,12 +30,18 @@ const toolItems = (part: Part): ReadonlyArray<Tool.Content> =>
? (part.state.content ?? [])
: []
function collect(data: MessageRow["data"] | InboxRow["payload"], hashes: Set<string>) {
/** A stored row, one `session_part` row, or the `session_part` rows of one assistant message. */
type Refs = MessageRow["data"] | InboxRow["payload"] | Part | { readonly content: ReadonlyArray<Part> }
function collect(data: Refs, hashes: Set<string>) {
if ("files" in data)
for (const file of data.files ?? []) if (isFileRef(file)) hashes.add(SessionStored.refHash(file.uri))
if ("content" in data)
// A corrupt row's non-array content is left for decoding to reject.
if ("content" in data && Array.isArray(data.content))
for (const part of data.content)
for (const item of toolItems(part)) if (isToolFileRef(item)) hashes.add(SessionStored.refHash(item.uri))
if ("type" in data && data.type === "tool")
for (const item of toolItems(data)) if (isToolFileRef(item)) hashes.add(SessionStored.refHash(item.uri))
}
/** A ref whose blob is gone reads as this text, so clients and the model never see a bare ref. */
@@ -97,13 +104,12 @@ function rewriteInbox(row: InboxRow, file: (file: SessionStored.File) => Session
}
const load = Effect.fnUntraced(function* (db: DatabaseService, hashes: ReadonlySet<string>) {
const all = [...hashes]
const blobs = new Map<string, string>()
for (let offset = 0; offset < all.length; offset += 500) {
for (const batch of DatabaseBatch.batches([...hashes], 1)) {
const found = yield* db
.select({ hash: BlobTable.hash, data: BlobTable.data })
.from(BlobTable)
.where(inArray(BlobTable.hash, all.slice(offset, offset + 500)))
.where(inArray(BlobTable.hash, batch))
.all()
.pipe(Effect.orDie)
for (const blob of found) blobs.set(blob.hash, view(blob.data).toString("base64"))
@@ -166,14 +172,13 @@ export const keepInboxRefs = Effect.fnUntraced(function* (db: DatabaseService, r
const loadMeta = Effect.fnUntraced(function* (db: DatabaseService, hashes: ReadonlySet<string>) {
const meta = new Map<string, Meta>()
const all = [...hashes]
for (let offset = 0; offset < all.length; offset += 500) {
for (const batch of DatabaseBatch.batches([...hashes], 1)) {
const found = yield* db
.select({ hash: BlobTable.hash, bytes: BlobTable.bytes, width: BlobTable.width, height: BlobTable.height })
.from(BlobTable)
.where(
and(
inArray(BlobTable.hash, all.slice(offset, offset + 500)),
inArray(BlobTable.hash, batch),
exists(
db
.select({ one: sql`1` })
@@ -313,13 +318,8 @@ export const save = Effect.fnUntraced(function* (
db: DatabaseService,
blobs: ReadonlyArray<typeof BlobTable.$inferInsert>,
) {
if (blobs.length === 0) return
yield* db
.insert(BlobTable)
.values([...blobs])
.onConflictDoNothing()
.run()
.pipe(Effect.orDie)
for (const batch of DatabaseBatch.insertBatches(BlobTable, blobs))
yield* db.insert(BlobTable).values(batch).onConflictDoNothing().run().pipe(Effect.orDie)
})
/** {@link externalize} for a user message or pending inbox user payload. */
@@ -351,7 +351,17 @@ export const externalize = Effect.fnUntraced(function* (
return { ...message, content }
})
const hashesOf = (data: MessageRow["data"] | InboxRow["payload"]) => {
/** {@link externalize} for one assistant part. Returns the same part when nothing moved. */
export const externalizePart = Effect.fnUntraced(function* (db: DatabaseService, part: AssistantPart) {
if (!(yield* enabled)) return part
const pending: Pending = new Map()
const stored = toolPartRefs(pending, part)
if (pending.size === 0) return part
yield* save(db, [...pending.values()])
return stored
})
const hashesOf = (data: Refs) => {
const hashes = new Set<string>()
collect(data, hashes)
return hashes
@@ -368,17 +378,17 @@ function storedPartRefs(pending: Pending, part: Part): Part {
* Moves a stored row's inline media into blobs whatever the flag says, for offline backfills. Returns the row to
* store (the same object when nothing moved), the blobs to insert, and every hash the row then names.
*/
export function split<T extends MessageRow["data"] | InboxRow["payload"]>(
data: T,
): { data: T; moved: boolean; blobs: Array<typeof BlobTable.$inferInsert>; hashes: Set<string> }
export function split(data: MessageRow["data"] | InboxRow["payload"]) {
export function split<T extends MessageRow["data"] | InboxRow["payload"] | Part>(data: T) {
const pending: Pending = new Map()
const stored =
const stored = (
"files" in data && data.files?.some((file) => !isFileRef(file))
? { ...data, files: data.files.map((file) => fileRef(pending, file)) }
: "content" in data
? { ...data, content: data.content.map((part) => storedPartRefs(pending, part)) }
: data
: "type" in data && data.type === "tool"
? storedPartRefs(pending, data)
: data
) as T
const result = pending.size > 0 ? stored : data
return { data: result, moved: pending.size > 0, blobs: [...pending.values()], hashes: hashesOf(result) }
}
@@ -395,11 +405,7 @@ export const externalizeRow = Effect.fnUntraced(function* (db: DatabaseService,
})
/** Records that `owner` references each blob its stored row names, so release keeps them until it lets go. */
export const recordRefs = Effect.fnUntraced(function* (
db: DatabaseService,
owner: Owner,
data: MessageRow["data"] | InboxRow["payload"],
) {
export const recordRefs = Effect.fnUntraced(function* (db: DatabaseService, owner: Owner, data: Refs) {
yield* recordHashes(db, owner, hashesOf(data))
})
@@ -412,34 +418,50 @@ export const recordHashes = Effect.fnUntraced(function* (
owner: Owner,
hashes: ReadonlySet<string>,
) {
if (hashes.size === 0) return
// A ref whose blob is already gone keeps nothing alive; reads render it as unavailable.
const found = yield* db
.select({ hash: BlobTable.hash })
.from(BlobTable)
.where(inArray(BlobTable.hash, [...hashes]))
.all()
.pipe(Effect.orDie)
if (found.length === 0) return
yield* db
.insert(BlobRefTable)
.values(
found.map(({ hash }) => ({ session_id: owner.sessionID, message_id: owner.messageID, hash, seq: owner.seq })),
const found: Array<{ hash: string }> = []
for (const batch of DatabaseBatch.batches([...hashes], 1))
found.push(
...(yield* db
.select({ hash: BlobTable.hash })
.from(BlobTable)
.where(inArray(BlobTable.hash, batch))
.all()
.pipe(Effect.orDie)),
)
.onConflictDoUpdate({
target: [BlobRefTable.session_id, BlobRefTable.message_id, BlobRefTable.hash],
set: { seq: owner.seq },
})
.run()
.pipe(Effect.orDie)
const refs = found.map(({ hash }) => ({
session_id: owner.sessionID,
message_id: owner.messageID,
hash,
seq: owner.seq,
}))
// The conflict update binds `seq` once more.
for (const batch of DatabaseBatch.insertBatches(BlobRefTable, refs, 1))
yield* db
.insert(BlobRefTable)
.values(batch)
.onConflictDoUpdate({
target: [BlobRefTable.session_id, BlobRefTable.message_id, BlobRefTable.hash],
set: { seq: owner.seq },
})
.run()
.pipe(Effect.orDie)
})
/** Records the refs a rewritten message names and releases the ones it no longer does. */
export const syncRefs = Effect.fnUntraced(function* (db: DatabaseService, owner: Owner, data: MessageRow["data"]) {
export const syncRefs = Effect.fnUntraced(function* (db: DatabaseService, owner: Owner, data: Refs) {
yield* recordRefs(db, owner, data)
const present = [...hashesOf(data)]
const present = hashesOf(data)
const own = [eq(BlobRefTable.session_id, owner.sessionID), eq(BlobRefTable.message_id, owner.messageID)] as const
yield* present.length === 0 ? release(db, ...own) : release(db, ...own, notInArray(BlobRefTable.hash, present))
if (present.size === 0) return yield* release(db, ...own)
const stale = (yield* db
.select({ hash: BlobRefTable.hash })
.from(BlobRefTable)
.where(and(...own))
.all()
.pipe(Effect.orDie)).flatMap((ref) => (present.has(ref.hash) ? [] : [ref.hash]))
for (const batch of DatabaseBatch.batches(stale, 1, own.length))
yield* release(db, ...own, inArray(BlobRefTable.hash, batch))
})
/** Deletes the matching refs, then every blob among theirs that no ref names anymore. */
@@ -450,13 +472,12 @@ const release = Effect.fnUntraced(function* (db: DatabaseService, ...where: read
.returning({ hash: BlobRefTable.hash })
.all()
.pipe(Effect.orDie)
const hashes = [...new Set(released.map((ref) => ref.hash))]
for (let offset = 0; offset < hashes.length; offset += 500)
for (const batch of DatabaseBatch.batches([...new Set(released.map((ref) => ref.hash))], 1))
yield* db
.delete(BlobTable)
.where(
and(
inArray(BlobTable.hash, hashes.slice(offset, offset + 500)),
inArray(BlobTable.hash, batch),
notExists(
db
.select({ one: sql`1` })
+23 -5
View File
@@ -4,6 +4,7 @@ import { Effect, Option, Schema } from "effect"
import type { Model } from "@opencode/schema/model"
import { SessionLegacy } from "@opencode/schema/session-legacy"
import { Database } from "../database/database.js"
import { DatabaseBatch } from "../database/batch.js"
import { MessageDecodeError } from "./error.js"
import { SessionMessage } from "./message.js"
import type { SessionSchema } from "./schema.js"
@@ -20,7 +21,8 @@ const decode = Schema.decodeUnknownEffect(SessionMessage.Info)
const decodeSync = Schema.decodeUnknownSync(SessionMessage.Info)
const decodeStoredUser = Schema.decodeUnknownEffect(SessionStored.User)
const decodeMedia = Schema.decodeUnknownEffect(SessionMessage.MediaInfo)
const PartQueryBatchSize = 500
const decodePartSync = Schema.decodeUnknownSync(SessionMessage.AssistantContent)
export const encodePart = Schema.encodeSync(SessionMessage.AssistantContent)
/**
* Which completed compactions bound a history read. Local summaries always do. Native
@@ -87,7 +89,9 @@ const latestCompaction = Effect.fnUntraced(function* (
/**
* Puts each assistant row's `session_part` rows into its `data.content`, in ordinal order. Rows without parts keep
* their inline content, so legacy rows read unchanged. Runs before blob hydration so media in parts restores too.
* their inline content, so legacy rows read unchanged. A row an older build wrote inline content to after its parts
* were split reads as the parts followed by that content, the order the projector moves them in. Runs before blob
* hydration so media in parts restores too.
*/
export const withParts = Effect.fnUntraced(function* (
db: DatabaseService,
@@ -96,11 +100,11 @@ export const withParts = Effect.fnUntraced(function* (
const assistantIDs = rows.flatMap((row) => (row.type === "assistant" ? [row.id] : []))
if (assistantIDs.length === 0) return rows
const partsByMessage = new Map<SessionMessage.ID, Array<(typeof SessionPartTable.$inferSelect)["data"]>>()
for (let index = 0; index < assistantIDs.length; index += PartQueryBatchSize) {
for (const ids of DatabaseBatch.batches(assistantIDs, 1)) {
const partRows = yield* db
.select()
.from(SessionPartTable)
.where(inArray(SessionPartTable.message_id, assistantIDs.slice(index, index + PartQueryBatchSize)))
.where(inArray(SessionPartTable.message_id, ids))
.orderBy(asc(SessionPartTable.message_id), asc(SessionPartTable.ordinal))
.all()
.pipe(Effect.orDie)
@@ -113,7 +117,9 @@ export const withParts = Effect.fnUntraced(function* (
if (partsByMessage.size === 0) return rows
return rows.map((row) => {
const parts = partsByMessage.get(row.id)
return parts && "content" in row.data ? { ...row, data: { ...row.data, content: parts } } : row
if (!parts) return row
const inline = "content" in row.data ? row.data.content : undefined
return { ...row, data: { ...row.data, content: Array.isArray(inline) ? [...parts, ...inline] : parts } }
})
})
@@ -124,6 +130,9 @@ const rowInput = (row: typeof SessionMessageTable.$inferSelect) => ({
type: row.type,
})
export const decodePartRowSync = (row: typeof SessionPartTable.$inferSelect) =>
decodePartSync(SessionLegacy.part({ ...row.data, type: row.type }))
export const decodeMessageRowSync = (row: typeof SessionMessageTable.$inferSelect) => decodeSync(rowInput(row))
const decodeError = (row: typeof SessionMessageTable.$inferSelect) => () =>
@@ -151,6 +160,15 @@ export const decodeMessageRows = Effect.fnUntraced(function* (
})
})
/** {@link decodeMessageRows} for one row. */
export const loadMessageRow = Effect.fnUntraced(function* (
db: DatabaseService,
row: typeof SessionMessageTable.$inferSelect,
) {
const [hydrated = row] = yield* SessionBlob.hydrate(db, yield* withParts(db, [row]))
return yield* decodeMessageRow(hydrated)
})
/** Decodes rows for clients that fetch media themselves, so files stored in `blob` stay `media:` refs. */
export const decodeMessageRowsWithRefs = Effect.fnUntraced(function* (
db: DatabaseService,
+102 -96
View File
@@ -12,6 +12,23 @@ export interface Adapter {
readonly getShell: (shellID: SessionMessage.Shell["shellID"]) => Effect.Effect<SessionMessage.Shell | undefined>
readonly getCompaction: () => Effect.Effect<SessionMessage.Compaction | undefined>
readonly updateAssistant: (assistant: SessionMessage.Assistant) => Effect.Effect<void>
readonly appendAssistantPart: (
messageID: SessionMessage.ID,
part: SessionMessage.AssistantContent,
) => Effect.Effect<void>
readonly updateAssistantPart: <T extends SessionMessage.AssistantContent["type"]>(
messageID: SessionMessage.ID,
match: {
readonly type: T
readonly callID?: string
readonly incompleteReasoning?: boolean
},
recipe: (draft: WritableDraft<Extract<SessionMessage.AssistantContent, { readonly type: T }>>) => void,
) => Effect.Effect<void>
readonly replaceAssistantParts: (
messageID: SessionMessage.ID,
parts: ReadonlyArray<SessionMessage.AssistantContent>,
) => Effect.Effect<void>
readonly updateShell: (shell: SessionMessage.Shell) => Effect.Effect<void>
readonly updateCompaction: (compaction: SessionMessage.Compaction) => Effect.Effect<void>
readonly appendMessage: (message: SessionMessage.Info) => Effect.Effect<void>
@@ -31,20 +48,8 @@ const projectTerminalSnapshot = (draft: DraftAssistant, event: SessionEvent.Step
}
export function update(adapter: Adapter, event: SessionEvent.DurableEvent) {
type DraftTool = WritableDraft<SessionMessage.AssistantTool>
type DraftText = WritableDraft<SessionMessage.AssistantText>
type DraftReasoning = WritableDraft<SessionMessage.AssistantReasoning>
const created = DateTime.makeUnsafe(event.created)
const latestTool = (assistant: DraftAssistant, id: string) =>
assistant.content.findLast((item): item is DraftTool => item.type === "tool" && item.id === id)
const latestText = (assistant: DraftAssistant) =>
assistant.content.findLast((item): item is DraftText => item.type === "text")
const latestReasoning = (assistant: DraftAssistant) =>
assistant.content.findLast((item): item is DraftReasoning => item.type === "reasoning" && !item.time?.completed)
const updateOwnedAssistant = (messageID: SessionMessage.ID, recipe: (draft: DraftAssistant) => void) =>
Effect.gen(function* () {
const assistant = yield* adapter.getAssistant(messageID)
@@ -86,9 +91,7 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) {
"session.archived": () => Effect.void,
"session.unarchived": () => Effect.void,
"session.message.content.updated": (event) =>
updateOwnedAssistant(event.data.messageID, (draft) => {
draft.content = castDraft(decodeAssistantContent(event.data.content))
}),
adapter.replaceAssistantParts(event.data.messageID, decodeAssistantContent(event.data.content)),
"session.usage.recorded": () => Effect.void,
"session.agent.selected": (event) =>
Effect.gen(function* () {
@@ -289,44 +292,43 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) {
})
},
"session.text.started": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
draft.content.push(castDraft(SessionMessage.AssistantText.make({ type: "text", text: "" })))
})
return adapter.appendAssistantPart(
event.data.assistantMessageID,
SessionMessage.AssistantText.make({ type: "text", text: "" }),
)
},
"session.text.ended": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
const match = latestText(draft)
if (match) {
match.text = event.data.text
match.native = castDraft(event.data.native)
}
return adapter.updateAssistantPart(event.data.assistantMessageID, { type: "text" }, (match) => {
match.text = event.data.text
match.native = castDraft(event.data.native)
})
},
"session.tool.input.started": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
draft.content.push(
castDraft(
SessionMessage.AssistantTool.make({
type: "tool",
id: event.data.id,
name: event.data.name,
time: { created },
state: SessionMessage.ToolStateStreaming.make({ status: "streaming", input: "" }),
}),
),
)
})
return adapter.appendAssistantPart(
event.data.assistantMessageID,
SessionMessage.AssistantTool.make({
type: "tool",
id: event.data.id,
name: event.data.name,
time: { created },
state: SessionMessage.ToolStateStreaming.make({ status: "streaming", input: "" }),
}),
)
},
"session.tool.input.ended": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
const match = latestTool(draft, event.data.id)
if (match && match.state.status === "streaming") match.state.input = event.data.text
})
return adapter.updateAssistantPart(
event.data.assistantMessageID,
{ type: "tool", callID: event.data.id },
(match) => {
if (match.state.status === "streaming") match.state.input = event.data.text
},
)
},
"session.tool.called": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
const match = latestTool(draft, event.data.id)
if (match) {
return adapter.updateAssistantPart(
event.data.assistantMessageID,
{ type: "tool", callID: event.data.id },
(match) => {
match.executed = event.data.executed
match.native = event.data.native
match.time.ran = created
@@ -337,71 +339,75 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) {
metadata: {},
}),
)
}
})
},
)
},
// Terminal tool events are self-contained; projection is a direct copy and
// never reaches into ephemeral progress history.
"session.tool.success": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
const match = latestTool(draft, event.data.id)
if (match && match.state.status === "running") {
match.executed = event.data.executed || match.executed === true
if (event.data.native) match.native = { ...match.native, ...event.data.native }
match.time.completed = created
match.state = castDraft(
SessionMessage.ToolStateCompleted.make({
status: "completed",
input: match.state.input,
content: event.data.content,
...(event.data.metadata === undefined ? {} : { metadata: event.data.metadata }),
}),
)
}
})
return adapter.updateAssistantPart(
event.data.assistantMessageID,
{ type: "tool", callID: event.data.id },
(match) => {
if (match.state.status === "running") {
match.executed = event.data.executed || match.executed === true
if (event.data.native) match.native = { ...match.native, ...event.data.native }
match.time.completed = created
match.state = castDraft(
SessionMessage.ToolStateCompleted.make({
status: "completed",
input: match.state.input,
content: event.data.content,
...(event.data.metadata === undefined ? {} : { metadata: event.data.metadata }),
}),
)
}
},
)
},
"session.tool.failed": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
const match = latestTool(draft, event.data.id)
if (match && (match.state.status === "streaming" || match.state.status === "running")) {
match.executed = event.data.executed || match.executed === true
if (event.data.native) match.native = { ...match.native, ...event.data.native }
match.time.completed = created
match.state = castDraft(
SessionMessage.ToolStateError.make({
status: "error",
error: event.data.error,
input: typeof match.state.input === "string" ? {} : match.state.input,
...(event.data.content === undefined ? {} : { content: event.data.content }),
...(event.data.metadata === undefined ? {} : { metadata: event.data.metadata }),
}),
)
}
})
return adapter.updateAssistantPart(
event.data.assistantMessageID,
{ type: "tool", callID: event.data.id },
(match) => {
if (match.state.status === "streaming" || match.state.status === "running") {
match.executed = event.data.executed || match.executed === true
if (event.data.native) match.native = { ...match.native, ...event.data.native }
match.time.completed = created
match.state = castDraft(
SessionMessage.ToolStateError.make({
status: "error",
error: event.data.error,
input: typeof match.state.input === "string" ? {} : match.state.input,
...(event.data.content === undefined ? {} : { content: event.data.content }),
...(event.data.metadata === undefined ? {} : { metadata: event.data.metadata }),
}),
)
}
},
)
},
"session.reasoning.started": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
draft.content.push(
castDraft(
SessionMessage.AssistantReasoning.make({
type: "reasoning",
text: "",
native: event.data.native,
time: { created },
}),
),
)
})
return adapter.appendAssistantPart(
event.data.assistantMessageID,
SessionMessage.AssistantReasoning.make({
type: "reasoning",
text: "",
native: event.data.native,
time: { created },
}),
)
},
"session.reasoning.ended": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
const match = latestReasoning(draft)
if (match) {
return adapter.updateAssistantPart(
event.data.assistantMessageID,
{ type: "reasoning", incompleteReasoning: true },
(match) => {
match.text = event.data.text
match.time = { created: match.time?.created ?? created, completed: created }
if (event.data.native !== undefined) match.native = event.data.native
}
})
},
)
},
"session.retry.scheduled": (event) => {
return updateOwnedAssistant(event.data.assistantMessageID, (draft) => {
+458 -125
View File
@@ -1,9 +1,12 @@
export * as SessionProjector from "./projector.js"
import { castDraft, produce, type WritableDraft } from "immer"
import { and, asc, desc, eq, gt, gte, inArray, isNull, lt, lte, or, sql } from "drizzle-orm"
import { DateTime, Effect, Layer, Schema, Stream } from "effect"
import { DateTime, Effect, Layer, Option, Schema, Stream } from "effect"
import path from "path"
import { Config } from "../config.js"
import { Database } from "../database/database.js"
import { DatabaseBatch } from "../database/batch.js"
import { Bus } from "../bus.js"
import { makeGlobalNode } from "@opencode/util/effect/app-node"
import { Model } from "@opencode/schema/model"
@@ -46,6 +49,13 @@ type Usage = {
const ForkBatchSize = 500
const insertParts = (db: DatabaseService, rows: ReadonlyArray<typeof SessionPartTable.$inferInsert>) =>
Effect.forEach(
DatabaseBatch.insertBatches(SessionPartTable, rows),
(batch) => db.insert(SessionPartTable).values(batch).run().pipe(Effect.orDie),
{ discard: true },
)
const forkTitle = (value?: string) => {
if (value === undefined) return
const match = value.match(/^(.+) \(fork #(\d+)\)$/)
@@ -190,7 +200,6 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* (
const media = yield* SessionBlob.ForkMedia
let cursor = -1
while (copiedSeq !== undefined) {
// A parent message whose parts live in session_part is copied with them inline, so the fork reads the same.
const rows = yield* db
.select()
.from(SessionMessageTable)
@@ -208,7 +217,7 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* (
.orderBy(asc(SessionMessageTable.seq))
.limit(ForkBatchSize)
.all()
.pipe(Effect.orDie, Effect.flatMap((rows) => SessionHistory.withParts(db, rows)))
.pipe(Effect.orDie)
if (rows.length === 0) break
const values = yield* Effect.forEach(rows, (row) =>
@@ -220,6 +229,7 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* (
seq: row.seq,
time_created: row.time_created,
time_updated: row.time_updated,
// Split parents' envelopes carry no content, so only unsplit parents' inline media moves here.
data: media ? yield* SessionBlob.externalizeRow(db, row.data) : row.data,
}
// A summary references the first message it kept, which the fork copied under a new ID.
@@ -238,7 +248,11 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* (
}
}),
)
yield* db.insert(SessionMessageTable).values(values).run().pipe(Effect.orDie)
yield* Effect.forEach(
DatabaseBatch.insertBatches(SessionMessageTable, values),
(batch) => db.insert(SessionMessageTable).values(batch).run().pipe(Effect.orDie),
{ discard: true },
)
yield* Effect.forEach(
values,
(value) =>
@@ -250,6 +264,48 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* (
{ discard: true },
)
const assistantRows = rows.filter((row) => row.type === "assistant")
if (assistantRows.length > 0) {
const idMap = new Map(assistantRows.map((row) => [row.id, forkedID(row.seq)]))
const storedParts = yield* Effect.forEach(DatabaseBatch.batches([...idMap.keys()], 1, 1), (ids) =>
db
.select()
.from(SessionPartTable)
.where(and(eq(SessionPartTable.session_id, event.data.parentID), inArray(SessionPartTable.message_id, ids)))
.orderBy(asc(SessionPartTable.message_id), asc(SessionPartTable.ordinal))
.all()
.pipe(Effect.orDie),
).pipe(Effect.map((batches) => batches.flat()))
const partValues: Array<typeof SessionPartTable.$inferInsert> = yield* Effect.forEach(storedParts, (part) =>
Effect.gen(function* () {
const data = media ? SessionBlob.split(part.data) : undefined
if (data?.moved) yield* SessionBlob.save(db, data.blobs)
return {
message_id: idMap.get(part.message_id)!,
session_id: event.data.sessionID,
ordinal: part.ordinal,
type: part.type,
call_id: part.call_id,
time_created: part.time_created,
time_updated: part.time_updated,
data: data?.data ?? part.data,
}
}),
)
yield* insertParts(db, partValues)
// Split parents' envelopes carry no content, so the parts' refs are recorded for each forked message here.
const partsByMessage = Map.groupBy(partValues, (part) => part.message_id)
yield* Effect.forEach(
assistantRows,
(row) => {
const messageID = idMap.get(row.id)!
const content = (partsByMessage.get(messageID) ?? []).map((part) => part.data)
return SessionBlob.recordRefs(db, { sessionID: event.data.sessionID, messageID, seq: row.seq }, { content })
},
{ discard: true },
)
}
cursor = rows.at(-1)!.seq
}
if (copiedSeq !== undefined) yield* Bus.reserveSequence(db, event.data.sessionID, copiedSeq)
@@ -257,86 +313,32 @@ const projectFork = Effect.fn("SessionProjector.projectFork")(function* (
yield* InstructionState.initialize(db, event.data.sessionID, event.durable.seq, event.data.instructions)
})
/** Reads `experimental.part_rows` from the Config in scope on every call; without one, it is off. */
const partRowsEnabled = Effect.gen(function* () {
const config = yield* Effect.serviceOption(Config.Service)
if (Option.isNone(config)) return false
return Config.latest(yield* config.value.entries(), "experimental")?.part_rows === true
})
type MessageRow = typeof SessionMessageTable.$inferSelect
type PartRow = typeof SessionPartTable.$inferSelect
type Writes = Pick<
SessionMessageUpdater.Adapter,
| "getCurrentAssistant"
| "getAssistant"
| "updateAssistant"
| "appendAssistantPart"
| "updateAssistantPart"
| "replaceAssistantParts"
| "updateShell"
| "updateCompaction"
>
function run(db: DatabaseService, event: MessageEvent) {
return Effect.gen(function* () {
const partRows = (messageID: SessionMessage.ID) =>
db
.select()
.from(SessionPartTable)
.where(eq(SessionPartTable.message_id, messageID))
.orderBy(asc(SessionPartTable.ordinal))
.all()
.pipe(Effect.orDie)
// Writes `content` over a message's existing part rows, touching only the rows that changed.
const writeParts = Effect.fnUntraced(function* (
messageID: SessionMessage.ID,
existing: ReadonlyArray<typeof SessionPartTable.$inferSelect>,
content: ReadonlyArray<SessionPartData>,
) {
let next = (existing.at(-1)?.ordinal ?? -1) + 1
for (const [index, part] of content.entries()) {
const current = existing[index]
const columns = { type: part.type, call_id: part.type === "tool" ? part.id : null, data: part }
if (current === undefined)
yield* db
.insert(SessionPartTable)
.values({
...columns,
message_id: messageID,
session_id: event.data.sessionID,
ordinal: next++,
time_created: event.created,
time_updated: event.created,
})
.run()
.pipe(Effect.orDie)
else if (JSON.stringify(current.data) !== JSON.stringify(part))
yield* db
.update(SessionPartTable)
.set({ ...columns, time_updated: event.created })
.where(and(eq(SessionPartTable.message_id, messageID), eq(SessionPartTable.ordinal, current.ordinal)))
.run()
.pipe(Effect.orDie)
}
const stale = existing.slice(content.length).map((part) => part.ordinal)
if (stale.length > 0)
yield* db
.delete(SessionPartTable)
.where(and(eq(SessionPartTable.message_id, messageID), inArray(SessionPartTable.ordinal, stale)))
.run()
.pipe(Effect.orDie)
})
const decodeAssistantRow = Effect.fnUntraced(function* (row: typeof SessionMessageTable.$inferSelect) {
const [withParts = row] = yield* SessionHistory.withParts(db, [row])
return SessionHistory.decodeMessageRowSync(withParts)
})
const updateMessage = Effect.fnUntraced(function* (message: SessionMessage.Info) {
const stored = yield* SessionBlob.externalize(db, message)
const { id: _, type, ...data } = encodeStored(stored)
// A newer release may have moved this message's parts to session_part. Reads prefer those rows, so updates
// go there too and the row keeps an empty envelope; messages without part rows are written inline as before.
const parts = message.type === "assistant" && "content" in data ? yield* partRows(message.id) : []
const updated = yield* db
.update(SessionMessageTable)
.set({
type,
time_created: DateTime.toEpochMillis(message.time.created),
data: parts.length > 0 && "content" in data ? { ...data, content: [] } : data,
})
.where(and(eq(SessionMessageTable.id, message.id), eq(SessionMessageTable.session_id, event.data.sessionID)))
.returning({ seq: SessionMessageTable.seq })
.get()
.pipe(Effect.orDie)
if (!updated) return
if (parts.length > 0 && "content" in data) yield* writeParts(message.id, parts, data.content)
const owner = { sessionID: event.data.sessionID, messageID: message.id, seq: updated.seq }
// Most updates only carry existing refs forward; a content rewrite may also drop some. `externalize` returns
// the same message when it extracted nothing, so streaming updates skip the ref tables entirely.
if (event.type === SessionEvent.MessageContentUpdated.type) yield* SessionBlob.syncRefs(db, owner, data)
else if (stored !== message) yield* SessionBlob.recordRefs(db, owner, data)
})
const appendMessage = (message: SessionMessage.Info) => insertMessage(db, event, message)
const writes = (yield* partRowsEnabled) ? partRowWrites(db, event) : messageWrites(db, event)
const adapter: SessionMessageUpdater.Adapter = {
...writes,
getAgent() {
return db
.select({ agent: SessionTable.agent })
@@ -386,43 +388,6 @@ function run(db: DatabaseService, event: MessageEvent) {
),
)
},
getCurrentAssistant() {
return Effect.gen(function* () {
// A newer step supersedes stale incomplete rows; never resume an older assistant projection.
const row = yield* db
.select()
.from(SessionMessageTable)
.where(
and(eq(SessionMessageTable.session_id, event.data.sessionID), eq(SessionMessageTable.type, "assistant")),
)
.orderBy(desc(SessionMessageTable.seq))
.limit(1)
.get()
.pipe(Effect.orDie)
if (!row || ("completed" in row.data.time && row.data.time.completed)) return
const message = yield* decodeAssistantRow(row)
return message.type === "assistant" && !message.time.completed ? message : undefined
})
},
getAssistant(messageID) {
return Effect.gen(function* () {
const row = yield* db
.select()
.from(SessionMessageTable)
.where(
and(
eq(SessionMessageTable.id, messageID),
eq(SessionMessageTable.session_id, event.data.sessionID),
eq(SessionMessageTable.type, "assistant"),
),
)
.get()
.pipe(Effect.orDie)
if (!row) return
const message = yield* decodeAssistantRow(row)
return message.type === "assistant" ? message : undefined
})
},
getShell(shellID) {
return Effect.gen(function* () {
const row = yield* db
@@ -465,15 +430,368 @@ function run(db: DatabaseService, event: MessageEvent) {
return message.type === "compaction" ? message : undefined
})
},
updateAssistant: updateMessage,
updateShell: updateMessage,
updateCompaction: updateMessage,
appendMessage,
appendMessage: (message) => insertMessage(db, event, message),
}
yield* SessionMessageUpdater.update(adapter, event)
})
}
/** Assistant reads shared by both write paths; `decode` decides whether part rows are folded into `content`. */
function assistantReads(
db: DatabaseService,
event: MessageEvent,
decode: (row: MessageRow) => Effect.Effect<SessionMessage.Info>,
): Pick<Writes, "getCurrentAssistant" | "getAssistant"> {
return {
getCurrentAssistant() {
return Effect.gen(function* () {
// A newer step supersedes stale incomplete rows; never resume an older assistant projection.
const row = yield* db
.select()
.from(SessionMessageTable)
.where(
and(eq(SessionMessageTable.session_id, event.data.sessionID), eq(SessionMessageTable.type, "assistant")),
)
.orderBy(desc(SessionMessageTable.seq))
.limit(1)
.get()
.pipe(Effect.orDie)
if (!row || ("completed" in row.data.time && row.data.time.completed)) return
const message = yield* decode(row)
return message.type === "assistant" && !message.time.completed ? message : undefined
})
},
getAssistant(messageID) {
return Effect.gen(function* () {
const row = yield* db
.select()
.from(SessionMessageTable)
.where(
and(
eq(SessionMessageTable.id, messageID),
eq(SessionMessageTable.session_id, event.data.sessionID),
eq(SessionMessageTable.type, "assistant"),
),
)
.get()
.pipe(Effect.orDie)
if (!row) return
const message = yield* decode(row)
return message.type === "assistant" ? message : undefined
})
},
}
}
/**
* The write path with `experimental.part_rows` off, unchanged from before part rows were written: every assistant
* event rewrites the whole message. A message whose parts already live in session_part (written with the flag on,
* or by a newer release) is written into those rows, and its row keeps an empty envelope.
*/
function messageWrites(db: DatabaseService, event: MessageEvent): Writes {
const partRows = (messageID: SessionMessage.ID) =>
db
.select()
.from(SessionPartTable)
.where(eq(SessionPartTable.message_id, messageID))
.orderBy(asc(SessionPartTable.ordinal))
.all()
.pipe(Effect.orDie)
// Writes `content` over a message's existing part rows, touching only the rows that changed.
const writeParts = Effect.fnUntraced(function* (
messageID: SessionMessage.ID,
existing: ReadonlyArray<PartRow>,
content: ReadonlyArray<SessionPartData>,
) {
let next = (existing.at(-1)?.ordinal ?? -1) + 1
for (const [index, part] of content.entries()) {
const current = existing[index]
const columns = { type: part.type, call_id: part.type === "tool" ? part.id : null, data: part }
if (current === undefined)
yield* db
.insert(SessionPartTable)
.values({
...columns,
message_id: messageID,
session_id: event.data.sessionID,
ordinal: next++,
time_created: event.created,
time_updated: event.created,
})
.run()
.pipe(Effect.orDie)
else if (JSON.stringify(current.data) !== JSON.stringify(part))
yield* db
.update(SessionPartTable)
.set({ ...columns, time_updated: event.created })
.where(and(eq(SessionPartTable.message_id, messageID), eq(SessionPartTable.ordinal, current.ordinal)))
.run()
.pipe(Effect.orDie)
}
const stale = existing.slice(content.length).map((part) => part.ordinal)
if (stale.length > 0)
yield* db
.delete(SessionPartTable)
.where(and(eq(SessionPartTable.message_id, messageID), inArray(SessionPartTable.ordinal, stale)))
.run()
.pipe(Effect.orDie)
})
const updateMessage = Effect.fnUntraced(function* (message: SessionMessage.Info) {
const stored = yield* SessionBlob.externalize(db, message)
const { id: _, type, ...data } = encodeStored(stored)
const parts = message.type === "assistant" && "content" in data ? yield* partRows(message.id) : []
const updated = yield* db
.update(SessionMessageTable)
.set({
type,
time_created: DateTime.toEpochMillis(message.time.created),
data: parts.length > 0 && "content" in data ? { ...data, content: [] } : data,
})
.where(and(eq(SessionMessageTable.id, message.id), eq(SessionMessageTable.session_id, event.data.sessionID)))
.returning({ seq: SessionMessageTable.seq })
.get()
.pipe(Effect.orDie)
if (!updated) return
if (parts.length > 0 && "content" in data) yield* writeParts(message.id, parts, data.content)
const owner = { sessionID: event.data.sessionID, messageID: message.id, seq: updated.seq }
// Most updates only carry existing refs forward; a content rewrite may also drop some. `externalize` returns
// the same message when it extracted nothing, so streaming updates skip the ref tables entirely.
if (event.type === SessionEvent.MessageContentUpdated.type) yield* SessionBlob.syncRefs(db, owner, data)
else if (stored !== message) yield* SessionBlob.recordRefs(db, owner, data)
})
const reads = assistantReads(db, event, (row) =>
SessionHistory.withParts(db, [row]).pipe(
Effect.map(([withParts = row]) => SessionHistory.decodeMessageRowSync(withParts)),
),
)
const editAssistant = (
messageID: SessionMessage.ID,
recipe: (draft: WritableDraft<SessionMessage.Assistant>) => void,
) =>
Effect.gen(function* () {
const assistant = yield* reads.getAssistant(messageID)
if (!assistant) return
yield* updateMessage(produce(assistant, recipe))
})
return {
...reads,
updateAssistant: updateMessage,
updateShell: updateMessage,
updateCompaction: updateMessage,
appendAssistantPart: (messageID, part) =>
editAssistant(messageID, (draft) => {
draft.content.push(castDraft(part))
}),
updateAssistantPart: (messageID, match, recipe) =>
editAssistant(messageID, (draft) => {
const found = draft.content.findLast(
(
item,
): item is WritableDraft<Extract<SessionMessage.AssistantContent, { readonly type: typeof match.type }>> =>
item.type === match.type &&
(match.callID === undefined || (item.type === "tool" && item.id === match.callID)) &&
(!match.incompleteReasoning || (item.type === "reasoning" && !item.time?.completed)),
)
if (found) recipe(found)
}),
replaceAssistantParts: (messageID, parts) =>
editAssistant(messageID, (draft) => {
draft.content = castDraft([...parts])
}),
}
}
/** The write path with `experimental.part_rows` on: each assistant part is its own session_part row. */
function partRowWrites(db: DatabaseService, event: MessageEvent): Writes {
/**
* The only place a legacy inline assistant row moves to session_part, called right before its parts are written.
* `carry: false` drops the inline parts instead of copying them, for a write that replaces them all. Returns the
* owner the message's part refs belong to.
*/
const prepareAssistantParts = Effect.fnUntraced(function* (messageID: SessionMessage.ID, carry: boolean) {
const row = yield* db
.select({
id: SessionMessageTable.id,
seq: SessionMessageTable.seq,
inlineCount: sql<number>`json_array_length(${SessionMessageTable.data}, '$.content')`,
})
.from(SessionMessageTable)
.where(
and(
eq(SessionMessageTable.id, messageID),
eq(SessionMessageTable.session_id, event.data.sessionID),
eq(SessionMessageTable.type, "assistant"),
),
)
.get()
.pipe(Effect.orDie)
if (!row) return
const full =
row.inlineCount > 0
? yield* db
.select()
.from(SessionMessageTable)
.where(eq(SessionMessageTable.id, messageID))
.get()
.pipe(Effect.orDie)
: undefined
const message = full ? SessionHistory.decodeMessageRowSync(full) : undefined
if (full && message?.type === "assistant") {
if (carry) {
// An older build that wrote inline content while part rows existed leaves both shapes on one message.
// Keep the rows and append the inline parts after them, so a downgrade and re-upgrade loses nothing.
const last = yield* db
.select({ ordinal: sql<number | null>`max(${SessionPartTable.ordinal})` })
.from(SessionPartTable)
.where(eq(SessionPartTable.message_id, messageID))
.get()
.pipe(Effect.orDie)
const first = (last?.ordinal ?? -1) + 1
const partRows = message.content.map((part, index) => ({
message_id: messageID,
session_id: event.data.sessionID,
ordinal: first + index,
type: part.type,
call_id: part.type === "tool" ? part.id : null,
time_created: full.time_created,
time_updated: event.created,
data: SessionHistory.encodePart(part),
}))
yield* insertParts(db, partRows)
}
const { id: _, type: __, ...data } = encodeMessage({ ...message, content: [] })
yield* db
.update(SessionMessageTable)
.set({ data })
.where(and(eq(SessionMessageTable.id, messageID), eq(SessionMessageTable.session_id, event.data.sessionID)))
.run()
.pipe(Effect.orDie)
}
return { sessionID: event.data.sessionID, messageID, seq: row.seq } satisfies SessionBlob.Owner
})
// Assistant content is written only by the part methods below, which also record its blob refs. Envelope updates
// keep whatever content the row already holds: inline for a legacy row that was never split, [] once it was.
const updateMessage = (message: SessionMessage.Info) => {
const {
id: _,
type: __,
...data
} = encodeMessage(message.type === "assistant" ? { ...message, content: [] } : message)
return db
.update(SessionMessageTable)
.set({
...(event.type === SessionEvent.Step.Started.type
? { time_created: DateTime.toEpochMillis(message.time.created) }
: {}),
data:
message.type === "assistant"
? sql`json_set(${JSON.stringify(data)}, '$.content', json(coalesce(${SessionMessageTable.data} -> '$.content', '[]')))`
: data,
})
.where(and(eq(SessionMessageTable.id, message.id), eq(SessionMessageTable.session_id, event.data.sessionID)))
.run()
.pipe(Effect.orDie)
}
const reads = assistantReads(db, event, (row) => Effect.sync(() => SessionHistory.decodeMessageRowSync(row)))
return {
...reads,
appendAssistantPart(messageID, part) {
return Effect.gen(function* () {
const owner = yield* prepareAssistantParts(messageID, true)
if (!owner) return
const data = SessionHistory.encodePart(yield* SessionBlob.externalizePart(db, part))
const next = yield* db
.select({ ordinal: sql<number>`coalesce(max(${SessionPartTable.ordinal}) + 1, 0)` })
.from(SessionPartTable)
.where(eq(SessionPartTable.message_id, messageID))
.get()
.pipe(Effect.orDie)
yield* db
.insert(SessionPartTable)
.values({
message_id: messageID,
session_id: event.data.sessionID,
ordinal: next?.ordinal ?? 0,
type: part.type,
call_id: part.type === "tool" ? part.id : null,
time_created: event.created,
time_updated: event.created,
data,
})
.run()
.pipe(Effect.orDie)
yield* SessionBlob.recordRefs(db, owner, { content: [data] })
})
},
updateAssistantPart(messageID, match, recipe) {
return Effect.gen(function* () {
const owner = yield* prepareAssistantParts(messageID, true)
if (!owner) return
const row = yield* db
.select()
.from(SessionPartTable)
.where(
and(
eq(SessionPartTable.message_id, messageID),
eq(SessionPartTable.session_id, event.data.sessionID),
eq(SessionPartTable.type, match.type),
match.callID !== undefined ? eq(SessionPartTable.call_id, match.callID) : undefined,
match.incompleteReasoning
? sql`json_extract(${SessionPartTable.data}, '$.time.completed') is null`
: undefined,
),
)
.orderBy(desc(SessionPartTable.ordinal))
.limit(1)
.get()
.pipe(Effect.orDie)
if (!row) return
const current = SessionHistory.decodePartRowSync(row) as Extract<
SessionMessage.AssistantContent,
{ readonly type: typeof match.type }
>
const updated = produce(current, recipe)
if (updated === current) return
// `externalizePart` returns the same part when it moved nothing, so streaming updates skip the ref tables.
const stored = yield* SessionBlob.externalizePart(db, updated)
const data = SessionHistory.encodePart(stored)
yield* db
.update(SessionPartTable)
.set({ data, time_updated: event.created })
.where(and(eq(SessionPartTable.message_id, messageID), eq(SessionPartTable.ordinal, row.ordinal)))
.run()
.pipe(Effect.orDie)
if (stored !== updated) yield* SessionBlob.recordRefs(db, owner, { content: [data] })
})
},
replaceAssistantParts(messageID, parts) {
return Effect.gen(function* () {
const owner = yield* prepareAssistantParts(messageID, false)
if (!owner) return
yield* db.delete(SessionPartTable).where(eq(SessionPartTable.message_id, messageID)).run().pipe(Effect.orDie)
const stored = yield* Effect.forEach(parts, (part) => SessionBlob.externalizePart(db, part))
const partRows = stored.map((part, ordinal) => ({
message_id: messageID,
session_id: event.data.sessionID,
ordinal,
type: part.type,
call_id: part.type === "tool" ? part.id : null,
time_created: event.created,
time_updated: event.created,
data: SessionHistory.encodePart(part),
}))
yield* insertParts(db, partRows)
// A content rewrite may drop media, so refs the new parts no longer name are released.
yield* SessionBlob.syncRefs(db, owner, { content: partRows.map((part) => part.data) })
})
},
updateAssistant: updateMessage,
updateShell: updateMessage,
updateCompaction: updateMessage,
}
}
const encodeStored = (message: SessionMessage.Info | SessionStored.User) =>
message.type === "user" ? encodeStoredUser(message) : encodeMessage(message)
@@ -482,7 +800,11 @@ const insertMessage = Effect.fnUntraced(function* (
event: SessionEvent.DurableEvent,
message: SessionMessage.Info | SessionStored.User,
) {
const { id: _, type, ...data } = encodeStored(yield* SessionBlob.externalize(db, message))
const stored = yield* SessionBlob.externalize(db, message)
const { id: _, type, ...data } = encodeStored(stored)
const timeCreated = DateTime.toEpochMillis(message.time.created)
// Assistant content goes to session_part rows; the message row keeps an empty envelope.
const parts = stored.type === "assistant" ? stored.content : []
yield* db
.insert(SessionMessageTable)
.values({
@@ -490,11 +812,22 @@ const insertMessage = Effect.fnUntraced(function* (
session_id: event.data.sessionID,
type,
seq: event.durable.seq,
time_created: DateTime.toEpochMillis(message.time.created),
data,
time_created: timeCreated,
data: "content" in data && parts.length > 0 ? { ...data, content: [] } : data,
})
.run()
.pipe(Effect.orDie)
const partRows = parts.map((part, ordinal) => ({
message_id: message.id,
session_id: event.data.sessionID,
ordinal,
type: part.type,
call_id: part.type === "tool" ? part.id : null,
time_created: timeCreated,
time_updated: event.created,
data: SessionHistory.encodePart(part),
}))
yield* insertParts(db, partRows)
yield* SessionBlob.recordRefs(
db,
{ sessionID: event.data.sessionID, messageID: message.id, seq: event.durable.seq },
+8 -7
View File
@@ -11,6 +11,7 @@ import { SessionEvent } from "@opencode/schema/session-event"
import { ToolMode } from "@opencode/schema/session-stats"
import { TokenUsage } from "@opencode/schema/token-usage"
import { Database } from "../database/database.js"
import { DatabaseBatch } from "../database/batch.js"
import { EventTable } from "../event/sql.js"
import { SessionMessageTable, SessionPartTable, SessionTable } from "./sql.js"
@@ -212,7 +213,7 @@ export const get = Effect.fn("SessionStats.get")(function* (input: Input = {}) {
AND message.time_created >= ${range.from}
AND message.time_created < ${range.to}
AND (session.fork_session_id IS NULL OR message.time_created >= session.time_created)
AND NOT EXISTS (SELECT 1 FROM ${SessionPartTable} AS part WHERE part.message_id = message.id)
AND json_array_length(message.data, '$.content') > 0
AND json_extract(content.value, '$.type') = 'tool'
${project}
)
@@ -273,7 +274,7 @@ export const get = Effect.fn("SessionStats.get")(function* (input: Input = {}) {
AND message.time_created >= ${range.from}
AND message.time_created < ${range.to}
AND (session.fork_session_id IS NULL OR message.time_created >= session.time_created)
AND NOT EXISTS (SELECT 1 FROM ${SessionPartTable} AS part WHERE part.message_id = message.id)
AND json_array_length(message.data, '$.content') > 0
AND json_extract(content.value, '$.type') = 'tool'
${project}
`,
@@ -312,7 +313,11 @@ export const get = Effect.fn("SessionStats.get")(function* (input: Input = {}) {
.all()
.pipe(Effect.orDie)
const events = (yield* Effect.forEach(
batches(ids.map((row) => row.id)),
DatabaseBatch.batches(
ids.map((row) => row.id),
1,
3,
),
(batch) =>
db
.select({ data: EventTable.data })
@@ -384,10 +389,6 @@ function windows(from: number, to: number) {
}))
}
function batches(ids: string[]) {
return Array.from({ length: Math.ceil(ids.length / 500) }, (_, index) => ids.slice(index * 500, (index + 1) * 500))
}
function rowTokens(row: MessageRow): Tokens {
return {
input: row.input ?? 0,
+3 -1
View File
@@ -11,6 +11,7 @@ import { makeGlobalNode } from "@opencode/util/effect/app-node"
import { App } from "../app.js"
import { Bus } from "../bus.js"
import { Database } from "../database/database.js"
import { DatabaseBatch } from "../database/batch.js"
import { Instance } from "../instance/service.js"
import { Location } from "../location.js"
import { Project } from "../project.js"
@@ -129,7 +130,8 @@ const layer = Layer.effect(
const stored = media
? messages.map((message, i) => ({ ...message, data: split[i]!.data }))
: messages
yield* db.insert(SessionMessageTable).values(stored).run().pipe(Effect.orDie)
for (const batch of DatabaseBatch.insertBatches(SessionMessageTable, stored))
yield* db.insert(SessionMessageTable).values(batch).run().pipe(Effect.orDie)
if (media)
yield* Effect.forEach(
stored,
@@ -553,6 +553,7 @@ describe("ConfigNormalize", () => {
todo: true,
background_compaction: true,
media_blob: true,
part_rows: true,
image_generate: { model: "openai/gpt-image-2" },
subagent_depth: 0,
location_idle_timeout: "2 hours",
@@ -565,6 +566,7 @@ describe("ConfigNormalize", () => {
todo: true,
background_compaction: true,
media_blob: true,
part_rows: true,
image_generate: { model: { providerID: "openai", model: "gpt-image-2" } },
subagent_depth: 0,
location_idle_timeout: "7200000 millis",
@@ -6,6 +6,8 @@ import type { DurableObjectStorage } from "@opencode/core/database/sqlite.worker
// Cloudflare runtime dependencies.
export const makeDurableObjectStorage = (): DurableObjectStorage => {
const native = new Database(":memory:")
// Durable Object SQLite compiles with SQLITE_DEFAULT_FOREIGN_KEYS=1 and rejects toggling it via PRAGMA.
native.run("PRAGMA foreign_keys = ON")
const toSqlStorageValue = (value: unknown) => {
if (!(value instanceof Uint8Array)) return value as ArrayBuffer | string | number | null
const buffer = new ArrayBuffer(value.byteLength)
@@ -15,6 +17,9 @@ export const makeDurableObjectStorage = (): DurableObjectStorage => {
return {
sql: {
exec(query: string, ...bindings: Array<unknown>) {
// Durable Object SQLite rejects a statement with more than 100 bound parameters.
if (bindings.length > 100) throw new Error(`too many SQL variables: ${bindings.length}`)
if (/^\s*pragma\s+(?:\w+\.)?foreign_keys\b/i.test(query)) throw new Error("not authorized")
const statement = native.query(query)
const rows = (statement.values(...(bindings as never[])) ?? []).map((row) => row.map(toSqlStorageValue))
const columnNames = statement.columnNames
+60 -7
View File
@@ -42,6 +42,8 @@ import { offlineModels } from "./fixture/models"
import { promptLocationNode } from "./fixture/prompt-location"
import { globalProjectNode } from "./lib/project"
import { tmpdirScoped } from "./fixture/tmpdir"
import { makeDurableObjectStorage } from "./fixture/durable-object-storage"
import { sqliteLayer } from "@opencode/core/database/sqlite.workerd"
const it = testEffect(
AppNodeBuilder.build(
@@ -62,6 +64,29 @@ const it = testEffect(
],
),
)
// Durable Object SQLite, which rejects statements that bind more than 100 parameters.
const durableIt = testEffect(
AppNodeBuilder.build(
LayerNode.group([
Database.node,
Bus.node,
SessionProjector.node,
SessionStore.node,
Session.node,
SessionTransfer.node,
InstructionEntry.node,
]),
[
Bus.node.replace(Bus.configured({ persist: true })),
Project.node.replace(globalProjectNode),
LocationServiceMap.node.replace(promptLocationNode),
SessionExecution.node.replace(SessionExecution.noopLayer),
Database.node.replace(
Database.configuredClient(Layer.suspend(() => sqliteLayer({ storage: makeDurableObjectStorage() }))),
),
],
),
)
const liveIt = testEffect(
AppNodeBuilder.build(
LayerNode.group([Database.node, Bus.node, Project.node, SessionProjector.node, SessionStore.node, Session.node]),
@@ -358,9 +383,7 @@ describe("Session.create", () => {
const created = yield* session.create({ location: { directory: AbsolutePath.make("/project/") } })
expect(created.location.directory).toBe(location.directory)
expect((yield* session.list({ directory: location.directory })).data.map((item) => item.id)).toEqual([
created.id,
])
expect((yield* session.list({ directory: location.directory })).data.map((item) => item.id)).toEqual([created.id])
}),
)
@@ -852,9 +875,10 @@ describe("Session.create", () => {
const session = yield* Session.Service
const parent = yield* session.create({ location })
expect(
yield* session.fork({ sessionID: parent.id }).pipe(Effect.flip),
).toMatchObject({ _tag: "Session.ForkEmptyError", sessionID: parent.id })
expect(yield* session.fork({ sessionID: parent.id }).pipe(Effect.flip)).toMatchObject({
_tag: "Session.ForkEmptyError",
sessionID: parent.id,
})
}),
)
@@ -969,7 +993,12 @@ describe("Session.create", () => {
const input = { id, location }
const created = yield* session.create(input)
yield* db.update(SessionTable).set({ agent: Agent.defaultID }).where(eq(SessionTable.id, id)).run().pipe(Effect.orDie)
yield* db
.update(SessionTable)
.set({ agent: Agent.defaultID })
.where(eq(SessionTable.id, id))
.run()
.pipe(Effect.orDie)
expect(yield* session.create(input)).toMatchObject({ id: created.id, agent: "build" })
}),
@@ -1564,4 +1593,28 @@ describe("SessionTransfer", () => {
expect(imported.outcome).toBe("succeeded")
}),
)
durableIt.effect("imports a long transcript in statements under 100 bound parameters", () =>
Effect.gen(function* () {
const session = yield* Session.Service
const transfer = yield* SessionTransfer.Service
const template = yield* session.create({ location })
const sessionID = Session.ID.create()
const messages = Array.from({ length: 30 }, (_, index) =>
SessionMessage.Assistant.make({
id: SessionMessage.ID.create(),
type: "assistant",
agent: Agent.defaultID,
model: { id: Model.ID.make("model"), providerID: Provider.ID.make("provider") },
content: Array.from({ length: 30 }, (_, ordinal) => ({ type: "text" as const, text: `${index}:${ordinal}` })),
time: { created: DateTime.makeUnsafe(index), completed: DateTime.makeUnsafe(index) },
}),
)
yield* transfer.import({ data: { info: { ...template, id: sessionID }, messages }, location })
expect(yield* session.messages({ sessionID, order: "asc" })).toMatchObject(
messages.map((message) => ({ id: message.id, content: message.content })),
)
}),
)
})
File diff suppressed because it is too large Load Diff
+153 -2
View File
@@ -9,14 +9,18 @@ import { SessionEvent } from "@opencode/schema/session-event"
import { SessionMessage } from "@opencode/schema/session-message"
import { AbsolutePath } from "@opencode/core/schema"
import { Bus } from "@opencode/core/bus"
import { DatabaseBatch } from "@opencode/core/database/batch"
import { Database } from "@opencode/core/database/database"
import { sqliteLayer } from "@opencode/core/database/sqlite.workerd"
import { ProjectTable } from "@opencode/core/project/sql"
import { SessionMessageTable, SessionTable } from "@opencode/core/session/sql"
import { SessionMessageTable, SessionPartTable, SessionTable } from "@opencode/core/session/sql"
import { SessionHistory } from "@opencode/core/session/history"
import { SessionStats } from "@opencode/core/session/stats"
import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder"
import { LayerNode } from "@opencode/util/effect/layer-node"
import { DateTime, Effect, Schema } from "effect"
import { DateTime, Effect, Layer, Schema } from "effect"
import { TestClock } from "effect/testing"
import { makeDurableObjectStorage } from "./fixture/durable-object-storage"
import { testEffect } from "./lib/effect"
const it = testEffect(
@@ -24,6 +28,14 @@ const it = testEffect(
Bus.node.replace(Bus.configured({ persist: true })),
]),
)
const durableIt = testEffect(
AppNodeBuilder.build(LayerNode.group([Database.node, Bus.node]), [
Bus.node.replace(Bus.configured({ persist: true })),
Database.node.replace(
Database.configuredClient(Layer.suspend(() => sqliteLayer({ storage: makeDurableObjectStorage() }))),
),
]),
)
const projectID = Project.ID.make("stats-project")
const otherProjectID = Project.ID.make("stats-other-project")
const sessionID = Session.ID.make("ses_stats_root")
@@ -246,6 +258,145 @@ describe("SessionStats", () => {
expect(future._tag).toBe("SessionStats.InvalidRangeError")
}),
)
it.effect("counts tool calls in both session_part rows and inline content an older build left on one message", () =>
Effect.gen(function* () {
const db = (yield* Database.Service).db
yield* db
.insert(ProjectTable)
.values({ id: projectID, worktree: AbsolutePath.make("/stats"), name: "stats", sandboxes: [] })
.run()
.pipe(Effect.orDie)
yield* db
.insert(SessionTable)
.values({ id: sessionID, project_id: projectID, slug: "root", directory: "/stats", version: "test" })
.run()
.pipe(Effect.orDie)
const created = Date.UTC(2026, 0, 2, 10)
const time = { created: DateTime.makeUnsafe(created), completed: DateTime.makeUnsafe(created + 1_000) }
const row = messageRow(
sessionID,
1,
assistant("msg_stats_hybrid", created, [
SessionMessage.AssistantTool.make({
type: "tool",
id: "call_inline",
name: "edit",
state: SessionMessage.ToolStateError.make({
status: "error",
input: {},
error: { type: "tool", message: "failed" },
}),
time,
}),
]),
)
yield* db.insert(SessionMessageTable).values(row).run().pipe(Effect.orDie)
yield* db
.insert(SessionPartTable)
.values({
message_id: row.id,
session_id: sessionID,
ordinal: 0,
type: "tool",
call_id: "call_part",
time_created: created,
time_updated: created,
data: SessionHistory.encodePart(
SessionMessage.AssistantTool.make({
type: "tool",
id: "call_part",
name: "read",
state: SessionMessage.ToolStateCompleted.make({
status: "completed",
input: {},
content: [{ type: "text", text: "ok" }],
}),
time,
}),
),
})
.run()
.pipe(Effect.orDie)
const range = { from: Date.UTC(2026, 0, 1), to: Date.UTC(2026, 1, 1), timezone: "UTC" }
expect((yield* SessionStats.get(range)).tools).toEqual({
mode: "summary",
totals: { calls: 2, succeeded: 1, failed: 1, unfinished: 0 },
})
const detail = (yield* SessionStats.get({ ...range, tools: "detail" })).tools
if (detail.mode !== "detail") throw new Error("Expected detailed tool statistics")
expect(detail.usage.map((tool) => tool.name).toSorted()).toEqual(["edit", "read"])
}),
)
durableIt.effect("aggregates compaction usage across more than 100 sessions under the 100-parameter limit", () =>
Effect.gen(function* () {
const db = (yield* Database.Service).db
const bus = yield* Bus.Service
yield* db
.insert(ProjectTable)
.values({ id: projectID, worktree: AbsolutePath.make("/stats"), name: "stats", sandboxes: [] })
.run()
.pipe(Effect.orDie)
const ids = Array.from({ length: 120 }, (_, i) => Session.ID.make(`ses_stats_${String(i).padStart(3, "0")}`))
const created = Date.UTC(2026, 0, 2, 10)
yield* Effect.forEach(
DatabaseBatch.insertBatches(
SessionTable,
ids.map((id, i) => ({
id,
project_id: projectID,
slug: `session-${i}`,
directory: "/stats",
version: "test",
})),
),
(batch) => db.insert(SessionTable).values(batch).run().pipe(Effect.orDie),
{ discard: true },
)
yield* Effect.forEach(
DatabaseBatch.insertBatches(
SessionMessageTable,
ids.map((id, i) => messageRow(id, 1, assistant(`msg_stats_${i}`, created, []))),
),
(batch) => db.insert(SessionMessageTable).values(batch).run().pipe(Effect.orDie),
{ discard: true },
)
yield* TestClock.setTime(created + 1_000)
yield* bus.publish(SessionEvent.UsageRecorded, {
sessionID: ids[0]!,
source: "compaction",
cost: Money.USD.make(0.25),
tokens: { input: 2, output: 1, reasoning: 0, cache: { read: 0, write: 0 } },
})
yield* bus.publish(SessionEvent.UsageRecorded, {
sessionID: ids.at(-1)!,
source: "compaction",
cost: Money.USD.make(0.75),
tokens: { input: 3, output: 4, reasoning: 0, cache: { read: 0, write: 0 } },
})
const stats = yield* SessionStats.get({
from: Date.UTC(2026, 0, 1),
to: Date.UTC(2026, 1, 1),
projectID,
timezone: "UTC",
tools: "none",
})
expect(stats.sessions).toBe(120)
expect(stats.steps).toBe(120)
expect(stats.tokens).toEqual({
input: 120 * 10 + 5,
output: 120 * 5 + 5,
reasoning: 120 * 2,
cache: { read: 120 * 4, write: 120 },
})
expect(stats.cost).toBe(Money.USD.make(120 * 1.5 + 1))
}),
)
})
function assistant(
+124 -73
View File
@@ -2,14 +2,17 @@ import { describe, expect } from "bun:test"
import { asc, eq } from "drizzle-orm"
import { Effect } from "effect"
import { Bus } from "@opencode/core/bus"
import { Config } from "@opencode/core/config"
import { Database } from "@opencode/core/database/database"
import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder"
import { ProjectTable } from "@opencode/core/project/sql"
import { SessionHistory } from "@opencode/core/session/history"
import { SessionProjector } from "@opencode/core/session/projector"
import { SessionHistory } from "@opencode/core/session/history"
import { SessionMessageTable, SessionPartTable } from "@opencode/core/session/sql"
import { SessionStore } from "@opencode/core/session/store"
import { Agent } from "@opencode/schema/agent"
import { Document, Info } from "@opencode/schema/config"
import { ConfigExperimental } from "@opencode/schema/config/experimental"
import { Event } from "@opencode/schema/event"
import { Model } from "@opencode/schema/model"
import { Project } from "@opencode/schema/project"
@@ -26,6 +29,12 @@ const it = testEffect(
Bus.node.replace(Bus.configured({ persist: true })),
]),
)
const partRows = Config.testLayer([
new Document({
type: "document",
info: new Info({ experimental: new ConfigExperimental.Info({ part_rows: true }) }),
}),
])
const seedSessions = (rows: { id: string; updated: number }[]) =>
Effect.gen(function* () {
@@ -194,107 +203,142 @@ describe("SessionStore", () => {
}),
)
it.effect("hydrates assistant parts from session_part when present and falls back to inline content", () =>
it.effect("hydrates and settles assistant messages across session_part rows and legacy inline content rows", () =>
Effect.gen(function* () {
const sessionID = Session.ID.make("ses_part_read_fallback")
yield* seedSessions([{ id: sessionID, updated: 0 }])
const sessionID = Session.ID.make("ses_dual_read")
const bus = yield* seedSessions([{ id: sessionID, updated: 0 }])
const database = yield* Database.Service
const store = yield* SessionStore.Service
const agent = Agent.ID.make("build")
const model = { providerID: Provider.ID.make("anthropic"), id: Model.ID.make("claude-sonnet-4-5") }
const inlineOnlyID = SessionMessage.ID.make("msg_inline_only")
const splitID = SessionMessage.ID.make("msg_split_parts")
const legacyUntouchedID = SessionMessage.ID.make("msg_legacy_untouched")
const legacyMutatedID = SessionMessage.ID.make("msg_legacy_mutated")
const splitID = SessionMessage.ID.make("msg_split")
// 1. Assistant message with inline content only (no session_part rows) -> falls back to inline content
// 1. Untouched legacy row with inline content and no session_part rows
yield* database.db
.insert(SessionMessageTable)
.values({
id: inlineOnlyID,
id: legacyUntouchedID,
session_id: sessionID,
seq: 10,
seq: -2,
type: "assistant",
time_created: 10,
time_updated: 10,
data: {
agent,
model,
content: [{ type: "text", text: "inline fallback content" }],
content: [{ type: "text", text: "legacy inline only" }],
time: { created: 10, completed: 11 },
},
})
.run()
// 2. Assistant message with session_part rows -> hydrates parts in ordinal order
// 2. Legacy row with inline content that gets mutated via projector and split into session_part
yield* database.db
.insert(SessionMessageTable)
.values({
id: splitID,
id: legacyMutatedID,
session_id: sessionID,
seq: 20,
seq: -1,
type: "assistant",
time_created: 20,
time_updated: 20,
data: {
agent,
model,
content: [],
time: { created: 20, completed: 25 },
content: [{ type: "reasoning", text: "legacy reasoning" }],
time: { created: 20 },
},
})
.run()
yield* database.db
.insert(SessionPartTable)
.values([
{
message_id: splitID,
session_id: sessionID,
ordinal: 0,
type: "reasoning",
call_id: null,
time_created: 20,
time_updated: 21,
data: { type: "reasoning", text: "first reasoning", time: { created: 20, completed: 21 } },
},
{
message_id: splitID,
session_id: sessionID,
ordinal: 1,
type: "tool",
call_id: "call_read",
time_created: 21,
time_updated: 23,
data: {
type: "tool",
id: "call_read",
name: "read",
executed: true,
state: {
status: "completed",
input: { path: "src/index.ts" },
content: [{ type: "text", text: "export {}" }],
},
time: { created: 21, ran: 22, completed: 23 },
},
},
{
message_id: splitID,
session_id: sessionID,
ordinal: 2,
type: "text",
call_id: null,
time_created: 23,
time_updated: 25,
data: { type: "text", text: "hydrated from session_part" },
},
])
.run()
yield* bus.publish(SessionEvent.Text.Started, {
sessionID,
assistantMessageID: legacyMutatedID,
ordinal: 1,
})
yield* bus.publish(SessionEvent.Text.Ended, {
sessionID,
assistantMessageID: legacyMutatedID,
ordinal: 1,
text: "appended into session_part",
})
expect(yield* store.message(inlineOnlyID)).toMatchObject({
// 3. Fresh step projected directly into session_part
yield* bus.publish(SessionEvent.Step.Started, {
sessionID,
assistantMessageID: splitID,
agent,
model,
started: 30,
})
yield* bus.publish(SessionEvent.Reasoning.Started, { sessionID, assistantMessageID: splitID, ordinal: 0 })
yield* bus.publish(SessionEvent.Reasoning.Ended, {
sessionID,
assistantMessageID: splitID,
ordinal: 0,
text: "step reasoning",
})
yield* bus.publish(SessionEvent.Tool.Input.Started, {
sessionID,
assistantMessageID: splitID,
id: "call_1",
name: "bash",
})
yield* bus.publish(SessionEvent.Tool.Called, {
sessionID,
assistantMessageID: splitID,
id: "call_1",
input: { command: "ls" },
executed: true,
})
yield* bus.publish(SessionEvent.Tool.Success, {
sessionID,
assistantMessageID: splitID,
id: "call_1",
content: [{ type: "text", text: "ok" }],
executed: true,
})
yield* bus.publish(SessionEvent.Text.Started, { sessionID, assistantMessageID: splitID, ordinal: 2 })
yield* bus.publish(SessionEvent.Text.Ended, {
sessionID,
assistantMessageID: splitID,
ordinal: 2,
text: "step answer",
})
// Verify raw storage split on the mutated legacy row
const mutatedRaw = yield* database.db
.select()
.from(SessionMessageTable)
.where(eq(SessionMessageTable.id, legacyMutatedID))
.get()
expect(mutatedRaw?.data).toMatchObject({ content: [] })
const mutatedParts = yield* database.db
.select()
.from(SessionPartTable)
.where(eq(SessionPartTable.message_id, legacyMutatedID))
.orderBy(asc(SessionPartTable.ordinal))
.all()
expect(mutatedParts).toHaveLength(2)
// Settle reads (single-message lookup)
expect(yield* store.message(legacyUntouchedID)).toMatchObject({
sessionID,
message: {
id: inlineOnlyID,
content: [{ type: "text", text: "inline fallback content" }],
id: legacyUntouchedID,
content: [{ type: "text", text: "legacy inline only" }],
},
})
expect(yield* store.message(legacyMutatedID)).toMatchObject({
sessionID,
message: {
id: legacyMutatedID,
content: [
{ type: "reasoning", text: "legacy reasoning" },
{ type: "text", text: "appended into session_part" },
],
},
})
expect(yield* store.message(splitID)).toMatchObject({
@@ -302,26 +346,33 @@ describe("SessionStore", () => {
message: {
id: splitID,
content: [
{ type: "reasoning", text: "first reasoning" },
{ type: "tool", id: "call_read", state: { status: "completed" } },
{ type: "text", text: "hydrated from session_part" },
{ type: "reasoning", text: "step reasoning" },
{ type: "tool", id: "call_1", state: { status: "completed" } },
{ type: "text", text: "step answer" },
],
},
})
// Hydrate reads (paged store.messages and full SessionHistory.get)
const hydrated = yield* store.messages({ sessionID, order: "asc" })
expect(hydrated).toMatchObject([
{ id: inlineOnlyID, content: [{ type: "text", text: "inline fallback content" }] },
{ id: legacyUntouchedID, content: [{ type: "text", text: "legacy inline only" }] },
{
id: legacyMutatedID,
content: [
{ type: "reasoning", text: "legacy reasoning" },
{ type: "text", text: "appended into session_part" },
],
},
{
id: splitID,
content: [
{ type: "reasoning", text: "first reasoning" },
{ type: "tool", id: "call_read", state: { status: "completed" } },
{ type: "text", text: "hydrated from session_part" },
{ type: "reasoning", text: "step reasoning" },
{ type: "tool", id: "call_1", state: { status: "completed" } },
{ type: "text", text: "step answer" },
],
},
])
const rows = yield* database.db
.select()
.from(SessionMessageTable)
@@ -329,6 +380,6 @@ describe("SessionStore", () => {
.orderBy(asc(SessionMessageTable.seq))
.all()
expect(yield* SessionHistory.decodeMessageRows(database.db, rows)).toEqual(hydrated)
}),
}).pipe(Effect.provide(partRows)),
)
})
+14
View File
@@ -93,5 +93,19 @@ describe("sqlite.workerd", () => {
.map((row) => row.name)
expect(names).toContain("migration")
expect(names).toContain("session_v2")
expect(() => storage.sql.exec("PRAGMA foreign_keys = OFF")).toThrow("not authorized")
await run(
storage,
Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient
yield* sql`CREATE TABLE _parent (id TEXT PRIMARY KEY)`
yield* sql`CREATE TABLE _child (parent_id TEXT NOT NULL REFERENCES _parent(id) ON DELETE CASCADE)`
yield* sql`INSERT INTO _parent (id) VALUES (${"p1"})`
yield* sql`INSERT INTO _child (parent_id) VALUES (${"p1"})`
yield* sql`DELETE FROM _parent WHERE id = ${"p1"}`
expect(yield* sql`SELECT * FROM _child`).toEqual([])
}),
)
})
})
+3
View File
@@ -16664,6 +16664,9 @@
"media_blob": {
"type": "boolean"
},
"part_rows": {
"type": "boolean"
},
"image_generate": {
"anyOf": [
{
@@ -20,6 +20,10 @@ export class Info extends Schema.Class<Info>("ConfigExperimental.Info")({
description:
"Store new session media once in a deduplicated blob table instead of inline in every message row. Defaults to false.",
}),
part_rows: Schema.Boolean.pipe(optional).annotate({
description:
"Write each assistant part as its own session_part row instead of rewriting the whole message on every event. Defaults to false.",
}),
image_generate: Schema.Union([
Schema.Boolean,
Schema.Struct({