mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-21 04:12:08 +03:00
fix(core): batch instruction blob writes and reads under the Durable Object parameter limit
A session's instruction state writes every changed value as one blob row and reads them back with an IN list. Both stayed one statement however many values there were, so a session with more than 50 entries exceeded Durable Object SQLite's 100 bound parameters and every turn failed while rendering the baseline. Batch both through DatabaseBatch like the rest of the runtime.
This commit is contained in:
@@ -2,6 +2,7 @@ export * as InstructionState from "./instruction-state.js"
|
||||
|
||||
import { and, eq, gte, inArray, sql } from "drizzle-orm"
|
||||
import { Effect, Option, Schema } from "effect"
|
||||
import { DatabaseBatch } from "../database/batch.js"
|
||||
import type { Database } from "../database/database.js"
|
||||
import type { Bus } from "../bus.js"
|
||||
import { Instructions } from "../instructions/index.js"
|
||||
@@ -221,17 +222,14 @@ const find = Effect.fnUntraced(function* (db: DatabaseService, sessionID: Sessio
|
||||
|
||||
const insertBlobs = Effect.fnUntraced(function* (db: DatabaseService, blobs: Readonly<Record<string, Schema.Json>>) {
|
||||
const rows = Object.entries(blobs).map(([hash, value]) => ({ hash: Instructions.Hash.make(hash), value }))
|
||||
if (rows.length === 0) return
|
||||
yield* db.insert(InstructionBlobTable).values(rows).onConflictDoNothing().run().pipe(Effect.orDie)
|
||||
for (const batch of DatabaseBatch.insertBatches(InstructionBlobTable, rows))
|
||||
yield* db.insert(InstructionBlobTable).values(batch).onConflictDoNothing().run().pipe(Effect.orDie)
|
||||
})
|
||||
|
||||
const loadBlobs = Effect.fnUntraced(function* (db: DatabaseService, values: ReadonlyArray<Instructions.Hash>) {
|
||||
const hashes = [...new Set(values)]
|
||||
const batches = Array.from({ length: Math.ceil(hashes.length / 500) }, (_, index) =>
|
||||
hashes.slice(index * 500, (index + 1) * 500),
|
||||
)
|
||||
const rows = (yield* Effect.forEach(
|
||||
batches,
|
||||
DatabaseBatch.batches(hashes, 1),
|
||||
(batch) =>
|
||||
db.select().from(InstructionBlobTable).where(inArray(InstructionBlobTable.hash, batch)).all().pipe(Effect.orDie),
|
||||
{ concurrency: 4 },
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import { and, asc, eq } from "drizzle-orm"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { Effect, Layer, Schema } from "effect"
|
||||
import { Database } from "@opencode/core/database/database"
|
||||
import { sqliteLayer } from "@opencode/core/database/sqlite.workerd"
|
||||
import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder"
|
||||
import { LayerNode } from "@opencode/util/effect/layer-node"
|
||||
import { Bus } from "@opencode/core/bus"
|
||||
@@ -22,6 +23,7 @@ import {
|
||||
SessionTable,
|
||||
} from "@opencode/core/session/sql"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { makeDurableObjectStorage } from "./fixture/durable-object-storage"
|
||||
|
||||
const it = testEffect(
|
||||
AppNodeBuilder.build(LayerNode.group([Database.node, Bus.node, SessionProjector.node, InstructionDiscovery.node]), [
|
||||
@@ -29,6 +31,16 @@ const it = testEffect(
|
||||
]),
|
||||
)
|
||||
|
||||
// Durable Object SQLite, which rejects statements that bind more than 100 parameters.
|
||||
const durableIt = testEffect(
|
||||
AppNodeBuilder.build(LayerNode.group([Database.node, Bus.node, SessionProjector.node, InstructionDiscovery.node]), [
|
||||
Bus.node.replace(Bus.configured({ persist: true })),
|
||||
Database.node.replace(
|
||||
Database.configuredClient(Layer.suspend(() => sqliteLayer({ storage: makeDurableObjectStorage() }))),
|
||||
),
|
||||
]),
|
||||
)
|
||||
|
||||
const source = (name: string, read: Effect.Effect<string | Instructions.Unavailable | Instructions.Removed>) =>
|
||||
Instructions.make({
|
||||
key: Instructions.Key.make(name),
|
||||
@@ -534,4 +546,19 @@ describe("InstructionState", () => {
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
durableIt.effect("initializes and renders more values than one statement may bind", () =>
|
||||
Effect.gen(function* () {
|
||||
const sessionID = SessionSchema.ID.make("ses_instruction_many_values")
|
||||
const { db, events } = yield* setup(sessionID)
|
||||
// One source per key, so a session with many API entries reaches this size in practice.
|
||||
const instructions = Instructions.combine(
|
||||
Array.from({ length: 150 }, (_, index) => source(`test/many/${index}`, Effect.succeed(`value ${index}`))),
|
||||
)
|
||||
yield* InstructionState.prepare(db, events, instructions, sessionID)
|
||||
const initial = yield* InstructionState.initial(db, sessionID, instructions)
|
||||
expect(initial).toHaveLength(150)
|
||||
expect(initial[149]).toEqual({ key: Instructions.Key.make("test/many/149"), text: "value 149" })
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user