diff --git a/packages/core/src/database/drizzle/effect-sqlite/session.ts b/packages/core/src/database/drizzle/effect-sqlite/session.ts index 9297bc7b677..7b91544bc14 100644 --- a/packages/core/src/database/drizzle/effect-sqlite/session.ts +++ b/packages/core/src/database/drizzle/effect-sqlite/session.ts @@ -5,6 +5,7 @@ import * as Exit from "effect/Exit" import * as Scope from "effect/Scope" import type { SqlClient } from "effect/sql/SqlClient" import type { SqlError } from "effect/sql/SqlError" +import { StorageFailure } from "../../storage-failure.js" import type { EffectCacheShape } from "drizzle-orm/cache/core/cache-effect" import type { WithCacheConfig } from "drizzle-orm/cache/core/types" import type { EffectDrizzleQueryError } from "drizzle-orm/effect-core/errors" @@ -186,13 +187,23 @@ export class EffectSQLiteSession extends SQLite ), ) : this.executeTransactionStatement(connection, `release savepoint effect_sql_${id}`) - : id === 0 - ? this.executeTransactionStatement(connection, "rollback") - : this.executeTransactionStatement(connection, `rollback to savepoint effect_sql_${id}`).pipe( - Effect.andThen( - this.executeTransactionStatement(connection, `release savepoint effect_sql_${id}`), - ), - ) + : (id === 0 + ? this.executeTransactionStatement(connection, "rollback") + : this.executeTransactionStatement(connection, `rollback to savepoint effect_sql_${id}`).pipe( + Effect.andThen( + this.executeTransactionStatement(connection, `release savepoint effect_sql_${id}`), + ), + ) + ).pipe( + // After errors such as SQLITE_FULL, SQLite rolls the whole transaction back itself, so this + // rollback has nothing to undo and fails. Report the body's error, not that one. Any other + // rollback failure still fails. + Effect.catch((error) => + Exit.isFailure(exit) && StorageFailure.rolledBack(exit.cause) + ? Effect.void + : Effect.fail(error), + ), + ) return finalize.pipe(Effect.flatMap(() => exit)) }), diff --git a/packages/core/src/database/sqlite.bun.ts b/packages/core/src/database/sqlite.bun.ts index 4958299b3a0..6a8e9996bb0 100644 --- a/packages/core/src/database/sqlite.bun.ts +++ b/packages/core/src/database/sqlite.bun.ts @@ -1,3 +1,4 @@ +import { SQLiteError } from "bun:sqlite" import { SqliteClient } from "@effect/sql-sqlite-bun" import { Layer } from "effect" import type { SqlClient } from "effect/sql" @@ -11,3 +12,6 @@ export const supportsForeignKeyToggle = true // busyTimeout; Database.layer treats both as fatal. export const sqliteLayer = (config: SqliteClient.SqliteClientConfig): Layer.Layer => Layer.orDie(SqliteClient.layer(config)) + +// bun:sqlite throws SQLiteError, whose errno is SQLite's extended result code. +export const resultCode = (cause: unknown) => (cause instanceof SQLiteError ? cause.errno : undefined) diff --git a/packages/core/src/database/sqlite.node.ts b/packages/core/src/database/sqlite.node.ts index 0e589917a71..77a3a189238 100644 --- a/packages/core/src/database/sqlite.node.ts +++ b/packages/core/src/database/sqlite.node.ts @@ -1,5 +1,5 @@ import { SqliteClient } from "@effect/sql-sqlite-node" -import { Layer } from "effect" +import { Layer, Option, Schema } from "effect" import type { SqlClient } from "effect/sql" export const supportsTuningPragmas = true @@ -11,3 +11,15 @@ export const supportsForeignKeyToggle = true // busyTimeout; Database.layer treats both as fatal. export const sqliteLayer = (config: SqliteClient.SqliteClientConfig): Layer.Layer => Layer.orDie(SqliteClient.layer(config)) + +// node:sqlite has no error class: SQLite failures are an Error marked ERR_SQLITE_ERROR that carries SQLite's extended +// result code in errcode. +const decodeDriverError = Schema.decodeUnknownOption( + Schema.Struct({ code: Schema.Literal("ERR_SQLITE_ERROR"), errcode: Schema.Number }), +) + +export const resultCode = (cause: unknown) => + decodeDriverError(cause).pipe( + Option.map((error) => error.errcode), + Option.getOrUndefined, + ) diff --git a/packages/core/src/database/sqlite.workerd.ts b/packages/core/src/database/sqlite.workerd.ts index 5ddf5e2cb50..73acde76dc2 100644 --- a/packages/core/src/database/sqlite.workerd.ts +++ b/packages/core/src/database/sqlite.workerd.ts @@ -221,3 +221,6 @@ export const sqliteLayer = (config: Config): Layer.Layer => ), ), ) + +// Durable Object SQLite throws a plain Error without SQLite result codes. +export const resultCode = (_cause: unknown): number | undefined => undefined diff --git a/packages/core/src/database/storage-failure.ts b/packages/core/src/database/storage-failure.ts new file mode 100644 index 00000000000..747de5134a6 --- /dev/null +++ b/packages/core/src/database/storage-failure.ts @@ -0,0 +1,58 @@ +export * as StorageFailure from "./storage-failure.js" + +import { resultCode } from "#sqlite" +import { EffectDrizzleQueryError } from "drizzle-orm/effect-core/errors" +import { Cause } from "effect" +import { isSqlError, type SqlError } from "effect/sql/SqlError" + +// SQLite primary result codes that mean storage itself can't be used, as opposed to a bad statement or a constraint, +// each with a message that doesn't name the database engine, since callers only know about "storage". BUSY and LOCKED +// only surface once a write has already waited out `busy_timeout`, so the write is lost just like on a full disk. +const messages = new Map([ + [5, "Storage is in use by another process"], // SQLITE_BUSY + [6, "Storage is in use by another process"], // SQLITE_LOCKED + [8, "Storage is read-only"], // SQLITE_READONLY + [10, "Storage couldn't be read or written"], // SQLITE_IOERR + [11, "Storage is damaged"], // SQLITE_CORRUPT + [13, "Storage is full"], // SQLITE_FULL + [14, "Storage couldn't be opened"], // SQLITE_CANTOPEN + [26, "Storage is damaged"], // SQLITE_NOTADB +]) + +/** The readable message for the storage failure that failed an effect, if that is what failed it. */ +export function message(cause: Cause.Cause) { + return sqlErrors(cause) + .map(fromSqlError) + .find((found) => found !== undefined) +} + +/** The readable message for a SqlError the driver reported as a storage failure. */ +export function fromSqlError(error: SqlError) { + const code = resultCode(error.reason.cause) + // An extended result code (SQLITE_IOERR_WRITE) keeps its primary code in the low byte. + return code === undefined ? undefined : messages.get(code & 0xff) +} + +// Result codes after which SQLite may have rolled the whole transaction back itself +// (https://www.sqlite.org/lang_transaction.html#response_to_errors_within_a_transaction): SQLITE_FULL, SQLITE_IOERR, +// SQLITE_BUSY, SQLITE_NOMEM and SQLITE_INTERRUPT. +const autoRollback = new Set([5, 7, 9, 10, 13]) + +/** Whether SQLite may have already rolled back the transaction this cause failed, so a rollback has nothing to undo. */ +export function rolledBack(cause: Cause.Cause) { + return sqlErrors(cause).some((error) => { + const code = resultCode(error.reason.cause) + return code !== undefined && autoRollback.has(code & 0xff) + }) +} + +// A failed transaction boundary is a SqlError; a failed query is a query error holding the Cause of its SqlError. +function sqlErrors(cause: Cause.Cause): SqlError[] { + return cause.reasons.flatMap((reason) => { + if (Cause.isInterruptReason(reason)) return [] + const error = Cause.isFailReason(reason) ? reason.error : reason.defect + if (isSqlError(error)) return [error] + if (error instanceof EffectDrizzleQueryError && Cause.isCause(error.cause)) return sqlErrors(error.cause) + return [] + }) +} diff --git a/packages/core/test/fixture/sqlite-node-client.ts b/packages/core/test/fixture/sqlite-node-client.ts index bb78be33301..1dab3b8687b 100644 --- a/packages/core/test/fixture/sqlite-node-client.ts +++ b/packages/core/test/fixture/sqlite-node-client.ts @@ -9,6 +9,7 @@ import { SqlClient } from "effect/sql" import { isSqlError } from "effect/sql/SqlError" import { Database } from "../../src/database/database" import { sqliteLayer } from "../../src/database/sqlite.node" +import { StorageFailure } from "../../src/database/storage-failure" const typed = sqliteTable("typed", { id: integer().primaryKey(), name: text().notNull() }) @@ -63,6 +64,10 @@ const program = Effect.gen(function* () { const leakedTransaction = yield* sweep((client) => client.withTransaction(client`SELECT 2`)) const leakedReserve = yield* sweep((client) => Effect.scoped(Effect.andThen(client.reserve, Effect.yieldNow))) + + // SQLite never lowers the limit below the current size, so 1 means the database may not grow. + yield* db.run(sql`PRAGMA max_page_count = 1`) + const full = yield* db.run(sql`INSERT INTO probe (id) VALUES (${"x".repeat(60_000)})`).pipe(Effect.flip) return { journal: journal?.journal_mode, reason: isSqlError(cause) ? cause.reason._tag : "not a SqlError", @@ -73,6 +78,10 @@ const program = Effect.gen(function* () { raw: raw?.count, leakedTransaction, leakedReserve, + storageFailure: { + full: StorageFailure.message(Cause.fail(full)), + constraint: StorageFailure.message(Cause.fail(error)) ?? null, + }, } }) diff --git a/packages/core/test/sqlite-node.test.ts b/packages/core/test/sqlite-node.test.ts index 12e7a5f6d52..a6ad1902459 100644 --- a/packages/core/test/sqlite-node.test.ts +++ b/packages/core/test/sqlite-node.test.ts @@ -10,7 +10,7 @@ const node = Bun.which("node") // Skipped on Windows for the same Bun.build panic as sqlite-bundle.test.ts. describe.skipIf(process.platform === "win32" || !node)("node sqlite client", () => { - test("boots migrations, commits drizzle transactions and types constraint errors", async () => { + test("boots migrations, commits drizzle transactions and types constraint and storage errors", async () => { const dir = await mkdtemp(join(tmpdir(), "sqlite-node-")) try { const build = await Bun.build({ @@ -42,6 +42,7 @@ describe.skipIf(process.platform === "win32" || !node)("node sqlite client", () raw: 1, leakedTransaction: [], leakedReserve: [], + storageFailure: { full: "Storage is full", constraint: null }, }) } finally { await rm(dir, { recursive: true, force: true }) diff --git a/packages/core/test/storage-failure.test.ts b/packages/core/test/storage-failure.test.ts new file mode 100644 index 00000000000..a76882b580f --- /dev/null +++ b/packages/core/test/storage-failure.test.ts @@ -0,0 +1,88 @@ +import path from "node:path" +import { Database } from "bun:sqlite" +import { expect } from "bun:test" +import { sql } from "drizzle-orm" +import { Cause, Effect } from "effect" +import { SqlClient } from "effect/sql" +import { EffectDrizzleSqlite } from "@opencode/core/database/drizzle" +import { sqliteLayer } from "@opencode/core/database/sqlite.bun" +import { StorageFailure } from "@opencode/core/database/storage-failure" +import { tmpdirScoped } from "./fixture/tmpdir" +import { it } from "./lib/effect" + +// The node:sqlite driver is covered by sqlite-node.test.ts, which runs under Node. +const large = "x".repeat(60_000) + +it.live("a transaction SQLite rolled back itself fails with the storage failure, not the rollback", () => + Effect.gen(function* () { + const client = yield* SqlClient.SqlClient + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + yield* client`CREATE TABLE probe (value TEXT)` + yield* client`PRAGMA max_page_count = 1` + + const cause = yield* db + .transaction(() => db.run(sql`INSERT INTO probe VALUES (${large})`), { behavior: "immediate" }) + .pipe(Effect.sandbox, Effect.flip) + expect(StorageFailure.message(cause)).toBe("Storage is full") + // Nothing was left open: the connection takes the next transaction. + yield* client`PRAGMA max_page_count = 1073741823` + yield* db.transaction(() => db.run(sql`INSERT INTO probe VALUES ('ok')`)) + }).pipe(Effect.provide(sqliteLayer({ filename: ":memory:" }))), +) + +it.live("a full database is a storage failure for a statement and for a query", () => + Effect.gen(function* () { + const client = yield* SqlClient.SqlClient + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + yield* client`CREATE TABLE probe (value TEXT)` + // SQLite never lowers the limit below the current size, so 1 means the database may not grow. + yield* client`PRAGMA max_page_count = 1` + + const statement = yield* client`INSERT INTO probe VALUES (${large})`.pipe(Effect.flip) + expect(StorageFailure.fromSqlError(statement)).toBe("Storage is full") + expect(StorageFailure.message(Cause.fail(statement))).toBe("Storage is full") + + const query = yield* db.run(sql`INSERT INTO probe VALUES (${large})`).pipe(Effect.flip) + expect(StorageFailure.message(Cause.die(query))).toBe("Storage is full") + }).pipe(Effect.provide(sqliteLayer({ filename: ":memory:" }))), +) + +it.live("an extended result code is a storage failure by its primary code", () => + Effect.gen(function* () { + const dir = yield* tmpdirScoped() + const filename = path.join(dir.path, "probe.db") + yield* Effect.gen(function* () { + const client = yield* SqlClient.SqlClient + yield* client`CREATE TABLE probe (value TEXT)` + // SQLITE_BUSY_SNAPSHOT (517): a WAL reader can't become the writer once another connection has committed. + yield* client`BEGIN` + yield* client`SELECT * FROM probe` + yield* Effect.sync(() => { + const other = new Database(filename) + other.run("INSERT INTO probe VALUES ('other')") + other.close() + }) + const error = yield* client`INSERT INTO probe VALUES ('mine')`.pipe(Effect.flip) + yield* client`ROLLBACK` + expect(error.reason.cause).toMatchObject({ errno: 517 }) + expect(StorageFailure.fromSqlError(error)).toBe("Storage is in use by another process") + }).pipe(Effect.provide(sqliteLayer({ filename }))) + }), +) + +it.live("a constraint, a bad statement and other defects are not storage failures", () => + Effect.gen(function* () { + const client = yield* SqlClient.SqlClient + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + yield* client`CREATE TABLE probe (id TEXT PRIMARY KEY)` + yield* client`INSERT INTO probe VALUES ('a')` + + const constraint = yield* client`INSERT INTO probe VALUES ('a')`.pipe(Effect.flip) + expect(StorageFailure.fromSqlError(constraint)).toBeUndefined() + const missing = yield* db.run(sql`SELECT * FROM missing`).pipe(Effect.flip) + expect(StorageFailure.message(Cause.die(missing))).toBeUndefined() + + expect(StorageFailure.message(Cause.die(new Error("database or disk is full")))).toBeUndefined() + expect(StorageFailure.message(Cause.interrupt())).toBeUndefined() + }).pipe(Effect.provide(sqliteLayer({ filename: ":memory:" }))), +) diff --git a/packages/server/src/middleware/defect-response.ts b/packages/server/src/middleware/defect-response.ts new file mode 100644 index 00000000000..968c5df3173 --- /dev/null +++ b/packages/server/src/middleware/defect-response.ts @@ -0,0 +1,39 @@ +import { StorageFailure } from "@opencode/core/database/storage-failure" +import { UnknownError } from "@opencode/protocol/errors" +import { Cause, Effect } from "effect" +import { HttpRouter, HttpServerError, HttpServerRequest, HttpServerResponse } from "effect/http" + +const respond = HttpServerResponse.schemaJson(UnknownError) + +// Effect answers a failure it has no response for with an empty 500. Replace exactly that response's body, like the +// encoding failure in schema-error.ts: the reason when it's one a person can act on (a storage failure), otherwise a +// ref to find it in the server log. Every cause Effect already answers (declared errors, route-not-found, defects that +// carry their own response, aborts) keeps its response. +export const defectResponseLayer = HttpRouter.middleware( + (effect) => + effect.pipe( + Effect.catchCause((cause) => + Effect.gen(function* () { + const [response] = yield* HttpServerError.causeResponse(cause) + if (response.status !== 500 || response.body._tag !== "Empty") return yield* Effect.failCause(cause) + const request = yield* HttpServerRequest.HttpServerRequest + const ref = `err_${crypto.randomUUID().slice(0, 8)}` + yield* Effect.logError("request failed", cause).pipe( + Effect.annotateLogs({ ref, route: `${request.method} ${request.url}` }), + ) + const body = yield* respond( + new UnknownError({ + message: + StorageFailure.message(cause) ?? "Server failed to handle the request. Check server logs for details.", + ref, + }), + { status: 500 }, + ).pipe(Effect.orDie) + // Fail with the response beside the original cause rather than succeed with it: Effect sends a response + // found in a cause, and the request's span and response log still record the failure. + return yield* Effect.failCause(Cause.combine(cause, Cause.die(body))) + }), + ), + ), + { global: true }, +) diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index 5cf2d3ffa6f..1b9f033a64a 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -47,6 +47,7 @@ import { CorsConfig } from "./cors" import { handlers } from "./handlers" import { authorizationLayer } from "./middleware/authorization" import { schemaErrorLayer } from "./middleware/schema-error" +import { defectResponseLayer } from "./middleware/defect-response" import { ServerPairing } from "./pairing" import { layer } from "./location" import { formLocationLayer } from "./middleware/form-location" @@ -200,6 +201,7 @@ function makeRoutes( Layer.provide(schemaErrorLayer), Layer.provide(auth), HttpRouter.provideRequest(requestServices), + Layer.merge(defectResponseLayer), Layer.provideMerge(services), Layer.provideMerge(HttpRouter.layer), ) diff --git a/packages/server/test/defect-response.test.ts b/packages/server/test/defect-response.test.ts new file mode 100644 index 00000000000..6f9c5455093 --- /dev/null +++ b/packages/server/test/defect-response.test.ts @@ -0,0 +1,83 @@ +import path from "node:path" +import { expect } from "bun:test" +import { OpenCode } from "@opencode/client" +import { Database } from "@opencode/core/database/database" +import { Global } from "@opencode/util/global" +import { Context, Effect } from "effect" +import { tempGlobalLayer } from "../../core/test/fixture/global" +import { tmpdirScoped } from "../../core/test/fixture/tmpdir" +import { it } from "../../core/test/lib/effect" +import { serveRoutes } from "./fixture/routes" + +const large = "x".repeat(60_000) + +const boot = Effect.gen(function* () { + const dir = yield* tmpdirScoped("opencode-defect-response-") + const server = yield* serveRoutes( + { + app: { version: "test" }, + database: { path: path.join(dir.path, "opencode.db") }, + models: { fetch: false }, + fs: { filewatcher: false }, + }, + [Global.node.replace(tempGlobalLayer)], + ) + const db = Context.get(server.context, Database.Service).db + return { + db, + location: { directory: dir.path }, + request: (method: string, route: string, body?: unknown) => + Effect.promise(async () => { + const response = await server.fetch(`http://opencode.local${route}`, { + method, + headers: { "content-type": "application/json" }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }) + return { status: response.status, body: await response.text() } + }), + api: OpenCode.make({ + baseUrl: "http://opencode.local", + fetch: Object.assign( + (input: string | URL | Request, init?: RequestInit) => server.handler(new Request(input, init)), + { preconnect: fetch.preconnect }, + ), + }), + } +}) + +it.live("a write on a full disk answers 500 with the storage reason instead of an empty body", () => + Effect.gen(function* () { + const server = yield* boot + const session = yield* Effect.promise(() => server.api.session.create({ location: server.location })) + // A real SQLITE_FULL: SQLite never lowers the limit below the current size, so 1 means the database may not grow + // and a write that needs a new page fails. + yield* server.db.run("PRAGMA max_page_count = 1").pipe(Effect.orDie) + + const prompt = yield* server.request("POST", `/api/session/${session.id}/prompt`, { text: large }) + expect(prompt.status).toBe(500) + expect(JSON.parse(prompt.body)).toEqual({ + _tag: "UnknownError", + message: "Storage is full", + ref: expect.stringMatching(/^err_[0-9a-f]{8}$/), + }) + }), +) + +it.live("other defects answer 500 with a ref, and unknown routes keep their 404", () => + Effect.gen(function* () { + const server = yield* boot + const session = yield* Effect.promise(() => server.api.session.create({ location: server.location })) + // A query that is simply wrong is a bug, not a storage failure: an empty table shadows the sessions. + yield* server.db.run("CREATE TEMP TABLE session_v2 (shadow text)").pipe(Effect.orDie) + const read = yield* server.request("GET", `/api/session/${session.id}`) + yield* server.db.run("DROP TABLE temp.session_v2").pipe(Effect.orDie) + expect(read.status).toBe(500) + expect(JSON.parse(read.body)).toEqual({ + _tag: "UnknownError", + message: "Server failed to handle the request. Check server logs for details.", + ref: expect.stringMatching(/^err_[0-9a-f]{8}$/), + }) + + expect((yield* server.request("GET", "/api/does-not-exist")).status).toBe(404) + }), +)