From c57733a6477856b35d50a600b5017886e31a5aa3 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Tue, 15 Sep 2026 23:05:06 -0500 Subject: [PATCH] refactor(codemode): before/after hooks for tool and extension calls (#49242) --- packages/codemode/README.md | 7 +- packages/codemode/interpreter-support.md | 2 + packages/codemode/src/codemode.ts | 16 +- packages/codemode/src/interpreter/execute.ts | 2 +- .../codemode/src/interpreter/extensions.ts | 35 ++- packages/codemode/src/tool-runtime.ts | 85 +++--- packages/codemode/test/codemode.test.ts | 258 +++++++++++------- packages/codemode/test/extensions.test.ts | 10 + packages/core/src/codemode/tool.ts | 57 ++-- 9 files changed, 282 insertions(+), 190 deletions(-) diff --git a/packages/codemode/README.md b/packages/codemode/README.md index e558abcff54..0e7fcb2f1d8 100644 --- a/packages/codemode/README.md +++ b/packages/codemode/README.md @@ -88,8 +88,11 @@ runtime.catalog() // structured tool descriptions runtime.execute(source) // Effect ``` -The Effect environment is inferred from the supplied tools. `onToolCallStart` observes admitted calls with decoded -input; `onToolCallEnd` observes settled outcomes and duration. Both hooks return Effects and must not fail. +The Effect environment is inferred from the supplied tools. `hooks` surround every call the program makes into the +host: `tool.before`/`tool.after` receive `{ name, input }` with decoded input, `extension.before`/`extension.after` +receive `{ extension, name, args }`. An `after` hook also receives how the call ended (`success` with its value, +`failure` with its error, or `interrupted`). A failing `before` hook denies the call, and the program catches the +failure as a thrown error. ### `Values` diff --git a/packages/codemode/interpreter-support.md b/packages/codemode/interpreter-support.md index 9cc75d124e3..b59d69519c4 100644 --- a/packages/codemode/interpreter-support.md +++ b/packages/codemode/interpreter-support.md @@ -467,6 +467,8 @@ Nothing is exposed unless a host provides it; extension calls are not tool calls - [x] A host function inside a result becomes a program function whose calls cross the same way, so a result can carry methods (`res.json()`) whose host closures keep the host state. Diagnostics name it by its path (`fetch.json`). Like any program function it vanishes at the data boundary. +- [x] Each call to an extension global runs inside the host's `extension.before`/`extension.after` hooks as + `{ extension, name, args }`, with the host's own error on failure; calls to functions inside results do not. - [x] A host `Promise` becomes a program promise. Whatever host code returns, resolves, throws, or rejects with crosses the same way, so `catch (e)` receives a copy of the thrown value (an `Error` of the matching type, or plain data). diff --git a/packages/codemode/src/codemode.ts b/packages/codemode/src/codemode.ts index 7c08d6b1846..7bd7496a21b 100644 --- a/packages/codemode/src/codemode.ts +++ b/packages/codemode/src/codemode.ts @@ -7,7 +7,14 @@ import { type Services, type ToolDescription, ToolRuntime } from "./tool-runtime import type { Tools } from "./tools.js" /** A tool call admitted during an execution. */ -export type { ToolCall, ToolCallEnded, ToolCallHooks, ToolCallStarted, ToolDescription } from "./tool-runtime.js" +export type { + CallResult, + ExtensionInvocation, + Hooks, + ToolCall, + ToolDescription, + ToolInvocation, +} from "./tool-runtime.js" /** Signature-construction helpers for host-owned catalog instructions. */ export { searchSignature, toolExpression } from "./tool-runtime.js" @@ -34,9 +41,11 @@ export type ResolvedExecutionLimits = { } /** Configuration shared by `CodeMode.make` and `CodeMode.execute`. */ -export type Options = {}> = ToolRuntime.ToolCallHooks> & { +export type Options = {}> = { /** Explicit tools exposed to the program as `tools`. */ tools?: Provided & Tools> + /** Hooks around every tool and extension call the program makes; see `Hooks`. */ + hooks?: ToolRuntime.Hooks> /** Host functions exposed as globals; see `Extension.make`. */ extensions?: ReadonlyArray /** Resource limits enforced on each execution. */ @@ -148,6 +157,7 @@ export const make = = {}>( } return { catalog: prepared.catalog, - execute: (code) => executeProgram(code, prepared, limits, options, (ctx) => extensionGlobals(ctx, extensions)), + execute: (code) => + executeProgram(code, prepared, limits, options.hooks ?? {}, (ctx) => extensionGlobals(ctx, extensions)), } } diff --git a/packages/codemode/src/interpreter/execute.ts b/packages/codemode/src/interpreter/execute.ts index 974ac463ad5..b21c649b190 100644 --- a/packages/codemode/src/interpreter/execute.ts +++ b/packages/codemode/src/interpreter/execute.ts @@ -16,7 +16,7 @@ export const executeProgram = ( code: string, prepared: ToolRuntime.Prepared, limits: ResolvedExecutionLimits, - hooks: ToolRuntime.ToolCallHooks, + hooks: ToolRuntime.Hooks, globals?: (ctx: Interpreter) => ReadonlyArray, ): Effect.Effect => { if (code.trim().length === 0) { diff --git a/packages/codemode/src/interpreter/extensions.ts b/packages/codemode/src/interpreter/extensions.ts index f276b9e659b..ca57d013fbe 100644 --- a/packages/codemode/src/interpreter/extensions.ts +++ b/packages/codemode/src/interpreter/extensions.ts @@ -1,6 +1,7 @@ import { Effect } from "effect" import type { Extension } from "../extension.js" import { coerceToString } from "../stdlib/value.js" +import { type ExtensionInvocation, hooked } from "../tool-runtime.js" import type { Interpreter } from "./interpreter.js" import { createErrorValue, isErrorType } from "./intrinsics.js" import { MAX_VALUE_DEPTH } from "./limits.js" @@ -11,7 +12,6 @@ import { define, entries, get, - type Native, Arr, Bytes, DateObj, @@ -128,26 +128,35 @@ export const extensionGlobals = ( throw typeError(`${label} produced ${describeHost(value)}, which the program cannot hold.`) } - // A host function as a program function: arguments cross in, and whatever it returns, resolves, throws, or - // rejects with crosses out, so the program catches what the author threw. - const wrap = (value: Function, label: string): Native => + // A host function as a program function. Arguments cross in; a global's call runs inside the host's extension + // hooks, which see the host's own error on failure; then whatever came back, or was thrown, crosses out so the + // program catches a copy. Functions inside results are part of a value's API and skip the hooks. + const wrap = (value: Function, label: string, describe?: (args: ReadonlyArray) => ExtensionInvocation) => fn(builtins, value.name, value.length, (_, values) => { - const converted = values.map((item, index) => toHost(item, `Argument ${index + 1} to ${label}`)) - const thrown = (reason: unknown) => new Throw(fromHost(reason, label)) + const args = values.map((item, index) => toHost(item, `Argument ${index + 1} to ${label}`)) + const hooks = ctx.tools.hooks + const settle = (run: Effect.Effect) => + (describe === undefined + ? run + : hooked(describe(args), hooks["extension.before"], hooks["extension.after"], run) + ).pipe( + Effect.mapError((reason) => new Throw(fromHost(reason, label))), + Effect.map((settled) => fromHost(settled, label)), + ) let result: unknown try { - result = value.apply(undefined, converted) + result = value.apply(undefined, args) } catch (reason) { - return Effect.fail(thrown(reason)) + return settle(Effect.fail(reason)) } - if (!(result instanceof Promise)) return fromHost(result, label) - return ctx.pending.create( - Effect.map(Effect.tryPromise({ try: () => result, catch: thrown }), (settled) => fromHost(settled, label)), - ) + if (!(result instanceof Promise)) return settle(Effect.succeed(result)) + return ctx.pending.create(settle(Effect.tryPromise({ try: () => result, catch: (reason) => reason }))) }) return extensions.flatMap((extension) => - Object.entries(extension.globals).map(([name, value]) => [name, wrap(value, name)] as const), + Object.entries(extension.globals).map( + ([name, value]) => [name, wrap(value, name, (args) => ({ extension: extension.name, name, args }))] as const, + ), ) } diff --git a/packages/codemode/src/tool-runtime.ts b/packages/codemode/src/tool-runtime.ts index f6fedee0029..5e68fc745c1 100644 --- a/packages/codemode/src/tool-runtime.ts +++ b/packages/codemode/src/tool-runtime.ts @@ -36,26 +36,48 @@ export type ToolCall = { readonly name: string } -export type ToolCallStarted = { - readonly index: number +/** A tool call the program is making, with its decoded input. */ +export type ToolInvocation = { readonly name: string; readonly input: unknown } + +/** A call the program is making to an extension global, with its arguments. */ +export type ExtensionInvocation = { + readonly extension: string readonly name: string - readonly input: unknown + readonly args: ReadonlyArray } -export type ToolCallEnded = { - readonly index: number - readonly name: string - readonly input: unknown - readonly durationMs: number - readonly outcome: "success" | "failure" | "interrupted" - readonly message?: string +/** How a call ended; `after` hooks observe it and cannot change it. */ +export type CallResult = + | { readonly status: "success"; readonly value: unknown } + | { readonly status: "failure"; readonly error: unknown } + | { readonly status: "interrupted" } + +/** Hooks around every call the program makes into the host. A failing `before` denies the call. */ +export type Hooks = { + readonly "tool.before"?: ((call: ToolInvocation) => Effect.Effect) | undefined + readonly "tool.after"?: ((call: ToolInvocation, result: CallResult) => Effect.Effect) | undefined + readonly "extension.before"?: ((call: ExtensionInvocation) => Effect.Effect) | undefined + readonly "extension.after"?: + | ((call: ExtensionInvocation, result: CallResult) => Effect.Effect) + | undefined } -export type ToolCallHooks = { - /** Observes decoded tool input immediately before tool execution. */ - readonly onToolCallStart?: ((call: ToolCallStarted) => Effect.Effect) | undefined - /** Observes each admitted tool call as it succeeds, fails, or is interrupted. */ - readonly onToolCallEnd?: ((call: ToolCallEnded) => Effect.Effect) | undefined +/** Runs `before`, then `run`, then `after` with how it ended, including when interrupted. */ +export const hooked = ( + call: Call, + before: ((call: Call) => Effect.Effect) | undefined, + after: ((call: Call, result: CallResult) => Effect.Effect) | undefined, + run: Effect.Effect, +): Effect.Effect => { + const observed = + after === undefined + ? run + : Effect.onExit(run, (exit) => { + if (Exit.isSuccess(exit)) return after(call, { status: "success", value: exit.value }) + if (Cause.hasInterruptsOnly(exit.cause)) return after(call, { status: "interrupted" }) + return after(call, { status: "failure", error: Cause.squash(exit.cause) }) + }) + return before === undefined ? observed : Effect.andThen(before(call), observed) } export type ToolDescription = { @@ -316,6 +338,7 @@ export class ToolRuntimeError extends Error { /** The tool bridge of one execution. Arguments arrive and results leave as JSON; program values never enter. */ export type ToolRuntime = { readonly calls: Array + readonly hooks: Hooks readonly execute: ( path: ReadonlyArray, args: Array, @@ -328,28 +351,12 @@ export type ToolRuntime = { export const make = ( prepared: Prepared, maxToolCalls: number | undefined, - hooks?: ToolCallHooks, + hooks: Hooks = {}, ): ToolRuntime => { const calls: Array = [] const root = prepared.root const searchTool = makeSearchTool(prepared.searchIndex) - const observeEnd = (effect: Effect.Effect, call: ToolCallStarted): Effect.Effect => { - const onEnd = hooks?.onToolCallEnd - if (onEnd === undefined) return effect - const startedAt = Date.now() - return effect.pipe( - Effect.onExit((exit) => { - const durationMs = Date.now() - startedAt - if (Exit.isSuccess(exit)) return onEnd({ ...call, durationMs, outcome: "success" }) - if (Cause.hasInterruptsOnly(exit.cause)) return onEnd({ ...call, durationMs, outcome: "interrupted" }) - const error = Cause.squash(exit.cause) - const message = error instanceof Error ? error.message : Cause.pretty(exit.cause) - return onEnd({ ...call, durationMs, outcome: "failure", message }) - }), - ) - } - const recordCall = (call: ToolCall): void => { if (maxToolCalls !== undefined && calls.length >= maxToolCalls) { throw new ToolRuntimeError("ToolCallLimitExceeded", `Execution exceeded its tool-call limit of ${maxToolCalls}.`) @@ -371,14 +378,12 @@ export const make = ( name === "search" ? [] : ["The signature may have changed. Use search to get the current signature."], ), }) - const index = yield* Effect.sync(() => { - recordCall({ name }) - return calls.length - 1 - }) - const call = { index, name, input } - return yield* observeEnd( + yield* Effect.sync(() => recordCall({ name })) + return yield* hooked( + { name, input }, + hooks["tool.before"], + hooks["tool.after"], Effect.gen(function* () { - if (hooks?.onToolCallStart !== undefined) yield* hooks.onToolCallStart(call) const raw = yield* Effect.suspend(() => tool.execute(input)).pipe( Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) return Effect.interrupt @@ -400,12 +405,12 @@ export const make = ( catch: (cause) => new ToolRuntimeError("InvalidToolOutput", `Invalid output from tool '${name}': ${cause}`), }) }), - call, ) }) return { calls, + hooks, keys: (path) => namespaceKeys(root, path), search: (args) => Effect.suspend(() => executeTool("search", searchTool, args)), execute: (path, args) => diff --git a/packages/codemode/test/codemode.test.ts b/packages/codemode/test/codemode.test.ts index 0fb371e4f88..dfdb7ca6797 100644 --- a/packages/codemode/test/codemode.test.ts +++ b/packages/codemode/test/codemode.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test" import { Cause, Effect, Schema } from "effect" -import { CodeMode, Tool, toolError } from "../src/index.js" +import { CodeMode, Extension, Tool, toolError } from "../src/index.js" const run = (tool: Tool.Tool) => Effect.runPromise(CodeMode.make({ tools: { host: { call: tool } } }).execute("return await tools.host.call({})")) @@ -160,77 +160,154 @@ describe("CodeMode host failure boundary", () => { }) }) -describe("CodeMode tool-call observation", () => { - test("reports the tools actually invoked with decoded input", async () => { - const calls: Array = [] +describe("CodeMode call hooks", () => { + const ended = (result: CodeMode.CallResult) => { + if (result.status !== "failure") return result.status + return `failure:${result.error instanceof Error ? result.error.message : String(result.error)}` + } + // Every hook logs its event, so the order across tools and extensions is visible. + const observe = (events: Array): CodeMode.Hooks => ({ + "tool.before": (call) => Effect.sync(() => void events.push(`tool.before ${JSON.stringify(call)}`)), + "tool.after": (call, result) => Effect.sync(() => void events.push(`tool.after ${call.name} ${ended(result)}`)), + "extension.before": (call) => Effect.sync(() => void events.push(`extension.before ${JSON.stringify(call)}`)), + "extension.after": (call, result) => + Effect.sync(() => void events.push(`extension.after ${call.name} ${ended(result)}`)), + }) + + test("surround the tools actually invoked, with decoded input", async () => { + const events: Array = [] const lookup = Tool.make({ description: "Look up a value", input: Schema.Struct({ query: Schema.String }), output: Schema.String, - execute: ({ query }) => Effect.succeed(query), + execute: ({ query }) => (query === "boom" ? Effect.fail(toolError("Lookup refused")) : Effect.succeed(query)), }) + const runtime = CodeMode.make({ tools: { context: { lookup } }, hooks: observe(events) }) - const result = await Effect.runPromise( - CodeMode.make({ - tools: { context: { lookup } }, - onToolCallStart: (call) => Effect.sync(() => calls.push(call)), - }).execute(` + const success = await Effect.runPromise( + runtime.execute(` if (false) await tools.context.lookup({ query: "not called" }) return await tools.context.lookup({ query: "deployment failure" }) `), ) - - expect(result.ok).toBe(true) - expect(calls).toStrictEqual([{ index: 0, name: "context.lookup", input: { query: "deployment failure" } }]) - }) - - test("observes settled calls with outcome and duration", async () => { - const events: Array<{ phase: string; index: number; name: string; outcome?: string; message?: string }> = [] - const lookup = Tool.make({ - description: "Look up a value", - input: Schema.Struct({ query: Schema.String }), - output: Schema.String, - execute: ({ query }) => - query === "boom" - ? Effect.fail(toolError("Lookup refused")) - : query === "defect" - ? Effect.die("broken") - : Effect.succeed(query), - }) - - const runtime = CodeMode.make({ - tools: { context: { lookup } }, - onToolCallStart: (call) => - Effect.sync(() => { - events.push({ phase: "start", index: call.index, name: call.name }) - }), - onToolCallEnd: (call) => - Effect.sync(() => { - expect(call.durationMs).toBeGreaterThanOrEqual(0) - events.push({ - phase: "end", - index: call.index, - name: call.name, - outcome: call.outcome, - ...(call.message === undefined ? {} : { message: call.message }), - }) - }), - }) - - const success = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "ok" })`)) - expect(success.ok).toBe(true) + expect(success).toMatchObject({ ok: true, value: "deployment failure" }) const failure = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "boom" })`)) expect(failure.ok).toBe(false) - const defect = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "defect" })`)) - expect(defect.ok).toBe(false) + expect(events).toEqual([ + 'tool.before {"name":"context.lookup","input":{"query":"deployment failure"}}', + "tool.after context.lookup success", + 'tool.before {"name":"context.lookup","input":{"query":"boom"}}', + "tool.after context.lookup failure:Lookup refused", + ]) + }) - expect(events).toStrictEqual([ - { phase: "start", index: 0, name: "context.lookup" }, - { phase: "end", index: 0, name: "context.lookup", outcome: "success" }, - { phase: "start", index: 0, name: "context.lookup" }, - { phase: "end", index: 0, name: "context.lookup", outcome: "failure", message: "Lookup refused" }, - { phase: "start", index: 0, name: "context.lookup" }, - { phase: "end", index: 0, name: "context.lookup", outcome: "failure", message: "broken" }, + test("surround each call to an extension global, with its arguments, but not functions inside results", async () => { + const events: Array = [] + const runtime = CodeMode.make({ + extensions: [ + Extension.make({ + name: "web", + globals: { + fetch: async (url: string, init?: { method?: string }) => ({ + url, + method: init?.method ?? "GET", + json: () => 1, + }), + fail: () => { + throw new RangeError("nope") + }, + }, + }), + ], + hooks: observe(events), + }) + const result = await Effect.runPromise( + runtime.execute(` + const res = await fetch("https://a.test/", { method: "POST" }) + res.json() + try { fail() } catch {} + return res.url + `), + ) + expect(result).toMatchObject({ ok: true, value: "https://a.test/", toolCalls: [] }) + expect(events).toEqual([ + 'extension.before {"extension":"web","name":"fetch","args":["https://a.test/",{"method":"POST"}]}', + "extension.after fetch success", + 'extension.before {"extension":"web","name":"fail","args":[]}', + "extension.after fail failure:nope", + ]) + }) + + test("a failing before hook denies the call; the program catches the host's error", async () => { + const runtime = CodeMode.make({ + tools: { + lookup: Tool.make({ + description: "Look up", + input: Schema.Struct({}), + output: Schema.String, + execute: () => Effect.succeed("ok"), + }), + }, + extensions: [Extension.make({ name: "web", globals: { fetch: async (url: string) => url } })], + hooks: { "extension.before": (call) => Effect.fail(new Error(`${call.name} is not allowed`)) }, + }) + expect( + await Effect.runPromise( + runtime.execute(` + const denied = await fetch("https://a.test/").catch((e) => [e instanceof Error, e.message]) + return [denied, await tools.lookup({})] + `), + ), + ).toMatchObject({ ok: true, value: [[true, "fetch is not allowed"], "ok"] }) + expect(await Effect.runPromise(runtime.execute(`await fetch("https://a.test/")`))).toMatchObject({ + ok: false, + error: { kind: "ExecutionFailure", message: "Error: fetch is not allowed" }, + }) + const refused = CodeMode.make({ + tools: { + lookup: Tool.make({ + description: "Look up", + input: Schema.Struct({}), + output: Schema.String, + execute: () => Effect.succeed("ok"), + }), + }, + hooks: { "tool.before": () => Effect.fail(toolError("lookup is not allowed")) }, + }) + expect( + await Effect.runPromise( + refused.execute(`try { await tools.lookup({}) } catch (e) { return [e.name, e.message] }`), + ), + ).toMatchObject({ ok: true, value: ["Error", "lookup is not allowed"], toolCalls: [{ name: "lookup" }] }) + expect(await Effect.runPromise(refused.execute(`await tools.lookup({})`))).toMatchObject({ + ok: false, + error: { kind: "ToolFailure", message: "lookup is not allowed" }, + }) + }) + + test("see a tool defect as the failure the program gets, and an interrupted extension call", async () => { + const events: Array = [] + const broken = Tool.make({ + description: "Broken", + input: Schema.Struct({}), + output: Schema.String, + execute: () => Effect.die("broken"), + }) + const runtime = CodeMode.make({ + tools: { broken }, + extensions: [Extension.make({ name: "slow", globals: { forever: () => new Promise(() => {}) } })], + hooks: observe(events), + }) + expect(await Effect.runPromise(runtime.execute(`await tools.broken({})`))).toMatchObject({ ok: false }) + expect(await Effect.runPromise(runtime.execute(`forever(); return "done"`))).toMatchObject({ + ok: true, + value: "done", + }) + expect(events).toEqual([ + 'tool.before {"name":"broken","input":{}}', + "tool.after broken failure:broken", + 'extension.before {"extension":"slow","name":"forever","args":[]}', + "extension.after forever interrupted", ]) }) @@ -243,15 +320,11 @@ describe("CodeMode tool-call observation", () => { execute: () => Effect.interrupt, }) const exit = await Effect.runPromiseExit( - CodeMode.make({ - tools: { host: { call } }, - onToolCallStart: () => Effect.sync(() => events.push("start")), - onToolCallEnd: (call) => Effect.sync(() => events.push(`end:${call.outcome}`)), - }).execute("return await tools.host.call({})"), + CodeMode.make({ tools: { host: { call } }, hooks: observe(events) }).execute("return await tools.host.call({})"), ) expect(exit._tag).toBe("Failure") - expect(events).toEqual(["start", "end:interrupted"]) + expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"]) }) test("observes running calls interrupted during completion", async () => { @@ -263,39 +336,17 @@ describe("CodeMode tool-call observation", () => { execute: () => Effect.never, }) const result = await Effect.runPromise( - CodeMode.make({ - tools: { host: { call } }, - onToolCallStart: () => Effect.sync(() => events.push("start")), - onToolCallEnd: (call) => Effect.sync(() => events.push(`end:${call.outcome}`)), - }).execute('tools.host.call({}); return "done"'), + CodeMode.make({ tools: { host: { call } }, hooks: observe(events) }).execute( + 'tools.host.call({}); return "done"', + ), ) expect(result).toMatchObject({ ok: true, value: "done" }) - expect(events).toEqual(["start", "end:interrupted"]) - }) - - test("ends calls interrupted during start observation", async () => { - const events: Array = [] - const call = Tool.make({ - description: "Unused", - input: Schema.Struct({}), - output: Schema.String, - execute: () => Effect.succeed("unused"), - }) - const exit = await Effect.runPromiseExit( - CodeMode.make({ - tools: { host: { call } }, - onToolCallStart: () => Effect.interrupt, - onToolCallEnd: (call) => Effect.sync(() => events.push(call.outcome)), - }).execute("return await tools.host.call({})"), - ) - - expect(exit._tag).toBe("Failure") - expect(events).toEqual(["interrupted"]) + expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"]) }) test("observes calls interrupted by the execution timeout", async () => { - const outcomes: Array = [] + const events: Array = [] const call = Tool.make({ description: "Pending", input: Schema.Struct({}), @@ -303,15 +354,13 @@ describe("CodeMode tool-call observation", () => { execute: () => Effect.never, }) const result = await Effect.runPromise( - CodeMode.make({ - tools: { host: { call } }, - limits: { timeoutMs: 10 }, - onToolCallEnd: (call) => Effect.sync(() => outcomes.push(call.outcome)), - }).execute("return await tools.host.call({})"), + CodeMode.make({ tools: { host: { call } }, limits: { timeoutMs: 10 }, hooks: observe(events) }).execute( + "return await tools.host.call({})", + ), ) expect(result).toMatchObject({ ok: false, error: { kind: "TimeoutExceeded" } }) - expect(outcomes).toEqual(["interrupted"]) + expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"]) }) }) @@ -840,20 +889,17 @@ describe("CodeMode public contract", () => { } }) - test("search is a counted tool call: it burns maxToolCalls and fires the hooks", async () => { - const started: Array = [] - const ended: Array = [] + test("search is a counted tool call: it burns maxToolCalls and is observed", async () => { + const observed: Array = [] const limited = CodeMode.make({ tools, limits: { maxToolCalls: 1 }, - onToolCallStart: (call) => Effect.sync(() => void started.push(call.name)), - onToolCallEnd: (call) => Effect.sync(() => void ended.push(`${call.name}:${call.outcome}`)), + hooks: { "tool.before": (call) => Effect.sync(() => void observed.push(call)) }, }) const result = await Effect.runPromise(limited.execute(`search({}); return search({})`)) expect(result.ok).toBe(false) if (!result.ok) expect(result.error.kind).toBe("ToolCallLimitExceeded") - expect(started).toEqual(["search"]) - expect(ended).toEqual(["search:success"]) + expect(observed).toEqual([{ name: "search", input: {} }]) }) test("search is an opaque, shadowable global like other built-ins", async () => { @@ -1089,7 +1135,7 @@ describe("CodeMode public contract", () => { }) const runtime = CodeMode.make({ tools: { math: { double: transformed } }, - onToolCallStart: (call) => Effect.sync(() => observed.push(call.input)), + hooks: { "tool.before": (call) => Effect.sync(() => void observed.push(call.input)) }, }) const success = await Effect.runPromise(runtime.execute(`return await tools.math.double({ value: "21" })`)) diff --git a/packages/codemode/test/extensions.test.ts b/packages/codemode/test/extensions.test.ts index fa7501c872d..df208af10cd 100644 --- a/packages/codemode/test/extensions.test.ts +++ b/packages/codemode/test/extensions.test.ts @@ -192,6 +192,16 @@ describe("values are converted at the boundary, never shared", () => { expect((await failure(`big()`, target)).message).toContain("big produced a bigint") }) + test("a value that cannot come out carries a location like any built-in error, sync or async", async () => { + const target = CodeMode.make({ + extensions: [ + Extension.make({ name: "odd", globals: { sym: () => Symbol("s"), later: async () => Symbol("s") } }), + ], + }) + expect((await failure(`sym()`, target)).location).toEqual((await failure(`JSON.parse("{")`, target)).location) + expect((await failure(`await later()`, target)).location).toBeDefined() + }) + test("a class instance cannot come out", async () => { class Other {} const target = CodeMode.make({ diff --git a/packages/core/src/codemode/tool.ts b/packages/core/src/codemode/tool.ts index b2f99b0201b..8514b9d50ac 100644 --- a/packages/core/src/codemode/tool.ts +++ b/packages/core/src/codemode/tool.ts @@ -82,9 +82,9 @@ export const create = ( const files = yield* Ref.make>([]) const calls = yield* Ref.make>([]) const lock = Semaphore.makeUnsafe(1) - const updateCalls = (update: (items: Array) => Array) => + const record = (update: (items: Array) => Array) => lock.withPermit( - Ref.updateAndGet(calls, update).pipe(Effect.flatMap((toolCalls) => context.progress({ toolCalls }))), + Ref.updateAndGet(calls, update).pipe(Effect.tap((toolCalls) => context.progress({ toolCalls }))), ) const result = yield* runtime( inventory, @@ -103,27 +103,7 @@ export const create = ( const text = content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join("\n") return text === "" ? null : text }), - { - onToolCallStart: ({ index, name, input }) => { - const shown = displayInput(input) - return updateCalls((items) => { - const next = [...items] - next[index] = { tool: name, status: "running", ...(shown ? { input: shown } : {}) } - return next - }) - }, - onToolCallEnd: ({ index, name, input, outcome }) => { - const shown = displayInput(input) - return updateCalls((items) => { - const next = [...items] - next[index] = { - ...(items[index] ?? { tool: name, ...(shown ? { input: shown } : {}) }), - status: outcome === "success" ? "completed" : "error", - } - return next - }) - }, - }, + progressHooks(record), ).execute(code) const toolCalls = yield* Ref.get(calls) const collected = (yield* Ref.get(files)) @@ -158,6 +138,33 @@ export const create = ( } satisfies Info } +// Rows appear in start order; the same call object arrives at both hooks, so a call finds its row again. +function progressHooks(record: (update: (items: Array) => Array) => Effect.Effect) { + const rows = new WeakMap() + const start = (call: object, entry: ExecuteCall) => + record((items) => { + rows.set(call, items.length) + return [...items, entry] + }) + const settle = (call: object, result: CodeMode.CallResult) => + record((items) => { + const index = rows.get(call) + if (index === undefined) return items + const next = [...items] + next[index] = { ...items[index], status: result.status === "success" ? "completed" : "error" } + return next + }) + return { + "tool.before": (call) => { + const shown = displayInput(call.input) + return start(call, { tool: call.name, status: "running", ...(shown ? { input: shown } : {}) }) + }, + "tool.after": settle, + "extension.before": (call) => start(call, { tool: call.name, status: "running" }), + "extension.after": settle, + } satisfies CodeMode.Hooks +} + export const catalog = (inventory: Inventory) => { const pinned = new Set( Array.from(inventory.tools.values()) @@ -204,7 +211,7 @@ function renderCatalog(root: CatalogNode): ReadonlyArray Effect.Effect, - hooks?: CodeMode.ToolCallHooks, + hooks?: CodeMode.Hooks, ) { // A path may carry namespace metadata, a callable tool, child tools, or all three. const root: ToolNode = { children: new Map() } @@ -219,7 +226,7 @@ function runtime( }) } const tools = renderTools(root) - return CodeMode.make({ tools, ...hooks }) + return CodeMode.make({ tools, hooks }) } function getNode(root: Node, path: string) {