refactor(codemode): before/after hooks for tool and extension calls (#49242)

This commit is contained in:
Aiden Cline
2026-09-15 23:05:06 -05:00
committed by GitHub
Unverified
parent 9bf5faa978
commit c57733a647
9 changed files with 282 additions and 190 deletions
+5 -2
View File
@@ -88,8 +88,11 @@ runtime.catalog() // structured tool descriptions
runtime.execute(source) // Effect<CodeMode.Result, never, ToolServices>
```
The Effect environment is inferred from the supplied tools. `onToolCallStart` observes admitted calls with decoded
input; `onToolCallEnd` observes settled outcomes and duration. Both hooks return Effects and must not fail.
The Effect environment is inferred from the supplied tools. `hooks` surround every call the program makes into the
host: `tool.before`/`tool.after` receive `{ name, input }` with decoded input, `extension.before`/`extension.after`
receive `{ extension, name, args }`. An `after` hook also receives how the call ended (`success` with its value,
`failure` with its error, or `interrupted`). A failing `before` hook denies the call, and the program catches the
failure as a thrown error.
### `Values`
+2
View File
@@ -467,6 +467,8 @@ Nothing is exposed unless a host provides it; extension calls are not tool calls
- [x] A host function inside a result becomes a program function whose calls cross the same way, so a result can
carry methods (`res.json()`) whose host closures keep the host state. Diagnostics name it by its path
(`fetch.json`). Like any program function it vanishes at the data boundary.
- [x] Each call to an extension global runs inside the host's `extension.before`/`extension.after` hooks as
`{ extension, name, args }`, with the host's own error on failure; calls to functions inside results do not.
- [x] A host `Promise` becomes a program promise. Whatever host code returns, resolves, throws, or rejects with
crosses the same way, so `catch (e)` receives a copy of the thrown value (an `Error` of the matching type, or
plain data).
+13 -3
View File
@@ -7,7 +7,14 @@ import { type Services, type ToolDescription, ToolRuntime } from "./tool-runtime
import type { Tools } from "./tools.js"
/** A tool call admitted during an execution. */
export type { ToolCall, ToolCallEnded, ToolCallHooks, ToolCallStarted, ToolDescription } from "./tool-runtime.js"
export type {
CallResult,
ExtensionInvocation,
Hooks,
ToolCall,
ToolDescription,
ToolInvocation,
} from "./tool-runtime.js"
/** Signature-construction helpers for host-owned catalog instructions. */
export { searchSignature, toolExpression } from "./tool-runtime.js"
@@ -34,9 +41,11 @@ export type ResolvedExecutionLimits = {
}
/** Configuration shared by `CodeMode.make` and `CodeMode.execute`. */
export type Options<Provided extends Record<string, unknown> = {}> = ToolRuntime.ToolCallHooks<Services<Provided>> & {
export type Options<Provided extends Record<string, unknown> = {}> = {
/** Explicit tools exposed to the program as `tools`. */
tools?: Provided & Tools<Services<Provided>>
/** Hooks around every tool and extension call the program makes; see `Hooks`. */
hooks?: ToolRuntime.Hooks<Services<Provided>>
/** Host functions exposed as globals; see `Extension.make`. */
extensions?: ReadonlyArray<Extension>
/** Resource limits enforced on each execution. */
@@ -148,6 +157,7 @@ export const make = <const Provided extends Record<string, unknown> = {}>(
}
return {
catalog: prepared.catalog,
execute: (code) => executeProgram(code, prepared, limits, options, (ctx) => extensionGlobals(ctx, extensions)),
execute: (code) =>
executeProgram(code, prepared, limits, options.hooks ?? {}, (ctx) => extensionGlobals(ctx, extensions)),
}
}
+1 -1
View File
@@ -16,7 +16,7 @@ export const executeProgram = <R>(
code: string,
prepared: ToolRuntime.Prepared<R>,
limits: ResolvedExecutionLimits,
hooks: ToolRuntime.ToolCallHooks<R>,
hooks: ToolRuntime.Hooks<R>,
globals?: (ctx: Interpreter<R>) => ReadonlyArray<readonly [string, unknown]>,
): Effect.Effect<Result, never, R> => {
if (code.trim().length === 0) {
+22 -13
View File
@@ -1,6 +1,7 @@
import { Effect } from "effect"
import type { Extension } from "../extension.js"
import { coerceToString } from "../stdlib/value.js"
import { type ExtensionInvocation, hooked } from "../tool-runtime.js"
import type { Interpreter } from "./interpreter.js"
import { createErrorValue, isErrorType } from "./intrinsics.js"
import { MAX_VALUE_DEPTH } from "./limits.js"
@@ -11,7 +12,6 @@ import {
define,
entries,
get,
type Native,
Arr,
Bytes,
DateObj,
@@ -128,26 +128,35 @@ export const extensionGlobals = <R>(
throw typeError(`${label} produced ${describeHost(value)}, which the program cannot hold.`)
}
// A host function as a program function: arguments cross in, and whatever it returns, resolves, throws, or
// rejects with crosses out, so the program catches what the author threw.
const wrap = (value: Function, label: string): Native<R> =>
// A host function as a program function. Arguments cross in; a global's call runs inside the host's extension
// hooks, which see the host's own error on failure; then whatever came back, or was thrown, crosses out so the
// program catches a copy. Functions inside results are part of a value's API and skip the hooks.
const wrap = (value: Function, label: string, describe?: (args: ReadonlyArray<unknown>) => ExtensionInvocation) =>
fn<R>(builtins, value.name, value.length, (_, values) => {
const converted = values.map((item, index) => toHost(item, `Argument ${index + 1} to ${label}`))
const thrown = (reason: unknown) => new Throw(fromHost(reason, label))
const args = values.map((item, index) => toHost(item, `Argument ${index + 1} to ${label}`))
const hooks = ctx.tools.hooks
const settle = (run: Effect.Effect<unknown, unknown, R>) =>
(describe === undefined
? run
: hooked(describe(args), hooks["extension.before"], hooks["extension.after"], run)
).pipe(
Effect.mapError((reason) => new Throw(fromHost(reason, label))),
Effect.map((settled) => fromHost(settled, label)),
)
let result: unknown
try {
result = value.apply(undefined, converted)
result = value.apply(undefined, args)
} catch (reason) {
return Effect.fail(thrown(reason))
return settle(Effect.fail(reason))
}
if (!(result instanceof Promise)) return fromHost(result, label)
return ctx.pending.create(
Effect.map(Effect.tryPromise({ try: () => result, catch: thrown }), (settled) => fromHost(settled, label)),
)
if (!(result instanceof Promise)) return settle(Effect.succeed(result))
return ctx.pending.create(settle(Effect.tryPromise({ try: () => result, catch: (reason) => reason })))
})
return extensions.flatMap((extension) =>
Object.entries(extension.globals).map(([name, value]) => [name, wrap(value, name)] as const),
Object.entries(extension.globals).map(
([name, value]) => [name, wrap(value, name, (args) => ({ extension: extension.name, name, args }))] as const,
),
)
}
+45 -40
View File
@@ -36,26 +36,48 @@ export type ToolCall = {
readonly name: string
}
export type ToolCallStarted = {
readonly index: number
/** A tool call the program is making, with its decoded input. */
export type ToolInvocation = { readonly name: string; readonly input: unknown }
/** A call the program is making to an extension global, with its arguments. */
export type ExtensionInvocation = {
readonly extension: string
readonly name: string
readonly input: unknown
readonly args: ReadonlyArray<unknown>
}
export type ToolCallEnded = {
readonly index: number
readonly name: string
readonly input: unknown
readonly durationMs: number
readonly outcome: "success" | "failure" | "interrupted"
readonly message?: string
/** How a call ended; `after` hooks observe it and cannot change it. */
export type CallResult =
| { readonly status: "success"; readonly value: unknown }
| { readonly status: "failure"; readonly error: unknown }
| { readonly status: "interrupted" }
/** Hooks around every call the program makes into the host. A failing `before` denies the call. */
export type Hooks<R = never> = {
readonly "tool.before"?: ((call: ToolInvocation) => Effect.Effect<void, unknown, R>) | undefined
readonly "tool.after"?: ((call: ToolInvocation, result: CallResult) => Effect.Effect<void, never, R>) | undefined
readonly "extension.before"?: ((call: ExtensionInvocation) => Effect.Effect<void, unknown, R>) | undefined
readonly "extension.after"?:
| ((call: ExtensionInvocation, result: CallResult) => Effect.Effect<void, never, R>)
| undefined
}
export type ToolCallHooks<R = never> = {
/** Observes decoded tool input immediately before tool execution. */
readonly onToolCallStart?: ((call: ToolCallStarted) => Effect.Effect<void, never, R>) | undefined
/** Observes each admitted tool call as it succeeds, fails, or is interrupted. */
readonly onToolCallEnd?: ((call: ToolCallEnded) => Effect.Effect<void, never, R>) | undefined
/** Runs `before`, then `run`, then `after` with how it ended, including when interrupted. */
export const hooked = <Call, A, R>(
call: Call,
before: ((call: Call) => Effect.Effect<void, unknown, R>) | undefined,
after: ((call: Call, result: CallResult) => Effect.Effect<void, never, R>) | undefined,
run: Effect.Effect<A, unknown, R>,
): Effect.Effect<A, unknown, R> => {
const observed =
after === undefined
? run
: Effect.onExit(run, (exit) => {
if (Exit.isSuccess(exit)) return after(call, { status: "success", value: exit.value })
if (Cause.hasInterruptsOnly(exit.cause)) return after(call, { status: "interrupted" })
return after(call, { status: "failure", error: Cause.squash(exit.cause) })
})
return before === undefined ? observed : Effect.andThen(before(call), observed)
}
export type ToolDescription = {
@@ -316,6 +338,7 @@ export class ToolRuntimeError extends Error {
/** The tool bridge of one execution. Arguments arrive and results leave as JSON; program values never enter. */
export type ToolRuntime<R = never> = {
readonly calls: Array<ToolCall>
readonly hooks: Hooks<R>
readonly execute: (
path: ReadonlyArray<string>,
args: Array<Json | undefined>,
@@ -328,28 +351,12 @@ export type ToolRuntime<R = never> = {
export const make = <R>(
prepared: Prepared<R>,
maxToolCalls: number | undefined,
hooks?: ToolCallHooks<R>,
hooks: Hooks<R> = {},
): ToolRuntime<R> => {
const calls: Array<ToolCall> = []
const root = prepared.root
const searchTool = makeSearchTool(prepared.searchIndex)
const observeEnd = <A, E>(effect: Effect.Effect<A, E, R>, call: ToolCallStarted): Effect.Effect<A, E, R> => {
const onEnd = hooks?.onToolCallEnd
if (onEnd === undefined) return effect
const startedAt = Date.now()
return effect.pipe(
Effect.onExit((exit) => {
const durationMs = Date.now() - startedAt
if (Exit.isSuccess(exit)) return onEnd({ ...call, durationMs, outcome: "success" })
if (Cause.hasInterruptsOnly(exit.cause)) return onEnd({ ...call, durationMs, outcome: "interrupted" })
const error = Cause.squash(exit.cause)
const message = error instanceof Error ? error.message : Cause.pretty(exit.cause)
return onEnd({ ...call, durationMs, outcome: "failure", message })
}),
)
}
const recordCall = (call: ToolCall): void => {
if (maxToolCalls !== undefined && calls.length >= maxToolCalls) {
throw new ToolRuntimeError("ToolCallLimitExceeded", `Execution exceeded its tool-call limit of ${maxToolCalls}.`)
@@ -371,14 +378,12 @@ export const make = <R>(
name === "search" ? [] : ["The signature may have changed. Use search to get the current signature."],
),
})
const index = yield* Effect.sync(() => {
recordCall({ name })
return calls.length - 1
})
const call = { index, name, input }
return yield* observeEnd(
yield* Effect.sync(() => recordCall({ name }))
return yield* hooked(
{ name, input },
hooks["tool.before"],
hooks["tool.after"],
Effect.gen(function* () {
if (hooks?.onToolCallStart !== undefined) yield* hooks.onToolCallStart(call)
const raw = yield* Effect.suspend(() => tool.execute(input)).pipe(
Effect.catchCause((cause) => {
if (Cause.hasInterruptsOnly(cause)) return Effect.interrupt
@@ -400,12 +405,12 @@ export const make = <R>(
catch: (cause) => new ToolRuntimeError("InvalidToolOutput", `Invalid output from tool '${name}': ${cause}`),
})
}),
call,
)
})
return {
calls,
hooks,
keys: (path) => namespaceKeys(root, path),
search: (args) => Effect.suspend(() => executeTool("search", searchTool, args)),
execute: (path, args) =>
+152 -106
View File
@@ -1,6 +1,6 @@
import { describe, expect, test } from "bun:test"
import { Cause, Effect, Schema } from "effect"
import { CodeMode, Tool, toolError } from "../src/index.js"
import { CodeMode, Extension, Tool, toolError } from "../src/index.js"
const run = (tool: Tool.Tool<never>) =>
Effect.runPromise(CodeMode.make({ tools: { host: { call: tool } } }).execute("return await tools.host.call({})"))
@@ -160,77 +160,154 @@ describe("CodeMode host failure boundary", () => {
})
})
describe("CodeMode tool-call observation", () => {
test("reports the tools actually invoked with decoded input", async () => {
const calls: Array<unknown> = []
describe("CodeMode call hooks", () => {
const ended = (result: CodeMode.CallResult) => {
if (result.status !== "failure") return result.status
return `failure:${result.error instanceof Error ? result.error.message : String(result.error)}`
}
// Every hook logs its event, so the order across tools and extensions is visible.
const observe = (events: Array<string>): CodeMode.Hooks => ({
"tool.before": (call) => Effect.sync(() => void events.push(`tool.before ${JSON.stringify(call)}`)),
"tool.after": (call, result) => Effect.sync(() => void events.push(`tool.after ${call.name} ${ended(result)}`)),
"extension.before": (call) => Effect.sync(() => void events.push(`extension.before ${JSON.stringify(call)}`)),
"extension.after": (call, result) =>
Effect.sync(() => void events.push(`extension.after ${call.name} ${ended(result)}`)),
})
test("surround the tools actually invoked, with decoded input", async () => {
const events: Array<string> = []
const lookup = Tool.make({
description: "Look up a value",
input: Schema.Struct({ query: Schema.String }),
output: Schema.String,
execute: ({ query }) => Effect.succeed(query),
execute: ({ query }) => (query === "boom" ? Effect.fail(toolError("Lookup refused")) : Effect.succeed(query)),
})
const runtime = CodeMode.make({ tools: { context: { lookup } }, hooks: observe(events) })
const result = await Effect.runPromise(
CodeMode.make({
tools: { context: { lookup } },
onToolCallStart: (call) => Effect.sync(() => calls.push(call)),
}).execute(`
const success = await Effect.runPromise(
runtime.execute(`
if (false) await tools.context.lookup({ query: "not called" })
return await tools.context.lookup({ query: "deployment failure" })
`),
)
expect(result.ok).toBe(true)
expect(calls).toStrictEqual([{ index: 0, name: "context.lookup", input: { query: "deployment failure" } }])
})
test("observes settled calls with outcome and duration", async () => {
const events: Array<{ phase: string; index: number; name: string; outcome?: string; message?: string }> = []
const lookup = Tool.make({
description: "Look up a value",
input: Schema.Struct({ query: Schema.String }),
output: Schema.String,
execute: ({ query }) =>
query === "boom"
? Effect.fail(toolError("Lookup refused"))
: query === "defect"
? Effect.die("broken")
: Effect.succeed(query),
})
const runtime = CodeMode.make({
tools: { context: { lookup } },
onToolCallStart: (call) =>
Effect.sync(() => {
events.push({ phase: "start", index: call.index, name: call.name })
}),
onToolCallEnd: (call) =>
Effect.sync(() => {
expect(call.durationMs).toBeGreaterThanOrEqual(0)
events.push({
phase: "end",
index: call.index,
name: call.name,
outcome: call.outcome,
...(call.message === undefined ? {} : { message: call.message }),
})
}),
})
const success = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "ok" })`))
expect(success.ok).toBe(true)
expect(success).toMatchObject({ ok: true, value: "deployment failure" })
const failure = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "boom" })`))
expect(failure.ok).toBe(false)
const defect = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "defect" })`))
expect(defect.ok).toBe(false)
expect(events).toEqual([
'tool.before {"name":"context.lookup","input":{"query":"deployment failure"}}',
"tool.after context.lookup success",
'tool.before {"name":"context.lookup","input":{"query":"boom"}}',
"tool.after context.lookup failure:Lookup refused",
])
})
expect(events).toStrictEqual([
{ phase: "start", index: 0, name: "context.lookup" },
{ phase: "end", index: 0, name: "context.lookup", outcome: "success" },
{ phase: "start", index: 0, name: "context.lookup" },
{ phase: "end", index: 0, name: "context.lookup", outcome: "failure", message: "Lookup refused" },
{ phase: "start", index: 0, name: "context.lookup" },
{ phase: "end", index: 0, name: "context.lookup", outcome: "failure", message: "broken" },
test("surround each call to an extension global, with its arguments, but not functions inside results", async () => {
const events: Array<string> = []
const runtime = CodeMode.make({
extensions: [
Extension.make({
name: "web",
globals: {
fetch: async (url: string, init?: { method?: string }) => ({
url,
method: init?.method ?? "GET",
json: () => 1,
}),
fail: () => {
throw new RangeError("nope")
},
},
}),
],
hooks: observe(events),
})
const result = await Effect.runPromise(
runtime.execute(`
const res = await fetch("https://a.test/", { method: "POST" })
res.json()
try { fail() } catch {}
return res.url
`),
)
expect(result).toMatchObject({ ok: true, value: "https://a.test/", toolCalls: [] })
expect(events).toEqual([
'extension.before {"extension":"web","name":"fetch","args":["https://a.test/",{"method":"POST"}]}',
"extension.after fetch success",
'extension.before {"extension":"web","name":"fail","args":[]}',
"extension.after fail failure:nope",
])
})
test("a failing before hook denies the call; the program catches the host's error", async () => {
const runtime = CodeMode.make({
tools: {
lookup: Tool.make({
description: "Look up",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.succeed("ok"),
}),
},
extensions: [Extension.make({ name: "web", globals: { fetch: async (url: string) => url } })],
hooks: { "extension.before": (call) => Effect.fail(new Error(`${call.name} is not allowed`)) },
})
expect(
await Effect.runPromise(
runtime.execute(`
const denied = await fetch("https://a.test/").catch((e) => [e instanceof Error, e.message])
return [denied, await tools.lookup({})]
`),
),
).toMatchObject({ ok: true, value: [[true, "fetch is not allowed"], "ok"] })
expect(await Effect.runPromise(runtime.execute(`await fetch("https://a.test/")`))).toMatchObject({
ok: false,
error: { kind: "ExecutionFailure", message: "Error: fetch is not allowed" },
})
const refused = CodeMode.make({
tools: {
lookup: Tool.make({
description: "Look up",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.succeed("ok"),
}),
},
hooks: { "tool.before": () => Effect.fail(toolError("lookup is not allowed")) },
})
expect(
await Effect.runPromise(
refused.execute(`try { await tools.lookup({}) } catch (e) { return [e.name, e.message] }`),
),
).toMatchObject({ ok: true, value: ["Error", "lookup is not allowed"], toolCalls: [{ name: "lookup" }] })
expect(await Effect.runPromise(refused.execute(`await tools.lookup({})`))).toMatchObject({
ok: false,
error: { kind: "ToolFailure", message: "lookup is not allowed" },
})
})
test("see a tool defect as the failure the program gets, and an interrupted extension call", async () => {
const events: Array<string> = []
const broken = Tool.make({
description: "Broken",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.die("broken"),
})
const runtime = CodeMode.make({
tools: { broken },
extensions: [Extension.make({ name: "slow", globals: { forever: () => new Promise(() => {}) } })],
hooks: observe(events),
})
expect(await Effect.runPromise(runtime.execute(`await tools.broken({})`))).toMatchObject({ ok: false })
expect(await Effect.runPromise(runtime.execute(`forever(); return "done"`))).toMatchObject({
ok: true,
value: "done",
})
expect(events).toEqual([
'tool.before {"name":"broken","input":{}}',
"tool.after broken failure:broken",
'extension.before {"extension":"slow","name":"forever","args":[]}',
"extension.after forever interrupted",
])
})
@@ -243,15 +320,11 @@ describe("CodeMode tool-call observation", () => {
execute: () => Effect.interrupt,
})
const exit = await Effect.runPromiseExit(
CodeMode.make({
tools: { host: { call } },
onToolCallStart: () => Effect.sync(() => events.push("start")),
onToolCallEnd: (call) => Effect.sync(() => events.push(`end:${call.outcome}`)),
}).execute("return await tools.host.call({})"),
CodeMode.make({ tools: { host: { call } }, hooks: observe(events) }).execute("return await tools.host.call({})"),
)
expect(exit._tag).toBe("Failure")
expect(events).toEqual(["start", "end:interrupted"])
expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"])
})
test("observes running calls interrupted during completion", async () => {
@@ -263,39 +336,17 @@ describe("CodeMode tool-call observation", () => {
execute: () => Effect.never,
})
const result = await Effect.runPromise(
CodeMode.make({
tools: { host: { call } },
onToolCallStart: () => Effect.sync(() => events.push("start")),
onToolCallEnd: (call) => Effect.sync(() => events.push(`end:${call.outcome}`)),
}).execute('tools.host.call({}); return "done"'),
CodeMode.make({ tools: { host: { call } }, hooks: observe(events) }).execute(
'tools.host.call({}); return "done"',
),
)
expect(result).toMatchObject({ ok: true, value: "done" })
expect(events).toEqual(["start", "end:interrupted"])
})
test("ends calls interrupted during start observation", async () => {
const events: Array<string> = []
const call = Tool.make({
description: "Unused",
input: Schema.Struct({}),
output: Schema.String,
execute: () => Effect.succeed("unused"),
})
const exit = await Effect.runPromiseExit(
CodeMode.make({
tools: { host: { call } },
onToolCallStart: () => Effect.interrupt,
onToolCallEnd: (call) => Effect.sync(() => events.push(call.outcome)),
}).execute("return await tools.host.call({})"),
)
expect(exit._tag).toBe("Failure")
expect(events).toEqual(["interrupted"])
expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"])
})
test("observes calls interrupted by the execution timeout", async () => {
const outcomes: Array<string> = []
const events: Array<string> = []
const call = Tool.make({
description: "Pending",
input: Schema.Struct({}),
@@ -303,15 +354,13 @@ describe("CodeMode tool-call observation", () => {
execute: () => Effect.never,
})
const result = await Effect.runPromise(
CodeMode.make({
tools: { host: { call } },
limits: { timeoutMs: 10 },
onToolCallEnd: (call) => Effect.sync(() => outcomes.push(call.outcome)),
}).execute("return await tools.host.call({})"),
CodeMode.make({ tools: { host: { call } }, limits: { timeoutMs: 10 }, hooks: observe(events) }).execute(
"return await tools.host.call({})",
),
)
expect(result).toMatchObject({ ok: false, error: { kind: "TimeoutExceeded" } })
expect(outcomes).toEqual(["interrupted"])
expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"])
})
})
@@ -840,20 +889,17 @@ describe("CodeMode public contract", () => {
}
})
test("search is a counted tool call: it burns maxToolCalls and fires the hooks", async () => {
const started: Array<string> = []
const ended: Array<string> = []
test("search is a counted tool call: it burns maxToolCalls and is observed", async () => {
const observed: Array<CodeMode.ToolInvocation> = []
const limited = CodeMode.make({
tools,
limits: { maxToolCalls: 1 },
onToolCallStart: (call) => Effect.sync(() => void started.push(call.name)),
onToolCallEnd: (call) => Effect.sync(() => void ended.push(`${call.name}:${call.outcome}`)),
hooks: { "tool.before": (call) => Effect.sync(() => void observed.push(call)) },
})
const result = await Effect.runPromise(limited.execute(`search({}); return search({})`))
expect(result.ok).toBe(false)
if (!result.ok) expect(result.error.kind).toBe("ToolCallLimitExceeded")
expect(started).toEqual(["search"])
expect(ended).toEqual(["search:success"])
expect(observed).toEqual([{ name: "search", input: {} }])
})
test("search is an opaque, shadowable global like other built-ins", async () => {
@@ -1089,7 +1135,7 @@ describe("CodeMode public contract", () => {
})
const runtime = CodeMode.make({
tools: { math: { double: transformed } },
onToolCallStart: (call) => Effect.sync(() => observed.push(call.input)),
hooks: { "tool.before": (call) => Effect.sync(() => void observed.push(call.input)) },
})
const success = await Effect.runPromise(runtime.execute(`return await tools.math.double({ value: "21" })`))
+10
View File
@@ -192,6 +192,16 @@ describe("values are converted at the boundary, never shared", () => {
expect((await failure(`big()`, target)).message).toContain("big produced a bigint")
})
test("a value that cannot come out carries a location like any built-in error, sync or async", async () => {
const target = CodeMode.make({
extensions: [
Extension.make({ name: "odd", globals: { sym: () => Symbol("s"), later: async () => Symbol("s") } }),
],
})
expect((await failure(`sym()`, target)).location).toEqual((await failure(`JSON.parse("{")`, target)).location)
expect((await failure(`await later()`, target)).location).toBeDefined()
})
test("a class instance cannot come out", async () => {
class Other {}
const target = CodeMode.make({
+32 -25
View File
@@ -82,9 +82,9 @@ export const create = (
const files = yield* Ref.make<Array<CollectedFiles>>([])
const calls = yield* Ref.make<Array<ExecuteCall>>([])
const lock = Semaphore.makeUnsafe(1)
const updateCalls = (update: (items: Array<ExecuteCall>) => Array<ExecuteCall>) =>
const record = (update: (items: Array<ExecuteCall>) => Array<ExecuteCall>) =>
lock.withPermit(
Ref.updateAndGet(calls, update).pipe(Effect.flatMap((toolCalls) => context.progress({ toolCalls }))),
Ref.updateAndGet(calls, update).pipe(Effect.tap((toolCalls) => context.progress({ toolCalls }))),
)
const result = yield* runtime(
inventory,
@@ -103,27 +103,7 @@ export const create = (
const text = content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join("\n")
return text === "" ? null : text
}),
{
onToolCallStart: ({ index, name, input }) => {
const shown = displayInput(input)
return updateCalls((items) => {
const next = [...items]
next[index] = { tool: name, status: "running", ...(shown ? { input: shown } : {}) }
return next
})
},
onToolCallEnd: ({ index, name, input, outcome }) => {
const shown = displayInput(input)
return updateCalls((items) => {
const next = [...items]
next[index] = {
...(items[index] ?? { tool: name, ...(shown ? { input: shown } : {}) }),
status: outcome === "success" ? "completed" : "error",
}
return next
})
},
},
progressHooks(record),
).execute(code)
const toolCalls = yield* Ref.get(calls)
const collected = (yield* Ref.get(files))
@@ -158,6 +138,33 @@ export const create = (
} satisfies Info
}
// Rows appear in start order; the same call object arrives at both hooks, so a call finds its row again.
function progressHooks(record: (update: (items: Array<ExecuteCall>) => Array<ExecuteCall>) => Effect.Effect<unknown>) {
const rows = new WeakMap<object, number>()
const start = (call: object, entry: ExecuteCall) =>
record((items) => {
rows.set(call, items.length)
return [...items, entry]
})
const settle = (call: object, result: CodeMode.CallResult) =>
record((items) => {
const index = rows.get(call)
if (index === undefined) return items
const next = [...items]
next[index] = { ...items[index], status: result.status === "success" ? "completed" : "error" }
return next
})
return {
"tool.before": (call) => {
const shown = displayInput(call.input)
return start(call, { tool: call.name, status: "running", ...(shown ? { input: shown } : {}) })
},
"tool.after": settle,
"extension.before": (call) => start(call, { tool: call.name, status: "running" }),
"extension.after": settle,
} satisfies CodeMode.Hooks
}
export const catalog = (inventory: Inventory) => {
const pinned = new Set(
Array.from(inventory.tools.values())
@@ -204,7 +211,7 @@ function renderCatalog(root: CatalogNode): ReadonlyArray<CodeModeCatalog.Tool |
function runtime(
inventory: Inventory,
executeTool: (name: string, tool: Info, input: unknown) => Effect.Effect<unknown, unknown>,
hooks?: CodeMode.ToolCallHooks,
hooks?: CodeMode.Hooks,
) {
// A path may carry namespace metadata, a callable tool, child tools, or all three.
const root: ToolNode = { children: new Map() }
@@ -219,7 +226,7 @@ function runtime(
})
}
const tools = renderTools(root)
return CodeMode.make<typeof tools>({ tools, ...hooks })
return CodeMode.make<typeof tools>({ tools, hooks })
}
function getNode<T>(root: Node<T>, path: string) {