mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-17 23:24:00 +03:00
refactor(codemode): before/after hooks for tool and extension calls (#49242)
This commit is contained in:
committed by
GitHub
Unverified
parent
9bf5faa978
commit
c57733a647
@@ -88,8 +88,11 @@ runtime.catalog() // structured tool descriptions
|
||||
runtime.execute(source) // Effect<CodeMode.Result, never, ToolServices>
|
||||
```
|
||||
|
||||
The Effect environment is inferred from the supplied tools. `onToolCallStart` observes admitted calls with decoded
|
||||
input; `onToolCallEnd` observes settled outcomes and duration. Both hooks return Effects and must not fail.
|
||||
The Effect environment is inferred from the supplied tools. `hooks` surround every call the program makes into the
|
||||
host: `tool.before`/`tool.after` receive `{ name, input }` with decoded input, `extension.before`/`extension.after`
|
||||
receive `{ extension, name, args }`. An `after` hook also receives how the call ended (`success` with its value,
|
||||
`failure` with its error, or `interrupted`). A failing `before` hook denies the call, and the program catches the
|
||||
failure as a thrown error.
|
||||
|
||||
### `Values`
|
||||
|
||||
|
||||
@@ -467,6 +467,8 @@ Nothing is exposed unless a host provides it; extension calls are not tool calls
|
||||
- [x] A host function inside a result becomes a program function whose calls cross the same way, so a result can
|
||||
carry methods (`res.json()`) whose host closures keep the host state. Diagnostics name it by its path
|
||||
(`fetch.json`). Like any program function it vanishes at the data boundary.
|
||||
- [x] Each call to an extension global runs inside the host's `extension.before`/`extension.after` hooks as
|
||||
`{ extension, name, args }`, with the host's own error on failure; calls to functions inside results do not.
|
||||
- [x] A host `Promise` becomes a program promise. Whatever host code returns, resolves, throws, or rejects with
|
||||
crosses the same way, so `catch (e)` receives a copy of the thrown value (an `Error` of the matching type, or
|
||||
plain data).
|
||||
|
||||
@@ -7,7 +7,14 @@ import { type Services, type ToolDescription, ToolRuntime } from "./tool-runtime
|
||||
import type { Tools } from "./tools.js"
|
||||
|
||||
/** A tool call admitted during an execution. */
|
||||
export type { ToolCall, ToolCallEnded, ToolCallHooks, ToolCallStarted, ToolDescription } from "./tool-runtime.js"
|
||||
export type {
|
||||
CallResult,
|
||||
ExtensionInvocation,
|
||||
Hooks,
|
||||
ToolCall,
|
||||
ToolDescription,
|
||||
ToolInvocation,
|
||||
} from "./tool-runtime.js"
|
||||
/** Signature-construction helpers for host-owned catalog instructions. */
|
||||
export { searchSignature, toolExpression } from "./tool-runtime.js"
|
||||
|
||||
@@ -34,9 +41,11 @@ export type ResolvedExecutionLimits = {
|
||||
}
|
||||
|
||||
/** Configuration shared by `CodeMode.make` and `CodeMode.execute`. */
|
||||
export type Options<Provided extends Record<string, unknown> = {}> = ToolRuntime.ToolCallHooks<Services<Provided>> & {
|
||||
export type Options<Provided extends Record<string, unknown> = {}> = {
|
||||
/** Explicit tools exposed to the program as `tools`. */
|
||||
tools?: Provided & Tools<Services<Provided>>
|
||||
/** Hooks around every tool and extension call the program makes; see `Hooks`. */
|
||||
hooks?: ToolRuntime.Hooks<Services<Provided>>
|
||||
/** Host functions exposed as globals; see `Extension.make`. */
|
||||
extensions?: ReadonlyArray<Extension>
|
||||
/** Resource limits enforced on each execution. */
|
||||
@@ -148,6 +157,7 @@ export const make = <const Provided extends Record<string, unknown> = {}>(
|
||||
}
|
||||
return {
|
||||
catalog: prepared.catalog,
|
||||
execute: (code) => executeProgram(code, prepared, limits, options, (ctx) => extensionGlobals(ctx, extensions)),
|
||||
execute: (code) =>
|
||||
executeProgram(code, prepared, limits, options.hooks ?? {}, (ctx) => extensionGlobals(ctx, extensions)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ export const executeProgram = <R>(
|
||||
code: string,
|
||||
prepared: ToolRuntime.Prepared<R>,
|
||||
limits: ResolvedExecutionLimits,
|
||||
hooks: ToolRuntime.ToolCallHooks<R>,
|
||||
hooks: ToolRuntime.Hooks<R>,
|
||||
globals?: (ctx: Interpreter<R>) => ReadonlyArray<readonly [string, unknown]>,
|
||||
): Effect.Effect<Result, never, R> => {
|
||||
if (code.trim().length === 0) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Effect } from "effect"
|
||||
import type { Extension } from "../extension.js"
|
||||
import { coerceToString } from "../stdlib/value.js"
|
||||
import { type ExtensionInvocation, hooked } from "../tool-runtime.js"
|
||||
import type { Interpreter } from "./interpreter.js"
|
||||
import { createErrorValue, isErrorType } from "./intrinsics.js"
|
||||
import { MAX_VALUE_DEPTH } from "./limits.js"
|
||||
@@ -11,7 +12,6 @@ import {
|
||||
define,
|
||||
entries,
|
||||
get,
|
||||
type Native,
|
||||
Arr,
|
||||
Bytes,
|
||||
DateObj,
|
||||
@@ -128,26 +128,35 @@ export const extensionGlobals = <R>(
|
||||
throw typeError(`${label} produced ${describeHost(value)}, which the program cannot hold.`)
|
||||
}
|
||||
|
||||
// A host function as a program function: arguments cross in, and whatever it returns, resolves, throws, or
|
||||
// rejects with crosses out, so the program catches what the author threw.
|
||||
const wrap = (value: Function, label: string): Native<R> =>
|
||||
// A host function as a program function. Arguments cross in; a global's call runs inside the host's extension
|
||||
// hooks, which see the host's own error on failure; then whatever came back, or was thrown, crosses out so the
|
||||
// program catches a copy. Functions inside results are part of a value's API and skip the hooks.
|
||||
const wrap = (value: Function, label: string, describe?: (args: ReadonlyArray<unknown>) => ExtensionInvocation) =>
|
||||
fn<R>(builtins, value.name, value.length, (_, values) => {
|
||||
const converted = values.map((item, index) => toHost(item, `Argument ${index + 1} to ${label}`))
|
||||
const thrown = (reason: unknown) => new Throw(fromHost(reason, label))
|
||||
const args = values.map((item, index) => toHost(item, `Argument ${index + 1} to ${label}`))
|
||||
const hooks = ctx.tools.hooks
|
||||
const settle = (run: Effect.Effect<unknown, unknown, R>) =>
|
||||
(describe === undefined
|
||||
? run
|
||||
: hooked(describe(args), hooks["extension.before"], hooks["extension.after"], run)
|
||||
).pipe(
|
||||
Effect.mapError((reason) => new Throw(fromHost(reason, label))),
|
||||
Effect.map((settled) => fromHost(settled, label)),
|
||||
)
|
||||
let result: unknown
|
||||
try {
|
||||
result = value.apply(undefined, converted)
|
||||
result = value.apply(undefined, args)
|
||||
} catch (reason) {
|
||||
return Effect.fail(thrown(reason))
|
||||
return settle(Effect.fail(reason))
|
||||
}
|
||||
if (!(result instanceof Promise)) return fromHost(result, label)
|
||||
return ctx.pending.create(
|
||||
Effect.map(Effect.tryPromise({ try: () => result, catch: thrown }), (settled) => fromHost(settled, label)),
|
||||
)
|
||||
if (!(result instanceof Promise)) return settle(Effect.succeed(result))
|
||||
return ctx.pending.create(settle(Effect.tryPromise({ try: () => result, catch: (reason) => reason })))
|
||||
})
|
||||
|
||||
return extensions.flatMap((extension) =>
|
||||
Object.entries(extension.globals).map(([name, value]) => [name, wrap(value, name)] as const),
|
||||
Object.entries(extension.globals).map(
|
||||
([name, value]) => [name, wrap(value, name, (args) => ({ extension: extension.name, name, args }))] as const,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -36,26 +36,48 @@ export type ToolCall = {
|
||||
readonly name: string
|
||||
}
|
||||
|
||||
export type ToolCallStarted = {
|
||||
readonly index: number
|
||||
/** A tool call the program is making, with its decoded input. */
|
||||
export type ToolInvocation = { readonly name: string; readonly input: unknown }
|
||||
|
||||
/** A call the program is making to an extension global, with its arguments. */
|
||||
export type ExtensionInvocation = {
|
||||
readonly extension: string
|
||||
readonly name: string
|
||||
readonly input: unknown
|
||||
readonly args: ReadonlyArray<unknown>
|
||||
}
|
||||
|
||||
export type ToolCallEnded = {
|
||||
readonly index: number
|
||||
readonly name: string
|
||||
readonly input: unknown
|
||||
readonly durationMs: number
|
||||
readonly outcome: "success" | "failure" | "interrupted"
|
||||
readonly message?: string
|
||||
/** How a call ended; `after` hooks observe it and cannot change it. */
|
||||
export type CallResult =
|
||||
| { readonly status: "success"; readonly value: unknown }
|
||||
| { readonly status: "failure"; readonly error: unknown }
|
||||
| { readonly status: "interrupted" }
|
||||
|
||||
/** Hooks around every call the program makes into the host. A failing `before` denies the call. */
|
||||
export type Hooks<R = never> = {
|
||||
readonly "tool.before"?: ((call: ToolInvocation) => Effect.Effect<void, unknown, R>) | undefined
|
||||
readonly "tool.after"?: ((call: ToolInvocation, result: CallResult) => Effect.Effect<void, never, R>) | undefined
|
||||
readonly "extension.before"?: ((call: ExtensionInvocation) => Effect.Effect<void, unknown, R>) | undefined
|
||||
readonly "extension.after"?:
|
||||
| ((call: ExtensionInvocation, result: CallResult) => Effect.Effect<void, never, R>)
|
||||
| undefined
|
||||
}
|
||||
|
||||
export type ToolCallHooks<R = never> = {
|
||||
/** Observes decoded tool input immediately before tool execution. */
|
||||
readonly onToolCallStart?: ((call: ToolCallStarted) => Effect.Effect<void, never, R>) | undefined
|
||||
/** Observes each admitted tool call as it succeeds, fails, or is interrupted. */
|
||||
readonly onToolCallEnd?: ((call: ToolCallEnded) => Effect.Effect<void, never, R>) | undefined
|
||||
/** Runs `before`, then `run`, then `after` with how it ended, including when interrupted. */
|
||||
export const hooked = <Call, A, R>(
|
||||
call: Call,
|
||||
before: ((call: Call) => Effect.Effect<void, unknown, R>) | undefined,
|
||||
after: ((call: Call, result: CallResult) => Effect.Effect<void, never, R>) | undefined,
|
||||
run: Effect.Effect<A, unknown, R>,
|
||||
): Effect.Effect<A, unknown, R> => {
|
||||
const observed =
|
||||
after === undefined
|
||||
? run
|
||||
: Effect.onExit(run, (exit) => {
|
||||
if (Exit.isSuccess(exit)) return after(call, { status: "success", value: exit.value })
|
||||
if (Cause.hasInterruptsOnly(exit.cause)) return after(call, { status: "interrupted" })
|
||||
return after(call, { status: "failure", error: Cause.squash(exit.cause) })
|
||||
})
|
||||
return before === undefined ? observed : Effect.andThen(before(call), observed)
|
||||
}
|
||||
|
||||
export type ToolDescription = {
|
||||
@@ -316,6 +338,7 @@ export class ToolRuntimeError extends Error {
|
||||
/** The tool bridge of one execution. Arguments arrive and results leave as JSON; program values never enter. */
|
||||
export type ToolRuntime<R = never> = {
|
||||
readonly calls: Array<ToolCall>
|
||||
readonly hooks: Hooks<R>
|
||||
readonly execute: (
|
||||
path: ReadonlyArray<string>,
|
||||
args: Array<Json | undefined>,
|
||||
@@ -328,28 +351,12 @@ export type ToolRuntime<R = never> = {
|
||||
export const make = <R>(
|
||||
prepared: Prepared<R>,
|
||||
maxToolCalls: number | undefined,
|
||||
hooks?: ToolCallHooks<R>,
|
||||
hooks: Hooks<R> = {},
|
||||
): ToolRuntime<R> => {
|
||||
const calls: Array<ToolCall> = []
|
||||
const root = prepared.root
|
||||
const searchTool = makeSearchTool(prepared.searchIndex)
|
||||
|
||||
const observeEnd = <A, E>(effect: Effect.Effect<A, E, R>, call: ToolCallStarted): Effect.Effect<A, E, R> => {
|
||||
const onEnd = hooks?.onToolCallEnd
|
||||
if (onEnd === undefined) return effect
|
||||
const startedAt = Date.now()
|
||||
return effect.pipe(
|
||||
Effect.onExit((exit) => {
|
||||
const durationMs = Date.now() - startedAt
|
||||
if (Exit.isSuccess(exit)) return onEnd({ ...call, durationMs, outcome: "success" })
|
||||
if (Cause.hasInterruptsOnly(exit.cause)) return onEnd({ ...call, durationMs, outcome: "interrupted" })
|
||||
const error = Cause.squash(exit.cause)
|
||||
const message = error instanceof Error ? error.message : Cause.pretty(exit.cause)
|
||||
return onEnd({ ...call, durationMs, outcome: "failure", message })
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
const recordCall = (call: ToolCall): void => {
|
||||
if (maxToolCalls !== undefined && calls.length >= maxToolCalls) {
|
||||
throw new ToolRuntimeError("ToolCallLimitExceeded", `Execution exceeded its tool-call limit of ${maxToolCalls}.`)
|
||||
@@ -371,14 +378,12 @@ export const make = <R>(
|
||||
name === "search" ? [] : ["The signature may have changed. Use search to get the current signature."],
|
||||
),
|
||||
})
|
||||
const index = yield* Effect.sync(() => {
|
||||
recordCall({ name })
|
||||
return calls.length - 1
|
||||
})
|
||||
const call = { index, name, input }
|
||||
return yield* observeEnd(
|
||||
yield* Effect.sync(() => recordCall({ name }))
|
||||
return yield* hooked(
|
||||
{ name, input },
|
||||
hooks["tool.before"],
|
||||
hooks["tool.after"],
|
||||
Effect.gen(function* () {
|
||||
if (hooks?.onToolCallStart !== undefined) yield* hooks.onToolCallStart(call)
|
||||
const raw = yield* Effect.suspend(() => tool.execute(input)).pipe(
|
||||
Effect.catchCause((cause) => {
|
||||
if (Cause.hasInterruptsOnly(cause)) return Effect.interrupt
|
||||
@@ -400,12 +405,12 @@ export const make = <R>(
|
||||
catch: (cause) => new ToolRuntimeError("InvalidToolOutput", `Invalid output from tool '${name}': ${cause}`),
|
||||
})
|
||||
}),
|
||||
call,
|
||||
)
|
||||
})
|
||||
|
||||
return {
|
||||
calls,
|
||||
hooks,
|
||||
keys: (path) => namespaceKeys(root, path),
|
||||
search: (args) => Effect.suspend(() => executeTool("search", searchTool, args)),
|
||||
execute: (path, args) =>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { Cause, Effect, Schema } from "effect"
|
||||
import { CodeMode, Tool, toolError } from "../src/index.js"
|
||||
import { CodeMode, Extension, Tool, toolError } from "../src/index.js"
|
||||
|
||||
const run = (tool: Tool.Tool<never>) =>
|
||||
Effect.runPromise(CodeMode.make({ tools: { host: { call: tool } } }).execute("return await tools.host.call({})"))
|
||||
@@ -160,77 +160,154 @@ describe("CodeMode host failure boundary", () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe("CodeMode tool-call observation", () => {
|
||||
test("reports the tools actually invoked with decoded input", async () => {
|
||||
const calls: Array<unknown> = []
|
||||
describe("CodeMode call hooks", () => {
|
||||
const ended = (result: CodeMode.CallResult) => {
|
||||
if (result.status !== "failure") return result.status
|
||||
return `failure:${result.error instanceof Error ? result.error.message : String(result.error)}`
|
||||
}
|
||||
// Every hook logs its event, so the order across tools and extensions is visible.
|
||||
const observe = (events: Array<string>): CodeMode.Hooks => ({
|
||||
"tool.before": (call) => Effect.sync(() => void events.push(`tool.before ${JSON.stringify(call)}`)),
|
||||
"tool.after": (call, result) => Effect.sync(() => void events.push(`tool.after ${call.name} ${ended(result)}`)),
|
||||
"extension.before": (call) => Effect.sync(() => void events.push(`extension.before ${JSON.stringify(call)}`)),
|
||||
"extension.after": (call, result) =>
|
||||
Effect.sync(() => void events.push(`extension.after ${call.name} ${ended(result)}`)),
|
||||
})
|
||||
|
||||
test("surround the tools actually invoked, with decoded input", async () => {
|
||||
const events: Array<string> = []
|
||||
const lookup = Tool.make({
|
||||
description: "Look up a value",
|
||||
input: Schema.Struct({ query: Schema.String }),
|
||||
output: Schema.String,
|
||||
execute: ({ query }) => Effect.succeed(query),
|
||||
execute: ({ query }) => (query === "boom" ? Effect.fail(toolError("Lookup refused")) : Effect.succeed(query)),
|
||||
})
|
||||
const runtime = CodeMode.make({ tools: { context: { lookup } }, hooks: observe(events) })
|
||||
|
||||
const result = await Effect.runPromise(
|
||||
CodeMode.make({
|
||||
tools: { context: { lookup } },
|
||||
onToolCallStart: (call) => Effect.sync(() => calls.push(call)),
|
||||
}).execute(`
|
||||
const success = await Effect.runPromise(
|
||||
runtime.execute(`
|
||||
if (false) await tools.context.lookup({ query: "not called" })
|
||||
return await tools.context.lookup({ query: "deployment failure" })
|
||||
`),
|
||||
)
|
||||
|
||||
expect(result.ok).toBe(true)
|
||||
expect(calls).toStrictEqual([{ index: 0, name: "context.lookup", input: { query: "deployment failure" } }])
|
||||
})
|
||||
|
||||
test("observes settled calls with outcome and duration", async () => {
|
||||
const events: Array<{ phase: string; index: number; name: string; outcome?: string; message?: string }> = []
|
||||
const lookup = Tool.make({
|
||||
description: "Look up a value",
|
||||
input: Schema.Struct({ query: Schema.String }),
|
||||
output: Schema.String,
|
||||
execute: ({ query }) =>
|
||||
query === "boom"
|
||||
? Effect.fail(toolError("Lookup refused"))
|
||||
: query === "defect"
|
||||
? Effect.die("broken")
|
||||
: Effect.succeed(query),
|
||||
})
|
||||
|
||||
const runtime = CodeMode.make({
|
||||
tools: { context: { lookup } },
|
||||
onToolCallStart: (call) =>
|
||||
Effect.sync(() => {
|
||||
events.push({ phase: "start", index: call.index, name: call.name })
|
||||
}),
|
||||
onToolCallEnd: (call) =>
|
||||
Effect.sync(() => {
|
||||
expect(call.durationMs).toBeGreaterThanOrEqual(0)
|
||||
events.push({
|
||||
phase: "end",
|
||||
index: call.index,
|
||||
name: call.name,
|
||||
outcome: call.outcome,
|
||||
...(call.message === undefined ? {} : { message: call.message }),
|
||||
})
|
||||
}),
|
||||
})
|
||||
|
||||
const success = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "ok" })`))
|
||||
expect(success.ok).toBe(true)
|
||||
expect(success).toMatchObject({ ok: true, value: "deployment failure" })
|
||||
const failure = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "boom" })`))
|
||||
expect(failure.ok).toBe(false)
|
||||
const defect = await Effect.runPromise(runtime.execute(`return await tools.context.lookup({ query: "defect" })`))
|
||||
expect(defect.ok).toBe(false)
|
||||
expect(events).toEqual([
|
||||
'tool.before {"name":"context.lookup","input":{"query":"deployment failure"}}',
|
||||
"tool.after context.lookup success",
|
||||
'tool.before {"name":"context.lookup","input":{"query":"boom"}}',
|
||||
"tool.after context.lookup failure:Lookup refused",
|
||||
])
|
||||
})
|
||||
|
||||
expect(events).toStrictEqual([
|
||||
{ phase: "start", index: 0, name: "context.lookup" },
|
||||
{ phase: "end", index: 0, name: "context.lookup", outcome: "success" },
|
||||
{ phase: "start", index: 0, name: "context.lookup" },
|
||||
{ phase: "end", index: 0, name: "context.lookup", outcome: "failure", message: "Lookup refused" },
|
||||
{ phase: "start", index: 0, name: "context.lookup" },
|
||||
{ phase: "end", index: 0, name: "context.lookup", outcome: "failure", message: "broken" },
|
||||
test("surround each call to an extension global, with its arguments, but not functions inside results", async () => {
|
||||
const events: Array<string> = []
|
||||
const runtime = CodeMode.make({
|
||||
extensions: [
|
||||
Extension.make({
|
||||
name: "web",
|
||||
globals: {
|
||||
fetch: async (url: string, init?: { method?: string }) => ({
|
||||
url,
|
||||
method: init?.method ?? "GET",
|
||||
json: () => 1,
|
||||
}),
|
||||
fail: () => {
|
||||
throw new RangeError("nope")
|
||||
},
|
||||
},
|
||||
}),
|
||||
],
|
||||
hooks: observe(events),
|
||||
})
|
||||
const result = await Effect.runPromise(
|
||||
runtime.execute(`
|
||||
const res = await fetch("https://a.test/", { method: "POST" })
|
||||
res.json()
|
||||
try { fail() } catch {}
|
||||
return res.url
|
||||
`),
|
||||
)
|
||||
expect(result).toMatchObject({ ok: true, value: "https://a.test/", toolCalls: [] })
|
||||
expect(events).toEqual([
|
||||
'extension.before {"extension":"web","name":"fetch","args":["https://a.test/",{"method":"POST"}]}',
|
||||
"extension.after fetch success",
|
||||
'extension.before {"extension":"web","name":"fail","args":[]}',
|
||||
"extension.after fail failure:nope",
|
||||
])
|
||||
})
|
||||
|
||||
test("a failing before hook denies the call; the program catches the host's error", async () => {
|
||||
const runtime = CodeMode.make({
|
||||
tools: {
|
||||
lookup: Tool.make({
|
||||
description: "Look up",
|
||||
input: Schema.Struct({}),
|
||||
output: Schema.String,
|
||||
execute: () => Effect.succeed("ok"),
|
||||
}),
|
||||
},
|
||||
extensions: [Extension.make({ name: "web", globals: { fetch: async (url: string) => url } })],
|
||||
hooks: { "extension.before": (call) => Effect.fail(new Error(`${call.name} is not allowed`)) },
|
||||
})
|
||||
expect(
|
||||
await Effect.runPromise(
|
||||
runtime.execute(`
|
||||
const denied = await fetch("https://a.test/").catch((e) => [e instanceof Error, e.message])
|
||||
return [denied, await tools.lookup({})]
|
||||
`),
|
||||
),
|
||||
).toMatchObject({ ok: true, value: [[true, "fetch is not allowed"], "ok"] })
|
||||
expect(await Effect.runPromise(runtime.execute(`await fetch("https://a.test/")`))).toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "ExecutionFailure", message: "Error: fetch is not allowed" },
|
||||
})
|
||||
const refused = CodeMode.make({
|
||||
tools: {
|
||||
lookup: Tool.make({
|
||||
description: "Look up",
|
||||
input: Schema.Struct({}),
|
||||
output: Schema.String,
|
||||
execute: () => Effect.succeed("ok"),
|
||||
}),
|
||||
},
|
||||
hooks: { "tool.before": () => Effect.fail(toolError("lookup is not allowed")) },
|
||||
})
|
||||
expect(
|
||||
await Effect.runPromise(
|
||||
refused.execute(`try { await tools.lookup({}) } catch (e) { return [e.name, e.message] }`),
|
||||
),
|
||||
).toMatchObject({ ok: true, value: ["Error", "lookup is not allowed"], toolCalls: [{ name: "lookup" }] })
|
||||
expect(await Effect.runPromise(refused.execute(`await tools.lookup({})`))).toMatchObject({
|
||||
ok: false,
|
||||
error: { kind: "ToolFailure", message: "lookup is not allowed" },
|
||||
})
|
||||
})
|
||||
|
||||
test("see a tool defect as the failure the program gets, and an interrupted extension call", async () => {
|
||||
const events: Array<string> = []
|
||||
const broken = Tool.make({
|
||||
description: "Broken",
|
||||
input: Schema.Struct({}),
|
||||
output: Schema.String,
|
||||
execute: () => Effect.die("broken"),
|
||||
})
|
||||
const runtime = CodeMode.make({
|
||||
tools: { broken },
|
||||
extensions: [Extension.make({ name: "slow", globals: { forever: () => new Promise(() => {}) } })],
|
||||
hooks: observe(events),
|
||||
})
|
||||
expect(await Effect.runPromise(runtime.execute(`await tools.broken({})`))).toMatchObject({ ok: false })
|
||||
expect(await Effect.runPromise(runtime.execute(`forever(); return "done"`))).toMatchObject({
|
||||
ok: true,
|
||||
value: "done",
|
||||
})
|
||||
expect(events).toEqual([
|
||||
'tool.before {"name":"broken","input":{}}',
|
||||
"tool.after broken failure:broken",
|
||||
'extension.before {"extension":"slow","name":"forever","args":[]}',
|
||||
"extension.after forever interrupted",
|
||||
])
|
||||
})
|
||||
|
||||
@@ -243,15 +320,11 @@ describe("CodeMode tool-call observation", () => {
|
||||
execute: () => Effect.interrupt,
|
||||
})
|
||||
const exit = await Effect.runPromiseExit(
|
||||
CodeMode.make({
|
||||
tools: { host: { call } },
|
||||
onToolCallStart: () => Effect.sync(() => events.push("start")),
|
||||
onToolCallEnd: (call) => Effect.sync(() => events.push(`end:${call.outcome}`)),
|
||||
}).execute("return await tools.host.call({})"),
|
||||
CodeMode.make({ tools: { host: { call } }, hooks: observe(events) }).execute("return await tools.host.call({})"),
|
||||
)
|
||||
|
||||
expect(exit._tag).toBe("Failure")
|
||||
expect(events).toEqual(["start", "end:interrupted"])
|
||||
expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"])
|
||||
})
|
||||
|
||||
test("observes running calls interrupted during completion", async () => {
|
||||
@@ -263,39 +336,17 @@ describe("CodeMode tool-call observation", () => {
|
||||
execute: () => Effect.never,
|
||||
})
|
||||
const result = await Effect.runPromise(
|
||||
CodeMode.make({
|
||||
tools: { host: { call } },
|
||||
onToolCallStart: () => Effect.sync(() => events.push("start")),
|
||||
onToolCallEnd: (call) => Effect.sync(() => events.push(`end:${call.outcome}`)),
|
||||
}).execute('tools.host.call({}); return "done"'),
|
||||
CodeMode.make({ tools: { host: { call } }, hooks: observe(events) }).execute(
|
||||
'tools.host.call({}); return "done"',
|
||||
),
|
||||
)
|
||||
|
||||
expect(result).toMatchObject({ ok: true, value: "done" })
|
||||
expect(events).toEqual(["start", "end:interrupted"])
|
||||
})
|
||||
|
||||
test("ends calls interrupted during start observation", async () => {
|
||||
const events: Array<string> = []
|
||||
const call = Tool.make({
|
||||
description: "Unused",
|
||||
input: Schema.Struct({}),
|
||||
output: Schema.String,
|
||||
execute: () => Effect.succeed("unused"),
|
||||
})
|
||||
const exit = await Effect.runPromiseExit(
|
||||
CodeMode.make({
|
||||
tools: { host: { call } },
|
||||
onToolCallStart: () => Effect.interrupt,
|
||||
onToolCallEnd: (call) => Effect.sync(() => events.push(call.outcome)),
|
||||
}).execute("return await tools.host.call({})"),
|
||||
)
|
||||
|
||||
expect(exit._tag).toBe("Failure")
|
||||
expect(events).toEqual(["interrupted"])
|
||||
expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"])
|
||||
})
|
||||
|
||||
test("observes calls interrupted by the execution timeout", async () => {
|
||||
const outcomes: Array<string> = []
|
||||
const events: Array<string> = []
|
||||
const call = Tool.make({
|
||||
description: "Pending",
|
||||
input: Schema.Struct({}),
|
||||
@@ -303,15 +354,13 @@ describe("CodeMode tool-call observation", () => {
|
||||
execute: () => Effect.never,
|
||||
})
|
||||
const result = await Effect.runPromise(
|
||||
CodeMode.make({
|
||||
tools: { host: { call } },
|
||||
limits: { timeoutMs: 10 },
|
||||
onToolCallEnd: (call) => Effect.sync(() => outcomes.push(call.outcome)),
|
||||
}).execute("return await tools.host.call({})"),
|
||||
CodeMode.make({ tools: { host: { call } }, limits: { timeoutMs: 10 }, hooks: observe(events) }).execute(
|
||||
"return await tools.host.call({})",
|
||||
),
|
||||
)
|
||||
|
||||
expect(result).toMatchObject({ ok: false, error: { kind: "TimeoutExceeded" } })
|
||||
expect(outcomes).toEqual(["interrupted"])
|
||||
expect(events).toEqual(['tool.before {"name":"host.call","input":{}}', "tool.after host.call interrupted"])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -840,20 +889,17 @@ describe("CodeMode public contract", () => {
|
||||
}
|
||||
})
|
||||
|
||||
test("search is a counted tool call: it burns maxToolCalls and fires the hooks", async () => {
|
||||
const started: Array<string> = []
|
||||
const ended: Array<string> = []
|
||||
test("search is a counted tool call: it burns maxToolCalls and is observed", async () => {
|
||||
const observed: Array<CodeMode.ToolInvocation> = []
|
||||
const limited = CodeMode.make({
|
||||
tools,
|
||||
limits: { maxToolCalls: 1 },
|
||||
onToolCallStart: (call) => Effect.sync(() => void started.push(call.name)),
|
||||
onToolCallEnd: (call) => Effect.sync(() => void ended.push(`${call.name}:${call.outcome}`)),
|
||||
hooks: { "tool.before": (call) => Effect.sync(() => void observed.push(call)) },
|
||||
})
|
||||
const result = await Effect.runPromise(limited.execute(`search({}); return search({})`))
|
||||
expect(result.ok).toBe(false)
|
||||
if (!result.ok) expect(result.error.kind).toBe("ToolCallLimitExceeded")
|
||||
expect(started).toEqual(["search"])
|
||||
expect(ended).toEqual(["search:success"])
|
||||
expect(observed).toEqual([{ name: "search", input: {} }])
|
||||
})
|
||||
|
||||
test("search is an opaque, shadowable global like other built-ins", async () => {
|
||||
@@ -1089,7 +1135,7 @@ describe("CodeMode public contract", () => {
|
||||
})
|
||||
const runtime = CodeMode.make({
|
||||
tools: { math: { double: transformed } },
|
||||
onToolCallStart: (call) => Effect.sync(() => observed.push(call.input)),
|
||||
hooks: { "tool.before": (call) => Effect.sync(() => void observed.push(call.input)) },
|
||||
})
|
||||
|
||||
const success = await Effect.runPromise(runtime.execute(`return await tools.math.double({ value: "21" })`))
|
||||
|
||||
@@ -192,6 +192,16 @@ describe("values are converted at the boundary, never shared", () => {
|
||||
expect((await failure(`big()`, target)).message).toContain("big produced a bigint")
|
||||
})
|
||||
|
||||
test("a value that cannot come out carries a location like any built-in error, sync or async", async () => {
|
||||
const target = CodeMode.make({
|
||||
extensions: [
|
||||
Extension.make({ name: "odd", globals: { sym: () => Symbol("s"), later: async () => Symbol("s") } }),
|
||||
],
|
||||
})
|
||||
expect((await failure(`sym()`, target)).location).toEqual((await failure(`JSON.parse("{")`, target)).location)
|
||||
expect((await failure(`await later()`, target)).location).toBeDefined()
|
||||
})
|
||||
|
||||
test("a class instance cannot come out", async () => {
|
||||
class Other {}
|
||||
const target = CodeMode.make({
|
||||
|
||||
@@ -82,9 +82,9 @@ export const create = (
|
||||
const files = yield* Ref.make<Array<CollectedFiles>>([])
|
||||
const calls = yield* Ref.make<Array<ExecuteCall>>([])
|
||||
const lock = Semaphore.makeUnsafe(1)
|
||||
const updateCalls = (update: (items: Array<ExecuteCall>) => Array<ExecuteCall>) =>
|
||||
const record = (update: (items: Array<ExecuteCall>) => Array<ExecuteCall>) =>
|
||||
lock.withPermit(
|
||||
Ref.updateAndGet(calls, update).pipe(Effect.flatMap((toolCalls) => context.progress({ toolCalls }))),
|
||||
Ref.updateAndGet(calls, update).pipe(Effect.tap((toolCalls) => context.progress({ toolCalls }))),
|
||||
)
|
||||
const result = yield* runtime(
|
||||
inventory,
|
||||
@@ -103,27 +103,7 @@ export const create = (
|
||||
const text = content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join("\n")
|
||||
return text === "" ? null : text
|
||||
}),
|
||||
{
|
||||
onToolCallStart: ({ index, name, input }) => {
|
||||
const shown = displayInput(input)
|
||||
return updateCalls((items) => {
|
||||
const next = [...items]
|
||||
next[index] = { tool: name, status: "running", ...(shown ? { input: shown } : {}) }
|
||||
return next
|
||||
})
|
||||
},
|
||||
onToolCallEnd: ({ index, name, input, outcome }) => {
|
||||
const shown = displayInput(input)
|
||||
return updateCalls((items) => {
|
||||
const next = [...items]
|
||||
next[index] = {
|
||||
...(items[index] ?? { tool: name, ...(shown ? { input: shown } : {}) }),
|
||||
status: outcome === "success" ? "completed" : "error",
|
||||
}
|
||||
return next
|
||||
})
|
||||
},
|
||||
},
|
||||
progressHooks(record),
|
||||
).execute(code)
|
||||
const toolCalls = yield* Ref.get(calls)
|
||||
const collected = (yield* Ref.get(files))
|
||||
@@ -158,6 +138,33 @@ export const create = (
|
||||
} satisfies Info
|
||||
}
|
||||
|
||||
// Rows appear in start order; the same call object arrives at both hooks, so a call finds its row again.
|
||||
function progressHooks(record: (update: (items: Array<ExecuteCall>) => Array<ExecuteCall>) => Effect.Effect<unknown>) {
|
||||
const rows = new WeakMap<object, number>()
|
||||
const start = (call: object, entry: ExecuteCall) =>
|
||||
record((items) => {
|
||||
rows.set(call, items.length)
|
||||
return [...items, entry]
|
||||
})
|
||||
const settle = (call: object, result: CodeMode.CallResult) =>
|
||||
record((items) => {
|
||||
const index = rows.get(call)
|
||||
if (index === undefined) return items
|
||||
const next = [...items]
|
||||
next[index] = { ...items[index], status: result.status === "success" ? "completed" : "error" }
|
||||
return next
|
||||
})
|
||||
return {
|
||||
"tool.before": (call) => {
|
||||
const shown = displayInput(call.input)
|
||||
return start(call, { tool: call.name, status: "running", ...(shown ? { input: shown } : {}) })
|
||||
},
|
||||
"tool.after": settle,
|
||||
"extension.before": (call) => start(call, { tool: call.name, status: "running" }),
|
||||
"extension.after": settle,
|
||||
} satisfies CodeMode.Hooks
|
||||
}
|
||||
|
||||
export const catalog = (inventory: Inventory) => {
|
||||
const pinned = new Set(
|
||||
Array.from(inventory.tools.values())
|
||||
@@ -204,7 +211,7 @@ function renderCatalog(root: CatalogNode): ReadonlyArray<CodeModeCatalog.Tool |
|
||||
function runtime(
|
||||
inventory: Inventory,
|
||||
executeTool: (name: string, tool: Info, input: unknown) => Effect.Effect<unknown, unknown>,
|
||||
hooks?: CodeMode.ToolCallHooks,
|
||||
hooks?: CodeMode.Hooks,
|
||||
) {
|
||||
// A path may carry namespace metadata, a callable tool, child tools, or all three.
|
||||
const root: ToolNode = { children: new Map() }
|
||||
@@ -219,7 +226,7 @@ function runtime(
|
||||
})
|
||||
}
|
||||
const tools = renderTools(root)
|
||||
return CodeMode.make<typeof tools>({ tools, ...hooks })
|
||||
return CodeMode.make<typeof tools>({ tools, hooks })
|
||||
}
|
||||
|
||||
function getNode<T>(root: Node<T>, path: string) {
|
||||
|
||||
Reference in New Issue
Block a user