fix(core): keep Code Mode catalog updates small and stable (#62176)

Co-authored-by: exe.dev user <exedev@anton.exe.xyz>
This commit is contained in:
Aiden Cline
2026-11-08 10:56:56 -06:00
committed by GitHub
Unverified
parent 84ea1bd71f
commit 61045f1a46
6 changed files with 343 additions and 199 deletions
+51 -68
View File
@@ -6,6 +6,7 @@ export type Tool = {
readonly type: "tool"
readonly name: string
readonly description: string
/** Read only for the tools the catalog shows. */
readonly signature: string
readonly pinned?: boolean
}
@@ -25,17 +26,19 @@ const Listing = Schema.Struct({
path: Schema.String,
line: Schema.String,
})
export type Listing = typeof Listing.Type
const NamespaceSummary = Schema.Struct({
name: Schema.String,
description: Schema.optionalKey(Schema.String),
count: Schema.Number,
/** Every tool's path, so an update can tell a new tool from one that was not shown. */
tools: Schema.Array(Schema.String),
entries: Schema.Array(Listing),
})
export type NamespaceSummary = typeof NamespaceSummary.Type
export const Summary = Schema.Struct({
total: Schema.Number,
shown: Schema.Number,
namespaces: Schema.Array(NamespaceSummary),
})
export type Summary = typeof Summary.Type
@@ -48,8 +51,8 @@ const DESCRIPTION_LIMIT = 120
const CHARACTERS_PER_TOKEN = 4
const INLINE_BUDGET = 2_000
// Keep every namespace visible, then select full listings one per namespace per round,
// considering shorter listings first until the inline budget is exhausted.
// Keep every namespace visible, then select full listings one per namespace per round, in path order, until the
// inline budget is exhausted. The selection depends only on the tools, and only selected signatures are rendered.
export function summarize(inventory: Inventory, options: Options = {}): Summary {
const budget = options.budget ?? INLINE_BUDGET
const flattened = flatten(inventory.tools)
@@ -61,32 +64,17 @@ export function summarize(inventory: Inventory, options: Options = {}): Summary
})
.map(([name, namespaceEntries]) => {
const description = flattened.namespaces.get(name)?.description
const listings = namespaceEntries
.map((entry) => {
const firstLine = entry.description.split("\n", 1)[0]?.trim() ?? ""
const description =
firstLine.length > DESCRIPTION_LIMIT ? firstLine.slice(0, DESCRIPTION_LIMIT - 3) + "..." : firstLine
const suffix = description.length === 0 ? "" : ` // ${description}`
return { path: entry.path, line: ` - ${entry.signature}${suffix}` }
})
.toSorted((left, right) => {
if (left.path < right.path) return -1
if (left.path > right.path) return 1
return 0
})
const ranked = rankListings(listings)
const pinned = new Set(
namespaceEntries
.filter((entry) => entry.pinned)
.map((entry) => listings.find((listing) => listing.path === entry.path))
.filter((listing) => listing !== undefined),
)
const sorted = namespaceEntries.toSorted((left, right) => {
if (left.path < right.path) return -1
if (left.path > right.path) return 1
return 0
})
return {
name,
...(description === undefined ? {} : { description }),
listings,
selectionOrder: ranked.filter((candidate) => !pinned.has(candidate.listing)),
selectedListings: pinned,
tools: sorted.map((entry) => entry.path),
selectionOrder: sorted.filter((entry) => !entry.tool.pinned),
selectedListings: sorted.filter((entry) => entry.tool.pinned).map(listing),
selectionIndex: 0,
}
})
@@ -102,54 +90,64 @@ export function summarize(inventory: Inventory, options: Options = {}): Summary
namespaceLine({
name: namespace.name,
...(namespace.description === undefined ? {} : { description: namespace.description }),
count: namespace.listings.length,
tools: namespace.tools,
entries: [],
}),
),
0,
) -
namespaces
.flatMap((namespace) => namespace.listings.filter((listing) => namespace.selectedListings.has(listing)))
.flatMap((namespace) => namespace.selectedListings)
.reduce((total, listing) => total + cost(listing.line), 0)
while (active.size > 0) {
for (const namespace of active) {
const candidate = namespace.selectionOrder[namespace.selectionIndex]
if (!candidate || candidate.cost > remaining) {
const entry = namespace.selectionOrder[namespace.selectionIndex]
const candidate = entry && listing(entry)
if (!candidate || cost(candidate.line) > remaining) {
active.delete(namespace)
continue
}
namespace.selectedListings.add(candidate.listing)
namespace.selectedListings.push(candidate)
namespace.selectionIndex += 1
remaining -= candidate.cost
remaining -= cost(candidate.line)
if (namespace.selectionIndex === namespace.selectionOrder.length) active.delete(namespace)
}
}
const namespaceSummaries = namespaces.map((namespace) => ({
name: namespace.name,
...(namespace.description === undefined ? {} : { description: namespace.description }),
count: namespace.listings.length,
entries: namespace.listings.filter((listing) => namespace.selectedListings.has(listing)),
}))
return {
total: flattened.tools.length,
shown: namespaceSummaries.reduce((total, namespace) => total + namespace.entries.length, 0),
namespaces: namespaceSummaries,
namespaces: namespaces.map((namespace) => ({
name: namespace.name,
...(namespace.description === undefined ? {} : { description: namespace.description }),
tools: namespace.tools,
entries: namespace.selectedListings,
})),
}
}
/** Looks up any tool's listing as the catalog would show it. */
export function lookup(inventory: Inventory) {
const tools = new Map(flatten(inventory.tools).tools.map((entry) => [entry.path, entry]))
return (path: string) => {
const entry = tools.get(path)
return entry === undefined ? undefined : listing(entry)
}
}
function listing(entry: { readonly path: string; readonly tool: Tool }) {
const firstLine = entry.tool.description.split("\n", 1)[0]?.trim() ?? ""
const description =
firstLine.length > DESCRIPTION_LIMIT ? firstLine.slice(0, DESCRIPTION_LIMIT - 3) + "..." : firstLine
const suffix = description.length === 0 ? "" : ` // ${description}`
return { path: entry.path, line: ` - ${entry.tool.signature}${suffix}` }
}
function flatten(entries: ReadonlyArray<Tool | Namespace>, path: ReadonlyArray<string> = []) {
const tools: Array<Omit<Tool, "name"> & { readonly path: string }> = []
const tools: Array<{ readonly path: string; readonly tool: Tool }> = []
const namespaces = new Map<string, Namespace>()
for (const entry of entries) {
if (entry.type === "tool") {
tools.push({
type: "tool",
path: [...path, entry.name].join("."),
description: entry.description,
signature: entry.signature,
...(entry.pinned === undefined ? {} : { pinned: entry.pinned }),
})
tools.push({ path: [...path, entry.name].join("."), tool: entry })
continue
}
const next = [...path, entry.name]
@@ -161,28 +159,13 @@ function flatten(entries: ReadonlyArray<Tool | Namespace>, path: ReadonlyArray<s
return { tools, namespaces }
}
export function namespaceLine(namespace: typeof NamespaceSummary.Type) {
const count = namespace.count === 1 ? "1 tool" : `${namespace.count} tools`
export function namespaceLine(namespace: NamespaceSummary, shown = namespace.entries.length) {
const count = namespace.tools.length === 1 ? "1 tool" : `${namespace.tools.length} tools`
const label =
namespace.entries.length === namespace.count
? count
: namespace.entries.length === 0
? `${count}, none shown`
: `${count}, ${namespace.entries.length} shown`
shown === namespace.tools.length ? count : shown === 0 ? `${count}, none shown` : `${count}, ${shown} shown`
return `- ${namespace.name} (${label})${namespace.description === undefined ? "" : ` // ${namespace.description}`}`
}
function rankListings(listings: ReadonlyArray<typeof Listing.Type>) {
return listings
.map((listing) => ({ listing, cost: cost(listing.line) }))
.toSorted((left, right) => {
if (left.cost !== right.cost) return left.cost - right.cost
if (left.listing.path < right.listing.path) return -1
if (left.listing.path > right.listing.path) return 1
return 0
})
}
function cost(text: string) {
export function cost(text: string) {
return Math.round(text.length / CHARACTERS_PER_TOKEN)
}
+91 -68
View File
@@ -16,6 +16,14 @@ ${hasMoreTools ? `The catalog is partial. Before calling a tool that is not list
## Code Mode tools (call only inside \`execute\`)`
// Sent once, when tools arrive that the catalog cannot list in full, so the model learns about `search`.
// prettier-ignore
const searchNote = `You can also look up tools that aren't listed with the global \`search(...)\` function inside \`execute\`. Before calling a tool that is not listed, find it this way. Do not guess tool names. Each result's \`path\` is the complete call expression: write it out as code, for example \`await tools.ns.tool_name({ ... })\`. Do not use it as a key such as \`tools[path]\`.
- ${searchSignature}`
const UPDATE_BUDGET = 1_000
export function render(catalog: CodeModeCatalog.Summary) {
if (catalog.total === 0)
return "No Code Mode tools are currently available. Later Code Mode catalog updates may add or remove tools. Do not call `execute` unless there is at least one available Code Mode tool."
@@ -24,105 +32,118 @@ export function render(catalog: CodeModeCatalog.Summary) {
return [CodeModeCatalog.namespaceLine(namespace), ...namespace.entries.map((entry) => entry.line)]
})
return `${prompt(catalog.shown < catalog.total)}
return `${prompt(!complete(catalog))}
${tools.join("\n")}`
}
export function update(previous: CodeModeCatalog.Summary, current: CodeModeCatalog.Summary) {
// An update names what changed for the model: namespaces and tools that came or went, and signatures it was shown
// that changed, even if they are no longer shown. Shown tools shifting because other namespaces came or went is left
// out. New signatures share a fixed budget while the catalog is partial, and the rest are counted.
export function update(
previous: CodeModeCatalog.Summary,
current: CodeModeCatalog.Summary,
lookup: (path: string) => CodeModeCatalog.Listing | undefined,
) {
const replacement = `The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.
${render(current)}`
if (current.total === 0) return replacement
const previousComplete = previous.shown === previous.total
const currentComplete = current.shown === current.total
if (previousComplete !== currentComplete) return replacement
if (current.total === 0 || previous.total === 0) return replacement
const descriptions = Instructions.diffByKey(
previous.namespaces.filter((namespace) => namespace.description !== undefined),
current.namespaces.filter((namespace) => namespace.description !== undefined),
const namespaces = Instructions.diffByKey(
previous.namespaces,
current.namespaces,
(namespace) => namespace.name,
(before, after) => before.description !== after.description,
)
if (descriptions.added.length > 0 || descriptions.removed.length > 0 || descriptions.changed.length > 0)
return replacement
const diff = Instructions.diffByKey(
previous.namespaces.flatMap((namespace) => namespace.entries),
current.namespaces.flatMap((namespace) => namespace.entries),
(entry) => entry.path,
(before, after) => before.line !== after.line,
// A namespace whose description changed is restated as if new.
const restated = [...namespaces.added, ...namespaces.changed.map((change) => change.current)]
const unchanged = new Set(
current.namespaces
.map((namespace) => namespace.name)
.filter((name) => !restated.some((item) => item.name === name)),
)
const entriesChanged = diff.added.length > 0 || diff.removed.length > 0 || diff.changed.length > 0
const kept = (path: string) => unchanged.has(path.split(".", 1)[0] ?? path)
const before = new Set(previous.namespaces.flatMap((namespace) => namespace.tools))
const after = new Set(current.namespaces.flatMap((namespace) => namespace.tools))
const added = [...after].filter((path) => kept(path) && !before.has(path))
const removed = [...before].filter((path) => kept(path) && !after.has(path))
const changed = previous.namespaces
.flatMap((namespace) => namespace.entries)
.filter((entry) => kept(entry.path) && after.has(entry.path))
.flatMap((entry) => {
const listing = lookup(entry.path)
return listing === undefined || listing.line === entry.line ? [] : [listing]
})
const budget = complete(current) ? Infinity : UPDATE_BUDGET / (restated.length + (added.length > 0 ? 1 : 0) || 1)
if (!currentComplete) {
if (entriesChanged) return replacement
const namespaces = Instructions.diffByKey(
previous.namespaces,
current.namespaces,
(namespace) => namespace.name,
(before, after) => before.count !== after.count,
)
const changed = namespaces.added.length > 0 || namespaces.removed.length > 0 || namespaces.changed.length > 0
if (!changed) return replacement
const parts = ["The Code Mode tool catalog has changed."]
if (namespaces.added.length > 0) {
parts.push(
`New tool namespaces are available: ${namespaces.added
.map((namespace) => `\`${namespace.name}\` (${namespace.count} tools)`)
.join(", ")}.`,
)
}
if (namespaces.changed.length > 0) {
parts.push(
`The following namespace inventories changed; search them again before relying on previous results: ${namespaces.changed
.map((change) => `\`${change.current.name}\` now has ${change.current.count} tools`)
.join(", ")}.`,
)
}
if (namespaces.removed.length > 0) {
parts.push(
`The following tool namespaces are no longer available and must not be used: ${namespaces.removed
.map((namespace) => `\`${namespace.name}\``)
.join(", ")}.`,
)
}
const delta = parts.join("\n\n")
if (delta.length < replacement.length) return delta
return replacement
}
if (!entriesChanged) return replacement
const parts = ["The Code Mode tool catalog has changed."]
if (diff.added.length > 0) {
if (restated.length > 0) {
parts.push(
[
"New tools are available:",
...restated.map((namespace) => {
const shown = fit(
namespace.tools.flatMap((path) => lookup(path) ?? []),
budget,
)
return [CodeModeCatalog.namespaceLine(namespace, shown.length), ...shown.map((entry) => entry.line)].join(
"\n",
)
}),
].join("\n"),
)
}
if (added.length > 0) {
const shown = fit(
added.flatMap((path) => lookup(path) ?? []),
budget,
)
const rest = added.length - shown.length
parts.push(
[
"New tools are available in addition to those previously listed:",
...diff.added.map((entry) => entry.line),
...shown.map((entry) => entry.line),
...(rest === 0 ? [] : [` - ${rest} more; search for them`]),
].join("\n"),
)
}
if (diff.changed.length > 0) {
if (complete(previous) && !complete(current)) parts.push(searchNote)
if (changed.length > 0) {
parts.push(
[
"Changed tool listings supersede the previously listed ones:",
...diff.changed.map((change) => change.current.line),
].join("\n"),
["Changed tool listings supersede the previously listed ones:", ...changed.map((entry) => entry.line)].join("\n"),
)
}
if (diff.removed.length > 0) {
if (removed.length > 0) {
parts.push(
`The following tools are no longer available and must not be called: ${diff.removed
.map((entry) => toolExpression(entry.path))
`The following tools are no longer available and must not be called: ${removed.map(toolExpression).join(", ")}.`,
)
}
if (namespaces.removed.length > 0) {
parts.push(
`The following tool namespaces are no longer available and must not be used: ${namespaces.removed
.map((namespace) => `\`${namespace.name}\``)
.join(", ")}.`,
)
}
if (parts.length === 1) return undefined
const delta = parts.join("\n\n")
if (delta.length < replacement.length) return delta
return replacement
}
function complete(catalog: CodeModeCatalog.Summary) {
return catalog.namespaces.every((namespace) => namespace.entries.length === namespace.tools.length)
}
/** The leading listings whose cost fits `budget`. */
function fit(listings: ReadonlyArray<CodeModeCatalog.Listing>, budget: number) {
const totals: number[] = []
for (const entry of listings) totals.push((totals.at(-1) ?? 0) + CodeModeCatalog.cost(entry.line))
const end = totals.findIndex((total) => total > budget)
return end === -1 ? listings : listings.slice(0, end)
}
const key = Instructions.Key.make("core/codemode")
const codec = Schema.toCodecJson(CodeModeCatalog.Summary)
@@ -134,7 +155,9 @@ export const make = (inventory?: CodeModeCatalog.Inventory): Instructions.List =
read: Effect.succeed(catalog),
render: {
initial: render,
changed: update,
// Rendered once, against the inventory that produced `current`, and stored with the update.
changed: (previous, current) =>
update(previous, current, inventory === undefined ? () => undefined : CodeModeCatalog.lookup(inventory)),
removed: () => "Code Mode tools are no longer available. Do not use any previously listed Code Mode tools.",
},
})
+4 -1
View File
@@ -232,7 +232,10 @@ export const catalog = (inventory: Inventory) => {
type: "tool",
name: tool.path.split(".").at(-1) ?? tool.path,
description: tool.description,
signature: tool.signature,
// Rendered only for the tools the catalog shows.
get signature() {
return tool.signature
},
pinned: pinned.has(tool.path),
}
return {
+121 -55
View File
@@ -27,19 +27,22 @@ const update = (
CodeModeInstructions.update(
CodeModeCatalog.summarize({ tools: previous }, budget === undefined ? {} : { budget }),
CodeModeCatalog.summarize({ tools: current }, budget === undefined ? {} : { budget }),
CodeModeCatalog.lookup({ tools: current }),
)
const shown = (catalog: CodeModeCatalog.Summary) =>
catalog.namespaces.reduce((total, namespace) => total + namespace.entries.length, 0)
describe("CodeModeCatalog.summarize", () => {
test("retains namespace inventory without retaining tools outside the inline budget", () => {
const catalog = CodeModeCatalog.summarize(
{ tools: Array.from({ length: 10_000 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`)) },
{ budget: 0 },
)
expect(catalog).toEqual({
total: 10_000,
shown: 0,
namespaces: [{ name: "bulk", count: 10_000, entries: [] }],
})
expect(catalog.total).toBe(10_000)
expect(catalog.namespaces).toHaveLength(1)
expect(catalog.namespaces[0]?.tools).toHaveLength(10_000)
expect(catalog.namespaces[0]?.entries).toEqual([])
})
test("retains every namespace when no full tool listing fits", () => {
@@ -58,7 +61,7 @@ describe("CodeModeCatalog.summarize", () => {
{ budget: 0 },
)
expect(catalog.shown).toBe(2)
expect(shown(catalog)).toBe(2)
expect(catalog.namespaces.flatMap((namespace) => namespace.entries.map((item) => item.path))).toEqual([
"alpha.first",
"beta.second",
@@ -71,17 +74,22 @@ describe("CodeModeCatalog.summarize", () => {
const pinCost = Math.round(` - ${pinned.signature} // Pinned`.length / 4)
const unpinnedCost = Math.round(` - ${unpinned.signature} // Unpinned`.length / 4)
const namespaceCost = [
{ name: "alpha", count: 1, entries: [] },
{ name: "beta", count: 1, entries: [] },
{ name: "alpha", tools: ["alpha.pinned"], entries: [] },
{ name: "beta", tools: ["beta.unpinned"], entries: [] },
].reduce((total, namespace) => total + Math.round(CodeModeCatalog.namespaceLine(namespace).length / 4), 0)
expect(
CodeModeCatalog.summarize({ tools: [pinned, unpinned] }, { budget: namespaceCost + pinCost + unpinnedCost })
.shown,
shown(
CodeModeCatalog.summarize({ tools: [pinned, unpinned] }, { budget: namespaceCost + pinCost + unpinnedCost }),
),
).toBe(2)
expect(
CodeModeCatalog.summarize({ tools: [pinned, unpinned] }, { budget: namespaceCost + pinCost + unpinnedCost - 1 })
.shown,
shown(
CodeModeCatalog.summarize(
{ tools: [pinned, unpinned] },
{ budget: namespaceCost + pinCost + unpinnedCost - 1 },
),
),
).toBe(1)
})
@@ -102,16 +110,42 @@ describe("CodeModeCatalog.summarize", () => {
test("always retains namespace descriptions and charges them before tool listings", () => {
const tool = entry("alpha.one", "One")
const listingCost = Math.round(` - ${tool.signature} // One`.length / 4)
const namespaceCost = Math.round(CodeModeCatalog.namespaceLine({ name: "alpha", count: 1, entries: [] }).length / 4)
const namespaceCost = Math.round(
CodeModeCatalog.namespaceLine({ name: "alpha", tools: ["alpha.one"], entries: [] }).length / 4,
)
const description = "A namespace description that stays visible beyond the available tool budget"
const namespace = { type: "namespace" as const, name: "alpha", description, tools: [tool] }
expect(CodeModeCatalog.summarize({ tools: [tool] }, { budget: namespaceCost + listingCost }).shown).toBe(1)
expect(shown(CodeModeCatalog.summarize({ tools: [tool] }, { budget: namespaceCost + listingCost }))).toBe(1)
const catalog = CodeModeCatalog.summarize({ tools: [namespace] }, { budget: namespaceCost + listingCost })
expect(catalog.shown).toBe(0)
expect(shown(catalog)).toBe(0)
expect(catalog.namespaces[0]?.description).toBe(description)
expect(CodeModeInstructions.render(catalog)).toContain(`- alpha (1 tool, none shown) // ${description}`)
})
test("selects tools in path order, whatever their cost", () => {
const costly = entry("alpha.a", "A", `tools.alpha.a({\n ${"long".repeat(40)}: string,\n}): Promise<string>`)
const catalog = CodeModeCatalog.summarize({ tools: [entry("alpha.b", "B"), costly] }, { budget: 70 })
expect(catalog.namespaces[0]?.entries.map((item) => item.path)).toEqual(["alpha.a"])
})
test("renders only the signatures it selects, and the next one it tries", () => {
const rendered = new Set<string>()
const tools = Array.from({ length: 500 }, (_, index): CodeModeCatalog.Tool => {
const path = `bulk.tool${String(index).padStart(3, "0")}`
return {
type: "tool",
name: path,
description: path,
get signature() {
rendered.add(path)
return `tools.${path}({ q: string }): Promise<string>`
},
}
})
const catalog = CodeModeCatalog.summarize({ tools }, { budget: 100 })
expect(rendered.size).toBe(shown(catalog) + 1)
})
})
describe("CodeModeInstructions.render", () => {
@@ -154,8 +188,8 @@ describe("CodeModeInstructions.render", () => {
// Round 1 places alpha.cheap and beta.cheap; in round 2 alpha.expensive does not fit,
// which marks only alpha done - it must NOT prevent other namespaces from inlining.
const namespaceCost = [
{ name: "alpha", count: 2, entries: [] },
{ name: "beta", count: 1, entries: [] },
{ name: "alpha", tools: ["alpha.cheap", "alpha.expensive"], entries: [] },
{ name: "beta", tools: ["beta.cheap"], entries: [] },
].reduce((total, namespace) => total + Math.round(CodeModeCatalog.namespaceLine(namespace).length / 4), 0)
const instructions = render([cheapAlpha, expensive, cheapBeta], 40 + namespaceCost)
expect(instructions).toContain("The catalog is partial.")
@@ -192,61 +226,93 @@ describe("CodeModeInstructions.update", () => {
const added = entry("notes.list", "List notes")
const unchanged = Array.from({ length: 5 }, (_, index) => entry(`stable.tool${index}`, `Stable ${index}`))
const text = update([echo, lookup, ...unchanged], [changed, added, ...unchanged])
expect(text).toContain("The Code Mode tool catalog has changed.")
expect(text).toContain(`New tools are available in addition to those previously listed:\n - ${added.signature}`)
expect(text).toContain(
`Changed tool listings supersede the previously listed ones:\n - ${changed.signature} // Echo text`,
expect(text).toBe(
[
"The Code Mode tool catalog has changed.",
`New tools are available in addition to those previously listed:\n - ${added.signature} // List notes`,
`Changed tool listings supersede the previously listed ones:\n - ${changed.signature} // Echo text`,
"The following tool namespaces are no longer available and must not be used: `orders`.",
].join("\n\n"),
)
expect(text).toContain("The following tools are no longer available and must not be called: tools.orders.lookup.")
expect(text).not.toContain("## Code Mode tools (call only inside `execute`)")
})
test("names removed tools with exact callable expressions including bracket notation", () => {
const dashed = entry("context7.resolve-library-id", "Resolve a library ID")
const text = update([echo, dashed], [echo])
expect(text).toContain(
const kept = entry("context7.query-docs", "Query docs")
expect(update([echo, dashed, kept], [echo, kept])).toContain(
'The following tools are no longer available and must not be called: tools.context7["resolve-library-id"].',
)
})
test("restates namespace descriptions when they change", () => {
const previous = CodeModeCatalog.summarize({
tools: [{ type: "namespace", name: "notes", description: "Old description", tools: [echo] }],
})
const current = CodeModeCatalog.summarize({
tools: [{ type: "namespace", name: "notes", description: "New description", tools: [echo] }],
})
const text = CodeModeInstructions.update(previous, current)
expect(text).toContain("This catalog supersedes the previous Code Mode tool catalog.")
expect(text).toContain("- notes (1 tool) // New description")
test("restates a namespace whose description changed", () => {
const text = update(
[{ type: "namespace", name: "notes", description: "Old description", tools: [echo] }],
[{ type: "namespace", name: "notes", description: "New description", tools: [echo] }],
)
expect(text).toContain("New tools are available:\n- notes (1 tool) // New description")
expect(text).not.toContain("Old description")
})
test("restates the full catalog when the rendering mode crosses full and compact", () => {
const wide = Array.from({ length: 40 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`))
const text = update([echo], [echo, ...wide], 30)
expect(text).toContain(
"The Code Mode tool catalog has changed. This catalog supersedes the previous Code Mode tool catalog.",
test("says nothing when only which tools are shown changes", () => {
const alpha = Array.from({ length: 20 }, (_, index) => entry(`alpha.tool${index}`, `Tool ${index}`))
const text = update(alpha, [...alpha, entry("beta.one", "One")], 120)
expect(text).toStartWith("The Code Mode tool catalog has changed.\n\nNew tools are available:\n- beta (1 tool")
expect(text).not.toContain("alpha")
expect(update([echo], [{ ...echo, pinned: true }])).toBeUndefined()
})
test("shows a new tool's signature even when the catalog would not", () => {
const alpha = Array.from({ length: 10 }, (_, index) => entry(`alpha.tool${index}`, `Tool ${index}`))
expect(update(alpha, [...alpha, entry("alpha.tool10", "Tool 10")], 0)).toContain(
"New tools are available in addition to those previously listed:\n - tools.alpha.tool10({",
)
expect(text).toContain("The catalog is partial.")
expect(text).toContain("## Code Mode tools (call only inside `execute`)")
})
test("counts new tools past the update budget", () => {
const bulk = Array.from({ length: 300 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`))
expect(update([bulk[0]!], bulk)).toMatch(/ {2}- \d+ more; search for them\n/)
})
test("shows a changed signature the model was shown even when the tool is no longer shown", () => {
const before = Array.from({ length: 200 }, (_, index) => entry(`ns.tool${String(index).padStart(3, "0")}`, "Tool"))
const last = CodeModeCatalog.summarize({ tools: before }).namespaces[0]!.entries.at(-1)!.path
const signature = `tools.${last}({\n key: string,\n}): Promise<string>`
// An arriving namespace takes the room the tool was shown in, in the same step its signature changes.
const arriving = Array.from({ length: 40 }, (_, index) => entry(`aa.tool${index}`, "Tool"))
const after = [...arriving, ...before.map((tool) => (tool.name === last ? { ...tool, signature } : tool))]
const current = CodeModeCatalog.summarize({ tools: after }).namespaces.find((namespace) => namespace.name === "ns")
expect(current?.entries.map((item) => item.path)).not.toContain(last)
expect(update(before, after)).toContain(
`Changed tool listings supersede the previously listed ones:\n - ${signature} // Tool`,
)
})
test("shows every new tool, without pointing to search, while the catalog lists every tool", () => {
const signature = (name: string) =>
`tools.wide.${name}({\n /** ${"A long parameter description. ".repeat(4)} */\n q: string,\n}): Promise<string>`
const wide = Array.from({ length: 30 }, (_, index) =>
entry(`wide.tool${index}`, `Tool ${index}`, signature(`tool${index}`)),
)
const text = update([echo], [echo, ...wide])!
expect(CodeModeCatalog.cost(text)).toBeGreaterThan(1_000)
expect(text).toContain("- wide (30 tools)\n")
expect(text).not.toContain("search")
})
test("introduces search once, when the catalog stops listing every tool", () => {
const wide = Array.from({ length: 200 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`))
const text = update([echo], [echo, ...wide])
expect(text).toStartWith("The Code Mode tool catalog has changed.\n\nNew tools are available:\n- bulk (200 tools")
expect(text).toContain("You can also look up tools that aren't listed with the global `search(...)` function")
expect(text).toContain("- search({")
expect(update([echo, ...wide], [echo, ...wide, entry("bulk.more", "More")])).not.toContain("search(")
expect(update([echo, ...wide], [echo])).not.toContain("search(")
})
test("falls back to full replacement when the delta is larger than the catalog", () => {
const previous = Array.from({ length: 200 }, (_, index) => entry(`bulk.tool${index}`, `Tool ${index}`))
const text = update([...previous, echo], [echo])
const removed = Array.from({ length: 200 }, (_, index) => entry(`a_long_removed_namespace_${index}.tool`, "Tool"))
const text = update([...removed, echo], [echo])
expect(text).toContain("This catalog supersedes the previous Code Mode tool catalog.")
expect(text).toContain("## Code Mode tools (call only inside `execute`)")
expect(text).toContain("The catalog is complete.")
expect(text).not.toContain("The following tools are no longer available")
})
test("renders namespace-only deltas without persisting hidden tool entries", () => {
const alpha = Array.from({ length: 10 }, (_, index) => entry(`alpha.tool${index}`, `Tool ${index}`))
const text = update(alpha, [...alpha, entry("alpha.tool10", "Tool 10")], 0)
expect(text).toContain("`alpha` now has 11 tools")
expect(text).toContain("search them again before relying on previous results")
expect(text).not.toContain("tools.alpha.tool10(input:")
expect(text).not.toContain("## Code Mode tools (call only inside `execute`)")
})
})
@@ -31,8 +31,9 @@ describe("CodeModeInstructions", () => {
"No Code Mode tools are currently available. Later Code Mode catalog updates may add or remove tools. Do not call `execute` unless there is at least one available Code Mode tool.",
)
// The empty notice never explained Code Mode, so the first tools restate the whole catalog.
const added = yield* readUpdate(CodeModeInstructions.make({ tools: [echo] }), initialized)
expect(added.text).toContain("New tools are available in addition to those previously listed:")
expect(added.text).toContain("This catalog supersedes the previous Code Mode tool catalog.")
expect(added.text).toContain(echo.signature)
expect(yield* readUpdate(CodeModeInstructions.make({ tools: [] }), { values: added.values })).toMatchObject({
@@ -54,15 +55,14 @@ describe("CodeModeInstructions", () => {
expect(initialized.text).toContain(` - ${echo.signature} // Echo text`)
const added = yield* readUpdate(CodeModeInstructions.make({ tools: [echo, lookup] }), initialized)
expect(added.text).toContain("The Code Mode tool catalog has changed.")
expect(added.text).toContain("New tools are available in addition to those previously listed:")
expect(added.text).toContain(` - ${lookup.signature} // Look up an order`)
expect(added.text).not.toContain("## Code Mode tools (call only inside `execute`)")
expect(added.text).toBe(
`The Code Mode tool catalog has changed.\n\nNew tools are available:\n- orders (1 tool)\n - ${lookup.signature} // Look up an order`,
)
const removed = yield* readUpdate(CodeModeInstructions.make({ tools: [echo] }), { values: added.values })
expect(removed.text).toBe(
"The Code Mode tool catalog has changed.\n\n" +
"The following tools are no longer available and must not be called: tools.orders.lookup.",
"The following tool namespaces are no longer available and must not be used: `orders`.",
)
expect(yield* readUpdate(CodeModeInstructions.make(), initialized)).toMatchObject({
+70 -1
View File
@@ -88,7 +88,7 @@ import { permissionLayer } from "./lib/permission"
import { agentHost, modelHost, host, noProviders } from "./plugin/host"
import { CodeModeInstructions } from "@opencode/core/codemode/instructions"
const emptyCodeMode = `${CodeModeInstructions.render({ total: 0, shown: 0, namespaces: [] })}\n\n`
const emptyCodeMode = `${CodeModeInstructions.render({ total: 0, namespaces: [] })}\n\n`
type ToolBarrier = {
readonly count: number
readonly started: Deferred.Deferred<void>
@@ -2382,6 +2382,75 @@ describe("SessionRunnerLLM", () => {
])
})
scenario("narrates only the Code Mode tools that change, through compaction and fork", function* (s) {
const registry = yield* Tool.Service
const register = (name: string, count: number, from = 0) =>
Effect.gen(function* () {
const scope = yield* Scope.make()
const tools = Object.fromEntries(
Array.from({ length: count }, (_, offset) => [
`${name}_${from + offset}`,
{
name: `${name}_${from + offset}`,
description: `${name} tool ${from + offset}`,
input: Schema.Struct({ id: Schema.String }),
output: Schema.String,
execute: () => Effect.succeed({ output: "ok" }),
} satisfies ToolInfo,
]),
)
yield* transformTools(registry, tools, { namespace: name }).pipe(Scope.provide(scope))
return scope
})
const catalogOf = (request: LLMRequest | undefined) =>
request?.system.map((part) => part.text).find((text) => text.includes("# Code Mode")) ?? ""
const updates = (request: LLMRequest | undefined) =>
request ? systemTexts(request).filter((text) => text.startsWith("The Code Mode tool catalog has changed")) : []
const signature = (name: string, index: number) =>
` - tools.${name}.${name}_${index}({\n id: string,\n}): Promise<string> // ${name} tool ${index}`
yield* register("alpha", 12)
yield* s.runPrompt("First")
expect(catalogOf(s.requests.at(-1))).toContain(`- alpha (12 tools)\n${signature("alpha", 0)}`)
const beta = yield* register("beta", 1)
yield* s.runPrompt("Second")
expect(updates(s.requests.at(-1))).toEqual([
`The Code Mode tool catalog has changed.\n\nNew tools are available:\n- beta (1 tool)\n${signature("beta", 0)}`,
])
// The next step rebuilds the same catalog and has nothing to say.
yield* s.runPrompt("Again")
expect(updates(s.requests.at(-1))).toHaveLength(1)
yield* Scope.close(beta, Exit.void)
yield* s.runPrompt("Third")
expect(updates(s.requests.at(-1)).at(-1)).toBe(
"The Code Mode tool catalog has changed.\n\nThe following tool namespaces are no longer available and must not be used: `beta`.",
)
yield* register("alpha", 1, 12)
yield* s.runPrompt("Fourth")
expect(updates(s.requests.at(-1)).at(-1)).toBe(
`The Code Mode tool catalog has changed.\n\nNew tools are available in addition to those previously listed:\n${signature("alpha", 12)}`,
)
// Compaction restates the current catalog as the new baseline and has no update to send.
yield* s.llm.push(TestLLM.text("## Objective\n- summary", "catalog-summary"))
yield* s.session.compact({ sessionID })
yield* s.resume
const latest = yield* s.runPrompt("Fifth")
expect(catalogOf(s.requests.at(-1))).toContain(signature("alpha", 12))
expect(catalogOf(s.requests.at(-1))).not.toContain("- beta")
expect(updates(s.requests.at(-1))).toEqual([])
// A fork starts from the same catalog and has nothing to narrate.
const forked = yield* s.session.fork({ sessionID, before: latest.id })
yield* s.session.prompt({ sessionID: forked.id, text: "Forked", resume: false })
yield* s.session.resume(forked.id)
expect(catalogOf(s.requests.at(-1))).toContain(signature("alpha", 12))
expect(updates(s.requests.at(-1))).toEqual([])
})
scenario("moves the epoch at compaction and narrates later changes", function* (s) {
yield* s.runPrompt("First")
s.systemBaseline = "Changed before compaction"