From 59ed43f0b8da891f03a7333d540a7b0c428049ea Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:44:40 -0600 Subject: [PATCH] feat(codemode): rank tool search with name-weighted BM25 (#62984) --- packages/codemode/src/tool-runtime.ts | 277 ++++++++++++++++++++---- packages/codemode/test/codemode.test.ts | 77 ++++++- 2 files changed, 304 insertions(+), 50 deletions(-) diff --git a/packages/codemode/src/tool-runtime.ts b/packages/codemode/src/tool-runtime.ts index ee914c94dfb..bd79eeccd66 100644 --- a/packages/codemode/src/tool-runtime.ts +++ b/packages/codemode/src/tool-runtime.ts @@ -204,44 +204,227 @@ export type Prepared = { export type SearchEntry = { readonly description: ToolDescription - /** The path split into words, so `zones` matches `get_zones` as a word rather than as a substring of `timezones`. */ - readonly pathWords: ReadonlyArray - readonly searchText: string + /** Position in the full search index, for scoring through the word index. */ + readonly position: number + /** The tool's searchable text by field, read when the word index is built. */ + readonly text: () => Readonly>> + /** Words in the tool's own name (the last path segment). */ + readonly nameLength: number } -const tokenize = (query: string): Array => - query +type SearchField = "namespace" | "name" | "description" | "parameters" | "namespaceDescription" + +// Standard field-weighted BM25 (k1 1.2, b 0.75) plus a preference for short names. Compared against other rankers on +// model-written and real search queries over an ~11k-tool catalog (anomalyco/evals, dumber/evals/tool-search). +const fieldWeights: Record = { + name: 3, + namespace: 2, + description: 1, + namespaceDescription: 0.4, + parameters: 0.3, +} +const searchFields = Object.keys(fieldWeights) as ReadonlyArray +const k1 = 1.2 +const lengthNormalization = 0.75 +/** score × (1 + brevityBoost / words in the tool name): "users" prefers `list_users` over `list_user_groups`. */ +const brevityBoost = 0.5 +/** A query word of 4+ letters also matches longer words it starts with, at this weight: `term` → `terminal`. */ +const prefixWeight = 0.3 + +/** Words split at case changes and non-alphanumerics: `listUserGroups` → list, user, groups. */ +const splitWords = (text: string): Array => + text .replace(/([a-z0-9])([A-Z])/g, "$1 $2") .toLowerCase() .split(/[^a-z0-9]+/) - .filter((term) => term.length > 0 && term !== "*") + .filter((word) => word.length > 0) -const termForms = (term: string): Array => { - const forms = [term] - if (term.endsWith("es") && term.length > 3) forms.push(term.slice(0, -2)) - if (term.endsWith("s") && term.length > 2) forms.push(term.slice(0, -1)) - return forms +/** Plural endings only, applied the same everywhere: `settings` → `setting`, `statuses` → `status`, `vms` → `vm`. */ +const stem = (word: string) => { + if (word.length > 4 && word.endsWith("ies")) return word.slice(0, -3) + "y" + if (word.length > 4 && /(?:ss|sh|ch|x|z|us)es$/.test(word)) return word.slice(0, -2) + if (word.length > 2 && word.endsWith("s") && !/(?:ss|us|is)$/.test(word)) return word.slice(0, -1) + return word } -const rank = (entries: ReadonlyArray, query: string): Array => { - const terms = tokenize(query).map(termForms) - return entries - .map((entry) => { - const path = entry.description.path.toLowerCase() - const description = entry.description.description.toLowerCase() - const score = terms.reduce( - (total, forms) => - total + - (forms.some((form) => path === form || path.endsWith(`.${form}`)) ? 20 : 0) + - (forms.some((form) => entry.pathWords.includes(form)) ? 12 : 0) + - (forms.some((form) => path.includes(form)) ? 8 : 0) + - (forms.some((form) => description.includes(form)) ? 4 : 0) + - (forms.some((form) => entry.searchText.includes(form)) ? 2 : 0), - 0, - ) - return { entry, score } - }) - .filter(({ score }) => terms.length === 0 || score > 0) +/** Searchable words of a text: its split words, plus each camelCase word whole (`CronJob` → cron, job, cronjob). */ +const searchWords = (text: string) => + splitWords(text) + .concat((text.match(/[A-Za-z0-9]*[a-z0-9][A-Z][A-Za-z0-9]*/g) ?? []).map((word) => word.toLowerCase())) + .map(stem) + +const fieldWords = (text: string, cache: Map>) => { + const cached = cache.get(text) + if (cached) return cached + const words = searchWords(text) + cache.set(text, words) + return words +} + +/** For each word, the tools containing it and their field-weighted, length-normalized term frequency. */ +type WordIndex = { + readonly postings: ReadonlyMap + readonly documents: number + /** Sorted indexed words, for partial-word matching. */ + readonly words: ReadonlyArray +} + +// Built once per search index on the first query. +const wordIndexes = new WeakMap, WordIndex>() + +// Locations with the same tools (global MCP servers) share one word index, keyed by a hash of the indexed text. An +// index lives while some search index uses it; once none does it can be collected and its key is removed. A hash +// collision (about 1 in 37 trillion with 1,000 distinct catalogs alive) would only misrank results: they still come +// from the caller's own tools, and the tool count in the key keeps positions in range. +const shared = + typeof WeakRef === "undefined" || typeof FinalizationRegistry === "undefined" + ? undefined + : { + indexes: new Map>(), + cleanup: new FinalizationRegistry<{ key: string; ref: WeakRef }>(({ key, ref }) => { + // A newer index may have been stored under the same key since this one was dropped. + if (shared?.indexes.get(key) === ref) shared.indexes.delete(key) + }), + } + +const wordIndexFor = (index: ReadonlyArray): WordIndex => { + const cached = wordIndexes.get(index) + if (cached) return cached + const texts = index.map((entry) => entry.text()) + const key = shared ? textKey(texts) : undefined + const reused = key === undefined ? undefined : shared?.indexes.get(key)?.deref() + if (reused) { + wordIndexes.set(index, reused) + return reused + } + const built = buildWordIndex(texts) + wordIndexes.set(index, built) + if (shared && key !== undefined) { + const ref = new WeakRef(built) + shared.indexes.set(key, ref) + shared.cleanup.register(built, { key, ref }) + } + return built +} + +/** Tool count plus two independent 32-bit FNV-1a hashes of every indexed field, in index order. */ +const textKey = (texts: ReadonlyArray>>>) => { + let first = 0x811c9dc5 + let second = 0x9e3779b9 + const add = (text: string) => { + for (let i = 0; i < text.length; i++) { + const code = text.charCodeAt(i) + first = Math.imul(first ^ code, 0x01000193) + second = Math.imul(second ^ code, 0x5bd1e995) + second ^= second >>> 15 + } + // Separators keep field boundaries distinct: ["ab", "c"] and ["a", "bc"] hash differently. + first = Math.imul(first ^ 0x1f, 0x01000193) + second = Math.imul(second ^ 0x1f, 0x5bd1e995) + } + for (const text of texts) + for (const field of searchFields) { + for (const part of text[field]) add(part) + add("\u0000") + } + return `${texts.length}:${(first >>> 0).toString(36)}:${(second >>> 0).toString(36)}` +} + +const buildWordIndex = (texts: ReadonlyArray>>>): WordIndex => { + // Field texts are split once here and dropped afterwards; descriptions repeat across tools, so each text is split + // once and shared while building. + const cache = new Map>() + const words = (texts: ReadonlyArray) => texts.flatMap((text) => fieldWords(text, cache)) + const fields = texts.map((text) => { + return { + namespace: words(text.namespace), + name: words(text.name), + description: words(text.description), + parameters: words(text.parameters), + namespaceDescription: words(text.namespaceDescription), + } + }) + const averageLength = Object.fromEntries( + searchFields.map((field) => [ + field, + fields.reduce((sum, entry) => sum + entry[field].length, 0) / texts.length || 1, + ]), + ) as Record + const building = new Map; weights: Array }>() + const weights = new Map() + fields.forEach((entry, position) => { + weights.clear() + for (const field of searchFields) { + const weight = + fieldWeights[field] / + (1 - lengthNormalization + (lengthNormalization * entry[field].length) / averageLength[field]) + for (const word of entry[field]) weights.set(word, (weights.get(word) ?? 0) + weight) + } + for (const [word, weight] of weights) { + let posting = building.get(word) + if (!posting) building.set(word, (posting = { positions: [], weights: [] })) + posting.positions.push(position) + posting.weights.push(weight) + } + }) + const postings = new Map( + [...building].map(([word, posting]) => [ + word, + { positions: Int32Array.from(posting.positions), weights: Float32Array.from(posting.weights) }, + ]), + ) + return { postings, documents: texts.length, words: [...postings.keys()].sort() } +} + +/** Indexed words longer than a 4+ letter query word that start with it. */ +const prefixMatches = (words: WordIndex, term: string): Array => { + if (term.length < 4) return [] + let low = 0 + let high = words.words.length + while (low < high) { + const middle = (low + high) >> 1 + if (words.words[middle]! < term) low = middle + 1 + else high = middle + } + const matches: Array = [] + for (let i = low; i < words.words.length && words.words[i]!.startsWith(term); i++) + if (words.words[i] !== term) matches.push([words.words[i]!, prefixWeight]) + return matches +} + +const rank = ( + entries: ReadonlyArray, + query: string, + index: ReadonlyArray, +): Array => { + const terms = searchWords(query) + if (terms.length === 0) + return [...entries].sort((left, right) => compareText(left.description.path, right.description.path)) + const words = wordIndexFor(index) + const allowed = entries === index ? undefined : new Set(entries.map((entry) => entry.position)) + const scores = new Map() + for (const term of terms) { + // Each tool counts a query word once, through its best-scoring match. + const best = new Map() + for (const [word, factor] of [[term, 1] as const, ...prefixMatches(words, term)]) { + const posting = words.postings.get(word) + if (!posting) continue + const idf = Math.log(1 + (words.documents - posting.positions.length + 0.5) / (posting.positions.length + 0.5)) + for (let i = 0; i < posting.positions.length; i++) { + const position = posting.positions[i]! + if (allowed && !allowed.has(position)) continue + const weight = posting.weights[i]! + const score = (factor * idf * weight * (k1 + 1)) / (weight + k1) + if (score > (best.get(position) ?? 0)) best.set(position, score) + } + } + for (const [position, score] of best) scores.set(position, (scores.get(position) ?? 0) + score) + } + return [...scores] + .map(([position, score]) => ({ + entry: index[position]!, + score: score * (1 + brevityBoost / Math.max(1, index[position]!.nameLength)), + })) .sort( (left, right) => right.score - left.score || compareText(left.entry.description.path, right.entry.description.path), @@ -276,7 +459,7 @@ const makeSearchTool = (searchIndex: ReadonlyArray): Tool => ({ : scoped.find( (entry) => entry.description.path === pathQuery || toolExpression(entry.description.path) === trimmed, ) - const ranked = exact !== undefined ? [exact] : rank(scoped, query) + const ranked = exact !== undefined ? [exact] : rank(scoped, pathQuery, searchIndex) const items = ranked.slice(offset, offset + (request.limit ?? defaultSearchLimit)).map(({ description }) => ({ ...description, path: toolExpression(description.path), @@ -296,20 +479,23 @@ export const searchSignature = (() => { return `search(${inputTypeScript(tool, true)}): ${outputTypeScript(tool, true)}` })() -const toSearchEntry = (visible: VisibleTool): SearchEntry => ({ - description: describeTool(visible), - pathWords: tokenize(visible.path), - searchText: [ - visible.path, - visible.tool.description, - ...visible.namespaces.flatMap((namespace) => (namespace.description === undefined ? [] : [namespace.description])), - ...inputProperties(visible.tool).flatMap(({ name, description: property }) => - property === undefined ? [name] : [name, property], - ), - ] - .join("\n") - .toLowerCase(), -}) +const toSearchEntry = (visible: VisibleTool, position: number): SearchEntry => { + const segments = visible.path.split(".") + const name = segments.at(-1) ?? "" + return { + description: describeTool(visible), + position, + text: () => ({ + namespace: segments.slice(0, -1), + name: [name], + description: [visible.tool.description], + // Parameter names only: their descriptions are half of all words and didn't improve ranking. + parameters: inputProperties(visible.tool).map((property) => property.name), + namespaceDescription: visible.namespaces.map((namespace) => namespace.description ?? ""), + }), + nameLength: splitWords(name).length, + } +} export const prepare = (tools: Tools): Prepared => { const root = toolTrie(tools) @@ -351,6 +537,7 @@ const resolve = (root: ToolNode, path: ReadonlyArray, index: Reado const closest = rank( ns ? index.filter((entry) => entry.description.path.startsWith(`${ns}.`)) : index, ns ? segments.slice(1).join(" ") : name, + index, )[0] throw new ToolRuntimeError( "UnknownTool", diff --git a/packages/codemode/test/codemode.test.ts b/packages/codemode/test/codemode.test.ts index 121fba9d2bb..4258ffa9c83 100644 --- a/packages/codemode/test/codemode.test.ts +++ b/packages/codemode/test/codemode.test.ts @@ -1101,7 +1101,7 @@ describe("CodeMode public contract", () => { } }) - test("a query term that is a whole word of the path outranks a substring of it", async () => { + test("a query term matches whole words, not substrings", async () => { const simple = (description: string) => Tool.make({ description, @@ -1120,13 +1120,80 @@ describe("CodeMode public contract", () => { expect(ranked.ok).toBe(true) if (ranked.ok) { const value = ranked.value as { items: Array<{ path: string }> } - expect(value.items.map((item) => item.path)).toStrictEqual([ - "tools.cloudflare.get_zones", - "tools.cloudflare.get_timezones", - ]) + expect(value.items.map((item) => item.path)).toStrictEqual(["tools.cloudflare.get_zones"]) } }) + test("search indexes are shared only between runtimes with the same tool text", async () => { + const simple = (description: string) => + Tool.make({ + description, + input: Schema.Struct({}), + output: Schema.String, + execute: () => Effect.succeed("ok"), + }) + const top = async (runtime: ReturnType, query: string) => { + const result = await Effect.runPromise(runtime.execute(`return search({ query: ${JSON.stringify(query)} })`)) + expect(result.ok).toBe(true) + return result.ok ? (result.value as { items: Array<{ path: string }> }).items[0]?.path : undefined + } + // Same tools registered in a different order index identically. + const first = CodeMode.make({ tools: { mail: { send: simple("Send an email"), archive: simple("Archive a thread") } } }) + const reordered = CodeMode.make({ tools: { mail: { archive: simple("Archive a thread"), send: simple("Send an email") } } }) + // Same paths and count, different descriptions: must not reuse the other index. + const changed = CodeMode.make({ tools: { mail: { send: simple("Archive a thread"), archive: simple("Send an email") } } }) + + expect(await top(first, "email")).toBe("tools.mail.send") + expect(await top(reordered, "email")).toBe("tools.mail.send") + expect(await top(changed, "email")).toBe("tools.mail.archive") + expect(await top(first, "archive thread")).toBe("tools.mail.archive") + }) + + test("search ranks by rare words, prefers short names and matches partial words", async () => { + const simple = (description: string) => + Tool.make({ + description, + input: Schema.Struct({}), + output: Schema.String, + execute: () => Effect.succeed("ok"), + }) + const runtime = CodeMode.make({ + tools: { + gmail: { list_labels: simple("List the labels in the mailbox"), list_messages: simple("List messages") }, + slack: { + list_channels: simple("List channels"), + list_users: simple("List users"), + list_user_groups: simple("List user groups"), + get_schedules: simple("Get scheduled messages"), + }, + }, + }) + const paths = async (query: Record) => { + const result = await Effect.runPromise(runtime.execute(`return search(${JSON.stringify(query)})`)) + expect(result.ok).toBe(true) + return result.ok ? (result.value as { items: Array<{ path: string }> }).items.map((item) => item.path) : [] + } + + // "list" is in every tool; "labels" decides. + expect((await paths({ query: "list gmail labels" }))[0]).toBe("tools.gmail.list_labels") + // Both match "users"; the shorter, more general name wins. + expect((await paths({ query: "users", namespace: "slack" }))[0]).toBe("tools.slack.list_users") + // A partial word of 4+ letters that matches nothing whole finds the words it starts with. + expect(await paths({ query: "schedu" })).toStrictEqual(["tools.slack.get_schedules"]) + // Short plurals and stop-word-only queries still match. + const short = CodeMode.make({ + tools: { cloud: { describe_vm: simple("Describe a virtual machine"), list_all_issues: simple("List issues") } }, + }) + const top = async (query: string) => { + const result = await Effect.runPromise(short.execute(`return search(${JSON.stringify({ query })})`)) + return result.ok ? (result.value as { items: Array<{ path: string }> }).items.map((item) => item.path) : [] + } + expect((await top("vms"))[0]).toBe("tools.cloud.describe_vm") + expect(await top("all")).toStrictEqual(["tools.cloud.list_all_issues"]) + // A namespace filter is a hard filter. + expect(await paths({ query: "labels", namespace: "slack" })).toStrictEqual([]) + }) + test("a plural query term matches singular-only tool text", async () => { const simple = (description: string) => Tool.make({