diff --git a/bun.lock b/bun.lock index eb4c6e6fd5f..c4381ac73ba 100644 --- a/bun.lock +++ b/bun.lock @@ -46,7 +46,10 @@ "@tsconfig/bun": "catalog:", "@types/bun": "catalog:", "@typescript/native": "catalog:", + "@valibot/to-json-schema": "1.8.0", "typescript": "catalog:", + "valibot": "1.5.0", + "zod": "catalog:", }, }, "packages/app": { @@ -2662,6 +2665,8 @@ "@ungap/structured-clone": ["@ungap/structured-clone@1.3.3", "", {}, "sha512-60YRaenCQcVjYEKOcG824+DRGGIQ3VKErcBoAEDJZz5bKIs2ZG+X/H9Nk+Q6EVkwJk5QNApxbrc5QtBSwtrXAg=="], + "@valibot/to-json-schema": ["@valibot/to-json-schema@1.8.0", "", { "peerDependencies": { "valibot": "^1.5.0" } }, "sha512-a0M+uwCuQZEPAo65NYkFSJ14O5c213KoSZmPdoCfuFvUhfULi4T2Z6Tpjv6lTVMW9RoLm74RpRAjNEa43KcN+g=="], + "@volar/kit": ["@volar/kit@2.4.28", "", { "dependencies": { "@volar/language-service": "2.4.28", "@volar/typescript": "2.4.28", "typesafe-path": "^0.2.2", "vscode-languageserver-textdocument": "^1.0.11", "vscode-uri": "^3.0.8" }, "peerDependencies": { "typescript": "*" } }, "sha512-cKX4vK9dtZvDRaAzeoUdaAJEew6IdxHNCRrdp5Kvcl6zZOqb6jTOfk3kXkIkG3T7oTFXguEMt5+9ptyqYR84Pg=="], "@volar/language-core": ["@volar/language-core@2.4.28", "", { "dependencies": { "@volar/source-map": "2.4.28" } }, "sha512-w4qhIJ8ZSitgLAkVay6AbcnC7gP3glYM3fYwKV3srj8m494E3xtrCv6E+bWviiK/8hs6e6t1ij1s2Endql7vzQ=="], @@ -4656,6 +4661,8 @@ "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + "valibot": ["valibot@1.5.0", "", { "peerDependencies": { "typescript": ">=5" }, "optionalPeers": ["typescript"] }, "sha512-nil6AkP2TChWL43Z5uJ6GTxX01CUA+g8LWUM+N/rB9NBbkUMaUsi9PUNzlUPgoKASgmx9f7eGOYpJ04/fSa6FQ=="], + "validate-npm-package-name": ["validate-npm-package-name@7.0.2", "", {}, "sha512-hVDIBwsRruT73PbK7uP5ebUt+ezEtCmzZz3F59BSr2F6OVFnJ/6h8liuvdLrQ88Xmnk6/+xGGuq+pG9WwTuy3A=="], "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], diff --git a/packages/ai/README.md b/packages/ai/README.md index 36c53c3b3e5..4fc3cff0f32 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -85,6 +85,22 @@ for await (const event of events) { } ``` +`input` and structured-output `schema` also take a [Standard Schema](https://standardschema.dev), such as one from Zod +4.2+ or ArkType, which validates and types the arguments. The JSON Schema sent to the model comes from the schema's +[Standard JSON Schema](https://standardschema.dev/json-schema) export; a schema without one (a plain Valibot schema; +wrap it in `toStandardJsonSchema` from `@valibot/to-json-schema`) throws when the tool is built: + +```ts +import { z } from "zod" + +const weather = tool({ + name: "weather", + description: "Get the weather.", + input: z.object({ city: z.string() }), + execute: ({ city }) => getWeather(city), +}) +``` + ### Freeform tools A tool with exactly one required string parameter can set `freeform`, so the model writes that parameter as raw text diff --git a/packages/ai/package.json b/packages/ai/package.json index 3c89a6c7629..3bb72149a41 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -149,7 +149,10 @@ "@tsconfig/bun": "catalog:", "@types/bun": "catalog:", "@typescript/native": "catalog:", - "typescript": "catalog:" + "@valibot/to-json-schema": "1.8.0", + "typescript": "catalog:", + "valibot": "1.5.0", + "zod": "catalog:" }, "dependencies": { "@aws-sdk/credential-providers": "3.1057.0", diff --git a/packages/ai/src/llm.ts b/packages/ai/src/llm.ts index 858f2704d66..af8ea7bb6ba 100644 --- a/packages/ai/src/llm.ts +++ b/packages/ai/src/llm.ts @@ -1,4 +1,4 @@ -import { Effect, JsonSchema, Schema, Stream } from "effect" +import { Effect, Stream } from "effect" import { tryRequest } from "./media-model.js" import { LLMClient, Service, type StreamOptions } from "./route/client.js" import { @@ -20,7 +20,7 @@ import { type ToolEntryInput, withPartial, } from "./schema/index.js" -import { tool, type ExecutableTool, type Tool, type ToolSchema } from "./tool.js" +import { tool, type ExecutableTool, type Tool, type ToolInput, type ToolInputType } from "./tool.js" import { ToolRuntime } from "./tool-runtime.js" /** Input accepted by `LLM.request`, normalized into the canonical `LLMRequest` class. */ @@ -40,10 +40,10 @@ export type RequestInput | JsonSchema.JsonSchema +/** An Effect Schema or a Standard Schema with a JSON Schema export validates and types `object`; a raw JSON Schema leaves it `unknown`. */ +export type OutputSchema = ToolInput -export type OutputType = S extends Schema.Top ? Schema.Schema.Type : unknown +export type OutputType = ToolInputType export type ObjectInput< S extends OutputSchema, diff --git a/packages/ai/src/standard-schema.ts b/packages/ai/src/standard-schema.ts new file mode 100644 index 00000000000..a667feb5485 --- /dev/null +++ b/packages/ai/src/standard-schema.ts @@ -0,0 +1,56 @@ +// The Standard Schema and Standard JSON Schema interfaces (https://standardschema.dev), copied as the spec recommends so +// accepting them adds no dependency. Trimmed to the members this package reads. + +export interface StandardSchemaV1 { + readonly "~standard": StandardSchemaV1.Props +} + +export declare namespace StandardSchemaV1 { + export interface Props { + readonly version: 1 + readonly vendor: string + readonly validate: (value: unknown) => Result | Promise> + readonly types?: Types | undefined + } + + export type Result = { readonly value: Output; readonly issues?: undefined } | FailureResult + + export interface FailureResult { + readonly issues: ReadonlyArray + } + + export interface Issue { + readonly message: string + readonly path?: ReadonlyArray | undefined + } + + export interface Types { + readonly input: Input + readonly output: Output + } + + export type InferOutput = NonNullable["output"] +} + +export interface StandardJSONSchemaV1 { + readonly "~standard": StandardJSONSchemaV1.Props +} + +export declare namespace StandardJSONSchemaV1 { + export interface Props { + readonly version: 1 + readonly vendor: string + readonly types?: StandardSchemaV1.Types | undefined + readonly jsonSchema: Converter + } + + export interface Converter { + readonly input: (options: Options) => Record + readonly output: (options: Options) => Record + } + + export interface Options { + readonly target: "draft-2020-12" | "draft-07" | "openapi-3.0" | ({} & string) + readonly libraryOptions?: Record | undefined + } +} diff --git a/packages/ai/src/tool.ts b/packages/ai/src/tool.ts index 0f5858e1f58..ef272521fae 100644 --- a/packages/ai/src/tool.ts +++ b/packages/ai/src/tool.ts @@ -7,6 +7,7 @@ import type { ToolOutput as ToolOutputType, } from "./schema/index.js" import { ToolDefinition, ToolFailure, ToolOutput, type ToolDefinitionInput } from "./schema/index.js" +import type { StandardJSONSchemaV1, StandardSchemaV1 } from "./standard-schema.js" import { isRecord } from "./utils/record.js" /** @@ -16,11 +17,16 @@ import { isRecord } from "./utils/record.js" */ export type ToolSchema = Schema.Codec -/** A tool's `input`: an Effect Schema decodes and types it; a raw JSON Schema leaves it `unknown`. */ -export type ToolInput = ToolSchema | JsonSchema.JsonSchema +/** + * A tool's `input`: an Effect Schema or a Standard Schema (https://standardschema.dev, such as Zod 4, ArkType or Valibot) + * validates and types it; a raw JSON Schema leaves it `unknown`. A Standard Schema must also implement Standard JSON + * Schema, which gives the JSON Schema sent to the model. + */ +export type ToolInput = ToolSchema | StandardSchemaV1 | JsonSchema.JsonSchema /** The value `execute` receives for a tool `input`. */ -export type ToolInputType = Input extends ToolSchema ? T : unknown +export type ToolInputType = + Input extends ToolSchema ? T : Input extends StandardSchemaV1 ? StandardSchemaV1.InferOutput : unknown export interface ToolExecuteContext { readonly id: ToolCallPart["id"] @@ -83,7 +89,7 @@ export interface Tool readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown /** @internal */ - readonly _decode: (input: unknown) => Effect.Effect, Schema.SchemaError> + readonly _decode: (input: unknown) => Effect.Effect, Schema.SchemaError | Error> // Method syntax keeps these parameters bivariant, so a concrete tool stays assignable to `ExecutableTool`. /** @internal */ _encode(value: Schema.Schema.Type): Effect.Effect @@ -106,8 +112,9 @@ export type ExecutableTool validateStandard(input, value), + _encode: (value: unknown) => Effect.succeed(value ?? null), + _legacyResult: false, + _definition: definition({ + name: config.name, + description: config.description, + inputSchema: standardJsonSchema(input), + outputSchema: config.outputSchema, + freeform: config.freeform, + }), + } if (!Schema.isSchema(input)) return { ...shared, @@ -241,6 +264,39 @@ const toJsonSchema = (schema: Schema.Top): JsonSchema.JsonSchema => { return { ...document.schema, $defs: document.definitions } } +const isStandardSchema = (input: ToolInput): input is StandardSchemaV1 => "~standard" in input + +// Standard JSON Schema has no affordance for validation and Standard Schema none for JSON Schema, so ask the schema for +// both. Without a JSON Schema export there is nothing truthful to send the model, so fail rather than guess. +const standardJsonSchema = (schema: StandardSchemaV1): JsonSchema.JsonSchema => { + const props: StandardSchemaV1.Props & Partial = schema["~standard"] + if (props.jsonSchema === undefined) + throw new Error( + `This ${props.vendor} schema has no JSON Schema export. Use a Standard Schema that also implements Standard JSON Schema (https://standardschema.dev/json-schema), such as a Zod 4.2+ or ArkType schema or a Valibot schema wrapped in toStandardJsonSchema from @valibot/to-json-schema, or pass a raw JSON Schema.`, + ) + // Effect's JSON Schema export omits `$schema` too; providers only read the schema body. + const { $schema, ...body } = props.jsonSchema.input({ target: "draft-2020-12" }) + return body +} + +const validateStandard = (schema: StandardSchemaV1, value: unknown) => + Effect.suspend(() => { + const result = schema["~standard"].validate(value) + return result instanceof Promise ? Effect.promise(() => result) : Effect.succeed(result) + }).pipe( + Effect.flatMap((result) => + result.issues === undefined ? Effect.succeed(result.value) : Effect.fail(new Error(formatIssues(result.issues))), + ), + ) + +const formatIssues = (issues: ReadonlyArray) => + issues + .map((issue) => { + const path = (issue.path ?? []).map((segment) => String(typeof segment === "object" ? segment.key : segment)) + return path.length === 0 ? issue.message : `${path.join(".")}: ${issue.message}` + }) + .join("; ") + const project = ( toModelOutput: ((input: ToolModelOutputInput) => ReadonlyArray) | undefined, toStructuredOutput: ((output: unknown) => unknown) | undefined, diff --git a/packages/ai/test/standard-schema.test.ts b/packages/ai/test/standard-schema.test.ts new file mode 100644 index 00000000000..45498b37d03 --- /dev/null +++ b/packages/ai/test/standard-schema.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, test } from "bun:test" +import { toStandardJsonSchema } from "@valibot/to-json-schema" +import { Effect, Stream } from "effect" +import * as v from "valibot" +import { z } from "zod" +import { ai, ToolCallPart } from "../src/effect.js" +import { AI } from "../src/index.js" +import * as OpenAIChat from "../src/protocols/openai-chat.js" +import { Auth } from "../src/route.js" +import { TestLLM } from "../src/testing.js" +import { toDefinitions, tool } from "../src/tool.js" +import { ToolRuntime } from "../src/tool-runtime.js" +import { testEffect } from "./lib/effect.js" + +const model = OpenAIChat.route + .with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") }) + .model({ id: "gpt-4o-mini" }) + +const call = (name: string, input: unknown) => ToolCallPart.make({ id: "call_1", name, input }) + +describe("Standard Schema tool input", () => { + test("sends the schema's own JSON Schema and validates arguments with Zod", async () => { + const weather = tool({ + name: "weather", + description: "Get the weather.", + input: z.object({ city: z.string(), days: z.string().transform(Number) }), + execute: (input) => Effect.succeed(`${input.city}:${input.days + 1}`), + }) + + expect(toDefinitions([weather])[0]?.inputSchema).toEqual({ + type: "object", + properties: { city: { type: "string" }, days: { type: "string" } }, + required: ["city", "days"], + }) + expect( + (await Effect.runPromise(ToolRuntime.dispatch([weather], call("weather", { city: "Paris", days: "2" })))).result, + ).toEqual({ + type: "text", + value: "Paris:3", + }) + expect( + (await Effect.runPromise(ToolRuntime.dispatch([weather], call("weather", { city: 1, days: "2" })))).result, + ).toEqual({ + type: "error", + value: "Invalid tool input: city: Invalid input: expected string, received number", + }) + }) + + test("accepts a Valibot schema with a JSON Schema export in the Promise client", async () => { + const weather = AI.tool({ + name: "weather", + description: "Get the weather.", + input: toStandardJsonSchema(v.object({ city: v.string() })), + execute: (input) => input.city.toUpperCase(), + }) + + expect(toDefinitions([weather])[0]?.inputSchema).toMatchObject({ + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }) + expect( + (await Effect.runPromise(ToolRuntime.dispatch([weather], call("weather", { city: "Paris" })))).result, + ).toEqual({ + type: "text", + value: "PARIS", + }) + }) + + test("fails clearly when the schema has no JSON Schema export", () => { + expect(() => + tool({ name: "weather", description: "Get the weather.", input: v.object({ city: v.string() }) }), + ).toThrow("This valibot schema has no JSON Schema export.") + }) +}) + +describe("Standard Schema structured output", () => { + const scripted = testEffect(TestLLM.testLayer()) + + scripted.effect("validates the answer and sends the schema's JSON Schema", () => + Effect.gen(function* () { + const client = yield* TestLLM.Test + yield* client.push(TestLLM.toolCall("call-1", "generate_object", { city: "Paris" })) + + const events = Array.from( + yield* ai + .stream({ model, prompt: "Pick a city.", schema: z.object({ city: z.string() }) }) + .pipe(Stream.runCollect), + ) + + const finish = events.at(-1) + expect(finish?.type === "finish" && finish.object.city).toBe("Paris") + expect((yield* client.requests())[0]?.tools[0]?.inputSchema).toEqual({ + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }) + }), + ) + + scripted.effect("fails the request when the answer does not match", () => + Effect.gen(function* () { + const client = yield* TestLLM.Test + yield* client.push(TestLLM.toolCall("call-1", "generate_object", { city: 1 })) + + const error = yield* ai + .generate({ model, prompt: "Pick a city.", schema: z.object({ city: z.string() }) }) + .pipe(Effect.flip) + + expect(error.message).toContain("schema decode failed: city: Invalid input: expected string") + }), + ) +}) diff --git a/packages/ai/test/standard-schema.types.ts b/packages/ai/test/standard-schema.types.ts new file mode 100644 index 00000000000..5215fe3c554 --- /dev/null +++ b/packages/ai/test/standard-schema.types.ts @@ -0,0 +1,40 @@ +import { toStandardJsonSchema } from "@valibot/to-json-schema" +import { Effect } from "effect" +import * as v from "valibot" +import { z } from "zod" +import { ai, tool } from "../src/effect.js" +import { AI } from "../src/index.js" +import * as OpenAIChat from "../src/protocols/openai-chat.js" +import { Auth } from "../src/route.js" + +const model = OpenAIChat.route.with({ auth: Auth.bearer("fixture") }).model({ id: "gpt-4o-mini" }) + +tool({ + name: "weather", + description: "Get the weather.", + input: z.object({ days: z.string().transform(Number) }), + execute: (input) => Effect.succeed(input.days satisfies number), +}) + +AI.tool({ + name: "weather", + description: "Get the weather.", + input: toStandardJsonSchema(v.object({ city: v.string() })), + // @ts-expect-error `city` is a string. + execute: (input) => input.city satisfies number, +}) + +AI.tool({ + name: "run_code", + description: "Run JavaScript.", + input: z.object({ code: z.string() }), + freeform: { type: "text" }, +}) + +ai.generate({ model, prompt: "Pick a city.", schema: z.object({ city: z.string() }) }).pipe( + Effect.map((response) => response.object.city satisfies string), +) + +AI.ai + .generate({ model, prompt: "Pick a city.", schema: z.object({ city: z.string() }) }) + .then((response) => response.object.city satisfies string)