From 33432347ce82bd805f4d6bec5d252119170a92ce Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:06:11 -0600 Subject: [PATCH] feat(core): run LM Studio decision models (#63200) --- packages/ai/package.json | 1 + .../ai/src/providers/openai-compatible.ts | 15 +++++++ .../providers/openai-compatible/decisions.ts | 2 + .../test/provider/media-entrypoints.test.ts | 15 +++++++ packages/core/src/model-resolver.ts | 17 +++++++- packages/core/src/plugin/provider/lmstudio.ts | 6 ++- packages/core/src/plugin/provider/local.ts | 6 +++ packages/core/src/provider.ts | 4 ++ packages/core/test/generate.test.ts | 40 ++++++++++++++++++- .../test/plugin/provider-lmstudio.test.ts | 11 +++++ 10 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 packages/ai/src/providers/openai-compatible/decisions.ts diff --git a/packages/ai/package.json b/packages/ai/package.json index 65b6386dbec..fc81f06bcd7 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -99,6 +99,7 @@ "./providers/openai": "./src/providers/openai.ts", "./providers/openai-compatible": "./src/providers/openai-compatible.ts", "./providers/openai-compatible-responses": "./src/providers/openai-compatible-responses.ts", + "./providers/openai-compatible/decisions": "./src/providers/openai-compatible/decisions.ts", "./providers/openai-compatible/embeddings": "./src/providers/openai-compatible/embeddings.ts", "./providers/openai-compatible/images": "./src/providers/openai-compatible/images.ts", "./providers/openai-compatible/responses": "./src/providers/openai-compatible/responses.ts", diff --git a/packages/ai/src/providers/openai-compatible.ts b/packages/ai/src/providers/openai-compatible.ts index 633630c347e..6f9d6346357 100644 --- a/packages/ai/src/providers/openai-compatible.ts +++ b/packages/ai/src/providers/openai-compatible.ts @@ -8,6 +8,7 @@ import { import { OpenAICompatibleChat } from "../protocols/openai-compatible-chat.js" import { OpenAIEmbeddings, type OptionsInput as OpenAIEmbeddingOptionsInput } from "../protocols/openai-embeddings.js" import { OpenAIImages } from "../protocols/openai-images.js" +import { SystemOne } from "../protocols/system-one.js" import { MediaRoute } from "../route/media.js" import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js" import type { ProviderPackage } from "../provider-package.js" @@ -50,6 +51,13 @@ export const configure = (input: Config) => { headers: input.headers, http: input.http, }) + const decisionRoute = SystemOne.route.with({ + provider, + endpoint: { baseURL }, + auth, + headers: input.headers, + http: input.http, + }) return { id: ProviderID.make(provider), model: (modelID: string | ModelID) => route.model({ id: modelID }), @@ -67,6 +75,9 @@ export const configure = (input: Config) => { }), embedding: (modelID: string | ModelID, providerOptions?: ProviderOptions) => embeddingRoute.model({ id: modelID, providerOptions }), + // Compatible hosts follow llama.cpp's System One API, which takes inline media in root `images` (alias of `files`). + decision: (modelID: string | ModelID, providerOptions?: ProviderOptions) => + decisionRoute.model({ id: modelID, compatibility: { mediaInput: "root" }, providerOptions }), configure, } } @@ -86,6 +97,10 @@ export const embeddingModel = (modelID: string, settings: Settings) => { const input = config(settings) return configure(input).embedding(modelID, input.providerOptions) } +export const decisionModel = (modelID: string, settings: Settings) => { + const input = config(settings) + return configure(input).decision(modelID, input.providerOptions) +} function config({ provider, ...settings }: Settings) { return { ...toConfig(settings), provider } diff --git a/packages/ai/src/providers/openai-compatible/decisions.ts b/packages/ai/src/providers/openai-compatible/decisions.ts new file mode 100644 index 00000000000..8370306e568 --- /dev/null +++ b/packages/ai/src/providers/openai-compatible/decisions.ts @@ -0,0 +1,2 @@ +export { decisionModel as model } from "../openai-compatible.js" +export type { Settings } from "../openai-compatible.js" diff --git a/packages/ai/test/provider/media-entrypoints.test.ts b/packages/ai/test/provider/media-entrypoints.test.ts index 72fa291bcd7..063be80f7e5 100644 --- a/packages/ai/test/provider/media-entrypoints.test.ts +++ b/packages/ai/test/provider/media-entrypoints.test.ts @@ -264,6 +264,21 @@ const decisions: ReadonlyArray> = [ auth: ["authorization", "Bearer openrouter-key"], provider: "openrouter", }, + { + name: "openai-compatible", + load: async () => { + const { model } = await import("@opencode/ai/providers/openai-compatible/decisions") + return model("clef-flash", { + ...shared, + apiKey: "local-key", + baseURL: "http://127.0.0.1:1234/v1", + provider: "lmstudio", + }) + }, + url: "http://127.0.0.1:1234/v1/systemone", + auth: ["authorization", "Bearer local-key"], + provider: "lmstudio", + }, { name: "baseten", load: async () => { diff --git a/packages/core/src/model-resolver.ts b/packages/core/src/model-resolver.ts index 27d78497a14..48a339fe9ec 100644 --- a/packages/core/src/model-resolver.ts +++ b/packages/core/src/model-resolver.ts @@ -517,8 +517,20 @@ export const layer = Layer.effect( (model) => model.providerID === ref.providerID && model.id === ref.id, ) if (!info) return undefined - const { credential, runtimeInfo } = yield* prepare(info, ref.variant) - return yield* fromCatalogMedia(runtimeInfo, is, credential, dependencies) + const { provider, credential, runtimeInfo } = yield* prepare(info, ref.variant) + // Same no-key rule as chat in `load`. A built image or decision model can't have its auth swapped like chat, + // so the package gets `apiKey: false` up front and builds the model without auth. + const keyless = + provider?.activation === "enabled" && + credential === undefined && + !hasConfiguredAuth(runtimeInfo) && + usesAPIKeyAuth(runtimeInfo.package) + return yield* fromCatalogMedia( + keyless ? { ...runtimeInfo, settings: { ...runtimeInfo.settings, apiKey: false } } : runtimeInfo, + is, + credential, + dependencies, + ) }) const fallback = Effect.fn("ModelResolver.fallback")(function* () { const model = yield* models.default() @@ -576,6 +588,7 @@ const API_KEY_AUTH_PACKAGES: ReadonlySet = new Set([ "@opencode/ai/providers/deepseek", "@opencode/ai/providers/fireworks", "@opencode/ai/providers/openai-compatible", + "@opencode/ai/providers/openai-compatible/decisions", "@opencode/ai/providers/google", "@opencode/ai/providers/google/interactions", "@opencode/ai/providers/groq", diff --git a/packages/core/src/plugin/provider/lmstudio.ts b/packages/core/src/plugin/provider/lmstudio.ts index f8dcb81596e..e65943ee4bb 100644 --- a/packages/core/src/plugin/provider/lmstudio.ts +++ b/packages/core/src/plugin/provider/lmstudio.ts @@ -17,6 +17,7 @@ const RemoteModel = Schema.Struct({ vision: Schema.Boolean, trained_for_tool_use: Schema.Boolean, }).pipe(Schema.optional), + has_vision_adapter: Schema.Boolean.pipe(Schema.optional), }) const Response = Schema.Struct({ models: Schema.Array(RemoteModel) }) @@ -31,15 +32,16 @@ export const make = createLocalProviderPlugin({ Effect.gen(function* () { const response = yield* client.getJson("/api/v1/models", Response) return response.models - .filter((model) => model.type === "llm" && model.key.length > 0) + .filter((model) => (model.type === "llm" || model.type === "decision") && model.key.length > 0) .toSorted((a, b) => a.key.localeCompare(b.key)) }), model: (item) => ({ id: item.key, name: item.display_name || item.key, family: item.architecture, + type: item.type === "decision" ? "decision" : "chat", tools: item.capabilities?.trained_for_tool_use, - vision: item.capabilities?.vision, + vision: item.capabilities?.vision ?? item.has_vision_adapter, context: item.loaded_instances.length === 0 ? item.max_context_length diff --git a/packages/core/src/plugin/provider/local.ts b/packages/core/src/plugin/provider/local.ts index 65696cf37c9..cd6a5e6dd5f 100644 --- a/packages/core/src/plugin/provider/local.ts +++ b/packages/core/src/plugin/provider/local.ts @@ -18,6 +18,8 @@ export type LocalModel = { readonly tools?: boolean readonly vision?: boolean readonly context?: number | null + /** Decision models answer typed questions through the host's System One endpoint (`/v1/systemone`). */ + readonly type?: "chat" | "decision" } export type LocalDiscoveryClient = ReturnType @@ -86,6 +88,10 @@ export function createLocalProviderPlugin(input: { output: ["text"], } if (typeof spec.context === "number" && spec.context > 0) model.limit.context = spec.context + if (spec.type !== "decision") return + model.type = "decision" + model.package = "@opencode/ai/providers/openai-compatible/decisions" + model.capabilities.output = ["decision"] }) } }) diff --git a/packages/core/src/provider.ts b/packages/core/src/provider.ts index 312c3d60b1c..431cee79c47 100644 --- a/packages/core/src/provider.ts +++ b/packages/core/src/provider.ts @@ -131,6 +131,10 @@ const builtins = new Map Promise>([ () => import("@opencode/ai/providers/openai-compatible/responses"), ], ["@opencode/ai/providers/openai-compatible/images", () => import("@opencode/ai/providers/openai-compatible/images")], + [ + "@opencode/ai/providers/openai-compatible/decisions", + () => import("@opencode/ai/providers/openai-compatible/decisions"), + ], [ "@opencode/ai/providers/openai-compatible/embeddings", () => import("@opencode/ai/providers/openai-compatible/embeddings"), diff --git a/packages/core/test/generate.test.ts b/packages/core/test/generate.test.ts index 005240b4c66..6b73476558f 100644 --- a/packages/core/test/generate.test.ts +++ b/packages/core/test/generate.test.ts @@ -301,6 +301,11 @@ describe("image and decision generation", () => { type: "decision", package: "@opencode/ai/providers/openrouter/decisions", }), + Info.make({ + ...Info.default(Provider.ID.make("lmstudio"), ID.make("openjev")), + type: "decision", + package: "@opencode/ai/providers/openai-compatible/decisions", + }), ].concat( decisionCatalog(), (() => { @@ -318,7 +323,11 @@ describe("image and decision generation", () => { Provider.Info.make({ ...Provider.Info.empty(providerID), activation: "enabled", - settings: { apiKey: `${providerID}-key` }, + settings: + // A local host like LM Studio is enabled with no key. + providerID === "lmstudio" + ? { baseURL: "http://127.0.0.1:1234/v1", provider: "lmstudio", apiKey: "" } + : { apiKey: `${providerID}-key` }, }), ), }) @@ -398,6 +407,11 @@ describe("image and decision generation", () => { usage: { input_tokens: 9, output_tokens: 272 }, }), ) + if (request.url === "http://127.0.0.1:1234/v1/systemone") + return HttpClientResponse.fromWeb( + request, + Response.json({ model: "openjev", answers: { photo: { type: "noul", noul: 0.7 } } }), + ) if (request.url === "https://api.typesafe.ai/v1/systemone") return HttpClientResponse.fromWeb( request, @@ -469,6 +483,30 @@ describe("image and decision generation", () => { }), ) + mediaIt.effect("sends keyless local decisions without auth, with images at the body root", () => + Effect.gen(function* () { + requests.length = 0 + const generate = yield* Generate.Service + const result = yield* generate.decisions({ + model: ref("lmstudio", "openjev"), + input: [ + { type: "text", text: "A cat" }, + { type: "file", data: Buffer.from(PNG).toString("base64"), mime: "image/png" }, + ], + questions: [{ type: "predicate", name: "photo", instructions: "Is this a photo?" }], + }) + expect(requests[0]?.url).toBe("http://127.0.0.1:1234/v1/systemone") + expect(requests[0]?.headers.authorization).toBeUndefined() + expect(requests[0]?.body).toStrictEqual({ + model: "openjev", + state: "A cat", + images: [`data:image/png;base64,${Buffer.from(PNG).toString("base64")}`], + questions: { photo: { type: "noul", instructions: "Is this a photo?" } }, + }) + expect(result.answers).toEqual([{ type: "predicate", name: "photo", probability: 0.7 }]) + }), + ) + mediaIt.effect("answers decision questions and reports the provider's cost", () => Effect.gen(function* () { requests.length = 0 diff --git a/packages/core/test/plugin/provider-lmstudio.test.ts b/packages/core/test/plugin/provider-lmstudio.test.ts index b3219bc4f1f..9c33cf8795f 100644 --- a/packages/core/test/plugin/provider-lmstudio.test.ts +++ b/packages/core/test/plugin/provider-lmstudio.test.ts @@ -75,6 +75,7 @@ describe("LMStudioPlugin", () => { display_name: "Clef Flash", loaded_instances: [], max_context_length: null, + has_vision_adapter: true, }, ], }), @@ -114,6 +115,16 @@ describe("LMStudioPlugin", () => { }) expect(yield* modelState.get(providerID, Model.ID.make("nomic-embed"))).toBeUndefined() expect(yield* modelState.get(providerID, Model.ID.make("clef-flash"))).toBeUndefined() + expect( + (yield* modelState.available(["decision"])).find((model) => model.providerID === providerID), + ).toMatchObject({ + id: "clef-flash", + type: "decision", + name: "Clef Flash", + package: "@opencode/ai/providers/openai-compatible/decisions", + capabilities: { tools: false, input: ["text", "image"], output: ["decision"] }, + limit: { context: 200_000 }, + }) }), (server) => Effect.promise(() => server.stop(true)), ),