mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
feat(core): run LM Studio decision models (#63200)
This commit is contained in:
committed by
GitHub
Unverified
parent
59ed43f0b8
commit
33432347ce
@@ -99,6 +99,7 @@
|
||||
"./providers/openai": "./src/providers/openai.ts",
|
||||
"./providers/openai-compatible": "./src/providers/openai-compatible.ts",
|
||||
"./providers/openai-compatible-responses": "./src/providers/openai-compatible-responses.ts",
|
||||
"./providers/openai-compatible/decisions": "./src/providers/openai-compatible/decisions.ts",
|
||||
"./providers/openai-compatible/embeddings": "./src/providers/openai-compatible/embeddings.ts",
|
||||
"./providers/openai-compatible/images": "./src/providers/openai-compatible/images.ts",
|
||||
"./providers/openai-compatible/responses": "./src/providers/openai-compatible/responses.ts",
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
import { OpenAICompatibleChat } from "../protocols/openai-compatible-chat.js"
|
||||
import { OpenAIEmbeddings, type OptionsInput as OpenAIEmbeddingOptionsInput } from "../protocols/openai-embeddings.js"
|
||||
import { OpenAIImages } from "../protocols/openai-images.js"
|
||||
import { SystemOne } from "../protocols/system-one.js"
|
||||
import { MediaRoute } from "../route/media.js"
|
||||
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options.js"
|
||||
import type { ProviderPackage } from "../provider-package.js"
|
||||
@@ -50,6 +51,13 @@ export const configure = (input: Config) => {
|
||||
headers: input.headers,
|
||||
http: input.http,
|
||||
})
|
||||
const decisionRoute = SystemOne.route.with({
|
||||
provider,
|
||||
endpoint: { baseURL },
|
||||
auth,
|
||||
headers: input.headers,
|
||||
http: input.http,
|
||||
})
|
||||
return {
|
||||
id: ProviderID.make(provider),
|
||||
model: (modelID: string | ModelID) => route.model<OpenAIProviderOptionsInput>({ id: modelID }),
|
||||
@@ -67,6 +75,9 @@ export const configure = (input: Config) => {
|
||||
}),
|
||||
embedding: (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
|
||||
embeddingRoute.model<OpenAIEmbeddingOptionsInput>({ id: modelID, providerOptions }),
|
||||
// Compatible hosts follow llama.cpp's System One API, which takes inline media in root `images` (alias of `files`).
|
||||
decision: (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
|
||||
decisionRoute.model({ id: modelID, compatibility: { mediaInput: "root" }, providerOptions }),
|
||||
configure,
|
||||
}
|
||||
}
|
||||
@@ -86,6 +97,10 @@ export const embeddingModel = (modelID: string, settings: Settings) => {
|
||||
const input = config(settings)
|
||||
return configure(input).embedding(modelID, input.providerOptions)
|
||||
}
|
||||
export const decisionModel = (modelID: string, settings: Settings) => {
|
||||
const input = config(settings)
|
||||
return configure(input).decision(modelID, input.providerOptions)
|
||||
}
|
||||
|
||||
function config({ provider, ...settings }: Settings) {
|
||||
return { ...toConfig(settings), provider }
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
export { decisionModel as model } from "../openai-compatible.js"
|
||||
export type { Settings } from "../openai-compatible.js"
|
||||
@@ -264,6 +264,21 @@ const decisions: ReadonlyArray<Case<DecisionModel>> = [
|
||||
auth: ["authorization", "Bearer openrouter-key"],
|
||||
provider: "openrouter",
|
||||
},
|
||||
{
|
||||
name: "openai-compatible",
|
||||
load: async () => {
|
||||
const { model } = await import("@opencode/ai/providers/openai-compatible/decisions")
|
||||
return model("clef-flash", {
|
||||
...shared,
|
||||
apiKey: "local-key",
|
||||
baseURL: "http://127.0.0.1:1234/v1",
|
||||
provider: "lmstudio",
|
||||
})
|
||||
},
|
||||
url: "http://127.0.0.1:1234/v1/systemone",
|
||||
auth: ["authorization", "Bearer local-key"],
|
||||
provider: "lmstudio",
|
||||
},
|
||||
{
|
||||
name: "baseten",
|
||||
load: async () => {
|
||||
|
||||
@@ -517,8 +517,20 @@ export const layer = Layer.effect(
|
||||
(model) => model.providerID === ref.providerID && model.id === ref.id,
|
||||
)
|
||||
if (!info) return undefined
|
||||
const { credential, runtimeInfo } = yield* prepare(info, ref.variant)
|
||||
return yield* fromCatalogMedia(runtimeInfo, is, credential, dependencies)
|
||||
const { provider, credential, runtimeInfo } = yield* prepare(info, ref.variant)
|
||||
// Same no-key rule as chat in `load`. A built image or decision model can't have its auth swapped like chat,
|
||||
// so the package gets `apiKey: false` up front and builds the model without auth.
|
||||
const keyless =
|
||||
provider?.activation === "enabled" &&
|
||||
credential === undefined &&
|
||||
!hasConfiguredAuth(runtimeInfo) &&
|
||||
usesAPIKeyAuth(runtimeInfo.package)
|
||||
return yield* fromCatalogMedia(
|
||||
keyless ? { ...runtimeInfo, settings: { ...runtimeInfo.settings, apiKey: false } } : runtimeInfo,
|
||||
is,
|
||||
credential,
|
||||
dependencies,
|
||||
)
|
||||
})
|
||||
const fallback = Effect.fn("ModelResolver.fallback")(function* () {
|
||||
const model = yield* models.default()
|
||||
@@ -576,6 +588,7 @@ const API_KEY_AUTH_PACKAGES: ReadonlySet<string> = new Set([
|
||||
"@opencode/ai/providers/deepseek",
|
||||
"@opencode/ai/providers/fireworks",
|
||||
"@opencode/ai/providers/openai-compatible",
|
||||
"@opencode/ai/providers/openai-compatible/decisions",
|
||||
"@opencode/ai/providers/google",
|
||||
"@opencode/ai/providers/google/interactions",
|
||||
"@opencode/ai/providers/groq",
|
||||
|
||||
@@ -17,6 +17,7 @@ const RemoteModel = Schema.Struct({
|
||||
vision: Schema.Boolean,
|
||||
trained_for_tool_use: Schema.Boolean,
|
||||
}).pipe(Schema.optional),
|
||||
has_vision_adapter: Schema.Boolean.pipe(Schema.optional),
|
||||
})
|
||||
|
||||
const Response = Schema.Struct({ models: Schema.Array(RemoteModel) })
|
||||
@@ -31,15 +32,16 @@ export const make = createLocalProviderPlugin({
|
||||
Effect.gen(function* () {
|
||||
const response = yield* client.getJson("/api/v1/models", Response)
|
||||
return response.models
|
||||
.filter((model) => model.type === "llm" && model.key.length > 0)
|
||||
.filter((model) => (model.type === "llm" || model.type === "decision") && model.key.length > 0)
|
||||
.toSorted((a, b) => a.key.localeCompare(b.key))
|
||||
}),
|
||||
model: (item) => ({
|
||||
id: item.key,
|
||||
name: item.display_name || item.key,
|
||||
family: item.architecture,
|
||||
type: item.type === "decision" ? "decision" : "chat",
|
||||
tools: item.capabilities?.trained_for_tool_use,
|
||||
vision: item.capabilities?.vision,
|
||||
vision: item.capabilities?.vision ?? item.has_vision_adapter,
|
||||
context:
|
||||
item.loaded_instances.length === 0
|
||||
? item.max_context_length
|
||||
|
||||
@@ -18,6 +18,8 @@ export type LocalModel = {
|
||||
readonly tools?: boolean
|
||||
readonly vision?: boolean
|
||||
readonly context?: number | null
|
||||
/** Decision models answer typed questions through the host's System One endpoint (`/v1/systemone`). */
|
||||
readonly type?: "chat" | "decision"
|
||||
}
|
||||
|
||||
export type LocalDiscoveryClient = ReturnType<typeof createClient>
|
||||
@@ -86,6 +88,10 @@ export function createLocalProviderPlugin<Discovered>(input: {
|
||||
output: ["text"],
|
||||
}
|
||||
if (typeof spec.context === "number" && spec.context > 0) model.limit.context = spec.context
|
||||
if (spec.type !== "decision") return
|
||||
model.type = "decision"
|
||||
model.package = "@opencode/ai/providers/openai-compatible/decisions"
|
||||
model.capabilities.output = ["decision"]
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
@@ -131,6 +131,10 @@ const builtins = new Map<string, () => Promise<unknown>>([
|
||||
() => import("@opencode/ai/providers/openai-compatible/responses"),
|
||||
],
|
||||
["@opencode/ai/providers/openai-compatible/images", () => import("@opencode/ai/providers/openai-compatible/images")],
|
||||
[
|
||||
"@opencode/ai/providers/openai-compatible/decisions",
|
||||
() => import("@opencode/ai/providers/openai-compatible/decisions"),
|
||||
],
|
||||
[
|
||||
"@opencode/ai/providers/openai-compatible/embeddings",
|
||||
() => import("@opencode/ai/providers/openai-compatible/embeddings"),
|
||||
|
||||
@@ -301,6 +301,11 @@ describe("image and decision generation", () => {
|
||||
type: "decision",
|
||||
package: "@opencode/ai/providers/openrouter/decisions",
|
||||
}),
|
||||
Info.make({
|
||||
...Info.default(Provider.ID.make("lmstudio"), ID.make("openjev")),
|
||||
type: "decision",
|
||||
package: "@opencode/ai/providers/openai-compatible/decisions",
|
||||
}),
|
||||
].concat(
|
||||
decisionCatalog(),
|
||||
(() => {
|
||||
@@ -318,7 +323,11 @@ describe("image and decision generation", () => {
|
||||
Provider.Info.make({
|
||||
...Provider.Info.empty(providerID),
|
||||
activation: "enabled",
|
||||
settings: { apiKey: `${providerID}-key` },
|
||||
settings:
|
||||
// A local host like LM Studio is enabled with no key.
|
||||
providerID === "lmstudio"
|
||||
? { baseURL: "http://127.0.0.1:1234/v1", provider: "lmstudio", apiKey: "" }
|
||||
: { apiKey: `${providerID}-key` },
|
||||
}),
|
||||
),
|
||||
})
|
||||
@@ -398,6 +407,11 @@ describe("image and decision generation", () => {
|
||||
usage: { input_tokens: 9, output_tokens: 272 },
|
||||
}),
|
||||
)
|
||||
if (request.url === "http://127.0.0.1:1234/v1/systemone")
|
||||
return HttpClientResponse.fromWeb(
|
||||
request,
|
||||
Response.json({ model: "openjev", answers: { photo: { type: "noul", noul: 0.7 } } }),
|
||||
)
|
||||
if (request.url === "https://api.typesafe.ai/v1/systemone")
|
||||
return HttpClientResponse.fromWeb(
|
||||
request,
|
||||
@@ -469,6 +483,30 @@ describe("image and decision generation", () => {
|
||||
}),
|
||||
)
|
||||
|
||||
mediaIt.effect("sends keyless local decisions without auth, with images at the body root", () =>
|
||||
Effect.gen(function* () {
|
||||
requests.length = 0
|
||||
const generate = yield* Generate.Service
|
||||
const result = yield* generate.decisions({
|
||||
model: ref("lmstudio", "openjev"),
|
||||
input: [
|
||||
{ type: "text", text: "A cat" },
|
||||
{ type: "file", data: Buffer.from(PNG).toString("base64"), mime: "image/png" },
|
||||
],
|
||||
questions: [{ type: "predicate", name: "photo", instructions: "Is this a photo?" }],
|
||||
})
|
||||
expect(requests[0]?.url).toBe("http://127.0.0.1:1234/v1/systemone")
|
||||
expect(requests[0]?.headers.authorization).toBeUndefined()
|
||||
expect(requests[0]?.body).toStrictEqual({
|
||||
model: "openjev",
|
||||
state: "A cat",
|
||||
images: [`data:image/png;base64,${Buffer.from(PNG).toString("base64")}`],
|
||||
questions: { photo: { type: "noul", instructions: "Is this a photo?" } },
|
||||
})
|
||||
expect(result.answers).toEqual([{ type: "predicate", name: "photo", probability: 0.7 }])
|
||||
}),
|
||||
)
|
||||
|
||||
mediaIt.effect("answers decision questions and reports the provider's cost", () =>
|
||||
Effect.gen(function* () {
|
||||
requests.length = 0
|
||||
|
||||
@@ -75,6 +75,7 @@ describe("LMStudioPlugin", () => {
|
||||
display_name: "Clef Flash",
|
||||
loaded_instances: [],
|
||||
max_context_length: null,
|
||||
has_vision_adapter: true,
|
||||
},
|
||||
],
|
||||
}),
|
||||
@@ -114,6 +115,16 @@ describe("LMStudioPlugin", () => {
|
||||
})
|
||||
expect(yield* modelState.get(providerID, Model.ID.make("nomic-embed"))).toBeUndefined()
|
||||
expect(yield* modelState.get(providerID, Model.ID.make("clef-flash"))).toBeUndefined()
|
||||
expect(
|
||||
(yield* modelState.available(["decision"])).find((model) => model.providerID === providerID),
|
||||
).toMatchObject({
|
||||
id: "clef-flash",
|
||||
type: "decision",
|
||||
name: "Clef Flash",
|
||||
package: "@opencode/ai/providers/openai-compatible/decisions",
|
||||
capabilities: { tools: false, input: ["text", "image"], output: ["decision"] },
|
||||
limit: { context: 200_000 },
|
||||
})
|
||||
}),
|
||||
(server) => Effect.promise(() => server.stop(true)),
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user