diff --git a/packages/core/src/plugin/provider/lmstudio.ts b/packages/core/src/plugin/provider/lmstudio.ts index 1c87eec4487..255672f38d0 100644 --- a/packages/core/src/plugin/provider/lmstudio.ts +++ b/packages/core/src/plugin/provider/lmstudio.ts @@ -1,13 +1,5 @@ -import { define } from "@opencode/plugin/effect/plugin" -import type { Entry } from "@opencode/schema/config" -import { Duration, Effect, Schedule, Schema, Semaphore, Stream } from "effect" -import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" -import { Config } from "../../config.js" -import { Model } from "../../model.js" -import type { PluginInternal } from "../internal.js" -import { foldSettings } from "./configured.js" - -const providerID = "lmstudio" +import { Effect, Schema } from "effect" +import { createLocalProviderPlugin } from "./local.js" const RemoteModel = Schema.Struct({ type: Schema.Literals(["llm", "embedding"]), @@ -27,140 +19,31 @@ const RemoteModel = Schema.Struct({ }) const Response = Schema.Struct({ models: Schema.Array(RemoteModel) }) -const discovery = new Map() -const discoveryLock = Semaphore.makeUnsafe(1) -export function make(origin = "http://127.0.0.1:1234", interval: Duration.Input = "30 seconds") { - return define({ - id: "opencode.provider.lmstudio", - effect: Effect.fn(function* (ctx) { - const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient) - const config = yield* Config.Service - const source = { current: configured(yield* config.entries(), origin) } - const loaded = { models: [] as (typeof RemoteModel.Type)[], hash: "[]" } - - yield* ctx.integration.transform((integrations) => { - if (loaded.models.length === 0) return - integrations.remove(providerID) - }) - - yield* ctx.provider.transform((providers) => { - if (loaded.models.length === 0) return - for (const model of providers.get(providerID)?.models.values() ?? []) { - providers.models.remove(providerID, model.id) - } - providers.update(providerID, (provider) => { - provider.name = "LM Studio" - provider.activation = "enabled" - provider.package = "@opencode/ai/providers/openai-compatible" - provider.settings = { - baseURL: source.current.baseURL, - provider: providerID, - apiKey: source.current.apiKey ?? "", - } - provider.integrationID = undefined - }) - for (const item of loaded.models) { - providers.models.update(providerID, item.key, (model) => { - model.modelID = Model.ID.make(item.key) - model.name = item.display_name || item.key - model.family = item.architecture ? Model.Family.make(item.architecture) : undefined - model.capabilities = { - tools: item.capabilities?.trained_for_tool_use ?? false, - input: ["text", ...(item.capabilities?.vision ? ["image"] : [])], - output: ["text"], - } - const context = - item.loaded_instances.length === 0 - ? item.max_context_length - : Math.min(...item.loaded_instances.map((instance) => instance.config.context_length)) - if (context > 0) model.limit.context = context - }) - } - }) - - const discover = Effect.fn("LMStudioPlugin.discover")(function* () { - const current = source.current - if (!current.endpoint) return undefined - return yield* discoveryLock.withPermit( - Effect.gen(function* () { - const cached = discovery.get(current.endpoint) - if (cached && cached.apiKey === current.apiKey && Date.now() - cached.checked < Duration.toMillis(interval)) - return { source: current, models: cached.models } - discovery.set(current.endpoint, { - checked: Date.now(), - apiKey: current.apiKey, - models: cached && cached.apiKey === current.apiKey ? cached.models : undefined, - }) - const request = current.apiKey - ? HttpClientRequest.get(current.endpoint).pipe( - HttpClientRequest.acceptJson, - HttpClientRequest.bearerToken(current.apiKey), - ) - : HttpClientRequest.get(current.endpoint).pipe(HttpClientRequest.acceptJson) - const response = yield* http - .execute(request) - .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(Response)), Effect.timeout("1 second")) - const models = response.models - .filter((model) => model.type === "llm" && model.key.length > 0) - .toSorted((a, b) => a.key.localeCompare(b.key)) - discovery.set(current.endpoint, { checked: Date.now(), apiKey: current.apiKey, models }) - return { source: current, models } - }), - ) - }) - - const refresh = Effect.fn("LMStudioPlugin.refresh")(function* () { - const result = yield* discover() - if (!result?.models || result.source !== source.current) return - const hash = JSON.stringify(result.models) - if (hash === loaded.hash) return - loaded.models = result.models - loaded.hash = hash - yield* ctx.integration.reload() - yield* ctx.provider.reload() - }) - - // Keep the last successful inventory through transient outages instead of flickering model availability. - yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped) - const reload = Effect.fn("LMStudioPlugin.reload")(function* () { - const next = configured(yield* config.entries(), origin) - if ( - next.baseURL === source.current.baseURL && - next.apiKey === source.current.apiKey && - next.endpoint === source.current.endpoint - ) - return - source.current = next - loaded.models = [] - loaded.hash = "[]" - yield* ctx.integration.reload() - yield* ctx.provider.reload() - yield* refresh().pipe(Effect.ignore) - }) - yield* ctx.event.subscribe().pipe( - Stream.filter((event) => event.type === "config.updated"), - Stream.runForEach(reload), - Effect.forkScoped({ startImmediately: true }), - ) +export const make = createLocalProviderPlugin({ + id: "opencode.provider.lmstudio", + providerID: "lmstudio", + name: "LM Studio", + origin: "http://127.0.0.1:1234", + stripPathSuffix: /\/(?:api\/)?v1$/, + discover: (client) => + Effect.gen(function* () { + const response = yield* client.getJson("/api/v1/models", Response) + return response.models + .filter((model) => model.type === "llm" && model.key.length > 0) + .toSorted((a, b) => a.key.localeCompare(b.key)) }), - } satisfies PluginInternal.InternalPlugin) -} + model: (item) => ({ + id: item.key, + name: item.display_name || item.key, + family: item.architecture, + tools: item.capabilities?.trained_for_tool_use, + vision: item.capabilities?.vision, + context: + item.loaded_instances.length === 0 + ? item.max_context_length + : Math.min(...item.loaded_instances.map((instance) => instance.config.context_length)), + }), +}) export const LMStudioPlugin = make() - -function configured(entries: readonly Entry[], origin: string) { - const settings = foldSettings(entries, providerID, undefined) - const baseURL = ( - typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1` - ).replace(/\/+$/, "") - const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined - if (!URL.canParse(baseURL)) return { baseURL, apiKey } - const url = new URL(baseURL) - if (url.protocol !== "http:" && url.protocol !== "https:") return { baseURL, apiKey } - const prefix = url.pathname.endsWith("/v1") ? url.pathname.slice(0, -3) : url.pathname.replace(/\/+$/, "") - url.pathname = `${prefix}/api/v1/models` - url.search = "" - url.hash = "" - return { baseURL, apiKey, endpoint: url.toString() } -} diff --git a/packages/core/src/plugin/provider/local.ts b/packages/core/src/plugin/provider/local.ts new file mode 100644 index 00000000000..5b77dd8f5d9 --- /dev/null +++ b/packages/core/src/plugin/provider/local.ts @@ -0,0 +1,193 @@ +import { define } from "@opencode/plugin/effect/plugin" +import type { Entry } from "@opencode/schema/config" +import { Duration, Effect, Schedule, type Schema, Semaphore, Stream } from "effect" +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" +import { Config } from "../../config.js" +import { Model } from "../../model.js" +import type { PluginInternal } from "../internal.js" +import { foldSettings } from "./configured.js" + +export type LocalModel = { + readonly id: string + readonly name?: string + readonly family?: string | null + readonly tools?: boolean + readonly vision?: boolean + readonly context?: number | null +} + +export type LocalDiscoveryClient = ReturnType + +export function createLocalProviderPlugin(input: { + readonly id: string + readonly providerID: string + readonly name: string + readonly origin: string + readonly stripPathSuffix?: RegExp + readonly discover: (client: LocalDiscoveryClient) => Effect.Effect + readonly model: (item: NoInfer) => LocalModel +}) { + const discovery = new Map() + const discoveryLock = Semaphore.makeUnsafe(1) + const suffix = input.stripPathSuffix ?? /\/v1$/ + + return function make(origin = input.origin, interval: Duration.Input = "30 seconds") { + return define({ + id: input.id, + effect: Effect.fn(function* (ctx) { + const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient) + const config = yield* Config.Service + const source = { current: configured(yield* config.entries(), input.providerID, origin, suffix) } + const loaded = { models: [] as readonly Discovered[], hash: "[]" } + + yield* ctx.integration.transform((integrations) => { + if (loaded.models.length === 0) return + integrations.remove(input.providerID) + }) + + yield* ctx.provider.transform((providers) => { + if (loaded.models.length === 0) return + for (const model of providers.get(input.providerID)?.models.values() ?? []) { + providers.models.remove(input.providerID, model.id) + } + providers.update(input.providerID, (provider) => { + provider.name = input.name + provider.activation = "enabled" + provider.package = "@opencode/ai/providers/openai-compatible" + provider.settings = { + baseURL: source.current.baseURL, + provider: input.providerID, + apiKey: source.current.apiKey ?? "", + } + provider.integrationID = undefined + }) + for (const item of loaded.models) { + const spec = input.model(item) + providers.models.update(input.providerID, spec.id, (model) => { + model.modelID = Model.ID.make(spec.id) + model.name = spec.name || spec.id + model.family = spec.family ? Model.Family.make(spec.family) : undefined + model.capabilities = { + tools: spec.tools ?? false, + input: ["text", ...(spec.vision ? ["image"] : [])], + output: ["text"], + } + if (typeof spec.context === "number" && spec.context > 0) model.limit.context = spec.context + }) + } + }) + + const discover = Effect.fn("LocalProviderPlugin.discover")(function* () { + const current = source.current + if (!current.endpoint) return undefined + return yield* discoveryLock.withPermit( + Effect.gen(function* () { + const cached = discovery.get(current.root) + if ( + cached && + cached.apiKey === current.apiKey && + Date.now() - cached.checked < Duration.toMillis(interval) + ) + return { source: current, models: cached.models } + discovery.set(current.root, { + checked: Date.now(), + apiKey: current.apiKey, + models: cached && cached.apiKey === current.apiKey ? cached.models : undefined, + }) + const models = yield* input.discover(createClient(http, current.root, current.apiKey, current.endpoint)) + discovery.set(current.root, { checked: Date.now(), apiKey: current.apiKey, models }) + return { source: current, models } + }), + ) + }) + + const refresh = Effect.fn("LocalProviderPlugin.refresh")(function* () { + const result = yield* discover() + if (!result?.models || result.source !== source.current) return + const hash = JSON.stringify(result.models) + if (hash === loaded.hash) return + loaded.models = result.models + loaded.hash = hash + yield* ctx.integration.reload() + yield* ctx.provider.reload() + }) + + // Keep the last successful inventory through transient outages instead of flickering model availability. + yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped) + const reload = Effect.fn("LocalProviderPlugin.reload")(function* () { + const next = configured(yield* config.entries(), input.providerID, origin, suffix) + if ( + next.baseURL === source.current.baseURL && + next.apiKey === source.current.apiKey && + next.root === source.current.root + ) + return + source.current = next + loaded.models = [] + loaded.hash = "[]" + yield* ctx.integration.reload() + yield* ctx.provider.reload() + yield* refresh().pipe(Effect.ignore) + }) + yield* ctx.event.subscribe().pipe( + Stream.filter((event) => event.type === "config.updated"), + Stream.runForEach(reload), + Effect.forkScoped({ startImmediately: true }), + ) + }), + } satisfies PluginInternal.InternalPlugin) + } +} + +function createClient( + http: HttpClient.HttpClient, + root: string, + apiKey: string | undefined, + endpoint: (path: string) => string, +) { + const auth = (request: HttpClientRequest.HttpClientRequest) => + apiKey + ? request.pipe(HttpClientRequest.acceptJson, HttpClientRequest.bearerToken(apiKey)) + : request.pipe(HttpClientRequest.acceptJson) + return { + root, + apiKey, + get: (path: string) => http.execute(auth(HttpClientRequest.get(endpoint(path)))).pipe(Effect.timeout("1 second")), + getJson: (path: string, schema: Schema.Codec) => + http + .execute(auth(HttpClientRequest.get(endpoint(path)))) + .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)), Effect.timeout("1 second")), + postJson: ( + path: string, + bodySchema: Schema.Codec, + body: B, + responseSchema: Schema.Codec, + ) => + auth(HttpClientRequest.post(endpoint(path))).pipe( + HttpClientRequest.schemaBodyJson(bodySchema)(body), + Effect.flatMap(http.execute), + Effect.flatMap(HttpClientResponse.schemaBodyJson(responseSchema)), + Effect.timeout("1 second"), + ), + } +} + +function configured(entries: readonly Entry[], providerID: string, origin: string, suffix: RegExp) { + const settings = foldSettings(entries, providerID, undefined) + const baseURL = ( + typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1` + ).replace(/\/+$/, "") + const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined + if (!URL.canParse(baseURL)) return { baseURL, apiKey } + const url = new URL(baseURL) + if (url.protocol !== "http:" && url.protocol !== "https:") return { baseURL, apiKey } + const prefix = url.pathname.replace(/\/+$/, "").replace(suffix, "") + const endpoint = (path: string) => { + const next = new URL(url) + next.pathname = `${prefix}${path}` + next.search = "" + next.hash = "" + return next.toString() + } + return { baseURL, apiKey, root: endpoint(""), endpoint } +} diff --git a/packages/core/src/plugin/provider/ollama.ts b/packages/core/src/plugin/provider/ollama.ts index 690ec7229de..815edcecd7d 100644 --- a/packages/core/src/plugin/provider/ollama.ts +++ b/packages/core/src/plugin/provider/ollama.ts @@ -1,13 +1,5 @@ -import { define } from "@opencode/plugin/effect/plugin" -import type { Entry } from "@opencode/schema/config" -import { Duration, Effect, Schedule, Schema, Semaphore, Stream } from "effect" -import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" -import { Config } from "../../config.js" -import { Model } from "../../model.js" -import type { PluginInternal } from "../internal.js" -import { foldSettings } from "./configured.js" - -const providerID = "ollama" +import { Effect, Schema } from "effect" +import { createLocalProviderPlugin } from "./local.js" const Details = Schema.Struct({ parent_model: Schema.String.pipe(Schema.optional), @@ -42,183 +34,59 @@ const ShowResponse = Schema.Struct({ }) type DiscoveredModel = typeof RemoteModel.Type & { show: typeof ShowResponse.Type } -type Discovery = { - checked: number +type ShowCache = { apiKey?: string - models?: DiscoveredModel[] - shows: Map + entries: Map } -const discovery = new Map() -const discoveryLock = Semaphore.makeUnsafe(1) +const shows = new Map() -export function make(origin = "http://127.0.0.1:11434", interval: Duration.Input = "30 seconds") { - return define({ - id: "opencode.provider.ollama", - effect: Effect.fn(function* (ctx) { - const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient) - const config = yield* Config.Service - const source = { current: configured(yield* config.entries(), origin) } - const loaded = { models: [] as DiscoveredModel[], hash: "[]" } - - yield* ctx.integration.transform((integrations) => { - if (loaded.models.length === 0) return - integrations.remove(providerID) - }) - - yield* ctx.provider.transform((providers) => { - if (loaded.models.length === 0) return - for (const model of providers.get(providerID)?.models.values() ?? []) { - providers.models.remove(providerID, model.id) - } - providers.update(providerID, (provider) => { - provider.name = "Ollama" - provider.activation = "enabled" - provider.package = "@opencode/ai/providers/openai-compatible" - provider.settings = { - baseURL: source.current.baseURL, - provider: providerID, - apiKey: source.current.apiKey ?? "", - } - provider.integrationID = undefined - }) - for (const item of loaded.models) { - providers.models.update(providerID, item.model, (model) => { - model.modelID = Model.ID.make(item.model) - model.name = item.name || item.model - model.family = item.show.details?.family - ? Model.Family.make(item.show.details.family) - : item.details.family - ? Model.Family.make(item.details.family) - : undefined - model.capabilities = { - tools: item.show.capabilities?.includes("tools") ?? false, - input: ["text", ...(item.show.capabilities?.includes("vision") ? ["image"] : [])], - output: ["text"], - } - const context = Object.entries(item.show.model_info ?? {}).flatMap(([key, value]) => - key.endsWith(".context_length") && typeof value === "number" && value > 0 ? [value] : [], - )[0] - if (context !== undefined) model.limit.context = context - }) - } - }) - - const discover = Effect.fn("OllamaPlugin.discover")(function* () { - const current = source.current - if (!current.tagsEndpoint || !current.showEndpoint) return undefined - return yield* discoveryLock.withPermit( +export const make = createLocalProviderPlugin({ + id: "opencode.provider.ollama", + providerID: "ollama", + name: "Ollama", + origin: "http://127.0.0.1:11434", + stripPathSuffix: /\/(?:v1|api)$/, + discover: (client) => + Effect.gen(function* () { + const cached = shows.get(client.root) + const previous = cached && cached.apiKey === client.apiKey ? cached.entries : new Map() + shows.set(client.root, { apiKey: client.apiKey, entries: previous }) + const response = yield* client.getJson("/api/tags", TagsResponse) + const summaries = response.models + .filter((model) => model.model.length > 0) + .toSorted((a, b) => a.model.localeCompare(b.model)) + const entries = new Map() + const models = yield* Effect.forEach( + summaries, + (model) => Effect.gen(function* () { - const cached = discovery.get(current.tagsEndpoint) - if (cached && cached.apiKey === current.apiKey && Date.now() - cached.checked < Duration.toMillis(interval)) - return { source: current, models: cached.models } - const previous: Discovery = - cached && cached.apiKey === current.apiKey - ? cached - : { checked: 0, apiKey: current.apiKey, shows: new Map() } - discovery.set(current.tagsEndpoint, { ...previous, checked: Date.now(), apiKey: current.apiKey }) - const tagsRequest = current.apiKey - ? HttpClientRequest.get(current.tagsEndpoint).pipe( - HttpClientRequest.acceptJson, - HttpClientRequest.bearerToken(current.apiKey), - ) - : HttpClientRequest.get(current.tagsEndpoint).pipe(HttpClientRequest.acceptJson) - const response = yield* http - .execute(tagsRequest) - .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(TagsResponse)), Effect.timeout("1 second")) - const summaries = response.models - .filter((model) => model.model.length > 0) - .toSorted((a, b) => a.model.localeCompare(b.model)) - const shows = new Map() - const models = yield* Effect.forEach( - summaries, - (model) => - Effect.gen(function* () { - const saved = previous.shows.get(model.model) - const info = - saved?.digest === model.digest - ? saved.info - : yield* HttpClientRequest.post(current.showEndpoint).pipe( - HttpClientRequest.acceptJson, - current.apiKey ? HttpClientRequest.bearerToken(current.apiKey) : (request) => request, - HttpClientRequest.schemaBodyJson(ShowRequest)({ model: model.model }), - Effect.flatMap(http.execute), - Effect.flatMap(HttpClientResponse.schemaBodyJson(ShowResponse)), - Effect.timeout("1 second"), - ) - shows.set(model.model, { digest: model.digest, info }) - return { ...model, show: info } - }).pipe(Effect.orElseSucceed(() => undefined)), - { concurrency: 4 }, - ) - const filtered = models.filter( - (model): model is DiscoveredModel => - model !== undefined && (model.show.capabilities?.includes("completion") ?? false), - ) - discovery.set(current.tagsEndpoint, { - checked: Date.now(), - apiKey: current.apiKey, - models: filtered, - shows, - }) - return { source: current, models: filtered } - }), - ) - }) - - const refresh = Effect.fn("OllamaPlugin.refresh")(function* () { - const result = yield* discover() - if (!result?.models || result.source !== source.current) return - const hash = JSON.stringify(result.models) - if (hash === loaded.hash) return - loaded.models = result.models - loaded.hash = hash - yield* ctx.integration.reload() - yield* ctx.provider.reload() - }) - - // Keep the last successful inventory through transient outages instead of flickering model availability. - yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped) - const reload = Effect.fn("OllamaPlugin.reload")(function* () { - const next = configured(yield* config.entries(), origin) - if ( - next.baseURL === source.current.baseURL && - next.apiKey === source.current.apiKey && - next.tagsEndpoint === source.current.tagsEndpoint - ) - return - source.current = next - loaded.models = [] - loaded.hash = "[]" - yield* ctx.integration.reload() - yield* ctx.provider.reload() - yield* refresh().pipe(Effect.ignore) - }) - yield* ctx.event.subscribe().pipe( - Stream.filter((event) => event.type === "config.updated"), - Stream.runForEach(reload), - Effect.forkScoped({ startImmediately: true }), + const saved = previous.get(model.model) + const info = + saved?.digest === model.digest + ? saved.info + : yield* client.postJson("/api/show", ShowRequest, { model: model.model }, ShowResponse) + entries.set(model.model, { digest: model.digest, info }) + return { ...model, show: info } + }).pipe(Effect.orElseSucceed(() => undefined)), + { concurrency: 4 }, + ) + shows.set(client.root, { apiKey: client.apiKey, entries }) + return models.filter( + (model): model is DiscoveredModel => + model !== undefined && (model.show.capabilities?.includes("completion") ?? false), ) }), - } satisfies PluginInternal.InternalPlugin) -} + model: (item) => ({ + id: item.model, + name: item.name || item.model, + family: item.show.details?.family || item.details.family, + tools: item.show.capabilities?.includes("tools"), + vision: item.show.capabilities?.includes("vision"), + context: Object.entries(item.show.model_info ?? {}).flatMap(([key, value]) => + key.endsWith(".context_length") && typeof value === "number" && value > 0 ? [value] : [], + )[0], + }), +}) export const OllamaPlugin = make() - -function configured(entries: readonly Entry[], origin: string) { - const settings = foldSettings(entries, providerID, undefined) - const baseURL = ( - typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1` - ).replace(/\/+$/, "") - const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined - if (!URL.canParse(baseURL)) return { baseURL, apiKey } - const url = new URL(baseURL) - if (url.protocol !== "http:" && url.protocol !== "https:") return { baseURL, apiKey } - const prefix = url.pathname.endsWith("/v1") ? url.pathname.slice(0, -3) : url.pathname.replace(/\/+$/, "") - url.pathname = `${prefix}/api/tags` - url.search = "" - url.hash = "" - const tagsEndpoint = url.toString() - url.pathname = `${prefix}/api/show` - return { baseURL, apiKey, tagsEndpoint, showEndpoint: url.toString() } -} diff --git a/packages/core/src/plugin/provider/vllm.ts b/packages/core/src/plugin/provider/vllm.ts index e5c83065d8e..75a5ccfd941 100644 --- a/packages/core/src/plugin/provider/vllm.ts +++ b/packages/core/src/plugin/provider/vllm.ts @@ -1,11 +1,5 @@ -import { define } from "@opencode/plugin/effect/plugin" -import type { Entry } from "@opencode/schema/config" -import { Duration, Effect, Schedule, Schema, Semaphore, Stream } from "effect" -import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" -import { Config } from "../../config.js" -import { Model } from "../../model.js" -import type { PluginInternal } from "../internal.js" -import { foldSettings } from "./configured.js" +import { Effect, Schema } from "effect" +import { createLocalProviderPlugin } from "./local.js" const providerID = "vllm" @@ -16,142 +10,25 @@ const RemoteModel = Schema.Struct({ }) const Response = Schema.Struct({ data: Schema.Array(RemoteModel) }) -const discovery = new Map() -const discoveryLock = Semaphore.makeUnsafe(1) -export function make(origin = "http://127.0.0.1:8000", interval: Duration.Input = "30 seconds") { - return define({ - id: "opencode.provider.vllm", - effect: Effect.fn(function* (ctx) { - const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient) - const config = yield* Config.Service - const source = { current: configured(yield* config.entries(), origin) } - const loaded = { models: [] as (typeof RemoteModel.Type)[], hash: "[]" } - - yield* ctx.integration.transform((integrations) => { - if (loaded.models.length === 0) return - integrations.remove(providerID) - }) - - yield* ctx.provider.transform((providers) => { - if (loaded.models.length === 0) return - for (const model of providers.get(providerID)?.models.values() ?? []) { - providers.models.remove(providerID, model.id) - } - providers.update(providerID, (provider) => { - provider.name = "vLLM" - provider.package = "@opencode/ai/providers/openai-compatible" - provider.settings = { - baseURL: source.current.baseURL, - provider: providerID, - apiKey: source.current.apiKey ?? "", - } - provider.integrationID = undefined - provider.activation = "enabled" - }) - for (const item of loaded.models) { - providers.models.update(providerID, item.id, (model) => { - model.modelID = Model.ID.make(item.id) - model.name = item.id - // Tool calling depends on vLLM server flags and parsers that model discovery does not report. - model.capabilities = { tools: false, input: ["text"], output: ["text"] } - if (typeof item.max_model_len === "number" && item.max_model_len > 0) - model.limit.context = item.max_model_len - }) - } - }) - - const discover = Effect.fn("VLLMPlugin.discover")(function* () { - const current = source.current - if (!current.healthEndpoint || !current.modelsEndpoint) return undefined - return yield* discoveryLock.withPermit( - Effect.gen(function* () { - const endpoint = `${current.healthEndpoint}\n${current.modelsEndpoint}` - const cached = discovery.get(endpoint) - if (cached && cached.apiKey === current.apiKey && Date.now() - cached.checked < Duration.toMillis(interval)) - return { source: current, models: cached.models } - discovery.set(endpoint, { - checked: Date.now(), - apiKey: current.apiKey, - models: cached && cached.apiKey === current.apiKey ? cached.models : undefined, - }) - const request = (endpoint: string) => - current.apiKey - ? HttpClientRequest.get(endpoint).pipe( - HttpClientRequest.acceptJson, - HttpClientRequest.bearerToken(current.apiKey), - ) - : HttpClientRequest.get(endpoint).pipe(HttpClientRequest.acceptJson) - yield* http.execute(request(current.healthEndpoint)).pipe(Effect.timeout("1 second")) - const response = yield* http - .execute(request(current.modelsEndpoint)) - .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(Response)), Effect.timeout("1 second")) - const models = response.data - .filter((model) => model.owned_by === providerID && model.id.length > 0) - .toSorted((a, b) => a.id.localeCompare(b.id)) - discovery.set(endpoint, { checked: Date.now(), apiKey: current.apiKey, models }) - return { source: current, models } - }), - ) - }) - - const refresh = Effect.fn("VLLMPlugin.refresh")(function* () { - const result = yield* discover() - if (!result?.models || result.source !== source.current) return - const hash = JSON.stringify(result.models) - if (hash === loaded.hash) return - loaded.models = result.models - loaded.hash = hash - yield* ctx.integration.reload() - yield* ctx.provider.reload() - }) - - // Keep the last successful inventory through transient outages instead of flickering model availability. - yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped) - const reload = Effect.fn("VLLMPlugin.reload")(function* () { - const next = configured(yield* config.entries(), origin) - if ( - next.baseURL === source.current.baseURL && - next.apiKey === source.current.apiKey && - next.healthEndpoint === source.current.healthEndpoint && - next.modelsEndpoint === source.current.modelsEndpoint - ) - return - source.current = next - loaded.models = [] - loaded.hash = "[]" - yield* ctx.integration.reload() - yield* ctx.provider.reload() - yield* refresh().pipe(Effect.ignore) - }) - yield* ctx.event.subscribe().pipe( - Stream.filter((event) => event.type === "config.updated"), - Stream.runForEach(reload), - Effect.forkScoped({ startImmediately: true }), - ) +export const make = createLocalProviderPlugin({ + id: "opencode.provider.vllm", + providerID, + name: "vLLM", + origin: "http://127.0.0.1:8000", + discover: (client) => + Effect.gen(function* () { + yield* client.get("/health") + const response = yield* client.getJson("/v1/models", Response) + return response.data + .filter((model) => model.owned_by === providerID && model.id.length > 0) + .toSorted((a, b) => a.id.localeCompare(b.id)) }), - } satisfies PluginInternal.InternalPlugin) -} + // Tool calling depends on vLLM server flags and parsers that model discovery does not report. + model: (item) => ({ + id: item.id, + context: item.max_model_len, + }), +}) export const VLLMPlugin = make() - -function configured(entries: readonly Entry[], origin: string) { - const settings = foldSettings(entries, providerID, undefined) - const baseURL = ( - typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1` - ).replace(/\/+$/, "") - const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined - if (!URL.canParse(baseURL)) return { baseURL, apiKey } - const models = new URL(baseURL) - if (models.protocol !== "http:" && models.protocol !== "https:") return { baseURL, apiKey } - models.pathname = `${models.pathname.replace(/\/+$/, "")}/models` - models.search = "" - models.hash = "" - const health = new URL(baseURL) - const path = health.pathname.replace(/\/+$/, "") - const prefix = path.endsWith("/v1") ? path.slice(0, -3) : path - health.pathname = `${prefix}/health` - health.search = "" - health.hash = "" - return { baseURL, apiKey, healthEndpoint: health.toString(), modelsEndpoint: models.toString() } -} diff --git a/packages/core/test/plugin/provider-lmstudio.test.ts b/packages/core/test/plugin/provider-lmstudio.test.ts index ec3754c00e4..3a4af6e94f2 100644 --- a/packages/core/test/plugin/provider-lmstudio.test.ts +++ b/packages/core/test/plugin/provider-lmstudio.test.ts @@ -226,6 +226,13 @@ describe("LMStudioPlugin", () => { yield* bus.publish(Event.Updated, {}) yield* eventually(providers.get(providerID), (provider) => provider?.settings?.apiKey === "") expect(requests).toContainEqual({ authorization: null, path: "/proxy/api/v1/models" }) + + requests.splice(0) + const apiBaseURL = `${configured.url.origin}/proxy/api/v1` + yield* config.setEntries([configuration(apiBaseURL, "api-secret")]) + yield* bus.publish(Event.Updated, {}) + yield* eventually(providers.get(providerID), (provider) => provider?.settings?.baseURL === apiBaseURL) + expect(requests).toContainEqual({ authorization: "Bearer api-secret", path: "/proxy/api/v1/models" }) }), ({ initial, configured }) => Effect.promise(() => Promise.all([initial.stop(true), configured.stop(true)])), ), diff --git a/packages/core/test/plugin/provider-ollama.test.ts b/packages/core/test/plugin/provider-ollama.test.ts index 4c445c5b51a..ed4a8ced4db 100644 --- a/packages/core/test/plugin/provider-ollama.test.ts +++ b/packages/core/test/plugin/provider-ollama.test.ts @@ -70,7 +70,7 @@ describe("OllamaPlugin", () => { } : body.model === "unknown-context" ? show({ family: "unknown", capabilities: ["completion"], context: 0 }) - : show({ family: "nomic-bert", capabilities: ["embedding"], context: 8192 }), + : show({ family: "nomic-bert", capabilities: ["embedding"], context: 8192 }), ) }, }), @@ -299,6 +299,22 @@ describe("OllamaPlugin", () => { yield* eventually(providers.get(providerID), (provider) => provider?.settings?.apiKey === "") expect(requests).toContainEqual({ authorization: null, method: "GET", path: "/proxy/api/tags" }) expect(requests).toContainEqual({ authorization: null, method: "POST", path: "/proxy/api/show" }) + + requests.splice(0) + const apiBaseURL = `${configured.url.origin}/proxy/api` + yield* config.setEntries([configuration({ baseURL: apiBaseURL, apiKey: "api-secret" })]) + yield* bus.publish(Event.Updated, {}) + yield* eventually(providers.get(providerID), (provider) => provider?.settings?.baseURL === apiBaseURL) + expect(requests).toContainEqual({ + authorization: "Bearer api-secret", + method: "GET", + path: "/proxy/api/tags", + }) + expect(requests).toContainEqual({ + authorization: "Bearer api-secret", + method: "POST", + path: "/proxy/api/show", + }) }), ({ initial, configured }) => Effect.promise(() => Promise.all([initial.stop(true), configured.stop(true)])), ),