refactor(core): share local provider discovery plugin across vllm, lmstudio, and ollama (#53438)

This commit is contained in:
Kit Langton
2026-10-05 15:00:04 -07:00
committed by GitHub
Unverified
parent aa7c1f3ebd
commit 39105835c6
6 changed files with 312 additions and 468 deletions
+26 -143
View File
@@ -1,13 +1,5 @@
import { define } from "@opencode/plugin/effect/plugin"
import type { Entry } from "@opencode/schema/config"
import { Duration, Effect, Schedule, Schema, Semaphore, Stream } from "effect"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import { Config } from "../../config.js"
import { Model } from "../../model.js"
import type { PluginInternal } from "../internal.js"
import { foldSettings } from "./configured.js"
const providerID = "lmstudio"
import { Effect, Schema } from "effect"
import { createLocalProviderPlugin } from "./local.js"
const RemoteModel = Schema.Struct({
type: Schema.Literals(["llm", "embedding"]),
@@ -27,140 +19,31 @@ const RemoteModel = Schema.Struct({
})
const Response = Schema.Struct({ models: Schema.Array(RemoteModel) })
const discovery = new Map<string, { checked: number; apiKey?: string; models?: (typeof RemoteModel.Type)[] }>()
const discoveryLock = Semaphore.makeUnsafe(1)
export function make(origin = "http://127.0.0.1:1234", interval: Duration.Input = "30 seconds") {
return define({
id: "opencode.provider.lmstudio",
effect: Effect.fn(function* (ctx) {
const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient)
const config = yield* Config.Service
const source = { current: configured(yield* config.entries(), origin) }
const loaded = { models: [] as (typeof RemoteModel.Type)[], hash: "[]" }
yield* ctx.integration.transform((integrations) => {
if (loaded.models.length === 0) return
integrations.remove(providerID)
})
yield* ctx.provider.transform((providers) => {
if (loaded.models.length === 0) return
for (const model of providers.get(providerID)?.models.values() ?? []) {
providers.models.remove(providerID, model.id)
}
providers.update(providerID, (provider) => {
provider.name = "LM Studio"
provider.activation = "enabled"
provider.package = "@opencode/ai/providers/openai-compatible"
provider.settings = {
baseURL: source.current.baseURL,
provider: providerID,
apiKey: source.current.apiKey ?? "",
}
provider.integrationID = undefined
})
for (const item of loaded.models) {
providers.models.update(providerID, item.key, (model) => {
model.modelID = Model.ID.make(item.key)
model.name = item.display_name || item.key
model.family = item.architecture ? Model.Family.make(item.architecture) : undefined
model.capabilities = {
tools: item.capabilities?.trained_for_tool_use ?? false,
input: ["text", ...(item.capabilities?.vision ? ["image"] : [])],
output: ["text"],
}
const context =
item.loaded_instances.length === 0
? item.max_context_length
: Math.min(...item.loaded_instances.map((instance) => instance.config.context_length))
if (context > 0) model.limit.context = context
})
}
})
const discover = Effect.fn("LMStudioPlugin.discover")(function* () {
const current = source.current
if (!current.endpoint) return undefined
return yield* discoveryLock.withPermit(
Effect.gen(function* () {
const cached = discovery.get(current.endpoint)
if (cached && cached.apiKey === current.apiKey && Date.now() - cached.checked < Duration.toMillis(interval))
return { source: current, models: cached.models }
discovery.set(current.endpoint, {
checked: Date.now(),
apiKey: current.apiKey,
models: cached && cached.apiKey === current.apiKey ? cached.models : undefined,
})
const request = current.apiKey
? HttpClientRequest.get(current.endpoint).pipe(
HttpClientRequest.acceptJson,
HttpClientRequest.bearerToken(current.apiKey),
)
: HttpClientRequest.get(current.endpoint).pipe(HttpClientRequest.acceptJson)
const response = yield* http
.execute(request)
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(Response)), Effect.timeout("1 second"))
const models = response.models
.filter((model) => model.type === "llm" && model.key.length > 0)
.toSorted((a, b) => a.key.localeCompare(b.key))
discovery.set(current.endpoint, { checked: Date.now(), apiKey: current.apiKey, models })
return { source: current, models }
}),
)
})
const refresh = Effect.fn("LMStudioPlugin.refresh")(function* () {
const result = yield* discover()
if (!result?.models || result.source !== source.current) return
const hash = JSON.stringify(result.models)
if (hash === loaded.hash) return
loaded.models = result.models
loaded.hash = hash
yield* ctx.integration.reload()
yield* ctx.provider.reload()
})
// Keep the last successful inventory through transient outages instead of flickering model availability.
yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped)
const reload = Effect.fn("LMStudioPlugin.reload")(function* () {
const next = configured(yield* config.entries(), origin)
if (
next.baseURL === source.current.baseURL &&
next.apiKey === source.current.apiKey &&
next.endpoint === source.current.endpoint
)
return
source.current = next
loaded.models = []
loaded.hash = "[]"
yield* ctx.integration.reload()
yield* ctx.provider.reload()
yield* refresh().pipe(Effect.ignore)
})
yield* ctx.event.subscribe().pipe(
Stream.filter((event) => event.type === "config.updated"),
Stream.runForEach(reload),
Effect.forkScoped({ startImmediately: true }),
)
export const make = createLocalProviderPlugin({
id: "opencode.provider.lmstudio",
providerID: "lmstudio",
name: "LM Studio",
origin: "http://127.0.0.1:1234",
stripPathSuffix: /\/(?:api\/)?v1$/,
discover: (client) =>
Effect.gen(function* () {
const response = yield* client.getJson("/api/v1/models", Response)
return response.models
.filter((model) => model.type === "llm" && model.key.length > 0)
.toSorted((a, b) => a.key.localeCompare(b.key))
}),
} satisfies PluginInternal.InternalPlugin)
}
model: (item) => ({
id: item.key,
name: item.display_name || item.key,
family: item.architecture,
tools: item.capabilities?.trained_for_tool_use,
vision: item.capabilities?.vision,
context:
item.loaded_instances.length === 0
? item.max_context_length
: Math.min(...item.loaded_instances.map((instance) => instance.config.context_length)),
}),
})
export const LMStudioPlugin = make()
function configured(entries: readonly Entry[], origin: string) {
const settings = foldSettings(entries, providerID, undefined)
const baseURL = (
typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1`
).replace(/\/+$/, "")
const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined
if (!URL.canParse(baseURL)) return { baseURL, apiKey }
const url = new URL(baseURL)
if (url.protocol !== "http:" && url.protocol !== "https:") return { baseURL, apiKey }
const prefix = url.pathname.endsWith("/v1") ? url.pathname.slice(0, -3) : url.pathname.replace(/\/+$/, "")
url.pathname = `${prefix}/api/v1/models`
url.search = ""
url.hash = ""
return { baseURL, apiKey, endpoint: url.toString() }
}
+193
View File
@@ -0,0 +1,193 @@
import { define } from "@opencode/plugin/effect/plugin"
import type { Entry } from "@opencode/schema/config"
import { Duration, Effect, Schedule, type Schema, Semaphore, Stream } from "effect"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import { Config } from "../../config.js"
import { Model } from "../../model.js"
import type { PluginInternal } from "../internal.js"
import { foldSettings } from "./configured.js"
export type LocalModel = {
readonly id: string
readonly name?: string
readonly family?: string | null
readonly tools?: boolean
readonly vision?: boolean
readonly context?: number | null
}
export type LocalDiscoveryClient = ReturnType<typeof createClient>
export function createLocalProviderPlugin<Discovered>(input: {
readonly id: string
readonly providerID: string
readonly name: string
readonly origin: string
readonly stripPathSuffix?: RegExp
readonly discover: (client: LocalDiscoveryClient) => Effect.Effect<readonly Discovered[], unknown>
readonly model: (item: NoInfer<Discovered>) => LocalModel
}) {
const discovery = new Map<string, { checked: number; apiKey?: string; models?: readonly Discovered[] }>()
const discoveryLock = Semaphore.makeUnsafe(1)
const suffix = input.stripPathSuffix ?? /\/v1$/
return function make(origin = input.origin, interval: Duration.Input = "30 seconds") {
return define({
id: input.id,
effect: Effect.fn(function* (ctx) {
const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient)
const config = yield* Config.Service
const source = { current: configured(yield* config.entries(), input.providerID, origin, suffix) }
const loaded = { models: [] as readonly Discovered[], hash: "[]" }
yield* ctx.integration.transform((integrations) => {
if (loaded.models.length === 0) return
integrations.remove(input.providerID)
})
yield* ctx.provider.transform((providers) => {
if (loaded.models.length === 0) return
for (const model of providers.get(input.providerID)?.models.values() ?? []) {
providers.models.remove(input.providerID, model.id)
}
providers.update(input.providerID, (provider) => {
provider.name = input.name
provider.activation = "enabled"
provider.package = "@opencode/ai/providers/openai-compatible"
provider.settings = {
baseURL: source.current.baseURL,
provider: input.providerID,
apiKey: source.current.apiKey ?? "",
}
provider.integrationID = undefined
})
for (const item of loaded.models) {
const spec = input.model(item)
providers.models.update(input.providerID, spec.id, (model) => {
model.modelID = Model.ID.make(spec.id)
model.name = spec.name || spec.id
model.family = spec.family ? Model.Family.make(spec.family) : undefined
model.capabilities = {
tools: spec.tools ?? false,
input: ["text", ...(spec.vision ? ["image"] : [])],
output: ["text"],
}
if (typeof spec.context === "number" && spec.context > 0) model.limit.context = spec.context
})
}
})
const discover = Effect.fn("LocalProviderPlugin.discover")(function* () {
const current = source.current
if (!current.endpoint) return undefined
return yield* discoveryLock.withPermit(
Effect.gen(function* () {
const cached = discovery.get(current.root)
if (
cached &&
cached.apiKey === current.apiKey &&
Date.now() - cached.checked < Duration.toMillis(interval)
)
return { source: current, models: cached.models }
discovery.set(current.root, {
checked: Date.now(),
apiKey: current.apiKey,
models: cached && cached.apiKey === current.apiKey ? cached.models : undefined,
})
const models = yield* input.discover(createClient(http, current.root, current.apiKey, current.endpoint))
discovery.set(current.root, { checked: Date.now(), apiKey: current.apiKey, models })
return { source: current, models }
}),
)
})
const refresh = Effect.fn("LocalProviderPlugin.refresh")(function* () {
const result = yield* discover()
if (!result?.models || result.source !== source.current) return
const hash = JSON.stringify(result.models)
if (hash === loaded.hash) return
loaded.models = result.models
loaded.hash = hash
yield* ctx.integration.reload()
yield* ctx.provider.reload()
})
// Keep the last successful inventory through transient outages instead of flickering model availability.
yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped)
const reload = Effect.fn("LocalProviderPlugin.reload")(function* () {
const next = configured(yield* config.entries(), input.providerID, origin, suffix)
if (
next.baseURL === source.current.baseURL &&
next.apiKey === source.current.apiKey &&
next.root === source.current.root
)
return
source.current = next
loaded.models = []
loaded.hash = "[]"
yield* ctx.integration.reload()
yield* ctx.provider.reload()
yield* refresh().pipe(Effect.ignore)
})
yield* ctx.event.subscribe().pipe(
Stream.filter((event) => event.type === "config.updated"),
Stream.runForEach(reload),
Effect.forkScoped({ startImmediately: true }),
)
}),
} satisfies PluginInternal.InternalPlugin)
}
}
function createClient(
http: HttpClient.HttpClient,
root: string,
apiKey: string | undefined,
endpoint: (path: string) => string,
) {
const auth = (request: HttpClientRequest.HttpClientRequest) =>
apiKey
? request.pipe(HttpClientRequest.acceptJson, HttpClientRequest.bearerToken(apiKey))
: request.pipe(HttpClientRequest.acceptJson)
return {
root,
apiKey,
get: (path: string) => http.execute(auth(HttpClientRequest.get(endpoint(path)))).pipe(Effect.timeout("1 second")),
getJson: <A, I>(path: string, schema: Schema.Codec<A, I>) =>
http
.execute(auth(HttpClientRequest.get(endpoint(path))))
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)), Effect.timeout("1 second")),
postJson: <B, BI, A, AI>(
path: string,
bodySchema: Schema.Codec<B, BI>,
body: B,
responseSchema: Schema.Codec<A, AI>,
) =>
auth(HttpClientRequest.post(endpoint(path))).pipe(
HttpClientRequest.schemaBodyJson(bodySchema)(body),
Effect.flatMap(http.execute),
Effect.flatMap(HttpClientResponse.schemaBodyJson(responseSchema)),
Effect.timeout("1 second"),
),
}
}
function configured(entries: readonly Entry[], providerID: string, origin: string, suffix: RegExp) {
const settings = foldSettings(entries, providerID, undefined)
const baseURL = (
typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1`
).replace(/\/+$/, "")
const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined
if (!URL.canParse(baseURL)) return { baseURL, apiKey }
const url = new URL(baseURL)
if (url.protocol !== "http:" && url.protocol !== "https:") return { baseURL, apiKey }
const prefix = url.pathname.replace(/\/+$/, "").replace(suffix, "")
const endpoint = (path: string) => {
const next = new URL(url)
next.pathname = `${prefix}${path}`
next.search = ""
next.hash = ""
return next.toString()
}
return { baseURL, apiKey, root: endpoint(""), endpoint }
}
+49 -181
View File
@@ -1,13 +1,5 @@
import { define } from "@opencode/plugin/effect/plugin"
import type { Entry } from "@opencode/schema/config"
import { Duration, Effect, Schedule, Schema, Semaphore, Stream } from "effect"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import { Config } from "../../config.js"
import { Model } from "../../model.js"
import type { PluginInternal } from "../internal.js"
import { foldSettings } from "./configured.js"
const providerID = "ollama"
import { Effect, Schema } from "effect"
import { createLocalProviderPlugin } from "./local.js"
const Details = Schema.Struct({
parent_model: Schema.String.pipe(Schema.optional),
@@ -42,183 +34,59 @@ const ShowResponse = Schema.Struct({
})
type DiscoveredModel = typeof RemoteModel.Type & { show: typeof ShowResponse.Type }
type Discovery = {
checked: number
type ShowCache = {
apiKey?: string
models?: DiscoveredModel[]
shows: Map<string, { digest: string; info: typeof ShowResponse.Type }>
entries: Map<string, { digest: string; info: typeof ShowResponse.Type }>
}
const discovery = new Map<string, Discovery>()
const discoveryLock = Semaphore.makeUnsafe(1)
const shows = new Map<string, ShowCache>()
export function make(origin = "http://127.0.0.1:11434", interval: Duration.Input = "30 seconds") {
return define({
id: "opencode.provider.ollama",
effect: Effect.fn(function* (ctx) {
const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient)
const config = yield* Config.Service
const source = { current: configured(yield* config.entries(), origin) }
const loaded = { models: [] as DiscoveredModel[], hash: "[]" }
yield* ctx.integration.transform((integrations) => {
if (loaded.models.length === 0) return
integrations.remove(providerID)
})
yield* ctx.provider.transform((providers) => {
if (loaded.models.length === 0) return
for (const model of providers.get(providerID)?.models.values() ?? []) {
providers.models.remove(providerID, model.id)
}
providers.update(providerID, (provider) => {
provider.name = "Ollama"
provider.activation = "enabled"
provider.package = "@opencode/ai/providers/openai-compatible"
provider.settings = {
baseURL: source.current.baseURL,
provider: providerID,
apiKey: source.current.apiKey ?? "",
}
provider.integrationID = undefined
})
for (const item of loaded.models) {
providers.models.update(providerID, item.model, (model) => {
model.modelID = Model.ID.make(item.model)
model.name = item.name || item.model
model.family = item.show.details?.family
? Model.Family.make(item.show.details.family)
: item.details.family
? Model.Family.make(item.details.family)
: undefined
model.capabilities = {
tools: item.show.capabilities?.includes("tools") ?? false,
input: ["text", ...(item.show.capabilities?.includes("vision") ? ["image"] : [])],
output: ["text"],
}
const context = Object.entries(item.show.model_info ?? {}).flatMap(([key, value]) =>
key.endsWith(".context_length") && typeof value === "number" && value > 0 ? [value] : [],
)[0]
if (context !== undefined) model.limit.context = context
})
}
})
const discover = Effect.fn("OllamaPlugin.discover")(function* () {
const current = source.current
if (!current.tagsEndpoint || !current.showEndpoint) return undefined
return yield* discoveryLock.withPermit(
export const make = createLocalProviderPlugin({
id: "opencode.provider.ollama",
providerID: "ollama",
name: "Ollama",
origin: "http://127.0.0.1:11434",
stripPathSuffix: /\/(?:v1|api)$/,
discover: (client) =>
Effect.gen(function* () {
const cached = shows.get(client.root)
const previous = cached && cached.apiKey === client.apiKey ? cached.entries : new Map()
shows.set(client.root, { apiKey: client.apiKey, entries: previous })
const response = yield* client.getJson("/api/tags", TagsResponse)
const summaries = response.models
.filter((model) => model.model.length > 0)
.toSorted((a, b) => a.model.localeCompare(b.model))
const entries = new Map<string, { digest: string; info: typeof ShowResponse.Type }>()
const models = yield* Effect.forEach(
summaries,
(model) =>
Effect.gen(function* () {
const cached = discovery.get(current.tagsEndpoint)
if (cached && cached.apiKey === current.apiKey && Date.now() - cached.checked < Duration.toMillis(interval))
return { source: current, models: cached.models }
const previous: Discovery =
cached && cached.apiKey === current.apiKey
? cached
: { checked: 0, apiKey: current.apiKey, shows: new Map() }
discovery.set(current.tagsEndpoint, { ...previous, checked: Date.now(), apiKey: current.apiKey })
const tagsRequest = current.apiKey
? HttpClientRequest.get(current.tagsEndpoint).pipe(
HttpClientRequest.acceptJson,
HttpClientRequest.bearerToken(current.apiKey),
)
: HttpClientRequest.get(current.tagsEndpoint).pipe(HttpClientRequest.acceptJson)
const response = yield* http
.execute(tagsRequest)
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(TagsResponse)), Effect.timeout("1 second"))
const summaries = response.models
.filter((model) => model.model.length > 0)
.toSorted((a, b) => a.model.localeCompare(b.model))
const shows = new Map<string, { digest: string; info: typeof ShowResponse.Type }>()
const models = yield* Effect.forEach(
summaries,
(model) =>
Effect.gen(function* () {
const saved = previous.shows.get(model.model)
const info =
saved?.digest === model.digest
? saved.info
: yield* HttpClientRequest.post(current.showEndpoint).pipe(
HttpClientRequest.acceptJson,
current.apiKey ? HttpClientRequest.bearerToken(current.apiKey) : (request) => request,
HttpClientRequest.schemaBodyJson(ShowRequest)({ model: model.model }),
Effect.flatMap(http.execute),
Effect.flatMap(HttpClientResponse.schemaBodyJson(ShowResponse)),
Effect.timeout("1 second"),
)
shows.set(model.model, { digest: model.digest, info })
return { ...model, show: info }
}).pipe(Effect.orElseSucceed(() => undefined)),
{ concurrency: 4 },
)
const filtered = models.filter(
(model): model is DiscoveredModel =>
model !== undefined && (model.show.capabilities?.includes("completion") ?? false),
)
discovery.set(current.tagsEndpoint, {
checked: Date.now(),
apiKey: current.apiKey,
models: filtered,
shows,
})
return { source: current, models: filtered }
}),
)
})
const refresh = Effect.fn("OllamaPlugin.refresh")(function* () {
const result = yield* discover()
if (!result?.models || result.source !== source.current) return
const hash = JSON.stringify(result.models)
if (hash === loaded.hash) return
loaded.models = result.models
loaded.hash = hash
yield* ctx.integration.reload()
yield* ctx.provider.reload()
})
// Keep the last successful inventory through transient outages instead of flickering model availability.
yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped)
const reload = Effect.fn("OllamaPlugin.reload")(function* () {
const next = configured(yield* config.entries(), origin)
if (
next.baseURL === source.current.baseURL &&
next.apiKey === source.current.apiKey &&
next.tagsEndpoint === source.current.tagsEndpoint
)
return
source.current = next
loaded.models = []
loaded.hash = "[]"
yield* ctx.integration.reload()
yield* ctx.provider.reload()
yield* refresh().pipe(Effect.ignore)
})
yield* ctx.event.subscribe().pipe(
Stream.filter((event) => event.type === "config.updated"),
Stream.runForEach(reload),
Effect.forkScoped({ startImmediately: true }),
const saved = previous.get(model.model)
const info =
saved?.digest === model.digest
? saved.info
: yield* client.postJson("/api/show", ShowRequest, { model: model.model }, ShowResponse)
entries.set(model.model, { digest: model.digest, info })
return { ...model, show: info }
}).pipe(Effect.orElseSucceed(() => undefined)),
{ concurrency: 4 },
)
shows.set(client.root, { apiKey: client.apiKey, entries })
return models.filter(
(model): model is DiscoveredModel =>
model !== undefined && (model.show.capabilities?.includes("completion") ?? false),
)
}),
} satisfies PluginInternal.InternalPlugin)
}
model: (item) => ({
id: item.model,
name: item.name || item.model,
family: item.show.details?.family || item.details.family,
tools: item.show.capabilities?.includes("tools"),
vision: item.show.capabilities?.includes("vision"),
context: Object.entries(item.show.model_info ?? {}).flatMap(([key, value]) =>
key.endsWith(".context_length") && typeof value === "number" && value > 0 ? [value] : [],
)[0],
}),
})
export const OllamaPlugin = make()
function configured(entries: readonly Entry[], origin: string) {
const settings = foldSettings(entries, providerID, undefined)
const baseURL = (
typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1`
).replace(/\/+$/, "")
const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined
if (!URL.canParse(baseURL)) return { baseURL, apiKey }
const url = new URL(baseURL)
if (url.protocol !== "http:" && url.protocol !== "https:") return { baseURL, apiKey }
const prefix = url.pathname.endsWith("/v1") ? url.pathname.slice(0, -3) : url.pathname.replace(/\/+$/, "")
url.pathname = `${prefix}/api/tags`
url.search = ""
url.hash = ""
const tagsEndpoint = url.toString()
url.pathname = `${prefix}/api/show`
return { baseURL, apiKey, tagsEndpoint, showEndpoint: url.toString() }
}
+20 -143
View File
@@ -1,11 +1,5 @@
import { define } from "@opencode/plugin/effect/plugin"
import type { Entry } from "@opencode/schema/config"
import { Duration, Effect, Schedule, Schema, Semaphore, Stream } from "effect"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import { Config } from "../../config.js"
import { Model } from "../../model.js"
import type { PluginInternal } from "../internal.js"
import { foldSettings } from "./configured.js"
import { Effect, Schema } from "effect"
import { createLocalProviderPlugin } from "./local.js"
const providerID = "vllm"
@@ -16,142 +10,25 @@ const RemoteModel = Schema.Struct({
})
const Response = Schema.Struct({ data: Schema.Array(RemoteModel) })
const discovery = new Map<string, { checked: number; apiKey?: string; models?: (typeof RemoteModel.Type)[] }>()
const discoveryLock = Semaphore.makeUnsafe(1)
export function make(origin = "http://127.0.0.1:8000", interval: Duration.Input = "30 seconds") {
return define({
id: "opencode.provider.vllm",
effect: Effect.fn(function* (ctx) {
const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient)
const config = yield* Config.Service
const source = { current: configured(yield* config.entries(), origin) }
const loaded = { models: [] as (typeof RemoteModel.Type)[], hash: "[]" }
yield* ctx.integration.transform((integrations) => {
if (loaded.models.length === 0) return
integrations.remove(providerID)
})
yield* ctx.provider.transform((providers) => {
if (loaded.models.length === 0) return
for (const model of providers.get(providerID)?.models.values() ?? []) {
providers.models.remove(providerID, model.id)
}
providers.update(providerID, (provider) => {
provider.name = "vLLM"
provider.package = "@opencode/ai/providers/openai-compatible"
provider.settings = {
baseURL: source.current.baseURL,
provider: providerID,
apiKey: source.current.apiKey ?? "",
}
provider.integrationID = undefined
provider.activation = "enabled"
})
for (const item of loaded.models) {
providers.models.update(providerID, item.id, (model) => {
model.modelID = Model.ID.make(item.id)
model.name = item.id
// Tool calling depends on vLLM server flags and parsers that model discovery does not report.
model.capabilities = { tools: false, input: ["text"], output: ["text"] }
if (typeof item.max_model_len === "number" && item.max_model_len > 0)
model.limit.context = item.max_model_len
})
}
})
const discover = Effect.fn("VLLMPlugin.discover")(function* () {
const current = source.current
if (!current.healthEndpoint || !current.modelsEndpoint) return undefined
return yield* discoveryLock.withPermit(
Effect.gen(function* () {
const endpoint = `${current.healthEndpoint}\n${current.modelsEndpoint}`
const cached = discovery.get(endpoint)
if (cached && cached.apiKey === current.apiKey && Date.now() - cached.checked < Duration.toMillis(interval))
return { source: current, models: cached.models }
discovery.set(endpoint, {
checked: Date.now(),
apiKey: current.apiKey,
models: cached && cached.apiKey === current.apiKey ? cached.models : undefined,
})
const request = (endpoint: string) =>
current.apiKey
? HttpClientRequest.get(endpoint).pipe(
HttpClientRequest.acceptJson,
HttpClientRequest.bearerToken(current.apiKey),
)
: HttpClientRequest.get(endpoint).pipe(HttpClientRequest.acceptJson)
yield* http.execute(request(current.healthEndpoint)).pipe(Effect.timeout("1 second"))
const response = yield* http
.execute(request(current.modelsEndpoint))
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(Response)), Effect.timeout("1 second"))
const models = response.data
.filter((model) => model.owned_by === providerID && model.id.length > 0)
.toSorted((a, b) => a.id.localeCompare(b.id))
discovery.set(endpoint, { checked: Date.now(), apiKey: current.apiKey, models })
return { source: current, models }
}),
)
})
const refresh = Effect.fn("VLLMPlugin.refresh")(function* () {
const result = yield* discover()
if (!result?.models || result.source !== source.current) return
const hash = JSON.stringify(result.models)
if (hash === loaded.hash) return
loaded.models = result.models
loaded.hash = hash
yield* ctx.integration.reload()
yield* ctx.provider.reload()
})
// Keep the last successful inventory through transient outages instead of flickering model availability.
yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped)
const reload = Effect.fn("VLLMPlugin.reload")(function* () {
const next = configured(yield* config.entries(), origin)
if (
next.baseURL === source.current.baseURL &&
next.apiKey === source.current.apiKey &&
next.healthEndpoint === source.current.healthEndpoint &&
next.modelsEndpoint === source.current.modelsEndpoint
)
return
source.current = next
loaded.models = []
loaded.hash = "[]"
yield* ctx.integration.reload()
yield* ctx.provider.reload()
yield* refresh().pipe(Effect.ignore)
})
yield* ctx.event.subscribe().pipe(
Stream.filter((event) => event.type === "config.updated"),
Stream.runForEach(reload),
Effect.forkScoped({ startImmediately: true }),
)
export const make = createLocalProviderPlugin({
id: "opencode.provider.vllm",
providerID,
name: "vLLM",
origin: "http://127.0.0.1:8000",
discover: (client) =>
Effect.gen(function* () {
yield* client.get("/health")
const response = yield* client.getJson("/v1/models", Response)
return response.data
.filter((model) => model.owned_by === providerID && model.id.length > 0)
.toSorted((a, b) => a.id.localeCompare(b.id))
}),
} satisfies PluginInternal.InternalPlugin)
}
// Tool calling depends on vLLM server flags and parsers that model discovery does not report.
model: (item) => ({
id: item.id,
context: item.max_model_len,
}),
})
export const VLLMPlugin = make()
function configured(entries: readonly Entry[], origin: string) {
const settings = foldSettings(entries, providerID, undefined)
const baseURL = (
typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1`
).replace(/\/+$/, "")
const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined
if (!URL.canParse(baseURL)) return { baseURL, apiKey }
const models = new URL(baseURL)
if (models.protocol !== "http:" && models.protocol !== "https:") return { baseURL, apiKey }
models.pathname = `${models.pathname.replace(/\/+$/, "")}/models`
models.search = ""
models.hash = ""
const health = new URL(baseURL)
const path = health.pathname.replace(/\/+$/, "")
const prefix = path.endsWith("/v1") ? path.slice(0, -3) : path
health.pathname = `${prefix}/health`
health.search = ""
health.hash = ""
return { baseURL, apiKey, healthEndpoint: health.toString(), modelsEndpoint: models.toString() }
}
@@ -226,6 +226,13 @@ describe("LMStudioPlugin", () => {
yield* bus.publish(Event.Updated, {})
yield* eventually(providers.get(providerID), (provider) => provider?.settings?.apiKey === "")
expect(requests).toContainEqual({ authorization: null, path: "/proxy/api/v1/models" })
requests.splice(0)
const apiBaseURL = `${configured.url.origin}/proxy/api/v1`
yield* config.setEntries([configuration(apiBaseURL, "api-secret")])
yield* bus.publish(Event.Updated, {})
yield* eventually(providers.get(providerID), (provider) => provider?.settings?.baseURL === apiBaseURL)
expect(requests).toContainEqual({ authorization: "Bearer api-secret", path: "/proxy/api/v1/models" })
}),
({ initial, configured }) => Effect.promise(() => Promise.all([initial.stop(true), configured.stop(true)])),
),
@@ -70,7 +70,7 @@ describe("OllamaPlugin", () => {
}
: body.model === "unknown-context"
? show({ family: "unknown", capabilities: ["completion"], context: 0 })
: show({ family: "nomic-bert", capabilities: ["embedding"], context: 8192 }),
: show({ family: "nomic-bert", capabilities: ["embedding"], context: 8192 }),
)
},
}),
@@ -299,6 +299,22 @@ describe("OllamaPlugin", () => {
yield* eventually(providers.get(providerID), (provider) => provider?.settings?.apiKey === "")
expect(requests).toContainEqual({ authorization: null, method: "GET", path: "/proxy/api/tags" })
expect(requests).toContainEqual({ authorization: null, method: "POST", path: "/proxy/api/show" })
requests.splice(0)
const apiBaseURL = `${configured.url.origin}/proxy/api`
yield* config.setEntries([configuration({ baseURL: apiBaseURL, apiKey: "api-secret" })])
yield* bus.publish(Event.Updated, {})
yield* eventually(providers.get(providerID), (provider) => provider?.settings?.baseURL === apiBaseURL)
expect(requests).toContainEqual({
authorization: "Bearer api-secret",
method: "GET",
path: "/proxy/api/tags",
})
expect(requests).toContainEqual({
authorization: "Bearer api-secret",
method: "POST",
path: "/proxy/api/show",
})
}),
({ initial, configured }) => Effect.promise(() => Promise.all([initial.stop(true), configured.stop(true)])),
),