mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-19 08:39:02 +03:00
refactor(core): share local provider discovery plugin across vllm, lmstudio, and ollama (#53438)
This commit is contained in:
committed by
GitHub
Unverified
parent
aa7c1f3ebd
commit
39105835c6
@@ -1,13 +1,5 @@
|
||||
import { define } from "@opencode/plugin/effect/plugin"
|
||||
import type { Entry } from "@opencode/schema/config"
|
||||
import { Duration, Effect, Schedule, Schema, Semaphore, Stream } from "effect"
|
||||
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import { Config } from "../../config.js"
|
||||
import { Model } from "../../model.js"
|
||||
import type { PluginInternal } from "../internal.js"
|
||||
import { foldSettings } from "./configured.js"
|
||||
|
||||
const providerID = "lmstudio"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { createLocalProviderPlugin } from "./local.js"
|
||||
|
||||
const RemoteModel = Schema.Struct({
|
||||
type: Schema.Literals(["llm", "embedding"]),
|
||||
@@ -27,140 +19,31 @@ const RemoteModel = Schema.Struct({
|
||||
})
|
||||
|
||||
const Response = Schema.Struct({ models: Schema.Array(RemoteModel) })
|
||||
const discovery = new Map<string, { checked: number; apiKey?: string; models?: (typeof RemoteModel.Type)[] }>()
|
||||
const discoveryLock = Semaphore.makeUnsafe(1)
|
||||
|
||||
export function make(origin = "http://127.0.0.1:1234", interval: Duration.Input = "30 seconds") {
|
||||
return define({
|
||||
id: "opencode.provider.lmstudio",
|
||||
effect: Effect.fn(function* (ctx) {
|
||||
const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient)
|
||||
const config = yield* Config.Service
|
||||
const source = { current: configured(yield* config.entries(), origin) }
|
||||
const loaded = { models: [] as (typeof RemoteModel.Type)[], hash: "[]" }
|
||||
|
||||
yield* ctx.integration.transform((integrations) => {
|
||||
if (loaded.models.length === 0) return
|
||||
integrations.remove(providerID)
|
||||
})
|
||||
|
||||
yield* ctx.provider.transform((providers) => {
|
||||
if (loaded.models.length === 0) return
|
||||
for (const model of providers.get(providerID)?.models.values() ?? []) {
|
||||
providers.models.remove(providerID, model.id)
|
||||
}
|
||||
providers.update(providerID, (provider) => {
|
||||
provider.name = "LM Studio"
|
||||
provider.activation = "enabled"
|
||||
provider.package = "@opencode/ai/providers/openai-compatible"
|
||||
provider.settings = {
|
||||
baseURL: source.current.baseURL,
|
||||
provider: providerID,
|
||||
apiKey: source.current.apiKey ?? "",
|
||||
}
|
||||
provider.integrationID = undefined
|
||||
})
|
||||
for (const item of loaded.models) {
|
||||
providers.models.update(providerID, item.key, (model) => {
|
||||
model.modelID = Model.ID.make(item.key)
|
||||
model.name = item.display_name || item.key
|
||||
model.family = item.architecture ? Model.Family.make(item.architecture) : undefined
|
||||
model.capabilities = {
|
||||
tools: item.capabilities?.trained_for_tool_use ?? false,
|
||||
input: ["text", ...(item.capabilities?.vision ? ["image"] : [])],
|
||||
output: ["text"],
|
||||
}
|
||||
const context =
|
||||
item.loaded_instances.length === 0
|
||||
? item.max_context_length
|
||||
: Math.min(...item.loaded_instances.map((instance) => instance.config.context_length))
|
||||
if (context > 0) model.limit.context = context
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
const discover = Effect.fn("LMStudioPlugin.discover")(function* () {
|
||||
const current = source.current
|
||||
if (!current.endpoint) return undefined
|
||||
return yield* discoveryLock.withPermit(
|
||||
Effect.gen(function* () {
|
||||
const cached = discovery.get(current.endpoint)
|
||||
if (cached && cached.apiKey === current.apiKey && Date.now() - cached.checked < Duration.toMillis(interval))
|
||||
return { source: current, models: cached.models }
|
||||
discovery.set(current.endpoint, {
|
||||
checked: Date.now(),
|
||||
apiKey: current.apiKey,
|
||||
models: cached && cached.apiKey === current.apiKey ? cached.models : undefined,
|
||||
})
|
||||
const request = current.apiKey
|
||||
? HttpClientRequest.get(current.endpoint).pipe(
|
||||
HttpClientRequest.acceptJson,
|
||||
HttpClientRequest.bearerToken(current.apiKey),
|
||||
)
|
||||
: HttpClientRequest.get(current.endpoint).pipe(HttpClientRequest.acceptJson)
|
||||
const response = yield* http
|
||||
.execute(request)
|
||||
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(Response)), Effect.timeout("1 second"))
|
||||
const models = response.models
|
||||
.filter((model) => model.type === "llm" && model.key.length > 0)
|
||||
.toSorted((a, b) => a.key.localeCompare(b.key))
|
||||
discovery.set(current.endpoint, { checked: Date.now(), apiKey: current.apiKey, models })
|
||||
return { source: current, models }
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
const refresh = Effect.fn("LMStudioPlugin.refresh")(function* () {
|
||||
const result = yield* discover()
|
||||
if (!result?.models || result.source !== source.current) return
|
||||
const hash = JSON.stringify(result.models)
|
||||
if (hash === loaded.hash) return
|
||||
loaded.models = result.models
|
||||
loaded.hash = hash
|
||||
yield* ctx.integration.reload()
|
||||
yield* ctx.provider.reload()
|
||||
})
|
||||
|
||||
// Keep the last successful inventory through transient outages instead of flickering model availability.
|
||||
yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped)
|
||||
const reload = Effect.fn("LMStudioPlugin.reload")(function* () {
|
||||
const next = configured(yield* config.entries(), origin)
|
||||
if (
|
||||
next.baseURL === source.current.baseURL &&
|
||||
next.apiKey === source.current.apiKey &&
|
||||
next.endpoint === source.current.endpoint
|
||||
)
|
||||
return
|
||||
source.current = next
|
||||
loaded.models = []
|
||||
loaded.hash = "[]"
|
||||
yield* ctx.integration.reload()
|
||||
yield* ctx.provider.reload()
|
||||
yield* refresh().pipe(Effect.ignore)
|
||||
})
|
||||
yield* ctx.event.subscribe().pipe(
|
||||
Stream.filter((event) => event.type === "config.updated"),
|
||||
Stream.runForEach(reload),
|
||||
Effect.forkScoped({ startImmediately: true }),
|
||||
)
|
||||
export const make = createLocalProviderPlugin({
|
||||
id: "opencode.provider.lmstudio",
|
||||
providerID: "lmstudio",
|
||||
name: "LM Studio",
|
||||
origin: "http://127.0.0.1:1234",
|
||||
stripPathSuffix: /\/(?:api\/)?v1$/,
|
||||
discover: (client) =>
|
||||
Effect.gen(function* () {
|
||||
const response = yield* client.getJson("/api/v1/models", Response)
|
||||
return response.models
|
||||
.filter((model) => model.type === "llm" && model.key.length > 0)
|
||||
.toSorted((a, b) => a.key.localeCompare(b.key))
|
||||
}),
|
||||
} satisfies PluginInternal.InternalPlugin)
|
||||
}
|
||||
model: (item) => ({
|
||||
id: item.key,
|
||||
name: item.display_name || item.key,
|
||||
family: item.architecture,
|
||||
tools: item.capabilities?.trained_for_tool_use,
|
||||
vision: item.capabilities?.vision,
|
||||
context:
|
||||
item.loaded_instances.length === 0
|
||||
? item.max_context_length
|
||||
: Math.min(...item.loaded_instances.map((instance) => instance.config.context_length)),
|
||||
}),
|
||||
})
|
||||
|
||||
export const LMStudioPlugin = make()
|
||||
|
||||
function configured(entries: readonly Entry[], origin: string) {
|
||||
const settings = foldSettings(entries, providerID, undefined)
|
||||
const baseURL = (
|
||||
typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1`
|
||||
).replace(/\/+$/, "")
|
||||
const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined
|
||||
if (!URL.canParse(baseURL)) return { baseURL, apiKey }
|
||||
const url = new URL(baseURL)
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return { baseURL, apiKey }
|
||||
const prefix = url.pathname.endsWith("/v1") ? url.pathname.slice(0, -3) : url.pathname.replace(/\/+$/, "")
|
||||
url.pathname = `${prefix}/api/v1/models`
|
||||
url.search = ""
|
||||
url.hash = ""
|
||||
return { baseURL, apiKey, endpoint: url.toString() }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
import { define } from "@opencode/plugin/effect/plugin"
|
||||
import type { Entry } from "@opencode/schema/config"
|
||||
import { Duration, Effect, Schedule, type Schema, Semaphore, Stream } from "effect"
|
||||
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import { Config } from "../../config.js"
|
||||
import { Model } from "../../model.js"
|
||||
import type { PluginInternal } from "../internal.js"
|
||||
import { foldSettings } from "./configured.js"
|
||||
|
||||
export type LocalModel = {
|
||||
readonly id: string
|
||||
readonly name?: string
|
||||
readonly family?: string | null
|
||||
readonly tools?: boolean
|
||||
readonly vision?: boolean
|
||||
readonly context?: number | null
|
||||
}
|
||||
|
||||
export type LocalDiscoveryClient = ReturnType<typeof createClient>
|
||||
|
||||
export function createLocalProviderPlugin<Discovered>(input: {
|
||||
readonly id: string
|
||||
readonly providerID: string
|
||||
readonly name: string
|
||||
readonly origin: string
|
||||
readonly stripPathSuffix?: RegExp
|
||||
readonly discover: (client: LocalDiscoveryClient) => Effect.Effect<readonly Discovered[], unknown>
|
||||
readonly model: (item: NoInfer<Discovered>) => LocalModel
|
||||
}) {
|
||||
const discovery = new Map<string, { checked: number; apiKey?: string; models?: readonly Discovered[] }>()
|
||||
const discoveryLock = Semaphore.makeUnsafe(1)
|
||||
const suffix = input.stripPathSuffix ?? /\/v1$/
|
||||
|
||||
return function make(origin = input.origin, interval: Duration.Input = "30 seconds") {
|
||||
return define({
|
||||
id: input.id,
|
||||
effect: Effect.fn(function* (ctx) {
|
||||
const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient)
|
||||
const config = yield* Config.Service
|
||||
const source = { current: configured(yield* config.entries(), input.providerID, origin, suffix) }
|
||||
const loaded = { models: [] as readonly Discovered[], hash: "[]" }
|
||||
|
||||
yield* ctx.integration.transform((integrations) => {
|
||||
if (loaded.models.length === 0) return
|
||||
integrations.remove(input.providerID)
|
||||
})
|
||||
|
||||
yield* ctx.provider.transform((providers) => {
|
||||
if (loaded.models.length === 0) return
|
||||
for (const model of providers.get(input.providerID)?.models.values() ?? []) {
|
||||
providers.models.remove(input.providerID, model.id)
|
||||
}
|
||||
providers.update(input.providerID, (provider) => {
|
||||
provider.name = input.name
|
||||
provider.activation = "enabled"
|
||||
provider.package = "@opencode/ai/providers/openai-compatible"
|
||||
provider.settings = {
|
||||
baseURL: source.current.baseURL,
|
||||
provider: input.providerID,
|
||||
apiKey: source.current.apiKey ?? "",
|
||||
}
|
||||
provider.integrationID = undefined
|
||||
})
|
||||
for (const item of loaded.models) {
|
||||
const spec = input.model(item)
|
||||
providers.models.update(input.providerID, spec.id, (model) => {
|
||||
model.modelID = Model.ID.make(spec.id)
|
||||
model.name = spec.name || spec.id
|
||||
model.family = spec.family ? Model.Family.make(spec.family) : undefined
|
||||
model.capabilities = {
|
||||
tools: spec.tools ?? false,
|
||||
input: ["text", ...(spec.vision ? ["image"] : [])],
|
||||
output: ["text"],
|
||||
}
|
||||
if (typeof spec.context === "number" && spec.context > 0) model.limit.context = spec.context
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
const discover = Effect.fn("LocalProviderPlugin.discover")(function* () {
|
||||
const current = source.current
|
||||
if (!current.endpoint) return undefined
|
||||
return yield* discoveryLock.withPermit(
|
||||
Effect.gen(function* () {
|
||||
const cached = discovery.get(current.root)
|
||||
if (
|
||||
cached &&
|
||||
cached.apiKey === current.apiKey &&
|
||||
Date.now() - cached.checked < Duration.toMillis(interval)
|
||||
)
|
||||
return { source: current, models: cached.models }
|
||||
discovery.set(current.root, {
|
||||
checked: Date.now(),
|
||||
apiKey: current.apiKey,
|
||||
models: cached && cached.apiKey === current.apiKey ? cached.models : undefined,
|
||||
})
|
||||
const models = yield* input.discover(createClient(http, current.root, current.apiKey, current.endpoint))
|
||||
discovery.set(current.root, { checked: Date.now(), apiKey: current.apiKey, models })
|
||||
return { source: current, models }
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
const refresh = Effect.fn("LocalProviderPlugin.refresh")(function* () {
|
||||
const result = yield* discover()
|
||||
if (!result?.models || result.source !== source.current) return
|
||||
const hash = JSON.stringify(result.models)
|
||||
if (hash === loaded.hash) return
|
||||
loaded.models = result.models
|
||||
loaded.hash = hash
|
||||
yield* ctx.integration.reload()
|
||||
yield* ctx.provider.reload()
|
||||
})
|
||||
|
||||
// Keep the last successful inventory through transient outages instead of flickering model availability.
|
||||
yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped)
|
||||
const reload = Effect.fn("LocalProviderPlugin.reload")(function* () {
|
||||
const next = configured(yield* config.entries(), input.providerID, origin, suffix)
|
||||
if (
|
||||
next.baseURL === source.current.baseURL &&
|
||||
next.apiKey === source.current.apiKey &&
|
||||
next.root === source.current.root
|
||||
)
|
||||
return
|
||||
source.current = next
|
||||
loaded.models = []
|
||||
loaded.hash = "[]"
|
||||
yield* ctx.integration.reload()
|
||||
yield* ctx.provider.reload()
|
||||
yield* refresh().pipe(Effect.ignore)
|
||||
})
|
||||
yield* ctx.event.subscribe().pipe(
|
||||
Stream.filter((event) => event.type === "config.updated"),
|
||||
Stream.runForEach(reload),
|
||||
Effect.forkScoped({ startImmediately: true }),
|
||||
)
|
||||
}),
|
||||
} satisfies PluginInternal.InternalPlugin)
|
||||
}
|
||||
}
|
||||
|
||||
function createClient(
|
||||
http: HttpClient.HttpClient,
|
||||
root: string,
|
||||
apiKey: string | undefined,
|
||||
endpoint: (path: string) => string,
|
||||
) {
|
||||
const auth = (request: HttpClientRequest.HttpClientRequest) =>
|
||||
apiKey
|
||||
? request.pipe(HttpClientRequest.acceptJson, HttpClientRequest.bearerToken(apiKey))
|
||||
: request.pipe(HttpClientRequest.acceptJson)
|
||||
return {
|
||||
root,
|
||||
apiKey,
|
||||
get: (path: string) => http.execute(auth(HttpClientRequest.get(endpoint(path)))).pipe(Effect.timeout("1 second")),
|
||||
getJson: <A, I>(path: string, schema: Schema.Codec<A, I>) =>
|
||||
http
|
||||
.execute(auth(HttpClientRequest.get(endpoint(path))))
|
||||
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)), Effect.timeout("1 second")),
|
||||
postJson: <B, BI, A, AI>(
|
||||
path: string,
|
||||
bodySchema: Schema.Codec<B, BI>,
|
||||
body: B,
|
||||
responseSchema: Schema.Codec<A, AI>,
|
||||
) =>
|
||||
auth(HttpClientRequest.post(endpoint(path))).pipe(
|
||||
HttpClientRequest.schemaBodyJson(bodySchema)(body),
|
||||
Effect.flatMap(http.execute),
|
||||
Effect.flatMap(HttpClientResponse.schemaBodyJson(responseSchema)),
|
||||
Effect.timeout("1 second"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
function configured(entries: readonly Entry[], providerID: string, origin: string, suffix: RegExp) {
|
||||
const settings = foldSettings(entries, providerID, undefined)
|
||||
const baseURL = (
|
||||
typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1`
|
||||
).replace(/\/+$/, "")
|
||||
const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined
|
||||
if (!URL.canParse(baseURL)) return { baseURL, apiKey }
|
||||
const url = new URL(baseURL)
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return { baseURL, apiKey }
|
||||
const prefix = url.pathname.replace(/\/+$/, "").replace(suffix, "")
|
||||
const endpoint = (path: string) => {
|
||||
const next = new URL(url)
|
||||
next.pathname = `${prefix}${path}`
|
||||
next.search = ""
|
||||
next.hash = ""
|
||||
return next.toString()
|
||||
}
|
||||
return { baseURL, apiKey, root: endpoint(""), endpoint }
|
||||
}
|
||||
@@ -1,13 +1,5 @@
|
||||
import { define } from "@opencode/plugin/effect/plugin"
|
||||
import type { Entry } from "@opencode/schema/config"
|
||||
import { Duration, Effect, Schedule, Schema, Semaphore, Stream } from "effect"
|
||||
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import { Config } from "../../config.js"
|
||||
import { Model } from "../../model.js"
|
||||
import type { PluginInternal } from "../internal.js"
|
||||
import { foldSettings } from "./configured.js"
|
||||
|
||||
const providerID = "ollama"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { createLocalProviderPlugin } from "./local.js"
|
||||
|
||||
const Details = Schema.Struct({
|
||||
parent_model: Schema.String.pipe(Schema.optional),
|
||||
@@ -42,183 +34,59 @@ const ShowResponse = Schema.Struct({
|
||||
})
|
||||
|
||||
type DiscoveredModel = typeof RemoteModel.Type & { show: typeof ShowResponse.Type }
|
||||
type Discovery = {
|
||||
checked: number
|
||||
type ShowCache = {
|
||||
apiKey?: string
|
||||
models?: DiscoveredModel[]
|
||||
shows: Map<string, { digest: string; info: typeof ShowResponse.Type }>
|
||||
entries: Map<string, { digest: string; info: typeof ShowResponse.Type }>
|
||||
}
|
||||
|
||||
const discovery = new Map<string, Discovery>()
|
||||
const discoveryLock = Semaphore.makeUnsafe(1)
|
||||
const shows = new Map<string, ShowCache>()
|
||||
|
||||
export function make(origin = "http://127.0.0.1:11434", interval: Duration.Input = "30 seconds") {
|
||||
return define({
|
||||
id: "opencode.provider.ollama",
|
||||
effect: Effect.fn(function* (ctx) {
|
||||
const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient)
|
||||
const config = yield* Config.Service
|
||||
const source = { current: configured(yield* config.entries(), origin) }
|
||||
const loaded = { models: [] as DiscoveredModel[], hash: "[]" }
|
||||
|
||||
yield* ctx.integration.transform((integrations) => {
|
||||
if (loaded.models.length === 0) return
|
||||
integrations.remove(providerID)
|
||||
})
|
||||
|
||||
yield* ctx.provider.transform((providers) => {
|
||||
if (loaded.models.length === 0) return
|
||||
for (const model of providers.get(providerID)?.models.values() ?? []) {
|
||||
providers.models.remove(providerID, model.id)
|
||||
}
|
||||
providers.update(providerID, (provider) => {
|
||||
provider.name = "Ollama"
|
||||
provider.activation = "enabled"
|
||||
provider.package = "@opencode/ai/providers/openai-compatible"
|
||||
provider.settings = {
|
||||
baseURL: source.current.baseURL,
|
||||
provider: providerID,
|
||||
apiKey: source.current.apiKey ?? "",
|
||||
}
|
||||
provider.integrationID = undefined
|
||||
})
|
||||
for (const item of loaded.models) {
|
||||
providers.models.update(providerID, item.model, (model) => {
|
||||
model.modelID = Model.ID.make(item.model)
|
||||
model.name = item.name || item.model
|
||||
model.family = item.show.details?.family
|
||||
? Model.Family.make(item.show.details.family)
|
||||
: item.details.family
|
||||
? Model.Family.make(item.details.family)
|
||||
: undefined
|
||||
model.capabilities = {
|
||||
tools: item.show.capabilities?.includes("tools") ?? false,
|
||||
input: ["text", ...(item.show.capabilities?.includes("vision") ? ["image"] : [])],
|
||||
output: ["text"],
|
||||
}
|
||||
const context = Object.entries(item.show.model_info ?? {}).flatMap(([key, value]) =>
|
||||
key.endsWith(".context_length") && typeof value === "number" && value > 0 ? [value] : [],
|
||||
)[0]
|
||||
if (context !== undefined) model.limit.context = context
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
const discover = Effect.fn("OllamaPlugin.discover")(function* () {
|
||||
const current = source.current
|
||||
if (!current.tagsEndpoint || !current.showEndpoint) return undefined
|
||||
return yield* discoveryLock.withPermit(
|
||||
export const make = createLocalProviderPlugin({
|
||||
id: "opencode.provider.ollama",
|
||||
providerID: "ollama",
|
||||
name: "Ollama",
|
||||
origin: "http://127.0.0.1:11434",
|
||||
stripPathSuffix: /\/(?:v1|api)$/,
|
||||
discover: (client) =>
|
||||
Effect.gen(function* () {
|
||||
const cached = shows.get(client.root)
|
||||
const previous = cached && cached.apiKey === client.apiKey ? cached.entries : new Map()
|
||||
shows.set(client.root, { apiKey: client.apiKey, entries: previous })
|
||||
const response = yield* client.getJson("/api/tags", TagsResponse)
|
||||
const summaries = response.models
|
||||
.filter((model) => model.model.length > 0)
|
||||
.toSorted((a, b) => a.model.localeCompare(b.model))
|
||||
const entries = new Map<string, { digest: string; info: typeof ShowResponse.Type }>()
|
||||
const models = yield* Effect.forEach(
|
||||
summaries,
|
||||
(model) =>
|
||||
Effect.gen(function* () {
|
||||
const cached = discovery.get(current.tagsEndpoint)
|
||||
if (cached && cached.apiKey === current.apiKey && Date.now() - cached.checked < Duration.toMillis(interval))
|
||||
return { source: current, models: cached.models }
|
||||
const previous: Discovery =
|
||||
cached && cached.apiKey === current.apiKey
|
||||
? cached
|
||||
: { checked: 0, apiKey: current.apiKey, shows: new Map() }
|
||||
discovery.set(current.tagsEndpoint, { ...previous, checked: Date.now(), apiKey: current.apiKey })
|
||||
const tagsRequest = current.apiKey
|
||||
? HttpClientRequest.get(current.tagsEndpoint).pipe(
|
||||
HttpClientRequest.acceptJson,
|
||||
HttpClientRequest.bearerToken(current.apiKey),
|
||||
)
|
||||
: HttpClientRequest.get(current.tagsEndpoint).pipe(HttpClientRequest.acceptJson)
|
||||
const response = yield* http
|
||||
.execute(tagsRequest)
|
||||
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(TagsResponse)), Effect.timeout("1 second"))
|
||||
const summaries = response.models
|
||||
.filter((model) => model.model.length > 0)
|
||||
.toSorted((a, b) => a.model.localeCompare(b.model))
|
||||
const shows = new Map<string, { digest: string; info: typeof ShowResponse.Type }>()
|
||||
const models = yield* Effect.forEach(
|
||||
summaries,
|
||||
(model) =>
|
||||
Effect.gen(function* () {
|
||||
const saved = previous.shows.get(model.model)
|
||||
const info =
|
||||
saved?.digest === model.digest
|
||||
? saved.info
|
||||
: yield* HttpClientRequest.post(current.showEndpoint).pipe(
|
||||
HttpClientRequest.acceptJson,
|
||||
current.apiKey ? HttpClientRequest.bearerToken(current.apiKey) : (request) => request,
|
||||
HttpClientRequest.schemaBodyJson(ShowRequest)({ model: model.model }),
|
||||
Effect.flatMap(http.execute),
|
||||
Effect.flatMap(HttpClientResponse.schemaBodyJson(ShowResponse)),
|
||||
Effect.timeout("1 second"),
|
||||
)
|
||||
shows.set(model.model, { digest: model.digest, info })
|
||||
return { ...model, show: info }
|
||||
}).pipe(Effect.orElseSucceed(() => undefined)),
|
||||
{ concurrency: 4 },
|
||||
)
|
||||
const filtered = models.filter(
|
||||
(model): model is DiscoveredModel =>
|
||||
model !== undefined && (model.show.capabilities?.includes("completion") ?? false),
|
||||
)
|
||||
discovery.set(current.tagsEndpoint, {
|
||||
checked: Date.now(),
|
||||
apiKey: current.apiKey,
|
||||
models: filtered,
|
||||
shows,
|
||||
})
|
||||
return { source: current, models: filtered }
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
const refresh = Effect.fn("OllamaPlugin.refresh")(function* () {
|
||||
const result = yield* discover()
|
||||
if (!result?.models || result.source !== source.current) return
|
||||
const hash = JSON.stringify(result.models)
|
||||
if (hash === loaded.hash) return
|
||||
loaded.models = result.models
|
||||
loaded.hash = hash
|
||||
yield* ctx.integration.reload()
|
||||
yield* ctx.provider.reload()
|
||||
})
|
||||
|
||||
// Keep the last successful inventory through transient outages instead of flickering model availability.
|
||||
yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped)
|
||||
const reload = Effect.fn("OllamaPlugin.reload")(function* () {
|
||||
const next = configured(yield* config.entries(), origin)
|
||||
if (
|
||||
next.baseURL === source.current.baseURL &&
|
||||
next.apiKey === source.current.apiKey &&
|
||||
next.tagsEndpoint === source.current.tagsEndpoint
|
||||
)
|
||||
return
|
||||
source.current = next
|
||||
loaded.models = []
|
||||
loaded.hash = "[]"
|
||||
yield* ctx.integration.reload()
|
||||
yield* ctx.provider.reload()
|
||||
yield* refresh().pipe(Effect.ignore)
|
||||
})
|
||||
yield* ctx.event.subscribe().pipe(
|
||||
Stream.filter((event) => event.type === "config.updated"),
|
||||
Stream.runForEach(reload),
|
||||
Effect.forkScoped({ startImmediately: true }),
|
||||
const saved = previous.get(model.model)
|
||||
const info =
|
||||
saved?.digest === model.digest
|
||||
? saved.info
|
||||
: yield* client.postJson("/api/show", ShowRequest, { model: model.model }, ShowResponse)
|
||||
entries.set(model.model, { digest: model.digest, info })
|
||||
return { ...model, show: info }
|
||||
}).pipe(Effect.orElseSucceed(() => undefined)),
|
||||
{ concurrency: 4 },
|
||||
)
|
||||
shows.set(client.root, { apiKey: client.apiKey, entries })
|
||||
return models.filter(
|
||||
(model): model is DiscoveredModel =>
|
||||
model !== undefined && (model.show.capabilities?.includes("completion") ?? false),
|
||||
)
|
||||
}),
|
||||
} satisfies PluginInternal.InternalPlugin)
|
||||
}
|
||||
model: (item) => ({
|
||||
id: item.model,
|
||||
name: item.name || item.model,
|
||||
family: item.show.details?.family || item.details.family,
|
||||
tools: item.show.capabilities?.includes("tools"),
|
||||
vision: item.show.capabilities?.includes("vision"),
|
||||
context: Object.entries(item.show.model_info ?? {}).flatMap(([key, value]) =>
|
||||
key.endsWith(".context_length") && typeof value === "number" && value > 0 ? [value] : [],
|
||||
)[0],
|
||||
}),
|
||||
})
|
||||
|
||||
export const OllamaPlugin = make()
|
||||
|
||||
function configured(entries: readonly Entry[], origin: string) {
|
||||
const settings = foldSettings(entries, providerID, undefined)
|
||||
const baseURL = (
|
||||
typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1`
|
||||
).replace(/\/+$/, "")
|
||||
const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined
|
||||
if (!URL.canParse(baseURL)) return { baseURL, apiKey }
|
||||
const url = new URL(baseURL)
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") return { baseURL, apiKey }
|
||||
const prefix = url.pathname.endsWith("/v1") ? url.pathname.slice(0, -3) : url.pathname.replace(/\/+$/, "")
|
||||
url.pathname = `${prefix}/api/tags`
|
||||
url.search = ""
|
||||
url.hash = ""
|
||||
const tagsEndpoint = url.toString()
|
||||
url.pathname = `${prefix}/api/show`
|
||||
return { baseURL, apiKey, tagsEndpoint, showEndpoint: url.toString() }
|
||||
}
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
import { define } from "@opencode/plugin/effect/plugin"
|
||||
import type { Entry } from "@opencode/schema/config"
|
||||
import { Duration, Effect, Schedule, Schema, Semaphore, Stream } from "effect"
|
||||
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import { Config } from "../../config.js"
|
||||
import { Model } from "../../model.js"
|
||||
import type { PluginInternal } from "../internal.js"
|
||||
import { foldSettings } from "./configured.js"
|
||||
import { Effect, Schema } from "effect"
|
||||
import { createLocalProviderPlugin } from "./local.js"
|
||||
|
||||
const providerID = "vllm"
|
||||
|
||||
@@ -16,142 +10,25 @@ const RemoteModel = Schema.Struct({
|
||||
})
|
||||
|
||||
const Response = Schema.Struct({ data: Schema.Array(RemoteModel) })
|
||||
const discovery = new Map<string, { checked: number; apiKey?: string; models?: (typeof RemoteModel.Type)[] }>()
|
||||
const discoveryLock = Semaphore.makeUnsafe(1)
|
||||
|
||||
export function make(origin = "http://127.0.0.1:8000", interval: Duration.Input = "30 seconds") {
|
||||
return define({
|
||||
id: "opencode.provider.vllm",
|
||||
effect: Effect.fn(function* (ctx) {
|
||||
const http = HttpClient.filterStatusOk(yield* HttpClient.HttpClient)
|
||||
const config = yield* Config.Service
|
||||
const source = { current: configured(yield* config.entries(), origin) }
|
||||
const loaded = { models: [] as (typeof RemoteModel.Type)[], hash: "[]" }
|
||||
|
||||
yield* ctx.integration.transform((integrations) => {
|
||||
if (loaded.models.length === 0) return
|
||||
integrations.remove(providerID)
|
||||
})
|
||||
|
||||
yield* ctx.provider.transform((providers) => {
|
||||
if (loaded.models.length === 0) return
|
||||
for (const model of providers.get(providerID)?.models.values() ?? []) {
|
||||
providers.models.remove(providerID, model.id)
|
||||
}
|
||||
providers.update(providerID, (provider) => {
|
||||
provider.name = "vLLM"
|
||||
provider.package = "@opencode/ai/providers/openai-compatible"
|
||||
provider.settings = {
|
||||
baseURL: source.current.baseURL,
|
||||
provider: providerID,
|
||||
apiKey: source.current.apiKey ?? "",
|
||||
}
|
||||
provider.integrationID = undefined
|
||||
provider.activation = "enabled"
|
||||
})
|
||||
for (const item of loaded.models) {
|
||||
providers.models.update(providerID, item.id, (model) => {
|
||||
model.modelID = Model.ID.make(item.id)
|
||||
model.name = item.id
|
||||
// Tool calling depends on vLLM server flags and parsers that model discovery does not report.
|
||||
model.capabilities = { tools: false, input: ["text"], output: ["text"] }
|
||||
if (typeof item.max_model_len === "number" && item.max_model_len > 0)
|
||||
model.limit.context = item.max_model_len
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
const discover = Effect.fn("VLLMPlugin.discover")(function* () {
|
||||
const current = source.current
|
||||
if (!current.healthEndpoint || !current.modelsEndpoint) return undefined
|
||||
return yield* discoveryLock.withPermit(
|
||||
Effect.gen(function* () {
|
||||
const endpoint = `${current.healthEndpoint}\n${current.modelsEndpoint}`
|
||||
const cached = discovery.get(endpoint)
|
||||
if (cached && cached.apiKey === current.apiKey && Date.now() - cached.checked < Duration.toMillis(interval))
|
||||
return { source: current, models: cached.models }
|
||||
discovery.set(endpoint, {
|
||||
checked: Date.now(),
|
||||
apiKey: current.apiKey,
|
||||
models: cached && cached.apiKey === current.apiKey ? cached.models : undefined,
|
||||
})
|
||||
const request = (endpoint: string) =>
|
||||
current.apiKey
|
||||
? HttpClientRequest.get(endpoint).pipe(
|
||||
HttpClientRequest.acceptJson,
|
||||
HttpClientRequest.bearerToken(current.apiKey),
|
||||
)
|
||||
: HttpClientRequest.get(endpoint).pipe(HttpClientRequest.acceptJson)
|
||||
yield* http.execute(request(current.healthEndpoint)).pipe(Effect.timeout("1 second"))
|
||||
const response = yield* http
|
||||
.execute(request(current.modelsEndpoint))
|
||||
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(Response)), Effect.timeout("1 second"))
|
||||
const models = response.data
|
||||
.filter((model) => model.owned_by === providerID && model.id.length > 0)
|
||||
.toSorted((a, b) => a.id.localeCompare(b.id))
|
||||
discovery.set(endpoint, { checked: Date.now(), apiKey: current.apiKey, models })
|
||||
return { source: current, models }
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
const refresh = Effect.fn("VLLMPlugin.refresh")(function* () {
|
||||
const result = yield* discover()
|
||||
if (!result?.models || result.source !== source.current) return
|
||||
const hash = JSON.stringify(result.models)
|
||||
if (hash === loaded.hash) return
|
||||
loaded.models = result.models
|
||||
loaded.hash = hash
|
||||
yield* ctx.integration.reload()
|
||||
yield* ctx.provider.reload()
|
||||
})
|
||||
|
||||
// Keep the last successful inventory through transient outages instead of flickering model availability.
|
||||
yield* refresh().pipe(Effect.ignore, Effect.repeat(Schedule.spaced(interval)), Effect.forkScoped)
|
||||
const reload = Effect.fn("VLLMPlugin.reload")(function* () {
|
||||
const next = configured(yield* config.entries(), origin)
|
||||
if (
|
||||
next.baseURL === source.current.baseURL &&
|
||||
next.apiKey === source.current.apiKey &&
|
||||
next.healthEndpoint === source.current.healthEndpoint &&
|
||||
next.modelsEndpoint === source.current.modelsEndpoint
|
||||
)
|
||||
return
|
||||
source.current = next
|
||||
loaded.models = []
|
||||
loaded.hash = "[]"
|
||||
yield* ctx.integration.reload()
|
||||
yield* ctx.provider.reload()
|
||||
yield* refresh().pipe(Effect.ignore)
|
||||
})
|
||||
yield* ctx.event.subscribe().pipe(
|
||||
Stream.filter((event) => event.type === "config.updated"),
|
||||
Stream.runForEach(reload),
|
||||
Effect.forkScoped({ startImmediately: true }),
|
||||
)
|
||||
export const make = createLocalProviderPlugin({
|
||||
id: "opencode.provider.vllm",
|
||||
providerID,
|
||||
name: "vLLM",
|
||||
origin: "http://127.0.0.1:8000",
|
||||
discover: (client) =>
|
||||
Effect.gen(function* () {
|
||||
yield* client.get("/health")
|
||||
const response = yield* client.getJson("/v1/models", Response)
|
||||
return response.data
|
||||
.filter((model) => model.owned_by === providerID && model.id.length > 0)
|
||||
.toSorted((a, b) => a.id.localeCompare(b.id))
|
||||
}),
|
||||
} satisfies PluginInternal.InternalPlugin)
|
||||
}
|
||||
// Tool calling depends on vLLM server flags and parsers that model discovery does not report.
|
||||
model: (item) => ({
|
||||
id: item.id,
|
||||
context: item.max_model_len,
|
||||
}),
|
||||
})
|
||||
|
||||
export const VLLMPlugin = make()
|
||||
|
||||
function configured(entries: readonly Entry[], origin: string) {
|
||||
const settings = foldSettings(entries, providerID, undefined)
|
||||
const baseURL = (
|
||||
typeof settings?.baseURL === "string" ? settings.baseURL : `${origin.replace(/\/+$/, "")}/v1`
|
||||
).replace(/\/+$/, "")
|
||||
const apiKey = typeof settings?.apiKey === "string" ? settings.apiKey : undefined
|
||||
if (!URL.canParse(baseURL)) return { baseURL, apiKey }
|
||||
const models = new URL(baseURL)
|
||||
if (models.protocol !== "http:" && models.protocol !== "https:") return { baseURL, apiKey }
|
||||
models.pathname = `${models.pathname.replace(/\/+$/, "")}/models`
|
||||
models.search = ""
|
||||
models.hash = ""
|
||||
const health = new URL(baseURL)
|
||||
const path = health.pathname.replace(/\/+$/, "")
|
||||
const prefix = path.endsWith("/v1") ? path.slice(0, -3) : path
|
||||
health.pathname = `${prefix}/health`
|
||||
health.search = ""
|
||||
health.hash = ""
|
||||
return { baseURL, apiKey, healthEndpoint: health.toString(), modelsEndpoint: models.toString() }
|
||||
}
|
||||
|
||||
@@ -226,6 +226,13 @@ describe("LMStudioPlugin", () => {
|
||||
yield* bus.publish(Event.Updated, {})
|
||||
yield* eventually(providers.get(providerID), (provider) => provider?.settings?.apiKey === "")
|
||||
expect(requests).toContainEqual({ authorization: null, path: "/proxy/api/v1/models" })
|
||||
|
||||
requests.splice(0)
|
||||
const apiBaseURL = `${configured.url.origin}/proxy/api/v1`
|
||||
yield* config.setEntries([configuration(apiBaseURL, "api-secret")])
|
||||
yield* bus.publish(Event.Updated, {})
|
||||
yield* eventually(providers.get(providerID), (provider) => provider?.settings?.baseURL === apiBaseURL)
|
||||
expect(requests).toContainEqual({ authorization: "Bearer api-secret", path: "/proxy/api/v1/models" })
|
||||
}),
|
||||
({ initial, configured }) => Effect.promise(() => Promise.all([initial.stop(true), configured.stop(true)])),
|
||||
),
|
||||
|
||||
@@ -70,7 +70,7 @@ describe("OllamaPlugin", () => {
|
||||
}
|
||||
: body.model === "unknown-context"
|
||||
? show({ family: "unknown", capabilities: ["completion"], context: 0 })
|
||||
: show({ family: "nomic-bert", capabilities: ["embedding"], context: 8192 }),
|
||||
: show({ family: "nomic-bert", capabilities: ["embedding"], context: 8192 }),
|
||||
)
|
||||
},
|
||||
}),
|
||||
@@ -299,6 +299,22 @@ describe("OllamaPlugin", () => {
|
||||
yield* eventually(providers.get(providerID), (provider) => provider?.settings?.apiKey === "")
|
||||
expect(requests).toContainEqual({ authorization: null, method: "GET", path: "/proxy/api/tags" })
|
||||
expect(requests).toContainEqual({ authorization: null, method: "POST", path: "/proxy/api/show" })
|
||||
|
||||
requests.splice(0)
|
||||
const apiBaseURL = `${configured.url.origin}/proxy/api`
|
||||
yield* config.setEntries([configuration({ baseURL: apiBaseURL, apiKey: "api-secret" })])
|
||||
yield* bus.publish(Event.Updated, {})
|
||||
yield* eventually(providers.get(providerID), (provider) => provider?.settings?.baseURL === apiBaseURL)
|
||||
expect(requests).toContainEqual({
|
||||
authorization: "Bearer api-secret",
|
||||
method: "GET",
|
||||
path: "/proxy/api/tags",
|
||||
})
|
||||
expect(requests).toContainEqual({
|
||||
authorization: "Bearer api-secret",
|
||||
method: "POST",
|
||||
path: "/proxy/api/show",
|
||||
})
|
||||
}),
|
||||
({ initial, configured }) => Effect.promise(() => Promise.all([initial.stop(true), configured.stop(true)])),
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user