mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 21:15:57 +03:00
feat(build): fetch the models.dev catalog once per release and verify the bundled copy (#59173)
This commit is contained in:
committed by
GitHub
Unverified
parent
94353780cf
commit
d5bb993278
@@ -100,17 +100,44 @@ jobs:
|
||||
release: ${{ steps.version.outputs.release || steps.beta.outputs.release }}
|
||||
tag: ${{ steps.version.outputs.tag || steps.beta.outputs.tag }}
|
||||
|
||||
models-catalog:
|
||||
runs-on: blacksmith-4vcpu-ubuntu-2404
|
||||
timeout-minutes: 10
|
||||
if: github.repository == 'anomalyco/opencode'
|
||||
steps:
|
||||
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
|
||||
|
||||
- uses: ./.github/actions/setup-bun
|
||||
|
||||
- name: Fetch models.dev catalog
|
||||
run: bun packages/core/script/update-models-snapshot.ts .cache/models-catalog/api.json
|
||||
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: models-catalog
|
||||
path: .cache/models-catalog/api.json
|
||||
if-no-files-found: error
|
||||
|
||||
build-cli:
|
||||
needs: version
|
||||
needs:
|
||||
- version
|
||||
- models-catalog
|
||||
runs-on: blacksmith-4vcpu-ubuntu-2404
|
||||
timeout-minutes: 30
|
||||
# Beta builds only the CLIs its desktop app bundles, in build-cli-beta.
|
||||
if: github.repository == 'anomalyco/opencode' && github.ref_name != 'beta'
|
||||
env:
|
||||
OPENCODE_MODELS_SNAPSHOT: ${{ github.workspace }}/.cache/models-catalog/api.json
|
||||
steps:
|
||||
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
|
||||
with:
|
||||
fetch-tags: true
|
||||
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
name: models-catalog
|
||||
path: .cache/models-catalog
|
||||
|
||||
- uses: ./.github/actions/setup-bun
|
||||
with:
|
||||
bun-version: 1.4.4
|
||||
@@ -142,6 +169,9 @@ jobs:
|
||||
GH_REPO: ${{ github.repository }}
|
||||
GH_TOKEN: ${{ steps.committer.outputs.token }}
|
||||
|
||||
- name: Verify bundled models catalog
|
||||
run: bun packages/cli/script/catalog-smoke.ts packages/cli/dist/cli-linux-x64/bin/opencode
|
||||
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
if: github.ref_name != 'dev' && github.ref_name != 'beta'
|
||||
with:
|
||||
@@ -167,10 +197,14 @@ jobs:
|
||||
# Beta publishes only the desktop app, so it builds just the 6 CLIs the desktop bundles, each in its own job and
|
||||
# artifact. Desktop jobs start alongside and wait for their CLI only when they need it.
|
||||
build-cli-beta:
|
||||
needs: version
|
||||
needs:
|
||||
- version
|
||||
- models-catalog
|
||||
runs-on: blacksmith-4vcpu-ubuntu-2404
|
||||
timeout-minutes: 30
|
||||
if: github.repository == 'anomalyco/opencode' && github.ref_name == 'beta'
|
||||
env:
|
||||
OPENCODE_MODELS_SNAPSHOT: ${{ github.workspace }}/.cache/models-catalog/api.json
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -189,6 +223,11 @@ jobs:
|
||||
with:
|
||||
bun-version: 1.4.4
|
||||
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
name: models-catalog
|
||||
path: .cache/models-catalog
|
||||
|
||||
- name: Build CLI
|
||||
run: ./packages/cli/script/build.ts --target=opencode-${{ matrix.target }}
|
||||
env:
|
||||
@@ -198,6 +237,10 @@ jobs:
|
||||
GH_REPO: ${{ github.repository }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
- name: Verify bundled models catalog
|
||||
if: matrix.target == 'linux-x64-baseline'
|
||||
run: bun packages/cli/script/catalog-smoke.ts packages/cli/dist/cli-linux-x64-baseline/bin/opencode
|
||||
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: opencode-cli-unsigned-cli-${{ matrix.target }}
|
||||
@@ -286,6 +329,7 @@ jobs:
|
||||
build-node-cli:
|
||||
needs:
|
||||
- version
|
||||
- models-catalog
|
||||
- build-node-app-archive
|
||||
if: github.repository == 'anomalyco/opencode' && github.ref_name != 'beta' && !(github.ref_name == 'dev' && (inputs.bump || inputs.version))
|
||||
strategy:
|
||||
@@ -304,6 +348,8 @@ jobs:
|
||||
- target: windows-x64
|
||||
host: blacksmith-4vcpu-windows-2025
|
||||
runs-on: ${{ matrix.settings.host }}
|
||||
env:
|
||||
OPENCODE_MODELS_SNAPSHOT: ${{ github.workspace }}/.cache/models-catalog/api.json
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -323,6 +369,11 @@ jobs:
|
||||
name: opencode-node-app-archive
|
||||
path: packages/cli/.cache
|
||||
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
name: models-catalog
|
||||
path: .cache/models-catalog
|
||||
|
||||
- name: Build
|
||||
run: bun packages/cli/script/build-node.ts --target=${{ matrix.settings.target }} --skip-install --outdir=dist/node --app-archive=.cache/app-archive.bin
|
||||
env:
|
||||
@@ -335,6 +386,10 @@ jobs:
|
||||
working-directory: packages/cli
|
||||
run: bun run script/service-smoke.ts --node
|
||||
|
||||
- name: Verify bundled models catalog
|
||||
if: matrix.settings.target != 'windows-arm64'
|
||||
run: bun packages/cli/script/catalog-smoke.ts packages/cli/dist/node/cli-node-${{ matrix.settings.target }}/bin/opencode2-node${{ startsWith(matrix.settings.target, 'windows') && '.exe' || '' }}
|
||||
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: opencode-node-cli-${{ matrix.settings.target }}
|
||||
@@ -693,6 +748,7 @@ jobs:
|
||||
publish:
|
||||
needs:
|
||||
- version
|
||||
- models-catalog
|
||||
- build-cli
|
||||
- build-cli-beta
|
||||
- sign-cli-macos
|
||||
@@ -704,11 +760,18 @@ jobs:
|
||||
runs-on: blacksmith-4vcpu-ubuntu-2404
|
||||
# Versioned and beta releases wait for approval in the release environment; dev previews do not.
|
||||
environment: ${{ needs.version.outputs.release && 'release' || '' }}
|
||||
env:
|
||||
OPENCODE_MODELS_SNAPSHOT: ${{ github.workspace }}/.cache/models-catalog/api.json
|
||||
steps:
|
||||
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
|
||||
|
||||
- uses: ./.github/actions/setup-bun
|
||||
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
name: models-catalog
|
||||
path: .cache/models-catalog
|
||||
|
||||
# Beta only records the desktop release, so it skips the Docker, git and AUR setup.
|
||||
- name: Login to GitHub Container Registry
|
||||
if: github.ref_name != 'beta'
|
||||
|
||||
@@ -15,6 +15,7 @@ import { nodeExecArgv, nodeTarget, type NodeTarget } from "../src/node/target"
|
||||
import { buildAppArchive } from "./app-assets"
|
||||
import { verifyArtifact } from "./verify-artifact"
|
||||
import { NativeArch } from "../../../script/native-arch"
|
||||
import { bundledModelsSnapshot } from "../../core/script/models-snapshot"
|
||||
|
||||
const NODE_VERSION = "26.4.0"
|
||||
const dir = path.resolve(import.meta.dirname, "..")
|
||||
@@ -62,6 +63,7 @@ if (targets.length === 0) {
|
||||
if (!bundleOnly && targets.some((target) => target.platform === "darwin" && target.arch === "x64")) {
|
||||
throw new Error("Node 26.4 SEA does not support macOS x64")
|
||||
}
|
||||
const modelsSnapshot = await bundledModelsSnapshot()
|
||||
const appArchive = archivePath ? await Bun.file(archivePath).bytes() : await buildAppArchive(Script.channel)
|
||||
if (!bundleOnly) await rm(outdir, { recursive: true, force: true })
|
||||
const builder =
|
||||
@@ -97,6 +99,7 @@ for (const target of targets) {
|
||||
version: Script.version,
|
||||
channel: Script.channel,
|
||||
highlights,
|
||||
modelsSnapshot,
|
||||
assetHash,
|
||||
target,
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import { buildAppArchive } from "./app-assets"
|
||||
import { verifyArtifact, verifySimulationGraph } from "./verify-artifact"
|
||||
import { resolveOpencodePty } from "./opencode-pty"
|
||||
import { NativeArch } from "../../../script/native-arch"
|
||||
import { bundledModelsSnapshot, modelsSnapshotPlugin } from "../../core/script/models-snapshot"
|
||||
|
||||
const dir = path.resolve(import.meta.dirname, "..")
|
||||
const binary = "opencode"
|
||||
@@ -33,6 +34,7 @@ const skipWebUi = process.argv.includes("--skip-web-ui")
|
||||
const solidPlugin = createSolidTransformPlugin()
|
||||
const highlights = Script.preview ? undefined : embed(process.env.OPENCODE_RELEASE_NOTES)
|
||||
const releaseAssets = new Map<string, Promise<Map<string, string>>>()
|
||||
const modelsSnapshot = modelsSnapshotPlugin(await bundledModelsSnapshot())
|
||||
|
||||
const allTargets: {
|
||||
os: string
|
||||
@@ -164,6 +166,7 @@ export default { path: file, version: ${JSON.stringify(opencodePty.version)}, sh
|
||||
opencodePtyPlugin,
|
||||
pluginRuntimePlugin,
|
||||
simulationGraphPlugin,
|
||||
modelsSnapshot,
|
||||
],
|
||||
external: ["node-gyp"],
|
||||
format: "esm",
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import fs from "node:fs/promises"
|
||||
import os from "node:os"
|
||||
import path from "node:path"
|
||||
|
||||
const catalog = process.env.OPENCODE_MODELS_SNAPSHOT
|
||||
const binary = process.argv[2]
|
||||
if (!catalog || !binary) throw new Error("Usage: OPENCODE_MODELS_SNAPSHOT=<catalog> catalog-smoke.ts <binary>")
|
||||
|
||||
const providers: Record<string, { env?: string[] }> = await Bun.file(catalog).json()
|
||||
const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "opencode-catalog-smoke-")))
|
||||
// Credentials for every provider make the model list cover the whole catalog.
|
||||
const env = {
|
||||
...Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("OPENCODE_MODELS"))),
|
||||
...Object.fromEntries(
|
||||
Object.values(providers).flatMap((provider) => (provider.env ?? []).map((key) => [key, "smoke"])),
|
||||
),
|
||||
HOME: root,
|
||||
USERPROFILE: root,
|
||||
OPENCODE_DB: path.join(root, "opencode.db"),
|
||||
OPENCODE_DISABLE_MODELS_FETCH: "1",
|
||||
XDG_CACHE_HOME: path.join(root, "cache"),
|
||||
XDG_CONFIG_HOME: path.join(root, "config"),
|
||||
XDG_DATA_HOME: path.join(root, "data"),
|
||||
XDG_STATE_HOME: path.join(root, "state"),
|
||||
}
|
||||
|
||||
const bundled = models({})
|
||||
const fetched = models({ OPENCODE_MODELS_PATH: path.resolve(catalog) })
|
||||
await fs.rm(root, { recursive: true, force: true })
|
||||
if (bundled.length === 0) throw new Error(`${binary} listed no models`)
|
||||
if (bundled.join("\n") !== fetched.join("\n")) {
|
||||
const missing = fetched.filter((model) => !bundled.includes(model))
|
||||
const extra = bundled.filter((model) => !fetched.includes(model))
|
||||
throw new Error(
|
||||
`${binary} bundles a different models.dev catalog than ${catalog}: missing ${missing.slice(0, 5).join(", ")}; extra ${extra.slice(0, 5).join(", ")}`,
|
||||
)
|
||||
}
|
||||
console.log(`${binary} bundles ${catalog} (${bundled.length} models)`)
|
||||
|
||||
function models(extra: Record<string, string>) {
|
||||
const result = Bun.spawnSync([binary, "models", "--standalone"], { cwd: root, env: { ...env, ...extra } })
|
||||
if (result.exitCode !== 0) throw new Error(`${binary} models exited with ${result.exitCode}: ${result.stderr}`)
|
||||
return result.stdout.toString().trim().split(/\r?\n/)
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import { verifySimulationGraph } from "./script/verify-artifact"
|
||||
|
||||
const dir = import.meta.dirname
|
||||
|
||||
function rawTextPlugin(): Plugin {
|
||||
function rawTextPlugin(modelsSnapshot?: string): Plugin {
|
||||
return {
|
||||
name: "opencode:raw-text",
|
||||
// "pre" is load-bearing for .txt: Vite's built-in asset plugin claims
|
||||
@@ -18,7 +18,9 @@ function rawTextPlugin(): Plugin {
|
||||
enforce: "pre",
|
||||
async load(id) {
|
||||
if (!id.endsWith(".md") && !id.endsWith(".txt")) return
|
||||
return `export default ${JSON.stringify(await readFile(id, "utf8"))}`
|
||||
const text =
|
||||
modelsSnapshot && /models-dev[/\\]snapshot\.txt$/.test(id) ? modelsSnapshot : await readFile(id, "utf8")
|
||||
return `export default ${JSON.stringify(text)}`
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -252,6 +254,7 @@ export type NodeBuildInput = {
|
||||
readonly version: string
|
||||
readonly channel: string
|
||||
readonly highlights?: string
|
||||
readonly modelsSnapshot?: string
|
||||
readonly assetHash: string
|
||||
readonly target: NodeTarget
|
||||
}
|
||||
@@ -261,7 +264,7 @@ export function mainConfig(input: NodeBuildInput): UserConfig {
|
||||
root: dir,
|
||||
plugins: [
|
||||
appAssetsPlugin(),
|
||||
rawTextPlugin(),
|
||||
rawTextPlugin(input.modelsSnapshot),
|
||||
runtimeRequirePlugin(),
|
||||
fffNodePlugin(),
|
||||
simulationGraphPlugin(),
|
||||
|
||||
@@ -4,6 +4,7 @@ import { $ } from "bun"
|
||||
import { rm } from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
import { bundledModelsSnapshot, modelsSnapshotPlugin } from "./models-snapshot"
|
||||
|
||||
process.chdir(fileURLToPath(new URL("..", import.meta.url)))
|
||||
|
||||
@@ -29,6 +30,7 @@ const result = await Bun.build({
|
||||
"#v1-migration",
|
||||
],
|
||||
splitting: true,
|
||||
plugins: [modelsSnapshotPlugin(await bundledModelsSnapshot())],
|
||||
loader: {
|
||||
".txt": "text",
|
||||
".md": "text",
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { BunPlugin } from "bun"
|
||||
import { Effect } from "effect"
|
||||
import { ModelsDev } from "../src/models-dev"
|
||||
|
||||
// A floor, not equality: guards against an error page or a truncated body
|
||||
// that still parses as a small object.
|
||||
const MINIMUM_PROVIDERS = 100
|
||||
export const committedSnapshot = new URL("../src/models-dev/snapshot.txt", import.meta.url)
|
||||
|
||||
export async function fetchModelsSnapshot() {
|
||||
const source = process.env.OPENCODE_MODELS_URL || "https://models.opencode.ai"
|
||||
const response = await fetch(`${source}/api.json`, { signal: AbortSignal.timeout(30_000) })
|
||||
if (!response.ok) throw new Error(`${source}/api.json returned ${response.status} ${response.statusText}`)
|
||||
return validateModelsSnapshot(await response.text())
|
||||
}
|
||||
|
||||
async function validateModelsSnapshot(text: string) {
|
||||
const providers = (await Effect.runPromise(ModelsDev.decodeSnapshot(text))).length
|
||||
if (providers < MINIMUM_PROVIDERS)
|
||||
throw new Error(`models.dev catalog has ${providers} providers, expected at least ${MINIMUM_PROVIDERS}`)
|
||||
return { text, providers }
|
||||
}
|
||||
|
||||
export async function bundledModelsSnapshot() {
|
||||
const file = process.env.OPENCODE_MODELS_SNAPSHOT
|
||||
if (!file) return Bun.file(committedSnapshot).text()
|
||||
const snapshot = await validateModelsSnapshot(await Bun.file(file).text())
|
||||
console.log(`bundling models.dev catalog from ${file} (${snapshot.providers} providers)`)
|
||||
return snapshot.text
|
||||
}
|
||||
|
||||
export function modelsSnapshotPlugin(text: string): BunPlugin {
|
||||
return {
|
||||
name: "opencode-models-snapshot",
|
||||
setup(build) {
|
||||
build.onLoad({ filter: /models-dev[/\\]snapshot\.txt$/ }, () => ({ contents: text, loader: "text" }))
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,24 +1,14 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Refreshes the bundled models.dev catalog snapshot at src/models-dev/snapshot.txt.
|
||||
* Refreshes the bundled models.dev catalog snapshot at src/models-dev/snapshot.txt,
|
||||
* or writes it to the path given as the first argument.
|
||||
* The snapshot is the boot-time floor for the catalog when no cache entry exists
|
||||
* and fetching is disabled or unavailable; live fetch still refreshes on top.
|
||||
*/
|
||||
const source = process.env.OPENCODE_MODELS_URL || "https://models.opencode.ai"
|
||||
const response = await fetch(`${source}/api.json`)
|
||||
if (!response.ok) {
|
||||
console.error(`Failed to fetch ${source}/api.json: ${response.status} ${response.statusText}`)
|
||||
process.exit(1)
|
||||
}
|
||||
const text = await response.text()
|
||||
const parsed: unknown = JSON.parse(text)
|
||||
// A floor, not equality: guards against committing an error page or a
|
||||
// truncated body that still parses as a small object.
|
||||
const MINIMUM_PROVIDERS = 100
|
||||
if (typeof parsed !== "object" || parsed === null || Object.keys(parsed).length < MINIMUM_PROVIDERS) {
|
||||
console.error(`Fetched catalog has fewer than ${MINIMUM_PROVIDERS} providers; refusing to write snapshot`)
|
||||
process.exit(1)
|
||||
}
|
||||
const target = new URL("../src/models-dev/snapshot.txt", import.meta.url)
|
||||
await Bun.write(target, text)
|
||||
console.log(`Wrote ${Object.keys(parsed).length} providers (${text.length} bytes) to ${Bun.fileURLToPath(target)}`)
|
||||
import path from "node:path"
|
||||
import { committedSnapshot, fetchModelsSnapshot } from "./models-snapshot"
|
||||
|
||||
const target = process.argv[2] ? path.resolve(process.argv[2]) : Bun.fileURLToPath(committedSnapshot)
|
||||
const snapshot = await fetchModelsSnapshot()
|
||||
await Bun.write(target, snapshot.text)
|
||||
console.log(`Wrote ${snapshot.providers} providers (${snapshot.text.length} bytes) to ${target}`)
|
||||
|
||||
@@ -304,9 +304,12 @@ const CacheMeta = Schema.Struct({
|
||||
})
|
||||
const defaultSource = "https://models.opencode.ai"
|
||||
|
||||
export const decodeSnapshot = (text: string) => decodeCatalog(text).pipe(Effect.map(normalize))
|
||||
|
||||
// Bundled snapshot of https://models.opencode.ai/api.json, committed at
|
||||
// packages/core/src/models-dev/snapshot.txt and refreshed via
|
||||
// `bun run script/update-models-snapshot.ts`. Decoded and normalized once per
|
||||
// `bun run script/update-models-snapshot.ts`. Release builds bundle the live
|
||||
// catalog in its place when it can be fetched. Decoded and normalized once per
|
||||
// isolate: the snapshot is a multi-MB module-level constant and one isolate can
|
||||
// host many runtimes (Cloudflare colocates Durable Object instances), so
|
||||
// per-runtime decoding would multiply the cost.
|
||||
@@ -314,9 +317,9 @@ let bundledCache: readonly Snapshot[] | undefined
|
||||
const bundledSnapshot = Effect.suspend(() =>
|
||||
bundledCache
|
||||
? Effect.succeed(bundledCache)
|
||||
: decodeCatalog(snapshotText).pipe(
|
||||
Effect.map((catalog) => {
|
||||
bundledCache = normalize(catalog)
|
||||
: decodeSnapshot(snapshotText).pipe(
|
||||
Effect.map((snapshot) => {
|
||||
bundledCache = snapshot
|
||||
return bundledCache
|
||||
}),
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user