mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
fix(core): time warming from the model's cache.ttl by default (#62831)
Co-authored-by: Ryan Skidmore <4580448+ryanskidmore@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
Unverified
parent
e36e6ce718
commit
039765b033
@@ -5,7 +5,7 @@ import type { SessionHooks } from "@opencode/plugin/effect/session"
|
||||
import type { Provider } from "@opencode/schema/provider"
|
||||
import type { Session } from "@opencode/schema/session"
|
||||
import { ConfigWarming } from "@opencode/schema/config/warming"
|
||||
import { Clock, Duration, Effect, Option, Schema, Scope, Stream } from "effect"
|
||||
import { Clock, Data, Duration, Effect, Option, Schema, Scope, Stream } from "effect"
|
||||
import { Config } from "../config.js"
|
||||
import { resolveModel } from "./optimize.js"
|
||||
|
||||
@@ -13,16 +13,33 @@ const decodeWarming = Schema.decodeUnknownOption(ConfigWarming.Warming)
|
||||
|
||||
const defaults = {
|
||||
prompt: "This is a keep-alive request. Do not perform any work or use tools. Reply with exactly: OK",
|
||||
interval: Duration.minutes(4),
|
||||
interval: "auto" as "auto" | Duration.Duration,
|
||||
duration: Duration.minutes(30),
|
||||
}
|
||||
|
||||
type Settings = typeof defaults & { interval: Duration.Duration }
|
||||
|
||||
/** Whether a session is warmed after a request, and why not. */
|
||||
type Decision = Data.TaggedEnum<{
|
||||
Warm: { readonly settings: Settings }
|
||||
/** Neither the top-level `warming` nor the model's `cache.warming` turns it on. */
|
||||
Off: {}
|
||||
/** `"auto"` timing on a cache that lasts longer than `duration`, so no ping is needed before the window closes. */
|
||||
CacheOutlivesWindow: {}
|
||||
/** An explicit interval that isn't positive and shorter than a finite `duration`. */
|
||||
InvalidInterval: {}
|
||||
}>
|
||||
const Decision = Data.taggedEnum<Decision>()
|
||||
|
||||
// The interval for "auto" when the model has no cache.ttl, a little under the five minutes most explicit caches last.
|
||||
const fallbackInterval = Duration.minutes(4)
|
||||
|
||||
export const Plugin = define({
|
||||
id: "opencode.warming",
|
||||
effect: Effect.fn(function* (ctx) {
|
||||
const config = yield* Config.Service
|
||||
// A model's or provider's `cache.warming` overrides the top-level `warming`.
|
||||
const loadSettings = Effect.fn("WarmingPlugin.loadSettings")(function* (event: SessionHooks["context"]) {
|
||||
const decide = Effect.fn("WarmingPlugin.decide")(function* (event: SessionHooks["context"]) {
|
||||
// Warming must never fail the request it hooks, so an unreadable catalog falls back to the top-level setting.
|
||||
const cache = (yield* resolveModel(ctx, event).pipe(Effect.orElseSucceed(() => undefined)))?.settings?.cache
|
||||
const local = cache?.warming === undefined ? undefined : Option.getOrUndefined(decodeWarming(cache.warming))
|
||||
@@ -31,30 +48,38 @@ export const Plugin = define({
|
||||
model: event.model,
|
||||
})
|
||||
const global = Config.latest(yield* config.entries(), "warming")
|
||||
if (!(local ?? global)) return
|
||||
if (!(local ?? global)) return Decision.Off()
|
||||
// The keep-alive prompt is only set at the top level; a model's setting changes timing.
|
||||
const settings = {
|
||||
...defaults,
|
||||
...(typeof global === "object" ? global : {}),
|
||||
...(typeof local === "object" ? local : {}),
|
||||
}
|
||||
const interval = settings.interval === "auto" ? autoInterval(cache?.ttl) : settings.interval
|
||||
// "auto" doesn't warm a model without a cache.ttl.
|
||||
if (interval === undefined) return
|
||||
const { duration } = settings
|
||||
const interval = settings.interval === "auto" ? (autoInterval(cache?.ttl) ?? fallbackInterval) : settings.interval
|
||||
// A cache that outlives the window, such as "1h" with the 30-minute default, needs no keep-alive.
|
||||
if (
|
||||
settings.interval === "auto" &&
|
||||
Duration.isFinite(duration) &&
|
||||
Duration.isGreaterThanOrEqualTo(interval, duration)
|
||||
)
|
||||
return Decision.CacheOutlivesWindow()
|
||||
const valid =
|
||||
Duration.isFinite(duration) &&
|
||||
Duration.isGreaterThan(interval, Duration.zero) &&
|
||||
Duration.isLessThan(interval, duration)
|
||||
if (valid) return { ...settings, interval }
|
||||
if (valid) return Decision.Warm({ settings: { ...settings, interval } })
|
||||
yield* Effect.logWarning("warming interval must be positive and shorter than a finite duration", {
|
||||
interval,
|
||||
duration,
|
||||
})
|
||||
return Decision.InvalidInterval()
|
||||
})
|
||||
|
||||
const scope = yield* Scope.Scope
|
||||
const sessions = new Map<Session.ID, { last: number; expires: number; settings: typeof defaults }>()
|
||||
const sessions = new Map<Session.ID, { last: number; expires: number; settings: Settings }>()
|
||||
// Sessions already told their cache outlives the window; the check runs on every request.
|
||||
const outlived = new Set<Session.ID>()
|
||||
// A ping during a model request would compete with the request it keeps warm, which stalls single-slot local models.
|
||||
// A request counts from its context hook, because `session.step.started` waits for the first response chunk.
|
||||
const requesting = new Set<Session.ID>()
|
||||
@@ -80,36 +105,34 @@ export const Plugin = define({
|
||||
),
|
||||
Effect.forkScoped({ startImmediately: true }),
|
||||
)
|
||||
const loop: (sessionID: Session.ID) => Effect.Effect<void> = Effect.fn("WarmingPlugin.loop")(
|
||||
function* (sessionID) {
|
||||
const current = sessions.get(sessionID)
|
||||
if (!current) return
|
||||
const loop: (sessionID: Session.ID) => Effect.Effect<void> = Effect.fn("WarmingPlugin.loop")(function* (sessionID) {
|
||||
const current = sessions.get(sessionID)
|
||||
if (!current) return
|
||||
|
||||
const now = yield* Clock.currentTimeMillis
|
||||
const next = Math.min(current.last + Duration.toMillis(current.settings.interval), current.expires)
|
||||
if (now < next) {
|
||||
yield* Effect.sleep(Duration.millis(next - now))
|
||||
return yield* loop(sessionID)
|
||||
}
|
||||
if (now >= current.expires) {
|
||||
sessions.delete(sessionID)
|
||||
return
|
||||
}
|
||||
if (requesting.has(sessionID)) {
|
||||
yield* Effect.sleep(current.settings.interval)
|
||||
return yield* loop(sessionID)
|
||||
}
|
||||
|
||||
const last = current.last
|
||||
yield* Effect.logInfo("warming session", { sessionID, last })
|
||||
yield* ctx.session
|
||||
.generate({ sessionID, prompt: current.settings.prompt })
|
||||
.pipe(Effect.catchCause((cause) => Effect.logWarning("failed to warm session", { sessionID, cause })))
|
||||
const latest = sessions.get(sessionID)
|
||||
if (latest === current && latest.last === last) latest.last = yield* Clock.currentTimeMillis
|
||||
const now = yield* Clock.currentTimeMillis
|
||||
const next = Math.min(current.last + Duration.toMillis(current.settings.interval), current.expires)
|
||||
if (now < next) {
|
||||
yield* Effect.sleep(Duration.millis(next - now))
|
||||
return yield* loop(sessionID)
|
||||
},
|
||||
)
|
||||
}
|
||||
if (now >= current.expires) {
|
||||
sessions.delete(sessionID)
|
||||
return
|
||||
}
|
||||
if (requesting.has(sessionID)) {
|
||||
yield* Effect.sleep(current.settings.interval)
|
||||
return yield* loop(sessionID)
|
||||
}
|
||||
|
||||
const last = current.last
|
||||
yield* Effect.logInfo("warming session", { sessionID, last })
|
||||
yield* ctx.session
|
||||
.generate({ sessionID, prompt: current.settings.prompt })
|
||||
.pipe(Effect.catchCause((cause) => Effect.logWarning("failed to warm session", { sessionID, cause })))
|
||||
const latest = sessions.get(sessionID)
|
||||
if (latest === current && latest.last === last) latest.last = yield* Clock.currentTimeMillis
|
||||
return yield* loop(sessionID)
|
||||
})
|
||||
|
||||
const hook = (event: SessionHooks["context"]) =>
|
||||
Effect.gen(function* () {
|
||||
@@ -117,11 +140,31 @@ export const Plugin = define({
|
||||
if (session.parentID) return
|
||||
|
||||
const active = sessions.get(event.sessionID)
|
||||
const settings = yield* loadSettings(event)
|
||||
if (!settings) {
|
||||
const stop = Effect.sync(() => {
|
||||
sessions.delete(event.sessionID)
|
||||
return
|
||||
}
|
||||
return undefined
|
||||
})
|
||||
const settings = yield* Decision.$match(yield* decide(event), {
|
||||
Warm: (decision) =>
|
||||
Effect.sync(() => {
|
||||
outlived.delete(event.sessionID)
|
||||
return decision.settings
|
||||
}),
|
||||
Off: () => stop,
|
||||
InvalidInterval: () => stop,
|
||||
CacheOutlivesWindow: () =>
|
||||
Effect.gen(function* () {
|
||||
if (!outlived.has(event.sessionID)) {
|
||||
outlived.add(event.sessionID)
|
||||
yield* Effect.logInfo("not warming: the prompt cache outlives the warming window", {
|
||||
sessionID: event.sessionID,
|
||||
model: event.model,
|
||||
})
|
||||
}
|
||||
return yield* stop
|
||||
}),
|
||||
})
|
||||
if (!settings) return
|
||||
|
||||
// Once generate exposes request metadata to context hooks, tag warm requests instead of matching the prompt.
|
||||
const message = event.messages.at(-1)
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { describe, expect } from "bun:test"
|
||||
import type { SessionContext } from "@opencode/plugin/effect/session"
|
||||
import { Document, Info } from "@opencode/schema/config"
|
||||
import { Event } from "@opencode/schema/event"
|
||||
import { Money } from "@opencode/schema/money"
|
||||
import { Agent } from "@opencode/core/agent"
|
||||
import { Config } from "@opencode/core/config"
|
||||
import { Location } from "@opencode/core/location"
|
||||
import { Model } from "@opencode/core/model"
|
||||
import { WarmingPlugin } from "@opencode/core/plugin/warming"
|
||||
import { Project } from "@opencode/core/project"
|
||||
import { Provider } from "@opencode/core/provider"
|
||||
import { AbsolutePath } from "@opencode/core/schema"
|
||||
import { Session } from "@opencode/core/session"
|
||||
import { SessionEvent } from "@opencode/core/session/event"
|
||||
import { Clock, DateTime, Effect, Queue, Schema, Stream } from "effect"
|
||||
import { TestClock } from "effect/testing"
|
||||
import { it } from "../lib/effect"
|
||||
import { host } from "./host"
|
||||
|
||||
const sessionID = Session.ID.make("ses_warming_test")
|
||||
const directory = AbsolutePath.make("/project")
|
||||
const model = Model.Info.default(Provider.ID.make("test"), Model.ID.make("test-model"))
|
||||
|
||||
const session = Session.Info.make({
|
||||
id: sessionID,
|
||||
projectID: Project.ID.global,
|
||||
cost: Money.USD.zero,
|
||||
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
time: { created: DateTime.makeUnsafe(0), updated: DateTime.makeUnsafe(0) },
|
||||
location: Location.Ref.make({ directory }),
|
||||
})
|
||||
|
||||
const context: SessionContext = {
|
||||
sessionID,
|
||||
agent: Agent.ID.make("build"),
|
||||
model: { id: model.id, providerID: model.providerID },
|
||||
system: [],
|
||||
messages: [],
|
||||
tools: {},
|
||||
options: {},
|
||||
}
|
||||
|
||||
/** Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. */
|
||||
const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cache) {
|
||||
const warms: number[] = []
|
||||
const events = yield* Queue.unbounded<SessionEvent.Execution.Succeeded>()
|
||||
let contextHook: ((input: SessionContext) => Effect.Effect<void>) | undefined
|
||||
yield* WarmingPlugin.Plugin.effect(
|
||||
host({
|
||||
model: {
|
||||
list: () => Effect.succeed({ location: { directory }, data: [{ ...model, settings: { cache } }] }),
|
||||
default: () => Effect.die("unused model.default"),
|
||||
transform: () => Effect.die("unused model.transform"),
|
||||
reload: () => Effect.die("unused model.reload"),
|
||||
},
|
||||
event: { subscribe: () => Stream.fromQueue(events) },
|
||||
session: {
|
||||
get: () => Effect.succeed(session),
|
||||
generate: () =>
|
||||
Clock.currentTimeMillis.pipe(
|
||||
Effect.map((now) => {
|
||||
warms.push(now / 60_000)
|
||||
return {
|
||||
text: "OK",
|
||||
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
cost: Money.USD.zero,
|
||||
}
|
||||
}),
|
||||
),
|
||||
hook: (name, callback) => {
|
||||
if (name === "context") contextHook = callback as (input: SessionContext) => Effect.Effect<void>
|
||||
return Effect.succeed({ dispose: Effect.void })
|
||||
},
|
||||
},
|
||||
}),
|
||||
).pipe(
|
||||
Effect.provide(
|
||||
Config.testLayer([new Document({ type: "document", info: Schema.decodeUnknownSync(Info)({ warming }) })]),
|
||||
),
|
||||
)
|
||||
if (!contextHook) return yield* Effect.die("warming plugin did not register a context hook")
|
||||
const hook = contextHook
|
||||
/** One whole turn: the step's context hook, then the turn ending. */
|
||||
const turn = Effect.gen(function* () {
|
||||
yield* hook({ ...context })
|
||||
yield* Queue.offer(events, {
|
||||
id: Event.ID.create(),
|
||||
created: 0,
|
||||
durable: { aggregateID: sessionID, seq: Event.Seq.make(0), version: Event.Version.make(1) },
|
||||
type: "session.execution.succeeded",
|
||||
data: { sessionID },
|
||||
})
|
||||
yield* Effect.yieldNow
|
||||
})
|
||||
const at = (minutes: number) => TestClock.setTime(minutes * 60_000).pipe(Effect.andThen(Effect.yieldNow))
|
||||
return { warms, turn, at }
|
||||
})
|
||||
|
||||
describe("WarmingPlugin interval", () => {
|
||||
it.effect("warms every 4 minutes when the model has no cache.ttl", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true)
|
||||
yield* warming.turn
|
||||
for (const minutes of [4, 8]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([4, 8])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("warms at 90% of the model's cache.ttl by default", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, { ttl: "5m" })
|
||||
yield* warming.turn
|
||||
for (const minutes of [4, 4.5, 9]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([4.5, 9])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("doesn't warm a cache that outlives the warming window", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, { ttl: "1h" })
|
||||
yield* warming.turn
|
||||
for (const minutes of [4, 27, 54]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps an explicit interval", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run({ interval: "2 minutes" }, { ttl: "5m" })
|
||||
yield* warming.turn
|
||||
for (const minutes of [2, 4]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([2, 4])
|
||||
}),
|
||||
)
|
||||
})
|
||||
@@ -11,7 +11,7 @@ export class Info extends Schema.Class<Info>("Config.Warming")({
|
||||
.pipe(optional)
|
||||
.annotate({
|
||||
description:
|
||||
'Idle time between keep-alive requests (default: "4 minutes"). "auto" uses 90% of the model\'s cache.ttl and skips models without one.',
|
||||
'Idle time between keep-alive requests (default: "auto"). "auto" uses 90% of the model\'s cache.ttl, or 4 minutes when it has none.',
|
||||
}),
|
||||
duration: Schema.DurationFromString.pipe(optional).annotate({
|
||||
description: 'Time after the last active request to keep a session warm (default: "30 minutes")',
|
||||
|
||||
@@ -60,7 +60,7 @@ export const CacheWarming = Schema.Union([
|
||||
// Settings hold JSON, so durations stay strings here; the warming plugin parses them.
|
||||
interval: Schema.Union([Schema.Literal("auto"), Schema.String])
|
||||
.pipe(optional)
|
||||
.annotate({ description: 'Duration such as "4 minutes", or "auto" for 90% of the model\'s cache.ttl' }),
|
||||
.annotate({ description: 'Duration such as "4 minutes", or "auto" (default) for 90% of the model\'s cache.ttl' }),
|
||||
duration: Schema.String.pipe(optional).annotate({ description: 'Duration such as "30 minutes"' }),
|
||||
}),
|
||||
]).annotate({ identifier: "Provider.CacheWarming" })
|
||||
|
||||
@@ -396,14 +396,14 @@ budgeting and overflow recovery.
|
||||
### Warming
|
||||
|
||||
Keep recently active model sessions warm with periodic transient requests.
|
||||
Warming is disabled by default; set it to `true` to use the four-minute idle
|
||||
interval and 30-minute active window.
|
||||
Warming is disabled by default; set it to `true` to warm just before each
|
||||
model's cache expires, for up to 30 minutes.
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"warming": {
|
||||
"prompt": "Do not perform any work. Reply with exactly: OK",
|
||||
"interval": "4 minutes",
|
||||
"interval": "auto",
|
||||
"duration": "30 minutes",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -15,9 +15,11 @@ Enable warming in any [OpenCode configuration file](/config):
|
||||
}
|
||||
```
|
||||
|
||||
With this configuration, OpenCode warms a recently active session after four
|
||||
minutes without a model request. It continues every four minutes until 30
|
||||
minutes have passed since the latest non-warming request.
|
||||
With this configuration, OpenCode warms a recently active session just before
|
||||
its prompt cache expires: at 90% of the model's
|
||||
[`cache.ttl`](/models#cache-lifetime), or every four minutes when the model
|
||||
has none. It continues until 30 minutes have passed since the latest
|
||||
non-warming request.
|
||||
|
||||
## Options
|
||||
|
||||
@@ -38,7 +40,7 @@ window:
|
||||
| Field | Default | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `prompt` | Keep-alive instruction | Appended as a transient user message for each warming request. The default asks the model to do no work, use no tools, and reply with `OK`. |
|
||||
| `interval` | `"4 minutes"` | Idle time before the next warming request, or `"auto"`. |
|
||||
| `interval` | `"auto"` | Idle time before the next warming request. `"auto"` uses 90% of the model's `cache.ttl`, or four minutes without one. |
|
||||
| `duration` | `"30 minutes"` | Warming window measured from the latest non-warming model request. |
|
||||
|
||||
Omitted object fields keep their defaults. For example, this changes only the
|
||||
@@ -54,23 +56,23 @@ interval:
|
||||
|
||||
## Automatic interval
|
||||
|
||||
Set `interval` to `"auto"` to warm just before each model's prompt cache
|
||||
expires. OpenCode uses 90% of the model's [`cache.ttl`](/models#cache-lifetime) setting, so a
|
||||
`"5m"` cache is warmed every 4.5 minutes. Models without a `cache.ttl` are not
|
||||
warmed.
|
||||
By default, `interval` is `"auto"`, which warms just before each model's
|
||||
prompt cache expires. OpenCode uses 90% of the model's
|
||||
[`cache.ttl`](/models#cache-lifetime) setting, so a `"5m"` cache is warmed
|
||||
every 4.5 minutes. Models without a `cache.ttl` are warmed every four minutes.
|
||||
|
||||
```jsonc title="opencode.jsonc"
|
||||
{
|
||||
"warming": { "interval": "auto", "duration": "2 hours" },
|
||||
"warming": { "duration": "2 hours" },
|
||||
"providers": {
|
||||
"anthropic": { "settings": { "cache": { "ttl": "1h" } } },
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
The interval must be shorter than `duration`, so raise `duration` for long
|
||||
cache lifetimes such as `"1h"`. Otherwise OpenCode skips warming and logs a
|
||||
warning.
|
||||
A cache that lasts longer than `duration`, such as `"1h"` with the 30-minute
|
||||
default, is not warmed, since it outlives the warming window anyway. OpenCode
|
||||
logs this once per session. Raise `duration` to warm it.
|
||||
|
||||
## Per provider and model
|
||||
|
||||
@@ -83,7 +85,7 @@ provider setting.
|
||||
{
|
||||
"providers": {
|
||||
"anthropic": {
|
||||
"settings": { "cache": { "ttl": "5m", "warming": { "interval": "auto" } } },
|
||||
"settings": { "cache": { "ttl": "5m", "warming": true } },
|
||||
"models": {
|
||||
"claude-haiku-5-5": { "settings": { "cache": { "warming": false } } },
|
||||
},
|
||||
@@ -172,8 +174,9 @@ costs, count against rate limits, and fail like other model requests.
|
||||
- Longer durations allow more warming requests.
|
||||
- Enable warming only when its provider-side benefit is worth the added usage.
|
||||
|
||||
For example, the defaults allow up to seven warming attempts during one
|
||||
uninterrupted 30-minute window: at minutes 4, 8, 12, 16, 20, 24, and 28.
|
||||
For example, on a model without a `cache.ttl` the defaults allow up to seven
|
||||
warming attempts during one uninterrupted 30-minute window: at minutes 4, 8,
|
||||
12, 16, 20, 24, and 28.
|
||||
|
||||
## Disabling
|
||||
|
||||
|
||||
Reference in New Issue
Block a user