fix(core): time warming from the model's cache.ttl by default (#62831)

Co-authored-by: Ryan Skidmore <4580448+ryanskidmore@users.noreply.github.com>
This commit is contained in:
Aiden Cline
2026-11-09 14:37:02 -06:00
committed by GitHub
Unverified
parent e36e6ce718
commit 039765b033
6 changed files with 243 additions and 61 deletions
+84 -41
View File
@@ -5,7 +5,7 @@ import type { SessionHooks } from "@opencode/plugin/effect/session"
import type { Provider } from "@opencode/schema/provider"
import type { Session } from "@opencode/schema/session"
import { ConfigWarming } from "@opencode/schema/config/warming"
import { Clock, Duration, Effect, Option, Schema, Scope, Stream } from "effect"
import { Clock, Data, Duration, Effect, Option, Schema, Scope, Stream } from "effect"
import { Config } from "../config.js"
import { resolveModel } from "./optimize.js"
@@ -13,16 +13,33 @@ const decodeWarming = Schema.decodeUnknownOption(ConfigWarming.Warming)
const defaults = {
prompt: "This is a keep-alive request. Do not perform any work or use tools. Reply with exactly: OK",
interval: Duration.minutes(4),
interval: "auto" as "auto" | Duration.Duration,
duration: Duration.minutes(30),
}
type Settings = typeof defaults & { interval: Duration.Duration }
/** Whether a session is warmed after a request, and why not. */
type Decision = Data.TaggedEnum<{
Warm: { readonly settings: Settings }
/** Neither the top-level `warming` nor the model's `cache.warming` turns it on. */
Off: {}
/** `"auto"` timing on a cache that lasts longer than `duration`, so no ping is needed before the window closes. */
CacheOutlivesWindow: {}
/** An explicit interval that isn't positive and shorter than a finite `duration`. */
InvalidInterval: {}
}>
const Decision = Data.taggedEnum<Decision>()
// The interval for "auto" when the model has no cache.ttl, a little under the five minutes most explicit caches last.
const fallbackInterval = Duration.minutes(4)
export const Plugin = define({
id: "opencode.warming",
effect: Effect.fn(function* (ctx) {
const config = yield* Config.Service
// A model's or provider's `cache.warming` overrides the top-level `warming`.
const loadSettings = Effect.fn("WarmingPlugin.loadSettings")(function* (event: SessionHooks["context"]) {
const decide = Effect.fn("WarmingPlugin.decide")(function* (event: SessionHooks["context"]) {
// Warming must never fail the request it hooks, so an unreadable catalog falls back to the top-level setting.
const cache = (yield* resolveModel(ctx, event).pipe(Effect.orElseSucceed(() => undefined)))?.settings?.cache
const local = cache?.warming === undefined ? undefined : Option.getOrUndefined(decodeWarming(cache.warming))
@@ -31,30 +48,38 @@ export const Plugin = define({
model: event.model,
})
const global = Config.latest(yield* config.entries(), "warming")
if (!(local ?? global)) return
if (!(local ?? global)) return Decision.Off()
// The keep-alive prompt is only set at the top level; a model's setting changes timing.
const settings = {
...defaults,
...(typeof global === "object" ? global : {}),
...(typeof local === "object" ? local : {}),
}
const interval = settings.interval === "auto" ? autoInterval(cache?.ttl) : settings.interval
// "auto" doesn't warm a model without a cache.ttl.
if (interval === undefined) return
const { duration } = settings
const interval = settings.interval === "auto" ? (autoInterval(cache?.ttl) ?? fallbackInterval) : settings.interval
// A cache that outlives the window, such as "1h" with the 30-minute default, needs no keep-alive.
if (
settings.interval === "auto" &&
Duration.isFinite(duration) &&
Duration.isGreaterThanOrEqualTo(interval, duration)
)
return Decision.CacheOutlivesWindow()
const valid =
Duration.isFinite(duration) &&
Duration.isGreaterThan(interval, Duration.zero) &&
Duration.isLessThan(interval, duration)
if (valid) return { ...settings, interval }
if (valid) return Decision.Warm({ settings: { ...settings, interval } })
yield* Effect.logWarning("warming interval must be positive and shorter than a finite duration", {
interval,
duration,
})
return Decision.InvalidInterval()
})
const scope = yield* Scope.Scope
const sessions = new Map<Session.ID, { last: number; expires: number; settings: typeof defaults }>()
const sessions = new Map<Session.ID, { last: number; expires: number; settings: Settings }>()
// Sessions already told their cache outlives the window; the check runs on every request.
const outlived = new Set<Session.ID>()
// A ping during a model request would compete with the request it keeps warm, which stalls single-slot local models.
// A request counts from its context hook, because `session.step.started` waits for the first response chunk.
const requesting = new Set<Session.ID>()
@@ -80,36 +105,34 @@ export const Plugin = define({
),
Effect.forkScoped({ startImmediately: true }),
)
const loop: (sessionID: Session.ID) => Effect.Effect<void> = Effect.fn("WarmingPlugin.loop")(
function* (sessionID) {
const current = sessions.get(sessionID)
if (!current) return
const loop: (sessionID: Session.ID) => Effect.Effect<void> = Effect.fn("WarmingPlugin.loop")(function* (sessionID) {
const current = sessions.get(sessionID)
if (!current) return
const now = yield* Clock.currentTimeMillis
const next = Math.min(current.last + Duration.toMillis(current.settings.interval), current.expires)
if (now < next) {
yield* Effect.sleep(Duration.millis(next - now))
return yield* loop(sessionID)
}
if (now >= current.expires) {
sessions.delete(sessionID)
return
}
if (requesting.has(sessionID)) {
yield* Effect.sleep(current.settings.interval)
return yield* loop(sessionID)
}
const last = current.last
yield* Effect.logInfo("warming session", { sessionID, last })
yield* ctx.session
.generate({ sessionID, prompt: current.settings.prompt })
.pipe(Effect.catchCause((cause) => Effect.logWarning("failed to warm session", { sessionID, cause })))
const latest = sessions.get(sessionID)
if (latest === current && latest.last === last) latest.last = yield* Clock.currentTimeMillis
const now = yield* Clock.currentTimeMillis
const next = Math.min(current.last + Duration.toMillis(current.settings.interval), current.expires)
if (now < next) {
yield* Effect.sleep(Duration.millis(next - now))
return yield* loop(sessionID)
},
)
}
if (now >= current.expires) {
sessions.delete(sessionID)
return
}
if (requesting.has(sessionID)) {
yield* Effect.sleep(current.settings.interval)
return yield* loop(sessionID)
}
const last = current.last
yield* Effect.logInfo("warming session", { sessionID, last })
yield* ctx.session
.generate({ sessionID, prompt: current.settings.prompt })
.pipe(Effect.catchCause((cause) => Effect.logWarning("failed to warm session", { sessionID, cause })))
const latest = sessions.get(sessionID)
if (latest === current && latest.last === last) latest.last = yield* Clock.currentTimeMillis
return yield* loop(sessionID)
})
const hook = (event: SessionHooks["context"]) =>
Effect.gen(function* () {
@@ -117,11 +140,31 @@ export const Plugin = define({
if (session.parentID) return
const active = sessions.get(event.sessionID)
const settings = yield* loadSettings(event)
if (!settings) {
const stop = Effect.sync(() => {
sessions.delete(event.sessionID)
return
}
return undefined
})
const settings = yield* Decision.$match(yield* decide(event), {
Warm: (decision) =>
Effect.sync(() => {
outlived.delete(event.sessionID)
return decision.settings
}),
Off: () => stop,
InvalidInterval: () => stop,
CacheOutlivesWindow: () =>
Effect.gen(function* () {
if (!outlived.has(event.sessionID)) {
outlived.add(event.sessionID)
yield* Effect.logInfo("not warming: the prompt cache outlives the warming window", {
sessionID: event.sessionID,
model: event.model,
})
}
return yield* stop
}),
})
if (!settings) return
// Once generate exposes request metadata to context hooks, tag warm requests instead of matching the prompt.
const message = event.messages.at(-1)
+136
View File
@@ -0,0 +1,136 @@
import { describe, expect } from "bun:test"
import type { SessionContext } from "@opencode/plugin/effect/session"
import { Document, Info } from "@opencode/schema/config"
import { Event } from "@opencode/schema/event"
import { Money } from "@opencode/schema/money"
import { Agent } from "@opencode/core/agent"
import { Config } from "@opencode/core/config"
import { Location } from "@opencode/core/location"
import { Model } from "@opencode/core/model"
import { WarmingPlugin } from "@opencode/core/plugin/warming"
import { Project } from "@opencode/core/project"
import { Provider } from "@opencode/core/provider"
import { AbsolutePath } from "@opencode/core/schema"
import { Session } from "@opencode/core/session"
import { SessionEvent } from "@opencode/core/session/event"
import { Clock, DateTime, Effect, Queue, Schema, Stream } from "effect"
import { TestClock } from "effect/testing"
import { it } from "../lib/effect"
import { host } from "./host"
const sessionID = Session.ID.make("ses_warming_test")
const directory = AbsolutePath.make("/project")
const model = Model.Info.default(Provider.ID.make("test"), Model.ID.make("test-model"))
const session = Session.Info.make({
id: sessionID,
projectID: Project.ID.global,
cost: Money.USD.zero,
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
time: { created: DateTime.makeUnsafe(0), updated: DateTime.makeUnsafe(0) },
location: Location.Ref.make({ directory }),
})
const context: SessionContext = {
sessionID,
agent: Agent.ID.make("build"),
model: { id: model.id, providerID: model.providerID },
system: [],
messages: [],
tools: {},
options: {},
}
/** Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. */
const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cache) {
const warms: number[] = []
const events = yield* Queue.unbounded<SessionEvent.Execution.Succeeded>()
let contextHook: ((input: SessionContext) => Effect.Effect<void>) | undefined
yield* WarmingPlugin.Plugin.effect(
host({
model: {
list: () => Effect.succeed({ location: { directory }, data: [{ ...model, settings: { cache } }] }),
default: () => Effect.die("unused model.default"),
transform: () => Effect.die("unused model.transform"),
reload: () => Effect.die("unused model.reload"),
},
event: { subscribe: () => Stream.fromQueue(events) },
session: {
get: () => Effect.succeed(session),
generate: () =>
Clock.currentTimeMillis.pipe(
Effect.map((now) => {
warms.push(now / 60_000)
return {
text: "OK",
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
cost: Money.USD.zero,
}
}),
),
hook: (name, callback) => {
if (name === "context") contextHook = callback as (input: SessionContext) => Effect.Effect<void>
return Effect.succeed({ dispose: Effect.void })
},
},
}),
).pipe(
Effect.provide(
Config.testLayer([new Document({ type: "document", info: Schema.decodeUnknownSync(Info)({ warming }) })]),
),
)
if (!contextHook) return yield* Effect.die("warming plugin did not register a context hook")
const hook = contextHook
/** One whole turn: the step's context hook, then the turn ending. */
const turn = Effect.gen(function* () {
yield* hook({ ...context })
yield* Queue.offer(events, {
id: Event.ID.create(),
created: 0,
durable: { aggregateID: sessionID, seq: Event.Seq.make(0), version: Event.Version.make(1) },
type: "session.execution.succeeded",
data: { sessionID },
})
yield* Effect.yieldNow
})
const at = (minutes: number) => TestClock.setTime(minutes * 60_000).pipe(Effect.andThen(Effect.yieldNow))
return { warms, turn, at }
})
describe("WarmingPlugin interval", () => {
it.effect("warms every 4 minutes when the model has no cache.ttl", () =>
Effect.gen(function* () {
const warming = yield* run(true)
yield* warming.turn
for (const minutes of [4, 8]) yield* warming.at(minutes)
expect(warming.warms).toEqual([4, 8])
}),
)
it.effect("warms at 90% of the model's cache.ttl by default", () =>
Effect.gen(function* () {
const warming = yield* run(true, { ttl: "5m" })
yield* warming.turn
for (const minutes of [4, 4.5, 9]) yield* warming.at(minutes)
expect(warming.warms).toEqual([4.5, 9])
}),
)
it.effect("doesn't warm a cache that outlives the warming window", () =>
Effect.gen(function* () {
const warming = yield* run(true, { ttl: "1h" })
yield* warming.turn
for (const minutes of [4, 27, 54]) yield* warming.at(minutes)
expect(warming.warms).toEqual([])
}),
)
it.effect("keeps an explicit interval", () =>
Effect.gen(function* () {
const warming = yield* run({ interval: "2 minutes" }, { ttl: "5m" })
yield* warming.turn
for (const minutes of [2, 4]) yield* warming.at(minutes)
expect(warming.warms).toEqual([2, 4])
}),
)
})
+1 -1
View File
@@ -11,7 +11,7 @@ export class Info extends Schema.Class<Info>("Config.Warming")({
.pipe(optional)
.annotate({
description:
'Idle time between keep-alive requests (default: "4 minutes"). "auto" uses 90% of the model\'s cache.ttl and skips models without one.',
'Idle time between keep-alive requests (default: "auto"). "auto" uses 90% of the model\'s cache.ttl, or 4 minutes when it has none.',
}),
duration: Schema.DurationFromString.pipe(optional).annotate({
description: 'Time after the last active request to keep a session warm (default: "30 minutes")',
+1 -1
View File
@@ -60,7 +60,7 @@ export const CacheWarming = Schema.Union([
// Settings hold JSON, so durations stay strings here; the warming plugin parses them.
interval: Schema.Union([Schema.Literal("auto"), Schema.String])
.pipe(optional)
.annotate({ description: 'Duration such as "4 minutes", or "auto" for 90% of the model\'s cache.ttl' }),
.annotate({ description: 'Duration such as "4 minutes", or "auto" (default) for 90% of the model\'s cache.ttl' }),
duration: Schema.String.pipe(optional).annotate({ description: 'Duration such as "30 minutes"' }),
}),
]).annotate({ identifier: "Provider.CacheWarming" })
+3 -3
View File
@@ -396,14 +396,14 @@ budgeting and overflow recovery.
### Warming
Keep recently active model sessions warm with periodic transient requests.
Warming is disabled by default; set it to `true` to use the four-minute idle
interval and 30-minute active window.
Warming is disabled by default; set it to `true` to warm just before each
model's cache expires, for up to 30 minutes.
```jsonc
{
"warming": {
"prompt": "Do not perform any work. Reply with exactly: OK",
"interval": "4 minutes",
"interval": "auto",
"duration": "30 minutes",
},
}
+18 -15
View File
@@ -15,9 +15,11 @@ Enable warming in any [OpenCode configuration file](/config):
}
```
With this configuration, OpenCode warms a recently active session after four
minutes without a model request. It continues every four minutes until 30
minutes have passed since the latest non-warming request.
With this configuration, OpenCode warms a recently active session just before
its prompt cache expires: at 90% of the model's
[`cache.ttl`](/models#cache-lifetime), or every four minutes when the model
has none. It continues until 30 minutes have passed since the latest
non-warming request.
## Options
@@ -38,7 +40,7 @@ window:
| Field | Default | Purpose |
| --- | --- | --- |
| `prompt` | Keep-alive instruction | Appended as a transient user message for each warming request. The default asks the model to do no work, use no tools, and reply with `OK`. |
| `interval` | `"4 minutes"` | Idle time before the next warming request, or `"auto"`. |
| `interval` | `"auto"` | Idle time before the next warming request. `"auto"` uses 90% of the model's `cache.ttl`, or four minutes without one. |
| `duration` | `"30 minutes"` | Warming window measured from the latest non-warming model request. |
Omitted object fields keep their defaults. For example, this changes only the
@@ -54,23 +56,23 @@ interval:
## Automatic interval
Set `interval` to `"auto"` to warm just before each model's prompt cache
expires. OpenCode uses 90% of the model's [`cache.ttl`](/models#cache-lifetime) setting, so a
`"5m"` cache is warmed every 4.5 minutes. Models without a `cache.ttl` are not
warmed.
By default, `interval` is `"auto"`, which warms just before each model's
prompt cache expires. OpenCode uses 90% of the model's
[`cache.ttl`](/models#cache-lifetime) setting, so a `"5m"` cache is warmed
every 4.5 minutes. Models without a `cache.ttl` are warmed every four minutes.
```jsonc title="opencode.jsonc"
{
"warming": { "interval": "auto", "duration": "2 hours" },
"warming": { "duration": "2 hours" },
"providers": {
"anthropic": { "settings": { "cache": { "ttl": "1h" } } },
},
}
```
The interval must be shorter than `duration`, so raise `duration` for long
cache lifetimes such as `"1h"`. Otherwise OpenCode skips warming and logs a
warning.
A cache that lasts longer than `duration`, such as `"1h"` with the 30-minute
default, is not warmed, since it outlives the warming window anyway. OpenCode
logs this once per session. Raise `duration` to warm it.
## Per provider and model
@@ -83,7 +85,7 @@ provider setting.
{
"providers": {
"anthropic": {
"settings": { "cache": { "ttl": "5m", "warming": { "interval": "auto" } } },
"settings": { "cache": { "ttl": "5m", "warming": true } },
"models": {
"claude-haiku-5-5": { "settings": { "cache": { "warming": false } } },
},
@@ -172,8 +174,9 @@ costs, count against rate limits, and fail like other model requests.
- Longer durations allow more warming requests.
- Enable warming only when its provider-side benefit is worth the added usage.
For example, the defaults allow up to seven warming attempts during one
uninterrupted 30-minute window: at minutes 4, 8, 12, 16, 20, 24, and 28.
For example, on a model without a `cache.ttl` the defaults allow up to seven
warming attempts during one uninterrupted 30-minute window: at minutes 4, 8,
12, 16, 20, 24, and 28.
## Disabling