diff --git a/packages/core/src/plugin/warming.ts b/packages/core/src/plugin/warming.ts index ff00c9bf3ca..d37d5e8db08 100644 --- a/packages/core/src/plugin/warming.ts +++ b/packages/core/src/plugin/warming.ts @@ -5,7 +5,7 @@ import type { SessionHooks } from "@opencode/plugin/effect/session" import type { Provider } from "@opencode/schema/provider" import type { Session } from "@opencode/schema/session" import { ConfigWarming } from "@opencode/schema/config/warming" -import { Clock, Duration, Effect, Option, Schema, Scope, Stream } from "effect" +import { Clock, Data, Duration, Effect, Option, Schema, Scope, Stream } from "effect" import { Config } from "../config.js" import { resolveModel } from "./optimize.js" @@ -13,16 +13,33 @@ const decodeWarming = Schema.decodeUnknownOption(ConfigWarming.Warming) const defaults = { prompt: "This is a keep-alive request. Do not perform any work or use tools. Reply with exactly: OK", - interval: Duration.minutes(4), + interval: "auto" as "auto" | Duration.Duration, duration: Duration.minutes(30), } +type Settings = typeof defaults & { interval: Duration.Duration } + +/** Whether a session is warmed after a request, and why not. */ +type Decision = Data.TaggedEnum<{ + Warm: { readonly settings: Settings } + /** Neither the top-level `warming` nor the model's `cache.warming` turns it on. */ + Off: {} + /** `"auto"` timing on a cache that lasts longer than `duration`, so no ping is needed before the window closes. */ + CacheOutlivesWindow: {} + /** An explicit interval that isn't positive and shorter than a finite `duration`. */ + InvalidInterval: {} +}> +const Decision = Data.taggedEnum() + +// The interval for "auto" when the model has no cache.ttl, a little under the five minutes most explicit caches last. +const fallbackInterval = Duration.minutes(4) + export const Plugin = define({ id: "opencode.warming", effect: Effect.fn(function* (ctx) { const config = yield* Config.Service // A model's or provider's `cache.warming` overrides the top-level `warming`. - const loadSettings = Effect.fn("WarmingPlugin.loadSettings")(function* (event: SessionHooks["context"]) { + const decide = Effect.fn("WarmingPlugin.decide")(function* (event: SessionHooks["context"]) { // Warming must never fail the request it hooks, so an unreadable catalog falls back to the top-level setting. const cache = (yield* resolveModel(ctx, event).pipe(Effect.orElseSucceed(() => undefined)))?.settings?.cache const local = cache?.warming === undefined ? undefined : Option.getOrUndefined(decodeWarming(cache.warming)) @@ -31,30 +48,38 @@ export const Plugin = define({ model: event.model, }) const global = Config.latest(yield* config.entries(), "warming") - if (!(local ?? global)) return + if (!(local ?? global)) return Decision.Off() // The keep-alive prompt is only set at the top level; a model's setting changes timing. const settings = { ...defaults, ...(typeof global === "object" ? global : {}), ...(typeof local === "object" ? local : {}), } - const interval = settings.interval === "auto" ? autoInterval(cache?.ttl) : settings.interval - // "auto" doesn't warm a model without a cache.ttl. - if (interval === undefined) return const { duration } = settings + const interval = settings.interval === "auto" ? (autoInterval(cache?.ttl) ?? fallbackInterval) : settings.interval + // A cache that outlives the window, such as "1h" with the 30-minute default, needs no keep-alive. + if ( + settings.interval === "auto" && + Duration.isFinite(duration) && + Duration.isGreaterThanOrEqualTo(interval, duration) + ) + return Decision.CacheOutlivesWindow() const valid = Duration.isFinite(duration) && Duration.isGreaterThan(interval, Duration.zero) && Duration.isLessThan(interval, duration) - if (valid) return { ...settings, interval } + if (valid) return Decision.Warm({ settings: { ...settings, interval } }) yield* Effect.logWarning("warming interval must be positive and shorter than a finite duration", { interval, duration, }) + return Decision.InvalidInterval() }) const scope = yield* Scope.Scope - const sessions = new Map() + const sessions = new Map() + // Sessions already told their cache outlives the window; the check runs on every request. + const outlived = new Set() // A ping during a model request would compete with the request it keeps warm, which stalls single-slot local models. // A request counts from its context hook, because `session.step.started` waits for the first response chunk. const requesting = new Set() @@ -80,36 +105,34 @@ export const Plugin = define({ ), Effect.forkScoped({ startImmediately: true }), ) - const loop: (sessionID: Session.ID) => Effect.Effect = Effect.fn("WarmingPlugin.loop")( - function* (sessionID) { - const current = sessions.get(sessionID) - if (!current) return + const loop: (sessionID: Session.ID) => Effect.Effect = Effect.fn("WarmingPlugin.loop")(function* (sessionID) { + const current = sessions.get(sessionID) + if (!current) return - const now = yield* Clock.currentTimeMillis - const next = Math.min(current.last + Duration.toMillis(current.settings.interval), current.expires) - if (now < next) { - yield* Effect.sleep(Duration.millis(next - now)) - return yield* loop(sessionID) - } - if (now >= current.expires) { - sessions.delete(sessionID) - return - } - if (requesting.has(sessionID)) { - yield* Effect.sleep(current.settings.interval) - return yield* loop(sessionID) - } - - const last = current.last - yield* Effect.logInfo("warming session", { sessionID, last }) - yield* ctx.session - .generate({ sessionID, prompt: current.settings.prompt }) - .pipe(Effect.catchCause((cause) => Effect.logWarning("failed to warm session", { sessionID, cause }))) - const latest = sessions.get(sessionID) - if (latest === current && latest.last === last) latest.last = yield* Clock.currentTimeMillis + const now = yield* Clock.currentTimeMillis + const next = Math.min(current.last + Duration.toMillis(current.settings.interval), current.expires) + if (now < next) { + yield* Effect.sleep(Duration.millis(next - now)) return yield* loop(sessionID) - }, - ) + } + if (now >= current.expires) { + sessions.delete(sessionID) + return + } + if (requesting.has(sessionID)) { + yield* Effect.sleep(current.settings.interval) + return yield* loop(sessionID) + } + + const last = current.last + yield* Effect.logInfo("warming session", { sessionID, last }) + yield* ctx.session + .generate({ sessionID, prompt: current.settings.prompt }) + .pipe(Effect.catchCause((cause) => Effect.logWarning("failed to warm session", { sessionID, cause }))) + const latest = sessions.get(sessionID) + if (latest === current && latest.last === last) latest.last = yield* Clock.currentTimeMillis + return yield* loop(sessionID) + }) const hook = (event: SessionHooks["context"]) => Effect.gen(function* () { @@ -117,11 +140,31 @@ export const Plugin = define({ if (session.parentID) return const active = sessions.get(event.sessionID) - const settings = yield* loadSettings(event) - if (!settings) { + const stop = Effect.sync(() => { sessions.delete(event.sessionID) - return - } + return undefined + }) + const settings = yield* Decision.$match(yield* decide(event), { + Warm: (decision) => + Effect.sync(() => { + outlived.delete(event.sessionID) + return decision.settings + }), + Off: () => stop, + InvalidInterval: () => stop, + CacheOutlivesWindow: () => + Effect.gen(function* () { + if (!outlived.has(event.sessionID)) { + outlived.add(event.sessionID) + yield* Effect.logInfo("not warming: the prompt cache outlives the warming window", { + sessionID: event.sessionID, + model: event.model, + }) + } + return yield* stop + }), + }) + if (!settings) return // Once generate exposes request metadata to context hooks, tag warm requests instead of matching the prompt. const message = event.messages.at(-1) diff --git a/packages/core/test/plugin/warming.test.ts b/packages/core/test/plugin/warming.test.ts new file mode 100644 index 00000000000..3f9c32afe1a --- /dev/null +++ b/packages/core/test/plugin/warming.test.ts @@ -0,0 +1,136 @@ +import { describe, expect } from "bun:test" +import type { SessionContext } from "@opencode/plugin/effect/session" +import { Document, Info } from "@opencode/schema/config" +import { Event } from "@opencode/schema/event" +import { Money } from "@opencode/schema/money" +import { Agent } from "@opencode/core/agent" +import { Config } from "@opencode/core/config" +import { Location } from "@opencode/core/location" +import { Model } from "@opencode/core/model" +import { WarmingPlugin } from "@opencode/core/plugin/warming" +import { Project } from "@opencode/core/project" +import { Provider } from "@opencode/core/provider" +import { AbsolutePath } from "@opencode/core/schema" +import { Session } from "@opencode/core/session" +import { SessionEvent } from "@opencode/core/session/event" +import { Clock, DateTime, Effect, Queue, Schema, Stream } from "effect" +import { TestClock } from "effect/testing" +import { it } from "../lib/effect" +import { host } from "./host" + +const sessionID = Session.ID.make("ses_warming_test") +const directory = AbsolutePath.make("/project") +const model = Model.Info.default(Provider.ID.make("test"), Model.ID.make("test-model")) + +const session = Session.Info.make({ + id: sessionID, + projectID: Project.ID.global, + cost: Money.USD.zero, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: DateTime.makeUnsafe(0), updated: DateTime.makeUnsafe(0) }, + location: Location.Ref.make({ directory }), +}) + +const context: SessionContext = { + sessionID, + agent: Agent.ID.make("build"), + model: { id: model.id, providerID: model.providerID }, + system: [], + messages: [], + tools: {}, + options: {}, +} + +/** Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. */ +const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cache) { + const warms: number[] = [] + const events = yield* Queue.unbounded() + let contextHook: ((input: SessionContext) => Effect.Effect) | undefined + yield* WarmingPlugin.Plugin.effect( + host({ + model: { + list: () => Effect.succeed({ location: { directory }, data: [{ ...model, settings: { cache } }] }), + default: () => Effect.die("unused model.default"), + transform: () => Effect.die("unused model.transform"), + reload: () => Effect.die("unused model.reload"), + }, + event: { subscribe: () => Stream.fromQueue(events) }, + session: { + get: () => Effect.succeed(session), + generate: () => + Clock.currentTimeMillis.pipe( + Effect.map((now) => { + warms.push(now / 60_000) + return { + text: "OK", + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + cost: Money.USD.zero, + } + }), + ), + hook: (name, callback) => { + if (name === "context") contextHook = callback as (input: SessionContext) => Effect.Effect + return Effect.succeed({ dispose: Effect.void }) + }, + }, + }), + ).pipe( + Effect.provide( + Config.testLayer([new Document({ type: "document", info: Schema.decodeUnknownSync(Info)({ warming }) })]), + ), + ) + if (!contextHook) return yield* Effect.die("warming plugin did not register a context hook") + const hook = contextHook + /** One whole turn: the step's context hook, then the turn ending. */ + const turn = Effect.gen(function* () { + yield* hook({ ...context }) + yield* Queue.offer(events, { + id: Event.ID.create(), + created: 0, + durable: { aggregateID: sessionID, seq: Event.Seq.make(0), version: Event.Version.make(1) }, + type: "session.execution.succeeded", + data: { sessionID }, + }) + yield* Effect.yieldNow + }) + const at = (minutes: number) => TestClock.setTime(minutes * 60_000).pipe(Effect.andThen(Effect.yieldNow)) + return { warms, turn, at } +}) + +describe("WarmingPlugin interval", () => { + it.effect("warms every 4 minutes when the model has no cache.ttl", () => + Effect.gen(function* () { + const warming = yield* run(true) + yield* warming.turn + for (const minutes of [4, 8]) yield* warming.at(minutes) + expect(warming.warms).toEqual([4, 8]) + }), + ) + + it.effect("warms at 90% of the model's cache.ttl by default", () => + Effect.gen(function* () { + const warming = yield* run(true, { ttl: "5m" }) + yield* warming.turn + for (const minutes of [4, 4.5, 9]) yield* warming.at(minutes) + expect(warming.warms).toEqual([4.5, 9]) + }), + ) + + it.effect("doesn't warm a cache that outlives the warming window", () => + Effect.gen(function* () { + const warming = yield* run(true, { ttl: "1h" }) + yield* warming.turn + for (const minutes of [4, 27, 54]) yield* warming.at(minutes) + expect(warming.warms).toEqual([]) + }), + ) + + it.effect("keeps an explicit interval", () => + Effect.gen(function* () { + const warming = yield* run({ interval: "2 minutes" }, { ttl: "5m" }) + yield* warming.turn + for (const minutes of [2, 4]) yield* warming.at(minutes) + expect(warming.warms).toEqual([2, 4]) + }), + ) +}) diff --git a/packages/schema/src/config/warming.ts b/packages/schema/src/config/warming.ts index 65f629b490b..a1ec8ba95b5 100644 --- a/packages/schema/src/config/warming.ts +++ b/packages/schema/src/config/warming.ts @@ -11,7 +11,7 @@ export class Info extends Schema.Class("Config.Warming")({ .pipe(optional) .annotate({ description: - 'Idle time between keep-alive requests (default: "4 minutes"). "auto" uses 90% of the model\'s cache.ttl and skips models without one.', + 'Idle time between keep-alive requests (default: "auto"). "auto" uses 90% of the model\'s cache.ttl, or 4 minutes when it has none.', }), duration: Schema.DurationFromString.pipe(optional).annotate({ description: 'Time after the last active request to keep a session warm (default: "30 minutes")', diff --git a/packages/schema/src/provider.ts b/packages/schema/src/provider.ts index 9985e7402cf..7248d46cb40 100644 --- a/packages/schema/src/provider.ts +++ b/packages/schema/src/provider.ts @@ -60,7 +60,7 @@ export const CacheWarming = Schema.Union([ // Settings hold JSON, so durations stay strings here; the warming plugin parses them. interval: Schema.Union([Schema.Literal("auto"), Schema.String]) .pipe(optional) - .annotate({ description: 'Duration such as "4 minutes", or "auto" for 90% of the model\'s cache.ttl' }), + .annotate({ description: 'Duration such as "4 minutes", or "auto" (default) for 90% of the model\'s cache.ttl' }), duration: Schema.String.pipe(optional).annotate({ description: 'Duration such as "30 minutes"' }), }), ]).annotate({ identifier: "Provider.CacheWarming" }) diff --git a/services/site/src/docs/content/config.mdx b/services/site/src/docs/content/config.mdx index c8aa28427c4..0cded3ea2da 100644 --- a/services/site/src/docs/content/config.mdx +++ b/services/site/src/docs/content/config.mdx @@ -396,14 +396,14 @@ budgeting and overflow recovery. ### Warming Keep recently active model sessions warm with periodic transient requests. -Warming is disabled by default; set it to `true` to use the four-minute idle -interval and 30-minute active window. +Warming is disabled by default; set it to `true` to warm just before each +model's cache expires, for up to 30 minutes. ```jsonc { "warming": { "prompt": "Do not perform any work. Reply with exactly: OK", - "interval": "4 minutes", + "interval": "auto", "duration": "30 minutes", }, } diff --git a/services/site/src/docs/content/warming.mdx b/services/site/src/docs/content/warming.mdx index 912b19194a9..e13c1ac0323 100644 --- a/services/site/src/docs/content/warming.mdx +++ b/services/site/src/docs/content/warming.mdx @@ -15,9 +15,11 @@ Enable warming in any [OpenCode configuration file](/config): } ``` -With this configuration, OpenCode warms a recently active session after four -minutes without a model request. It continues every four minutes until 30 -minutes have passed since the latest non-warming request. +With this configuration, OpenCode warms a recently active session just before +its prompt cache expires: at 90% of the model's +[`cache.ttl`](/models#cache-lifetime), or every four minutes when the model +has none. It continues until 30 minutes have passed since the latest +non-warming request. ## Options @@ -38,7 +40,7 @@ window: | Field | Default | Purpose | | --- | --- | --- | | `prompt` | Keep-alive instruction | Appended as a transient user message for each warming request. The default asks the model to do no work, use no tools, and reply with `OK`. | -| `interval` | `"4 minutes"` | Idle time before the next warming request, or `"auto"`. | +| `interval` | `"auto"` | Idle time before the next warming request. `"auto"` uses 90% of the model's `cache.ttl`, or four minutes without one. | | `duration` | `"30 minutes"` | Warming window measured from the latest non-warming model request. | Omitted object fields keep their defaults. For example, this changes only the @@ -54,23 +56,23 @@ interval: ## Automatic interval -Set `interval` to `"auto"` to warm just before each model's prompt cache -expires. OpenCode uses 90% of the model's [`cache.ttl`](/models#cache-lifetime) setting, so a -`"5m"` cache is warmed every 4.5 minutes. Models without a `cache.ttl` are not -warmed. +By default, `interval` is `"auto"`, which warms just before each model's +prompt cache expires. OpenCode uses 90% of the model's +[`cache.ttl`](/models#cache-lifetime) setting, so a `"5m"` cache is warmed +every 4.5 minutes. Models without a `cache.ttl` are warmed every four minutes. ```jsonc title="opencode.jsonc" { - "warming": { "interval": "auto", "duration": "2 hours" }, + "warming": { "duration": "2 hours" }, "providers": { "anthropic": { "settings": { "cache": { "ttl": "1h" } } }, }, } ``` -The interval must be shorter than `duration`, so raise `duration` for long -cache lifetimes such as `"1h"`. Otherwise OpenCode skips warming and logs a -warning. +A cache that lasts longer than `duration`, such as `"1h"` with the 30-minute +default, is not warmed, since it outlives the warming window anyway. OpenCode +logs this once per session. Raise `duration` to warm it. ## Per provider and model @@ -83,7 +85,7 @@ provider setting. { "providers": { "anthropic": { - "settings": { "cache": { "ttl": "5m", "warming": { "interval": "auto" } } }, + "settings": { "cache": { "ttl": "5m", "warming": true } }, "models": { "claude-haiku-5-5": { "settings": { "cache": { "warming": false } } }, }, @@ -172,8 +174,9 @@ costs, count against rate limits, and fail like other model requests. - Longer durations allow more warming requests. - Enable warming only when its provider-side benefit is worth the added usage. -For example, the defaults allow up to seven warming attempts during one -uninterrupted 30-minute window: at minutes 4, 8, 12, 16, 20, 24, and 28. +For example, on a model without a `cache.ttl` the defaults allow up to seven +warming attempts during one uninterrupted 30-minute window: at minutes 4, 8, +12, 16, 20, 24, and 28. ## Disabling