From 183007831d9d1f87ac16c762d4000c3fe8809660 Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:55:54 -0600 Subject: [PATCH] fix(core): stop warming a session after a ping misses the cache (#62926) Co-authored-by: Ryan Skidmore <4580448+ryanskidmore@users.noreply.github.com> --- packages/core/src/plugin/warming.ts | 39 +++++++++++--- packages/core/test/plugin/warming.test.ts | 62 +++++++++++++++++++--- services/site/src/docs/content/warming.mdx | 10 ++++ 3 files changed, 97 insertions(+), 14 deletions(-) diff --git a/packages/core/src/plugin/warming.ts b/packages/core/src/plugin/warming.ts index d37d5e8db08..652583438d6 100644 --- a/packages/core/src/plugin/warming.ts +++ b/packages/core/src/plugin/warming.ts @@ -34,6 +34,14 @@ const Decision = Data.taggedEnum() // The interval for "auto" when the model has no cache.ttl, a little under the five minutes most explicit caches last. const fallbackInterval = Duration.minutes(4) +/** Whether a session's warming is sending pings. */ +type Status = Data.TaggedEnum<{ + Active: {} + /** A ping missed the cache, so pinging waits for the session's next non-warming request. */ + StoppedAfterMiss: {} +}> +const Status = Data.taggedEnum() + export const Plugin = define({ id: "opencode.warming", effect: Effect.fn(function* (ctx) { @@ -77,7 +85,7 @@ export const Plugin = define({ }) const scope = yield* Scope.Scope - const sessions = new Map() + const sessions = new Map() // Sessions already told their cache outlives the window; the check runs on every request. const outlived = new Set() // A ping during a model request would compete with the request it keeps warm, which stalls single-slot local models. @@ -126,11 +134,27 @@ export const Plugin = define({ const last = current.last yield* Effect.logInfo("warming session", { sessionID, last }) - yield* ctx.session + const result = yield* ctx.session .generate({ sessionID, prompt: current.settings.prompt }) - .pipe(Effect.catchCause((cause) => Effect.logWarning("failed to warm session", { sessionID, cause }))) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("failed to warm session", { sessionID, cause }).pipe(Effect.as(undefined)), + ), + ) const latest = sessions.get(sessionID) - if (latest === current && latest.last === last) latest.last = yield* Clock.currentTimeMillis + if (latest !== current || latest.last !== last) return yield* loop(sessionID) + // A ping that wrote more to the cache than it read rebuilt the cache instead of keeping it, so later pings + // would keep paying for that. Providers that report no cache usage never count as a miss. + if (result && result.tokens.cache.read < result.tokens.cache.write) { + latest.status = Status.StoppedAfterMiss() + yield* Effect.logInfo("stopped warming session after a cache miss", { + sessionID, + read: result.tokens.cache.read, + write: result.tokens.cache.write, + }) + return + } + latest.last = yield* Clock.currentTimeMillis return yield* loop(sessionID) }) @@ -180,14 +204,15 @@ export const Plugin = define({ const now = yield* Clock.currentTimeMillis const duration = Duration.toMillis(settings.duration) - if (active) { + if (active && Status.$is("Active")(active.status)) { active.last = now active.expires = now + duration active.settings = settings return } - sessions.set(event.sessionID, { last: now, expires: now + duration, settings }) - yield* Effect.logInfo("scheduled session warming", { + // A session stopped after a miss has no running loop, so its next real request starts one. + sessions.set(event.sessionID, { last: now, expires: now + duration, settings, status: Status.Active() }) + yield* Effect.logInfo(active ? "resumed session warming" : "scheduled session warming", { sessionID: event.sessionID, interval: settings.interval, expires: now + duration, diff --git a/packages/core/test/plugin/warming.test.ts b/packages/core/test/plugin/warming.test.ts index 3f9c32afe1a..0b78560c7e3 100644 --- a/packages/core/test/plugin/warming.test.ts +++ b/packages/core/test/plugin/warming.test.ts @@ -41,8 +41,19 @@ const context: SessionContext = { options: {}, } -/** Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. */ -const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cache) { +const hit = { read: 1000, write: 10 } +const miss = { read: 0, write: 1000 } +const unreported = { read: 0, write: 0 } + +/** + * Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. Each ping + * reports the next cache usage in `usage`. + */ +const run = Effect.fnUntraced(function* ( + warming: unknown, + cache?: Provider.Cache, + usage: ReadonlyArray = [], +) { const warms: number[] = [] const events = yield* Queue.unbounded() let contextHook: ((input: SessionContext) => Effect.Effect) | undefined @@ -60,12 +71,9 @@ const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cach generate: () => Clock.currentTimeMillis.pipe( Effect.map((now) => { + const cache = usage[warms.length] ?? unreported warms.push(now / 60_000) - return { - text: "OK", - tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, - cost: Money.USD.zero, - } + return { text: "OK", tokens: { input: 0, output: 1, reasoning: 0, cache }, cost: Money.USD.zero } }), ), hook: (name, callback) => { @@ -134,3 +142,43 @@ describe("WarmingPlugin interval", () => { }), ) }) + +describe("WarmingPlugin cache miss", () => { + it.effect("keeps warming after a ping hits the cache", () => + Effect.gen(function* () { + const warming = yield* run(true, undefined, [hit, hit, hit]) + yield* warming.turn + for (const minutes of [4, 8, 12]) yield* warming.at(minutes) + expect(warming.warms).toEqual([4, 8, 12]) + }), + ) + + it.effect("stops warming after a ping misses the cache", () => + Effect.gen(function* () { + const warming = yield* run(true, undefined, [hit, miss, hit]) + yield* warming.turn + for (const minutes of [4, 8, 12, 16]) yield* warming.at(minutes) + expect(warming.warms).toEqual([4, 8]) + }), + ) + + it.effect("resumes warming on the next real request after a miss", () => + Effect.gen(function* () { + const warming = yield* run(true, undefined, [miss, hit, hit]) + yield* warming.turn + for (const minutes of [4, 8]) yield* warming.at(minutes) + yield* warming.turn + for (const minutes of [12, 16]) yield* warming.at(minutes) + expect(warming.warms).toEqual([4, 12, 16]) + }), + ) + + it.effect("doesn't count a ping without cache usage as a miss", () => + Effect.gen(function* () { + const warming = yield* run(true, undefined, [unreported, unreported]) + yield* warming.turn + for (const minutes of [4, 8]) yield* warming.at(minutes) + expect(warming.warms).toEqual([4, 8]) + }), + ) +}) diff --git a/services/site/src/docs/content/warming.mdx b/services/site/src/docs/content/warming.mdx index e13c1ac0323..b4c9e0af6f1 100644 --- a/services/site/src/docs/content/warming.mdx +++ b/services/site/src/docs/content/warming.mdx @@ -165,6 +165,16 @@ window continues to count down. 00:08 next warming attempt ``` +A warming request that misses the cache, reading less from it than it writes, +stops warming for that session until its next non-warming request. Providers +that don't report cache usage never count as a miss. + +```text +00:04 warming request misses the cache; warming stops +00:10 normal model request; warming resumes +00:14 warming request +``` + ## Costs Warming requests are real provider requests. They can consume tokens, incur