fix(core): stop warming a session after a ping misses the cache (#62926)

Co-authored-by: Ryan Skidmore <4580448+ryanskidmore@users.noreply.github.com>
This commit is contained in:
Aiden Cline
2026-11-09 14:55:54 -06:00
committed by GitHub
Unverified
parent 1aee095479
commit 183007831d
3 changed files with 97 additions and 14 deletions
+32 -7
View File
@@ -34,6 +34,14 @@ const Decision = Data.taggedEnum<Decision>()
// The interval for "auto" when the model has no cache.ttl, a little under the five minutes most explicit caches last.
const fallbackInterval = Duration.minutes(4)
/** Whether a session's warming is sending pings. */
type Status = Data.TaggedEnum<{
Active: {}
/** A ping missed the cache, so pinging waits for the session's next non-warming request. */
StoppedAfterMiss: {}
}>
const Status = Data.taggedEnum<Status>()
export const Plugin = define({
id: "opencode.warming",
effect: Effect.fn(function* (ctx) {
@@ -77,7 +85,7 @@ export const Plugin = define({
})
const scope = yield* Scope.Scope
const sessions = new Map<Session.ID, { last: number; expires: number; settings: Settings }>()
const sessions = new Map<Session.ID, { last: number; expires: number; settings: Settings; status: Status }>()
// Sessions already told their cache outlives the window; the check runs on every request.
const outlived = new Set<Session.ID>()
// A ping during a model request would compete with the request it keeps warm, which stalls single-slot local models.
@@ -126,11 +134,27 @@ export const Plugin = define({
const last = current.last
yield* Effect.logInfo("warming session", { sessionID, last })
yield* ctx.session
const result = yield* ctx.session
.generate({ sessionID, prompt: current.settings.prompt })
.pipe(Effect.catchCause((cause) => Effect.logWarning("failed to warm session", { sessionID, cause })))
.pipe(
Effect.catchCause((cause) =>
Effect.logWarning("failed to warm session", { sessionID, cause }).pipe(Effect.as(undefined)),
),
)
const latest = sessions.get(sessionID)
if (latest === current && latest.last === last) latest.last = yield* Clock.currentTimeMillis
if (latest !== current || latest.last !== last) return yield* loop(sessionID)
// A ping that wrote more to the cache than it read rebuilt the cache instead of keeping it, so later pings
// would keep paying for that. Providers that report no cache usage never count as a miss.
if (result && result.tokens.cache.read < result.tokens.cache.write) {
latest.status = Status.StoppedAfterMiss()
yield* Effect.logInfo("stopped warming session after a cache miss", {
sessionID,
read: result.tokens.cache.read,
write: result.tokens.cache.write,
})
return
}
latest.last = yield* Clock.currentTimeMillis
return yield* loop(sessionID)
})
@@ -180,14 +204,15 @@ export const Plugin = define({
const now = yield* Clock.currentTimeMillis
const duration = Duration.toMillis(settings.duration)
if (active) {
if (active && Status.$is("Active")(active.status)) {
active.last = now
active.expires = now + duration
active.settings = settings
return
}
sessions.set(event.sessionID, { last: now, expires: now + duration, settings })
yield* Effect.logInfo("scheduled session warming", {
// A session stopped after a miss has no running loop, so its next real request starts one.
sessions.set(event.sessionID, { last: now, expires: now + duration, settings, status: Status.Active() })
yield* Effect.logInfo(active ? "resumed session warming" : "scheduled session warming", {
sessionID: event.sessionID,
interval: settings.interval,
expires: now + duration,
+55 -7
View File
@@ -41,8 +41,19 @@ const context: SessionContext = {
options: {},
}
/** Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. */
const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cache) {
const hit = { read: 1000, write: 10 }
const miss = { read: 0, write: 1000 }
const unreported = { read: 0, write: 0 }
/**
* Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. Each ping
* reports the next cache usage in `usage`.
*/
const run = Effect.fnUntraced(function* (
warming: unknown,
cache?: Provider.Cache,
usage: ReadonlyArray<typeof hit> = [],
) {
const warms: number[] = []
const events = yield* Queue.unbounded<SessionEvent.Execution.Succeeded>()
let contextHook: ((input: SessionContext) => Effect.Effect<void>) | undefined
@@ -60,12 +71,9 @@ const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cach
generate: () =>
Clock.currentTimeMillis.pipe(
Effect.map((now) => {
const cache = usage[warms.length] ?? unreported
warms.push(now / 60_000)
return {
text: "OK",
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
cost: Money.USD.zero,
}
return { text: "OK", tokens: { input: 0, output: 1, reasoning: 0, cache }, cost: Money.USD.zero }
}),
),
hook: (name, callback) => {
@@ -134,3 +142,43 @@ describe("WarmingPlugin interval", () => {
}),
)
})
describe("WarmingPlugin cache miss", () => {
it.effect("keeps warming after a ping hits the cache", () =>
Effect.gen(function* () {
const warming = yield* run(true, undefined, [hit, hit, hit])
yield* warming.turn
for (const minutes of [4, 8, 12]) yield* warming.at(minutes)
expect(warming.warms).toEqual([4, 8, 12])
}),
)
it.effect("stops warming after a ping misses the cache", () =>
Effect.gen(function* () {
const warming = yield* run(true, undefined, [hit, miss, hit])
yield* warming.turn
for (const minutes of [4, 8, 12, 16]) yield* warming.at(minutes)
expect(warming.warms).toEqual([4, 8])
}),
)
it.effect("resumes warming on the next real request after a miss", () =>
Effect.gen(function* () {
const warming = yield* run(true, undefined, [miss, hit, hit])
yield* warming.turn
for (const minutes of [4, 8]) yield* warming.at(minutes)
yield* warming.turn
for (const minutes of [12, 16]) yield* warming.at(minutes)
expect(warming.warms).toEqual([4, 12, 16])
}),
)
it.effect("doesn't count a ping without cache usage as a miss", () =>
Effect.gen(function* () {
const warming = yield* run(true, undefined, [unreported, unreported])
yield* warming.turn
for (const minutes of [4, 8]) yield* warming.at(minutes)
expect(warming.warms).toEqual([4, 8])
}),
)
})
@@ -165,6 +165,16 @@ window continues to count down.
00:08 next warming attempt
```
A warming request that misses the cache, reading less from it than it writes,
stops warming for that session until its next non-warming request. Providers
that don't report cache usage never count as a miss.
```text
00:04 warming request misses the cache; warming stops
00:10 normal model request; warming resumes
00:14 warming request
```
## Costs
Warming requests are real provider requests. They can consume tokens, incur