mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-20 01:08:31 +03:00
fix(core): stop warming a session after a ping misses the cache (#62926)
Co-authored-by: Ryan Skidmore <4580448+ryanskidmore@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
Unverified
parent
1aee095479
commit
183007831d
@@ -34,6 +34,14 @@ const Decision = Data.taggedEnum<Decision>()
|
||||
// The interval for "auto" when the model has no cache.ttl, a little under the five minutes most explicit caches last.
|
||||
const fallbackInterval = Duration.minutes(4)
|
||||
|
||||
/** Whether a session's warming is sending pings. */
|
||||
type Status = Data.TaggedEnum<{
|
||||
Active: {}
|
||||
/** A ping missed the cache, so pinging waits for the session's next non-warming request. */
|
||||
StoppedAfterMiss: {}
|
||||
}>
|
||||
const Status = Data.taggedEnum<Status>()
|
||||
|
||||
export const Plugin = define({
|
||||
id: "opencode.warming",
|
||||
effect: Effect.fn(function* (ctx) {
|
||||
@@ -77,7 +85,7 @@ export const Plugin = define({
|
||||
})
|
||||
|
||||
const scope = yield* Scope.Scope
|
||||
const sessions = new Map<Session.ID, { last: number; expires: number; settings: Settings }>()
|
||||
const sessions = new Map<Session.ID, { last: number; expires: number; settings: Settings; status: Status }>()
|
||||
// Sessions already told their cache outlives the window; the check runs on every request.
|
||||
const outlived = new Set<Session.ID>()
|
||||
// A ping during a model request would compete with the request it keeps warm, which stalls single-slot local models.
|
||||
@@ -126,11 +134,27 @@ export const Plugin = define({
|
||||
|
||||
const last = current.last
|
||||
yield* Effect.logInfo("warming session", { sessionID, last })
|
||||
yield* ctx.session
|
||||
const result = yield* ctx.session
|
||||
.generate({ sessionID, prompt: current.settings.prompt })
|
||||
.pipe(Effect.catchCause((cause) => Effect.logWarning("failed to warm session", { sessionID, cause })))
|
||||
.pipe(
|
||||
Effect.catchCause((cause) =>
|
||||
Effect.logWarning("failed to warm session", { sessionID, cause }).pipe(Effect.as(undefined)),
|
||||
),
|
||||
)
|
||||
const latest = sessions.get(sessionID)
|
||||
if (latest === current && latest.last === last) latest.last = yield* Clock.currentTimeMillis
|
||||
if (latest !== current || latest.last !== last) return yield* loop(sessionID)
|
||||
// A ping that wrote more to the cache than it read rebuilt the cache instead of keeping it, so later pings
|
||||
// would keep paying for that. Providers that report no cache usage never count as a miss.
|
||||
if (result && result.tokens.cache.read < result.tokens.cache.write) {
|
||||
latest.status = Status.StoppedAfterMiss()
|
||||
yield* Effect.logInfo("stopped warming session after a cache miss", {
|
||||
sessionID,
|
||||
read: result.tokens.cache.read,
|
||||
write: result.tokens.cache.write,
|
||||
})
|
||||
return
|
||||
}
|
||||
latest.last = yield* Clock.currentTimeMillis
|
||||
return yield* loop(sessionID)
|
||||
})
|
||||
|
||||
@@ -180,14 +204,15 @@ export const Plugin = define({
|
||||
|
||||
const now = yield* Clock.currentTimeMillis
|
||||
const duration = Duration.toMillis(settings.duration)
|
||||
if (active) {
|
||||
if (active && Status.$is("Active")(active.status)) {
|
||||
active.last = now
|
||||
active.expires = now + duration
|
||||
active.settings = settings
|
||||
return
|
||||
}
|
||||
sessions.set(event.sessionID, { last: now, expires: now + duration, settings })
|
||||
yield* Effect.logInfo("scheduled session warming", {
|
||||
// A session stopped after a miss has no running loop, so its next real request starts one.
|
||||
sessions.set(event.sessionID, { last: now, expires: now + duration, settings, status: Status.Active() })
|
||||
yield* Effect.logInfo(active ? "resumed session warming" : "scheduled session warming", {
|
||||
sessionID: event.sessionID,
|
||||
interval: settings.interval,
|
||||
expires: now + duration,
|
||||
|
||||
@@ -41,8 +41,19 @@ const context: SessionContext = {
|
||||
options: {},
|
||||
}
|
||||
|
||||
/** Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. */
|
||||
const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cache) {
|
||||
const hit = { read: 1000, write: 10 }
|
||||
const miss = { read: 0, write: 1000 }
|
||||
const unreported = { read: 0, write: 0 }
|
||||
|
||||
/**
|
||||
* Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. Each ping
|
||||
* reports the next cache usage in `usage`.
|
||||
*/
|
||||
const run = Effect.fnUntraced(function* (
|
||||
warming: unknown,
|
||||
cache?: Provider.Cache,
|
||||
usage: ReadonlyArray<typeof hit> = [],
|
||||
) {
|
||||
const warms: number[] = []
|
||||
const events = yield* Queue.unbounded<SessionEvent.Execution.Succeeded>()
|
||||
let contextHook: ((input: SessionContext) => Effect.Effect<void>) | undefined
|
||||
@@ -60,12 +71,9 @@ const run = Effect.fnUntraced(function* (warming: unknown, cache?: Provider.Cach
|
||||
generate: () =>
|
||||
Clock.currentTimeMillis.pipe(
|
||||
Effect.map((now) => {
|
||||
const cache = usage[warms.length] ?? unreported
|
||||
warms.push(now / 60_000)
|
||||
return {
|
||||
text: "OK",
|
||||
tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
|
||||
cost: Money.USD.zero,
|
||||
}
|
||||
return { text: "OK", tokens: { input: 0, output: 1, reasoning: 0, cache }, cost: Money.USD.zero }
|
||||
}),
|
||||
),
|
||||
hook: (name, callback) => {
|
||||
@@ -134,3 +142,43 @@ describe("WarmingPlugin interval", () => {
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
describe("WarmingPlugin cache miss", () => {
|
||||
it.effect("keeps warming after a ping hits the cache", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, undefined, [hit, hit, hit])
|
||||
yield* warming.turn
|
||||
for (const minutes of [4, 8, 12]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([4, 8, 12])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("stops warming after a ping misses the cache", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, undefined, [hit, miss, hit])
|
||||
yield* warming.turn
|
||||
for (const minutes of [4, 8, 12, 16]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([4, 8])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("resumes warming on the next real request after a miss", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, undefined, [miss, hit, hit])
|
||||
yield* warming.turn
|
||||
for (const minutes of [4, 8]) yield* warming.at(minutes)
|
||||
yield* warming.turn
|
||||
for (const minutes of [12, 16]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([4, 12, 16])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("doesn't count a ping without cache usage as a miss", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, undefined, [unreported, unreported])
|
||||
yield* warming.turn
|
||||
for (const minutes of [4, 8]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([4, 8])
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
@@ -165,6 +165,16 @@ window continues to count down.
|
||||
00:08 next warming attempt
|
||||
```
|
||||
|
||||
A warming request that misses the cache, reading less from it than it writes,
|
||||
stops warming for that session until its next non-warming request. Providers
|
||||
that don't report cache usage never count as a miss.
|
||||
|
||||
```text
|
||||
00:04 warming request misses the cache; warming stops
|
||||
00:10 normal model request; warming resumes
|
||||
00:14 warming request
|
||||
```
|
||||
|
||||
## Costs
|
||||
|
||||
Warming requests are real provider requests. They can consume tokens, incur
|
||||
|
||||
Reference in New Issue
Block a user