mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 05:01:03 +03:00
fix(core): count the warming interval from when the request was sent (#62941)
This commit is contained in:
committed by
GitHub
Unverified
parent
183007831d
commit
f7a0684bee
@@ -5,7 +5,7 @@ import type { SessionHooks } from "@opencode/plugin/effect/session"
|
||||
import type { Provider } from "@opencode/schema/provider"
|
||||
import type { Session } from "@opencode/schema/session"
|
||||
import { ConfigWarming } from "@opencode/schema/config/warming"
|
||||
import { Clock, Data, Duration, Effect, Option, Schema, Scope, Stream } from "effect"
|
||||
import { Clock, Data, Deferred, Duration, Effect, Option, Schema, Scope, Stream } from "effect"
|
||||
import { Config } from "../config.js"
|
||||
import { resolveModel } from "./optimize.js"
|
||||
|
||||
@@ -17,7 +17,8 @@ const defaults = {
|
||||
duration: Duration.minutes(30),
|
||||
}
|
||||
|
||||
type Settings = typeof defaults & { interval: Duration.Duration }
|
||||
/** `ttl` is how long the provider keeps a cache entry after the request that last sent it. */
|
||||
type Settings = typeof defaults & { interval: Duration.Duration; ttl: Duration.Duration }
|
||||
|
||||
/** Whether a session is warmed after a request, and why not. */
|
||||
type Decision = Data.TaggedEnum<{
|
||||
@@ -33,6 +34,7 @@ const Decision = Data.taggedEnum<Decision>()
|
||||
|
||||
// The interval for "auto" when the model has no cache.ttl, a little under the five minutes most explicit caches last.
|
||||
const fallbackInterval = Duration.minutes(4)
|
||||
const fallbackTTL = Duration.minutes(5)
|
||||
|
||||
/** Whether a session's warming is sending pings. */
|
||||
type Status = Data.TaggedEnum<{
|
||||
@@ -42,6 +44,20 @@ type Status = Data.TaggedEnum<{
|
||||
}>
|
||||
const Status = Data.taggedEnum<Status>()
|
||||
|
||||
/** What a session's warming loop does next. */
|
||||
type Next = Data.TaggedEnum<{
|
||||
/** The next ping isn't due yet. */
|
||||
Wait: { readonly until: number }
|
||||
/** The warming window has closed. */
|
||||
Expired: {}
|
||||
/** A ping is due while a model request runs; it waits for the request to finish rather than compete with it. */
|
||||
AwaitRequest: { readonly finished: Deferred.Deferred<void> }
|
||||
/** That request ran past the cache's lifetime, so a ping would only write a new cache, not keep the old one. */
|
||||
Cold: {}
|
||||
Ping: {}
|
||||
}>
|
||||
const Next = Data.taggedEnum<Next>()
|
||||
|
||||
export const Plugin = define({
|
||||
id: "opencode.warming",
|
||||
effect: Effect.fn(function* (ctx) {
|
||||
@@ -64,7 +80,10 @@ export const Plugin = define({
|
||||
...(typeof local === "object" ? local : {}),
|
||||
}
|
||||
const { duration } = settings
|
||||
const interval = settings.interval === "auto" ? (autoInterval(cache?.ttl) ?? fallbackInterval) : settings.interval
|
||||
const ttl = parseTTL(cache?.ttl)
|
||||
// "auto" pings at 90% of the cache's lifetime, so it lands before the cache expires.
|
||||
const interval =
|
||||
settings.interval === "auto" ? (ttl ? Duration.times(ttl, 0.9) : fallbackInterval) : settings.interval
|
||||
// A cache that outlives the window, such as "1h" with the 30-minute default, needs no keep-alive.
|
||||
if (
|
||||
settings.interval === "auto" &&
|
||||
@@ -76,7 +95,11 @@ export const Plugin = define({
|
||||
Duration.isFinite(duration) &&
|
||||
Duration.isGreaterThan(interval, Duration.zero) &&
|
||||
Duration.isLessThan(interval, duration)
|
||||
if (valid) return Decision.Warm({ settings: { ...settings, interval } })
|
||||
// Without a cache.ttl, an explicit interval longer than five minutes implies a cache that lasts at least that long.
|
||||
if (valid)
|
||||
return Decision.Warm({
|
||||
settings: { ...settings, interval, ttl: ttl ?? Duration.max(fallbackTTL, interval) },
|
||||
})
|
||||
yield* Effect.logWarning("warming interval must be positive and shorter than a finite duration", {
|
||||
interval,
|
||||
duration,
|
||||
@@ -90,7 +113,8 @@ export const Plugin = define({
|
||||
const outlived = new Set<Session.ID>()
|
||||
// A ping during a model request would compete with the request it keeps warm, which stalls single-slot local models.
|
||||
// A request counts from its context hook, because `session.step.started` waits for the first response chunk.
|
||||
const requesting = new Set<Session.ID>()
|
||||
// Each running request's entry completes when it ends, which wakes a loop waiting to ping.
|
||||
const requesting = new Map<Session.ID, Deferred.Deferred<void>>()
|
||||
// A turn always ends with an execution event, so a step that ends without its own event can't stay in flight.
|
||||
yield* ctx.event
|
||||
.subscribe({
|
||||
@@ -104,59 +128,79 @@ export const Plugin = define({
|
||||
})
|
||||
.pipe(
|
||||
Stream.runForEach((event) =>
|
||||
Effect.gen(function* () {
|
||||
Effect.suspend(() => {
|
||||
// The interval still counts from when the request was sent, because that is when it touched the cache.
|
||||
const finished = requesting.get(event.data.sessionID)
|
||||
requesting.delete(event.data.sessionID)
|
||||
// Count the next interval from the end of the response, not from when a long request started.
|
||||
const current = sessions.get(event.data.sessionID)
|
||||
if (current) current.last = yield* Clock.currentTimeMillis
|
||||
return finished ? Deferred.succeed(finished, undefined) : Effect.void
|
||||
}),
|
||||
),
|
||||
Effect.forkScoped({ startImmediately: true }),
|
||||
)
|
||||
const loop: (sessionID: Session.ID) => Effect.Effect<void> = Effect.fn("WarmingPlugin.loop")(function* (sessionID) {
|
||||
const current = sessions.get(sessionID)
|
||||
if (!current) return
|
||||
|
||||
const now = yield* Clock.currentTimeMillis
|
||||
const next = Math.min(current.last + Duration.toMillis(current.settings.interval), current.expires)
|
||||
if (now < next) {
|
||||
yield* Effect.sleep(Duration.millis(next - now))
|
||||
return yield* loop(sessionID)
|
||||
}
|
||||
if (now >= current.expires) {
|
||||
sessions.delete(sessionID)
|
||||
return
|
||||
}
|
||||
if (requesting.has(sessionID)) {
|
||||
yield* Effect.sleep(current.settings.interval)
|
||||
return yield* loop(sessionID)
|
||||
}
|
||||
|
||||
const last = current.last
|
||||
yield* Effect.logInfo("warming session", { sessionID, last })
|
||||
const result = yield* ctx.session
|
||||
.generate({ sessionID, prompt: current.settings.prompt })
|
||||
.pipe(
|
||||
Effect.catchCause((cause) =>
|
||||
Effect.logWarning("failed to warm session", { sessionID, cause }).pipe(Effect.as(undefined)),
|
||||
),
|
||||
)
|
||||
const latest = sessions.get(sessionID)
|
||||
if (latest !== current || latest.last !== last) return yield* loop(sessionID)
|
||||
// A ping that wrote more to the cache than it read rebuilt the cache instead of keeping it, so later pings
|
||||
// would keep paying for that. Providers that report no cache usage never count as a miss.
|
||||
if (result && result.tokens.cache.read < result.tokens.cache.write) {
|
||||
latest.status = Status.StoppedAfterMiss()
|
||||
yield* Effect.logInfo("stopped warming session after a cache miss", {
|
||||
sessionID,
|
||||
read: result.tokens.cache.read,
|
||||
write: result.tokens.cache.write,
|
||||
// `last` is when the latest request or ping was sent, since a provider's cache lifetime starts when a request reads or
|
||||
// writes it, not when its response ends.
|
||||
// `overran` marks a loop resumed by a request that was still running when the ping came due.
|
||||
const next = (
|
||||
current: { last: number; expires: number; settings: Settings },
|
||||
now: number,
|
||||
sessionID: Session.ID,
|
||||
overran: boolean,
|
||||
) => {
|
||||
if (now >= current.expires) return Next.Expired()
|
||||
const due = Math.min(current.last + Duration.toMillis(current.settings.interval), current.expires)
|
||||
if (now < due) return Next.Wait({ until: due })
|
||||
const finished = requesting.get(sessionID)
|
||||
if (finished) return Next.AwaitRequest({ finished })
|
||||
if (overran && now >= current.last + Duration.toMillis(current.settings.ttl)) return Next.Cold()
|
||||
return Next.Ping()
|
||||
}
|
||||
const loop: (sessionID: Session.ID, overran?: boolean) => Effect.Effect<void> = Effect.fn("WarmingPlugin.loop")(
|
||||
function* (sessionID, overran = false) {
|
||||
const current = sessions.get(sessionID)
|
||||
if (!current) return
|
||||
const now = yield* Clock.currentTimeMillis
|
||||
const stop = Effect.sync(() => {
|
||||
if (sessions.get(sessionID) === current) sessions.delete(sessionID)
|
||||
})
|
||||
return
|
||||
}
|
||||
latest.last = yield* Clock.currentTimeMillis
|
||||
return yield* loop(sessionID)
|
||||
})
|
||||
yield* Next.$match(next(current, now, sessionID, overran), {
|
||||
Wait: (next) => Effect.sleep(Duration.millis(next.until - now)).pipe(Effect.andThen(loop(sessionID))),
|
||||
Expired: () => stop,
|
||||
// Pinging as soon as the request ends keeps a cache that is about to expire, and the next interval counts from then.
|
||||
AwaitRequest: (next) => Deferred.await(next.finished).pipe(Effect.andThen(loop(sessionID, true))),
|
||||
// The next real request starts warming again.
|
||||
Cold: () => stop,
|
||||
Ping: () =>
|
||||
Effect.gen(function* () {
|
||||
// A ping touches the cache like any request, so the next interval counts from when it was sent.
|
||||
current.last = now
|
||||
yield* Effect.logInfo("warming session", { sessionID })
|
||||
const result = yield* ctx.session
|
||||
.generate({ sessionID, prompt: current.settings.prompt })
|
||||
.pipe(
|
||||
Effect.catchCause((cause) =>
|
||||
Effect.logWarning("failed to warm session", { sessionID, cause }).pipe(Effect.as(undefined)),
|
||||
),
|
||||
)
|
||||
// A session restarted during the ping has its own loop.
|
||||
if (sessions.get(sessionID) !== current) return
|
||||
// A real request sent during the ping restarted the interval, and the cache is judged by that request.
|
||||
if (current.last !== now) return yield* loop(sessionID)
|
||||
// A ping that wrote more to the cache than it read rebuilt the cache instead of keeping it, so later pings
|
||||
// would keep paying for that. Providers that report no cache usage never count as a miss.
|
||||
if (result && result.tokens.cache.read < result.tokens.cache.write) {
|
||||
current.status = Status.StoppedAfterMiss()
|
||||
yield* Effect.logInfo("stopped warming session after a cache miss", {
|
||||
sessionID,
|
||||
read: result.tokens.cache.read,
|
||||
write: result.tokens.cache.write,
|
||||
})
|
||||
return
|
||||
}
|
||||
return yield* loop(sessionID)
|
||||
}),
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
const hook = (event: SessionHooks["context"]) =>
|
||||
Effect.gen(function* () {
|
||||
@@ -225,7 +269,11 @@ export const Plugin = define({
|
||||
)
|
||||
})
|
||||
yield* ctx.session.hook("context", (event) =>
|
||||
Effect.sync(() => requesting.add(event.sessionID)).pipe(Effect.andThen(hook(event))),
|
||||
Effect.gen(function* () {
|
||||
// Steps of one turn share an entry, so the next end event wakes a waiting loop even if a step ended silently.
|
||||
if (!requesting.has(event.sessionID)) requesting.set(event.sessionID, yield* Deferred.make<void>())
|
||||
yield* hook(event)
|
||||
}),
|
||||
)
|
||||
yield* ctx.session.hook("compaction", hook)
|
||||
yield* ctx.session.hook("generate", hook)
|
||||
@@ -234,8 +282,7 @@ export const Plugin = define({
|
||||
|
||||
const units = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 }
|
||||
|
||||
/** 90% of a cache TTL such as "5m", so the ping lands before the cache expires. */
|
||||
function autoInterval(ttl: Provider.CacheTTL | undefined) {
|
||||
function parseTTL(ttl: Provider.CacheTTL | undefined) {
|
||||
const match = ttl?.match(/^([\d.]+)(ms|s|m|h|d)$/)
|
||||
if (match) return Duration.millis(Number(match[1]) * units[match[2] as keyof typeof units] * 0.9)
|
||||
if (match) return Duration.millis(Number(match[1]) * units[match[2] as keyof typeof units])
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import { Provider } from "@opencode/core/provider"
|
||||
import { AbsolutePath } from "@opencode/core/schema"
|
||||
import { Session } from "@opencode/core/session"
|
||||
import { SessionEvent } from "@opencode/core/session/event"
|
||||
import { Clock, DateTime, Effect, Queue, Schema, Stream } from "effect"
|
||||
import { Clock, DateTime, Duration, Effect, Queue, Schema, Stream } from "effect"
|
||||
import { TestClock } from "effect/testing"
|
||||
import { it } from "../lib/effect"
|
||||
import { host } from "./host"
|
||||
@@ -47,12 +47,13 @@ const unreported = { read: 0, write: 0 }
|
||||
|
||||
/**
|
||||
* Runs the warming plugin with the given top-level `warming` and a catalog holding the test model's `cache`. Each ping
|
||||
* reports the next cache usage in `usage`.
|
||||
* reports the next cache usage in `usage` and takes `ping` to respond.
|
||||
*/
|
||||
const run = Effect.fnUntraced(function* (
|
||||
warming: unknown,
|
||||
cache?: Provider.Cache,
|
||||
usage: ReadonlyArray<typeof hit> = [],
|
||||
ping = Duration.zero,
|
||||
) {
|
||||
const warms: number[] = []
|
||||
const events = yield* Queue.unbounded<SessionEvent.Execution.Succeeded>()
|
||||
@@ -75,6 +76,7 @@ const run = Effect.fnUntraced(function* (
|
||||
warms.push(now / 60_000)
|
||||
return { text: "OK", tokens: { input: 0, output: 1, reasoning: 0, cache }, cost: Money.USD.zero }
|
||||
}),
|
||||
Effect.tap(() => Effect.sleep(ping)),
|
||||
),
|
||||
hook: (name, callback) => {
|
||||
if (name === "context") contextHook = callback as (input: SessionContext) => Effect.Effect<void>
|
||||
@@ -89,9 +91,10 @@ const run = Effect.fnUntraced(function* (
|
||||
)
|
||||
if (!contextHook) return yield* Effect.die("warming plugin did not register a context hook")
|
||||
const hook = contextHook
|
||||
/** One whole turn: the step's context hook, then the turn ending. */
|
||||
const turn = Effect.gen(function* () {
|
||||
yield* hook({ ...context })
|
||||
/** A request being sent: the step's context hook. */
|
||||
const send = hook({ ...context })
|
||||
/** The turn ending. */
|
||||
const end = Effect.gen(function* () {
|
||||
yield* Queue.offer(events, {
|
||||
id: Event.ID.create(),
|
||||
created: 0,
|
||||
@@ -101,8 +104,10 @@ const run = Effect.fnUntraced(function* (
|
||||
})
|
||||
yield* Effect.yieldNow
|
||||
})
|
||||
/** One whole turn whose response ends right away. */
|
||||
const turn = send.pipe(Effect.andThen(end))
|
||||
const at = (minutes: number) => TestClock.setTime(minutes * 60_000).pipe(Effect.andThen(Effect.yieldNow))
|
||||
return { warms, turn, at }
|
||||
return { warms, send, end, turn, at }
|
||||
})
|
||||
|
||||
describe("WarmingPlugin interval", () => {
|
||||
@@ -181,4 +186,65 @@ describe("WarmingPlugin cache miss", () => {
|
||||
expect(warming.warms).toEqual([4, 8])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps warming when a request is sent while a missed ping responds", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, { ttl: "5m" }, [miss, hit], Duration.seconds(30))
|
||||
yield* warming.turn
|
||||
yield* warming.at(4.5)
|
||||
yield* warming.at(4.75)
|
||||
yield* warming.turn
|
||||
for (const minutes of [5, 9.25]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([4.5, 9.25])
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
describe("WarmingPlugin timing", () => {
|
||||
it.effect("counts the interval from when the request was sent, not when its response ended", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, { ttl: "5m" })
|
||||
yield* warming.send
|
||||
yield* warming.at(1)
|
||||
yield* warming.end
|
||||
for (const minutes of [4.5, 5.5]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([4.5])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("counts the next interval from when a ping was sent", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, { ttl: "5m" }, [], Duration.seconds(30))
|
||||
yield* warming.turn
|
||||
for (const minutes of [4.5, 9, 9.5]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([4.5, 9])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("waits for a request running at the due time, then pings while the cache lasts", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, { ttl: "5m" })
|
||||
yield* warming.send
|
||||
yield* warming.at(4.5)
|
||||
expect(warming.warms).toEqual([])
|
||||
yield* warming.at(4.75)
|
||||
yield* warming.end
|
||||
for (const minutes of [9, 9.25]) yield* warming.at(minutes)
|
||||
expect(warming.warms).toEqual([4.75, 9.25])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("stops warming when a request outlasts the cache, until the next request", () =>
|
||||
Effect.gen(function* () {
|
||||
const warming = yield* run(true, { ttl: "5m" })
|
||||
yield* warming.send
|
||||
yield* warming.at(6)
|
||||
yield* warming.end
|
||||
yield* warming.at(15)
|
||||
expect(warming.warms).toEqual([])
|
||||
yield* warming.turn
|
||||
yield* warming.at(19.5)
|
||||
expect(warming.warms).toEqual([19.5])
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
@@ -117,11 +117,14 @@ otherwise OpenCode skips warming and logs a warning.
|
||||
## Timing
|
||||
|
||||
Warming starts after a non-warming model request in a top-level session.
|
||||
Subagent sessions are not warmed. After a warming request
|
||||
finishes, the next idle interval starts, but the active window is not extended.
|
||||
Subagent sessions are not warmed. Each interval counts from when the latest
|
||||
request or warming request was sent, not from when its response finished,
|
||||
because a provider's cache lifetime starts when the request reaches it. A
|
||||
warming request does not extend the active window.
|
||||
|
||||
```text
|
||||
00:00 normal model request
|
||||
00:00 normal model request sent
|
||||
00:01 response finishes
|
||||
00:04 warming request
|
||||
00:08 warming request
|
||||
00:30 warming stops
|
||||
@@ -138,6 +141,18 @@ request is sent.
|
||||
00:50 warming stops
|
||||
```
|
||||
|
||||
OpenCode never sends a warming request while a model request is running. If one
|
||||
is still running when the interval comes due, warming waits for it to finish and
|
||||
then sends right away, as long as the cache from that request hasn't expired.
|
||||
If it has, warming stops until the next normal request.
|
||||
|
||||
```text
|
||||
00:00 normal model request sent
|
||||
00:04 interval due; request still running
|
||||
00:04:30 response finishes; warming request
|
||||
00:08:30 warming request
|
||||
```
|
||||
|
||||
## Requests
|
||||
|
||||
A warming request uses the session's current model, agent, instructions, and
|
||||
|
||||
Reference in New Issue
Block a user