Files
hermes-agent/tests/agent/test_prompt_cache_clock.py
Teknium 58886fc81e fix(cache): coverage deadlines, guaranteed vs estimated ends, durable generation
The clock modelled every route as "expires = last touch + tier", which is wrong for OpenAI
GPT-5.6+: a read refreshes coverage to at least 30 minutes after it and never restarts a
12h write. It also treated documented minimums as expiries and lost or revived saved state.

- Coverage deadlines per tier with per-route refresh rules: Anthropic family restarts the tier
  on read; OpenAI keeps max(existing, read + 30m) and a 12h deadline stays at write + 12h.
- Windows carry guaranteed_until (earliest deadline, so mixed tiers in one response are only as
  warm as the shorter) and estimated_until (OpenAI: last touch + 24h retention ceiling; pre-5.6
  in_memory 1h). cache_certainly_warm / cache_certainly_cold are the two consumer helpers.
  /context: warm until / may still be warm until / expired / unknown.
- Events bind to the session's durable context generation (rows a compaction or prune archived,
  SessionDB.prompt_cache_generation), kept as saved on restore; a resumed agent restores and
  validates the saved event before folding its first response.
- Route identity is provider + normalize_route_base_url (scheme, host, port, path) + model.
- Agentless session.context_breakdown reads the saved clock (stored_cache_window).
- Codex app-server activity is recorded as an explicit unknown (it sets its own cache options).
- MoA records the resolved aggregator route and the payload the MoA client sent.
- effective_cache_ttl applies the route ceiling on the measured-provider path too.
2026-11-15 12:04:54 -08:00

221 lines
13 KiB
Python

"""The prompt-cache clock: one shared answer to "until when is this session's cached prompt warm?".
Behaviour contracts per route: what was sent on the wire, what the response proved, how coverage
deadlines move on a read (Anthropic restarts the tier; OpenAI keeps max(existing, read + 30m) and
never restarts 12h), the guaranteed vs estimated ends, and how the window dies on a route switch
or compaction.
"""
from types import SimpleNamespace
import pytest
from agent.prompt_cache_clock import (
cache_certainly_cold, cache_certainly_warm, cache_window, record_attempt_sent, record_cache_response,
render_prompt_cache_line,
)
from agent.usage_pricing import normalize_usage
def _agent(provider, base_url, model, *, api_mode="chat_completions", cache_ttl="1h", session_id="s1"):
return SimpleNamespace(
provider=provider, base_url=base_url, model=model, api_mode=api_mode, session_id=session_id,
_cache_ttl=cache_ttl, _use_prompt_caching=True, _session_db=None,
context_compressor=SimpleNamespace(compression_count=0),
)
def _marked_request(ttl):
marker = {"type": "ephemeral", "ttl": "1h"} if ttl == "1h" else {"type": "ephemeral"}
return {"messages": [{"role": "system", "content": [{"type": "text", "text": "sys", "cache_control": marker}]},
{"role": "user", "content": "hi"}]}
def _openai_request(ttl):
return {"input": [], "extra_body": {"prompt_cache_options": {"ttl": ttl}}}
def _turn(agent, request, raw_usage, *, sent_at, monkeypatch):
import agent.prompt_cache_clock as clock
monkeypatch.setattr(clock.time, "time", lambda: sent_at)
record_attempt_sent(agent, request)
usage = normalize_usage(raw_usage, provider=agent.provider, api_mode=agent.api_mode)
return record_cache_response(agent, usage, model=agent.model, provider=agent.provider,
base_url=agent.base_url, api_mode=agent.api_mode)
def _ends(agent):
window = cache_window(agent)
return window.tier, window.confidence, window.guaranteed_until, window.estimated_until
def test_anthropic_split_proves_the_tier_and_a_read_restarts_it(monkeypatch):
agent = _agent("anthropic", "https://api.anthropic.com", "claude-sonnet-4-6", api_mode="anthropic_messages")
write = {"input_tokens": 5, "output_tokens": 1, "cache_creation_input_tokens": 4096,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 4096}}
_turn(agent, _marked_request("1h"), write, sent_at=1_000.0, monkeypatch=monkeypatch)
assert _ends(agent) == ("1h", "proven", 4_600.0, 4_600.0) # Anthropic tiers are exact: both ends agree
read = {"input_tokens": 5, "output_tokens": 1, "cache_read_input_tokens": 4096}
_turn(agent, _marked_request("1h"), read, sent_at=2_000.0, monkeypatch=monkeypatch)
assert _ends(agent) == ("1h", "proven", 5_600.0, 5_600.0)
# The server granted 5m although 1h was asked for: the clock believes the response.
downgraded = {**write, "cache_creation": {"ephemeral_5m_input_tokens": 4096, "ephemeral_1h_input_tokens": 0}}
_turn(agent, _marked_request("1h"), downgraded, sent_at=3_000.0, monkeypatch=monkeypatch)
assert _ends(agent)[0] == "5m" and _ends(agent)[2] == 3_300.0
def test_mixed_tiers_in_one_response_guarantee_only_the_shorter(monkeypatch):
agent = _agent("anthropic", "https://api.anthropic.com", "claude-sonnet-4-6", api_mode="anthropic_messages")
mixed = {"input_tokens": 5, "output_tokens": 1, "cache_creation_input_tokens": 6000,
"cache_creation": {"ephemeral_5m_input_tokens": 2000, "ephemeral_1h_input_tokens": 4000}}
_turn(agent, _marked_request("1h"), mixed, sent_at=1_000.0, monkeypatch=monkeypatch)
window = cache_window(agent)
assert window.coverage == {"5m": 1_300.0, "1h": 4_600.0} # both recorded
assert (window.tier, window.guaranteed_until, window.estimated_until) == ("5m", 1_300.0, 4_600.0)
def test_openai_12h_deadline_is_fixed_at_write_and_a_read_only_extends_the_30m_floor(monkeypatch):
"""OpenAI: a 30m read "preserves the later of the existing expiry and 30 minutes after the read.
It does not restart a 12-hour clock"."""
agent = _agent("openai-api", "https://api.openai.com/v1", "gpt-5.6-luna", api_mode="codex_responses")
write = {"input_tokens": 4000, "output_tokens": 2, "input_tokens_details": {
"cached_tokens": 0, "cache_write_tokens": 3640,
"cache_write_tokens_details": {"ttl_30m_tokens": 0, "ttl_12h_tokens": 3640}}}
_turn(agent, _openai_request("12h"), write, sent_at=1_000.0, monkeypatch=monkeypatch)
assert _ends(agent)[:3] == ("12h", "proven", 44_200.0)
read = {"input_tokens": 4000, "output_tokens": 2, "input_tokens_details": {"cached_tokens": 3640}}
_turn(agent, _openai_request("12h"), read, sent_at=2_000.0, monkeypatch=monkeypatch)
assert _ends(agent)[:3] == ("12h", "proven", 44_200.0)
_turn(agent, _openai_request("12h"), read, sent_at=44_000.0, monkeypatch=monkeypatch)
assert _ends(agent)[2] == 45_800.0 # near the 12h deadline the read's 30 minutes are the later one
def test_openai_minimum_is_not_an_expiry(monkeypatch):
"""GPT-5.6+ 30m is a documented MINIMUM ("may retain entries longer"; 24h ceiling): warm before it,
may still be warm after it, certainly cold only past the retention ceiling."""
agent = _agent("openai-api", "https://api.openai.com/v1", "gpt-5.6-luna", api_mode="codex_responses")
write = {"input_tokens": 4000, "output_tokens": 2, "input_tokens_details": {
"cached_tokens": 0, "cache_write_tokens": 3640, "cache_write_tokens_details": {"ttl_30m_tokens": 3640}}}
_turn(agent, {"input": []}, write, sent_at=1_000.0, monkeypatch=monkeypatch)
window = cache_window(agent)
assert (window.guaranteed_until, window.estimated_until) == (2_800.0, 87_400.0)
assert cache_certainly_warm(window, now=2_000.0) and not cache_certainly_cold(window, now=2_000.0)
assert not cache_certainly_warm(window, now=3_000.0) and not cache_certainly_cold(window, now=3_000.0)
assert cache_certainly_cold(window, now=87_400.0)
assert render_prompt_cache_line(dict(window.__dict__), now=3_000.0).startswith("Prompt cache: may still be warm until")
# Pre-5.6 retention: no guarantee at all, only a ceiling.
old = _agent("openai-api", "https://api.openai.com/v1", "gpt-5.4-nano", api_mode="codex_responses")
_turn(old, {"input": []}, {"input_tokens": 4000, "output_tokens": 2,
"input_tokens_details": {"cached_tokens": 2816}}, sent_at=1_000.0, monkeypatch=monkeypatch)
assert _ends(old)[1:] == ("assumed", None, 87_400.0)
assert not cache_certainly_warm(cache_window(old), now=1_001.0)
def test_the_cache_moves_with_the_full_route_and_dies_at_compaction(monkeypatch):
agent = _agent("custom", "http://10.0.0.5:8000/v1", "anthropic/claude-sonnet-4.6")
usage = {"prompt_tokens": 5000, "completion_tokens": 3,
"prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 4800}}
_turn(agent, _marked_request("5m"), usage, sent_at=1_000.0, monkeypatch=monkeypatch)
assert _ends(agent)[:2] == ("5m", "declared")
agent.context_compressor.compression_count = 1
assert (cache_window(agent).confidence, cache_window(agent).reason) == ("unknown", "compacted")
agent.context_compressor.compression_count = 0
for moved in ("http://10.0.0.5:8001/v1", "http://10.0.0.5:8000/other/v1", "https://10.0.0.5:8000/v1"):
agent.base_url = moved # same host, different server: a different cache
assert cache_window(agent).reason == "route_changed", moved
agent.base_url = "http://10.0.0.5:8000/v1/" # a trailing slash is the same endpoint
assert cache_window(agent).confidence == "declared"
# A response with no cache activity leaves the previous window standing.
_turn(agent, _marked_request("5m"), {"prompt_tokens": 10, "completion_tokens": 1}, sent_at=1_200.0,
monkeypatch=monkeypatch)
assert _ends(agent)[2] == 1_300.0
def test_nous_ceiling_is_data_on_the_wire_and_in_the_clock(tmp_path, monkeypatch):
"""Nous Portal drops 1h upstream: Hermes asks for 5m there and the clock says 5m. Lifting the
ceiling is a config entry, not a code change, and a ``custom`` route at the Portal host counts."""
from agent.prompt_caching import effective_cache_ttl
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
portal = "https://inference-api.nousresearch.com/v1"
for provider in ("nous", "custom"):
assert effective_cache_ttl("1h", provider=provider, model="anthropic/claude-sonnet-4.6", base_url=portal) == "5m"
assert effective_cache_ttl("1h", provider="openrouter", model="anthropic/claude-sonnet-4.6",
base_url="https://openrouter.ai/api/v1") == "1h"
agent = _agent("nous", portal, "anthropic/claude-sonnet-4.6")
usage = {"prompt_tokens": 5000, "completion_tokens": 3,
"prompt_tokens_details": {"cached_tokens": 4800, "cache_write_tokens": 0}}
_turn(agent, _marked_request("5m"), usage, sent_at=1_000.0, monkeypatch=monkeypatch)
assert _ends(agent) == ("5m", "declared", 1_300.0, 1_300.0)
assert cache_window(agent).reason == "route_ceiling_5m"
(tmp_path / "config.yaml").write_text("prompt_caching:\n route_ttl_ceilings:\n nous: 1h\n", encoding="utf-8")
assert effective_cache_ttl("1h", provider="nous", model="anthropic/claude-sonnet-4.6", base_url=portal) == "1h"
def test_a_configured_ceiling_applies_on_the_measured_provider_path_too(tmp_path, monkeypatch):
"""opencode-go returns early as a measured 1h route; a configured 5m ceiling must still win there."""
from agent.prompt_caching import effective_cache_ttl
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
assert effective_cache_ttl("1h", provider="opencode-go", model="qwen3.6-plus") == "1h"
(tmp_path / "config.yaml").write_text("prompt_caching:\n route_ttl_ceilings:\n opencode-go: 5m\n",
encoding="utf-8")
assert effective_cache_ttl("1h", provider="opencode-go", model="qwen3.6-plus") == "5m"
def test_moa_records_the_resolved_aggregator_route_and_payload(monkeypatch):
"""The MoA agent shows the virtual preset; the clock must use the aggregator's resolved route
(OpenRouter Claude: 1h declared) and the payload the MoA client really sent."""
agent = _agent("moa", "moa://local", "closed")
agent.client = SimpleNamespace(
last_aggregator_route={"provider": "openrouter", "model": "anthropic/claude-opus-4.8",
"base_url": "https://openrouter.ai/api/v1", "api_mode": "chat_completions"},
last_aggregator_attempt={"sent_at": 1_000.0, "marker_ttl": "1h", "openai_ttl": None, "retention": None})
usage = normalize_usage({"prompt_tokens": 5000, "completion_tokens": 3,
"prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 4800}},
provider="openrouter", api_mode="chat_completions")
event = record_cache_response(agent, usage, model="closed", provider="moa", base_url="moa://local",
api_mode="chat_completions")
assert (event.route_class, event.tier, event.confidence) == ("openrouter_claude", "1h", "declared")
assert _ends(agent)[2] == 4_600.0
def test_implicit_caches_report_unknown_rather_than_a_guess(monkeypatch):
agent = _agent("deepseek", "https://api.deepseek.com/v1", "deepseek-v4")
agent._use_prompt_caching = False
usage = {"prompt_tokens": 4000, "completion_tokens": 2, "prompt_cache_hit_tokens": 3000}
_turn(agent, {"messages": [{"role": "user", "content": "hi"}]}, usage, sent_at=1_000.0, monkeypatch=monkeypatch)
window = cache_window(agent)
assert (window.confidence, window.estimated_until) == ("unknown", None)
assert not cache_certainly_warm(window) and not cache_certainly_cold(window)
assert render_prompt_cache_line({"estimated_until": None, "confidence": "unknown", "reason": "implicit_cache"}) == (
"Prompt cache: unknown (this provider does not publish its cache retention).")
@pytest.mark.parametrize(("now", "expected"), [
(1_048.0, "Prompt cache: warm until 01:16 (59m 12s left; 1h, proven)."),
(5_500.0, "Prompt cache: expired 15m ago (1h, proven); the next message re-processes the full prompt."),
])
def test_context_line_shows_the_wall_clock_expiry_then_how_long_ago(now, expected, monkeypatch):
import agent.prompt_cache_clock as clock
from agent.context_breakdown import render_context_breakdown_lines
monkeypatch.setattr(clock.time, "localtime", lambda ts: clock.time.gmtime(ts))
window = {"guaranteed_until": 4_600.0, "estimated_until": 4_600.0, "tier": "1h", "confidence": "proven",
"reason": "anthropic_usage_split", "refreshed_at": 1_000.0, "coverage": {"1h": 4_600.0}}
payload = {"categories": [], "context_max": 200_000, "context_used": 1_000, "context_percent": 1,
"context_source": "provider_usage", "estimated_total": 0, "prompt_cache": window}
monkeypatch.setattr(clock.time, "time", lambda: now)
assert expected in render_context_breakdown_lines(payload, grid=False)
assert not any("Prompt cache" in line for line in render_context_breakdown_lines({**payload, "prompt_cache": None},
grid=False))