From 94e0252d2ca44623ea05abfddfd3c9c2dbd35f54 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:49:08 -0800 Subject: [PATCH] docs(caching): claim_edge, package layout, 24h/local-endpoint rule, migration outcome, sandboxed offload caveat Co-authored-by: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> --- agent/background_review.py | 3 ++- agent/turn_response_check.py | 4 ++-- cli-config.yaml.example | 5 +++-- .../context-compression-and-caching.md | 6 ++--- website/docs/user-guide/configuration.md | 22 ++++++++++--------- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/agent/background_review.py b/agent/background_review.py index f34180732e..fa767f1b93 100644 --- a/agent/background_review.py +++ b/agent/background_review.py @@ -1086,7 +1086,8 @@ def build_cache_parity_fork( if not _routed and write_origin == "background_review": _warn_ignored_reasoning_effort(agent, task_cfg) review_agent = AIAgent(**_fork_init_kwargs(agent, _rt, _routed, max_iterations, task_cfg)) - review_agent._skip_prompt_cache_warming = True # the parent already keeps this shared prefix warm + # A same-route fork shares the parent's warmed prefix; a routed fork's own prefix is never warmed. + review_agent._skip_prompt_cache_warming = True if _routed and getattr(review_agent, "_cache_ttl", None) == "1h": # A routed fork's cache is cold and nobody pauses between its calls: 1h writes buy nothing (#104168). review_agent._cache_ttl = "5m" diff --git a/agent/turn_response_check.py b/agent/turn_response_check.py index d08ed433ba..771ad21561 100644 --- a/agent/turn_response_check.py +++ b/agent/turn_response_check.py @@ -207,8 +207,8 @@ def check_api_response( if _usage_outcome.rearmed: _preflight_compression_blocked = False _last_preflight_pressure = None - # Arm a one-token replay of THIS request before the clock's guaranteed window ends, so a tool - # round longer than the TTL does not turn the next request into a full-price cache write. + # Arm a bounded replay (smallest output cap) of THIS request before the clock's guaranteed window + # ends, so a tool round longer than the TTL does not turn the next request into a full-price cache write. start_prompt_cache_warming(agent, api_kwargs, response) _retry.has_retried_429 = False diff --git a/cli-config.yaml.example b/cli-config.yaml.example index b9a971ab5f..69fa0150cd 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -883,10 +883,11 @@ compression: # Anthropic supports two TTL tiers for cached prefixes: "5m" and "1h". The # default "auto" picks per session: "1h" when a person types the turns (CLI, # TUI, Desktop, messaging), "5m" for subagents, cron, one-shots, webhooks, -# Kanban, the API server and batch runs. Other values are ignored and "5m" is used. +# Kanban, the API server and batch runs. "off" (or false/none/disabled/no) +# turns prompt caching off entirely; any other unrecognised value means "5m". # # prompt_caching: -# cache_ttl: "auto" # or pin "5m" / "1h" for every session +# cache_ttl: "auto" # or pin "5m" / "1h" for every session, or "off" # ============================================================================= # Auxiliary Models (Advanced — Experimental) diff --git a/website/docs/developer-guide/context-compression-and-caching.md b/website/docs/developer-guide/context-compression-and-caching.md index 8bc8759368..6dbb1f1927 100644 --- a/website/docs/developer-guide/context-compression-and-caching.md +++ b/website/docs/developer-guide/context-compression-and-caching.md @@ -338,7 +338,7 @@ auxiliary: | `in_place` | `true` | bool | Compact on the same session id instead of rotating to a new one (see below) | | `prepare_ahead` | `false` | bool | Opt-in: near the trigger, write the next summary in the background and splice it at the next automatic compaction when it still covers the same messages (see below) | -Cache-edge compaction is step 2 of the prompt-cache lifecycle (`agent/prompt_cache_lifecycle.py`, no key; see Prompt caching below). Every compaction path summarizes first and archives after. `CompactionWindowMixin._plan_compaction_window` (`agent/context_compressor_window.py`) is write-free and only sets the boundaries; `summary_view` puts the window's ORIGINAL tool results back for the summarizer (a `clarify` result keeps its filtered one-liner, which drops legacy headless notices), and `_commit_window` runs only once a summary exists: `tool_result_storage.archive_compaction_drops` writes each removed or shortened original byte-exact to `cache/pruned/` and points its stub at the file, `_offload_window_results` archives the window's whole results, and lean mode demotes stale tail results. The prepared pass (`agent/prepared_compaction.py`) summarizes `summary_view` rows and writes nothing; the splicing `compress()` archives. An aborted summary writes no archive; a failed write keeps that result whole and the compaction proceeds; non-local backends keep main's plain stubs. +Cache-edge compaction is step 2 of the prompt-cache lifecycle (`agent/prompt_cache_lifecycle/`, no key; see Prompt caching below). Every compaction path summarizes first and archives after. `CompactionWindowMixin._plan_compaction_window` (`agent/context_compressor_window.py`) is write-free and only sets the boundaries; `summary_view` puts the window's ORIGINAL tool results back for the summarizer (a `clarify` result keeps its filtered one-liner, which drops legacy headless notices), and `_commit_window` runs only once a summary exists: `tool_result_storage.archive_compaction_drops` writes each removed or shortened original byte-exact to `cache/pruned/` (short and duplicate string results too) and points its stub at the file, `_offload_window_results` archives the window's whole results, and lean mode demotes stale tail results. The prepared pass (`agent/prepared_compaction.py`) summarizes `summary_view` rows and writes nothing; the splicing `compress()` archives. An aborted summary writes no archive; a failed write keeps that result whole and the compaction proceeds; non-local backends (docker, ssh, modal, daytona, …) keep plain stubs, no archive (the model cannot read host files there). ### In-place compaction (single stable session id) @@ -874,10 +874,10 @@ prompt_caching: `"auto"` resolves once per session in `agent/agent_init.py::_init_prompt_cache_config` via `agent/prompt_caching.py::auto_cache_ttl_for_source`: `1h` for human-paced sources, `5m` for `MACHINE_PACED_SOURCES` (subagent, cron, oneshot, webhook, kanban, api, api_server, tool, batch, curator); under `auto` a side agent (`side_agent=True`: `/background`, `/btw`, `/refine`) takes `5m`, and a background-review or `/btw` fork routed to a different model is clamped to `5m` like a delegated child. Auxiliary/stub calls (`configured_cache_ttl()`) treat `auto` as `5m`. -The prompt-cache lifecycle (`agent/prompt_cache_lifecycle.py`, no config key) is one controller per agent (`PromptCacheWarmer`, from `get_prompt_cache_warmer`) driven by the shared clock (`agent/prompt_cache_clock.py`, the only TTL source). The clock follows the API spec (`classify_route`, two rows): a request with `cache_control` markers is `anthropic_spec` on any wire or gateway (5m/1h declared as asked, clamped by `route_ttl_ceiling`); every other cached request is `openai_spec` (declared `30m`, a read refreshes to `max(existing, read + 30m)`), whatever the vendor or model generation. Only the expiry estimate departs from the tier (`_estimated_end`): a request sent with `prompt_cache_retention: 24h` keeps `guaranteed_until` at 30m (the warming deadline) but `estimated_until` at `read + 24h`, and an `is_local_endpoint` host has `estimated_until=None` with a known confidence, so neither is cold-pruned on the clock while `route_changed` and compaction still offload. A provider profile with `prompt_cache_markers_from_catalog=True` (Nous) lets its `/models` catalog pick marker models (`prompt_caching.catalog_cache_markers`: a `cache_write` rate means markers); `agent/prompt_cache_routes.py` is the cross-session layer: `prompt_cache_clock._learn` feeds `observe` from every folded response (gap = send time minus the previous event's `sent_at`, against its tier), `PromptCacheWarmer.start`/`_refresh` apply `learned_window`, and `prompt_caching.learned_or_catalog_markers` lets a learned marker preference beat the catalog; rules need `MIN_SAMPLES` agreeing samples and expire after `EXPIRY_SECONDS`; the file lives under `get_hermes_home()` resolved per call. Pricing takes a catalog's `cache_write_1h` (else its single write rate) for 1h writes instead of the fixed 2x. At `cache_warming_delay_s` (5/6 of the TTL, at least 10 s before expiry) it decides once: **warm** (replay the last request with `replay_output_cap(model)`, 1 or 16 for OpenAI GPT-5/o-series, at most `WARM_BUDGET`=3 idle keep-alives per user message; while `agent._executing_tools` is set a keep-alive is booked on its run instead (`tool_warms`/`tool_spent_usd`), uncounted and bounded only by `evaluate_economics`' one-rewrite cap on that tool wait's spend; the first miss sets `_missed_route` (`_disable_warming`), so `start` refuses every later arm on that route for the session (`WARMING_DISABLED`, cleared by a route switch or `reset_session_state`, shown on `/context` as `warming_off`). The idle budget is one `_Budget` object keyed on `_user_turn_count` that every run of the turn shares, each keep-alive reserved against it at admission (`_admit`) and reconciled with its billed cost in `_record`, so a run replaced mid-flight still counts; a miss or the third keep-alive ends it (`_end_budget`), and a later tool-round reply carries the ended budget to the prompt the turn ends on (`_carry_ended_budget`). The replay is bounded per client: `replay_cap_honoured` admits the OpenAI/Anthropic SDKs as they are and any other client only through its `effective_output_cap` (native Gemini lifts thinking requests to 65,535, so they are not warmed); `bounded_chat_replay` drops `extra_body` output caps such as Gemini's long-decode `max_output_tokens`), then **compact** (`compaction_due` / `claim_compaction`: budget over, proven/declared window of `MIN_COMPACT_WINDOW_SECONDS` or more, context at least `COMPACT_CONTEXT_FRACTION` of `threshold_tokens`, growth guard; no warmth check, the summary is a fresh aux request), else **expire + offload** (`offload_due` / `claim_offload`: idle, cache certainly cold, once per stretch; the owner runs `turn_proactive_prune.offload_cold_history`). Nothing is sent after a compaction until the next user message. A budget ends at whichever comes first, `WARM_BUDGET` keep-alives, a miss, or the owner releasing the agent: `release_compaction_due` / `claim_compaction_at_release` run the same gates at the release moment and cancel the run; past the window the release offloads instead. Step 4 (cold return) is `agent/turn_proactive_prune.turn_start_cold_prune`; `prompt_cache_is_cold` also answers True on `route_changed`, so a `/model` or fallback switch offloads before the first request on the new route: `turn_proactive_prune.route_change_offload` runs before every request assembly, so a mid-turn fallback restart offloads too (the running turn is never shortened). Economics (`evaluate_economics`): read cost below the rewrite it prevents, a budget's total at most one rewrite, known prices, prefix at least `MIN_CACHEABLE_PREFIX_TOKENS`; no dollar floor. +The prompt-cache lifecycle (`agent/prompt_cache_lifecycle/`, no config key: `__init__` facade, `replay.py` replay shape and caps, `economics.py` `evaluate_economics`, `warmer.py` the controller, `edge.py` `EdgeDecision`, `human_paced` and the compaction/offload gates) is one controller per agent (`PromptCacheWarmer`, from `get_prompt_cache_warmer`) driven by the shared clock (`agent/prompt_cache_clock.py`, the only TTL source). The clock follows the API spec (`classify_route`, two rows): a request with `cache_control` markers is `anthropic_spec` on any wire or gateway (5m/1h declared as asked, clamped by `route_ttl_ceiling`); every other cached request is `openai_spec` (declared `30m`, a read refreshes to `max(existing, read + 30m)`), whatever the vendor or model generation. Only the expiry estimate departs from the tier (`_estimated_end`): a request sent with `prompt_cache_retention: 24h` keeps `guaranteed_until` at 30m (the warming deadline) but `estimated_until` at `read + 24h`, and an `is_local_endpoint` host has `estimated_until=None` with a known confidence, so neither is cold-pruned on the clock while `route_changed` and compaction still offload (chatgpt.com Codex strips the retention field: 30m stands). A provider profile with `prompt_cache_markers_from_catalog=True` (Nous) lets its `/models` catalog pick marker models (`prompt_caching.catalog_cache_markers`: a `cache_write` rate means markers); `agent/prompt_cache_routes.py` is the cross-session layer: `prompt_cache_clock._learn` feeds `observe` from every folded response (gap = send time minus the previous event's `sent_at`, against its tier), `PromptCacheWarmer.start`/`_refresh` apply `learned_window`, and `prompt_caching.learned_or_catalog_markers` lets a learned marker preference beat the catalog; rules need `MIN_SAMPLES` agreeing samples and expire after `EXPIRY_SECONDS`; the file lives under `get_hermes_home()` resolved per call. Pricing takes a catalog's `cache_write_1h` (else its single write rate) for 1h writes instead of the fixed 2x. At `cache_warming_delay_s` (5/6 of the TTL, at least 10 s before expiry) it decides once: **warm** (replay the last request with `replay_output_cap(model)`, 1 or 16 for OpenAI GPT-5/o-series, at most `WARM_BUDGET`=3 idle keep-alives per user message; while `agent._executing_tools` is set a keep-alive is booked on its run instead (`tool_warms`/`tool_spent_usd`), uncounted and bounded only by `evaluate_economics`' one-rewrite cap on that tool wait's spend; the first miss sets `_missed_route` (`_disable_warming`), so `start` refuses every later arm on that route for the session (`WARMING_DISABLED`, shown on `/context` as `warming_off`); `_MISSED_ROUTES` keeps it per session id across agent rebuilds and carries it to the compaction child; only a route switch or `/new` (`reset_session_state`, also dropping budget and latches) clears it. The idle budget is one `_Budget` object keyed on `_user_turn_count` that every run of the turn shares, each keep-alive reserved against it at admission (`_admit`) and reconciled with its billed cost in `_record`, so a run replaced mid-flight still counts; a miss, the third keep-alive or the money cap ends it (`_end_budget`; steps 2/3 decide either way), and a later tool-round reply carries the ended budget to the prompt the turn ends on (`_carry_ended_budget`). The replay is bounded per client: `replay_cap_honoured` admits the OpenAI/Anthropic SDKs as they are and any other client only through its `effective_output_cap` (native Gemini lifts thinking requests to 65,535, so they are not warmed); `bounded_chat_replay` drops `extra_body` output caps such as Gemini's long-decode `max_output_tokens`), then **compact** (budget over, proven/declared window of `MIN_COMPACT_WINDOW_SECONDS` or more, context at least `COMPACT_CONTEXT_FRACTION` of `threshold_tokens`, growth guard; no warmth check, the summary is a fresh aux request), else **expire + offload** (idle, cache certainly cold, once per stretch; the owner runs `turn_proactive_prune.offload_cold_history`). `PromptCacheWarmer.claim_edge(*, releasing=False, now=None) -> "compact" | "offload" | None` decides and latches steps 2/3 under the warmer lock (`edge_decision(...)` is the read-only `EdgeDecision(action, compact, offload)`); owners pass the action to their job as an argument. Nothing is sent after a compaction until the next user message. A budget ends at whichever comes first, `WARM_BUDGET` keep-alives, a miss, or the owner releasing the agent: `claim_edge(releasing=True)` runs the same gates at the release moment and cancels the live run; past the window the release offloads instead. Step 4 (cold return) is `agent/turn_proactive_prune.turn_start_cold_prune`; `prompt_cache_is_cold` also answers True on `route_changed`, so a `/model` or fallback switch offloads before the first request on the new route: `turn_proactive_prune.route_change_offload` runs before every request assembly, so a mid-turn fallback restart offloads too (the running turn is never shortened). Economics (`evaluate_economics`): read cost, plus the replay's real output cap (`replay_output_cap`, 16 on GPT-5/o-series) and any per-request fee, below the rewrite it prevents, a budget's total at most one rewrite, known prices, prefix at least `MIN_CACHEABLE_PREFIX_TOKENS`; no dollar floor. `write-on-hit` is learned only from keep-alive replays. - Arming: `turn_response_check.check_api_response` calls `start_prompt_cache_warming` after each response is folded into the clock; `turn_api_request.build_request_payload` cancels the run (the real request refreshes the entry). `conversation_loop.run_conversation` calls `end_turn_prompt_cache_warming` on every turn exit: `human_paced` agents keep their budget and warm while idle; subagents, side agents (`/bg`, `/btw`), cache-parity forks (`_skip_prompt_cache_warming`) and `MACHINE_PACED_SOURCES` stop. -- Step 2/3 owners poll `claim_compaction` (then `claim_offload`) every `CHECK_INTERVAL_SECONDS` and run `compress_now(..., trigger="cache_expiry")` (normal compression model; a ready `prepare_ahead` summary is reused because the trigger is not `force`) or the cold offload: `CLISessionMixin._maybe_compact_before_cache_expiry` (idle tick), `tui_gateway/session_reaper.py` (ticker, bounded parallel jobs, profile scope per session, `_cache_edge_action` picks compact or `_offload_expired_cache`) and `GatewaySessionWatchersMixin._compact_idle_sessions_before_cache_expiry` (housekeeping; `claim_gate` held across load→commit, `_profile_scope_for_source`, detached Desktop chats via `route_entry`, per-session bounded jobs). No owner sends anything after a compaction. Release owners: `GatewayAgentCacheMixin._sweep_idle_cached_agents` and `_enforce_agent_cache_cap` hand each evicted agent to `GatewaySessionWatchersMixin._compact_then_release` (decided on the gateway loop; a due chat runs the claim-gated cache-edge job, compact or offload, then the agent is released), and the TUI reapers (`_reap_idle_sessions`, `_enforce_session_cap`) go through `_compact_then_close` (compact or offload, then close if still reapable). A queued TUI job whose chat closed meanwhile is refused at `_manual_compress_turn` (`_closing`). +- Step 2/3 owners poll `claim_edge()` every `CHECK_INTERVAL_SECONDS` and run `compress_now(..., trigger="cache_expiry")` (normal compression model; a ready `prepare_ahead` summary is reused because the trigger is not `force`) or the cold offload: `CLISessionMixin._maybe_compact_before_cache_expiry` (idle tick; adopts on `CompressResult.committed`), `tui_gateway/session_reaper.py` (ticker and reapers queue jobs on at most 3 daemon workers, `_cache_expiry_submit`; profile scope per session; the action picks compaction or `_offload_expired_cache`) and `GatewaySessionWatchersMixin._compact_idle_sessions_before_cache_expiry` (housekeeping; the due-scan runs in `asyncio.to_thread`; `claim_gate` held across load→commit, `_profile_scope_for_source`, detached Desktop chats via `route_entry`, per-session bounded jobs). No owner sends anything after a compaction. Release owners: `GatewayAgentCacheMixin._sweep_idle_cached_agents` and `_enforce_agent_cache_cap` hand each evicted agent to `GatewaySessionWatchersMixin._compact_then_release` (release claim in `asyncio.to_thread`; a due chat runs its compact or offload job, then the agent is released exactly once: task done-callback, plus `_drain_pending_cache_releases` at shutdown), and the TUI reapers (`_reap_idle_sessions`, `_enforce_session_cap`) go through `_compact_then_close` (compact or offload, then close if still reapable). A queued TUI job whose chat closed meanwhile is refused at `_manual_compress_turn` (`_closing`). - Archives: `_plan_compaction_window` is write-free (the prepared pass and every tool batch plan through it); `compress()` archives once, after its summary exists (`_commit_window`). `_assemble_with_archive_list` appends an `ARCHIVE_LIST_HEADING` list of every `cache/pruned` file a replaced row named that the kept rows and summary text do not, so `referenced_archive_refs` (the retention sweep) still sees each one. - Pricing: a keep-alive reply is priced at the tier its own request declared (`prompt_cache_clock.request_write_tier` + `turn_usage.with_write_tier`), as `record_response_usage` prices loop responses, so an OpenRouter 1h write with no tier split bills at 2x input. - Guards: each keep-alive is bound to `context_witness` at arm time and rechecked before send and before its reply is folded into the clock; `_admit` is the single admission under the lock `cancel` takes; the reset epoch and `session_totals_lock` fence the running totals; every send is bracketed by `pre/post_auxiliary_call` (`aux_task="cache_warm"`); `evaluate_economics` requires the read to cost less than the rewrite it prevents and keeps an idle budget's (or one tool wait's) billed total under that rewrite; unknown cache prices refuse arming; budget-based thinking is never replayed; usage is booked to the session and route snapshotted at arm time and folded into session totals only, never the compressor or usage anchors. diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index d1946ba9c9..a4c364f3e5 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1161,7 +1161,7 @@ The value is the **first rung** of an escalating ladder, not a fixed interval: c `proactive_prune_tokens` enables a deterministic, no-LLM prune of old tool-result payloads that runs independently of `threshold`. On large-window models the `threshold` compaction (≈50% of the window) rarely fires, so bulky tool outputs (terminal dumps, file reads, web extracts) ride along in history and get re-sent on every subsequent turn. When re-sent history exceeds `proactive_prune_tokens` (default `0` = off), the prune dedupes identical results and summarizes older oversized tool results, protecting the most recent `protect_last_n` messages and never calling the model. Tool-call arguments are execution records and are never rewritten by pruning; the summary model has a separate bounded serializer for copies included in its prompt. The pressure pass likewise demotes tool-result bodies only. During full semantic compaction the carried head/tail rows keep those tool-result demotions (so an oversized tail can still compress), while tool-call arguments stay byte-exact. Each committed prune rewrites already-sent history and breaks the prompt cache, so `proactive_prune_min_reclaim_tokens` (default `4096`) keeps those commits episodic; `proactive_prune_min_result_chars` (default `8000`, clamped to ≥ 200) sets the size below which a tool result is left untouched. -**When the cache has already expired** (on by default, with no setting of its own): once the session's prompt-cache clock (the same window `/context` shows) says the cached prompt is gone, rewriting old history costs nothing extra. That is true past the end of a proven or declared window (a Claude tier ends at its TTL after the last cache hit; every other model is treated as 30 minutes after its last cache hit, or 24 hours when the request asks for 24-hour retention, as on Meta and Bedrock Mantle; a local or private-network endpoint such as llama.cpp, vLLM or Ollama never expires on the clock), and right after a model or provider switch (`/model`, a fallback), because the new route has never seen this prompt. A window that may still be warm, or one the clock only assumes (such as a session saved before the clock existed), never counts as expired. Hermes then prunes old tool output: at the start of the next turn, after a tool round that outlasted the window, and, for a chat you type into, as soon as the window passes while the chat sits idle (so it is already small when you come back) or when the idle chat is released from memory. It runs without needing `proactive_prune_tokens`, and only when it frees at least 16,000 tokens (or `proactive_prune_min_reclaim_tokens`, if the warm prune is on with a lower value). It never touches the warm prune's regrowth wait, and what it frees counts toward the next warm gate. Turning `compression.enabled` off turns it off too. Nothing is lost: before a result is shortened, its full original is written to `$HERMES_HOME/cache/pruned/___.json` (`.txt` when the output is not JSON), read back and compared byte-for-byte, and the one-line note ends with `— full result: ` so the model reads the exact output back with `read_file` (which pages a long one-line archive too) instead of re-running the tool. If that file cannot be written, or the terminal backend is anything other than `local` (docker, modal, ssh, daytona, vercel_sandbox, plugin backends: `read_file` there opens the sandbox's files, not the host's), the result is left whole. The protected recent tail never starts in the middle of one assistant step's parallel tool results (each older result is still judged on its own size), and the results of the turn in progress are never shortened; if Hermes cannot tell where that turn starts, it skips the prune for that round. The originals also stay in `state.db` (archived, still found by `session_search`). +**When the cache has already expired** (on by default, with no setting of its own): once the session's prompt-cache clock (the same window `/context` shows) says the cached prompt is gone, rewriting old history costs nothing extra. That is true past the end of a proven or declared window (a Claude tier ends at its TTL after the last cache hit; every other model is treated as 30 minutes after its last cache hit, or 24 hours when the request asks for 24-hour retention, as on Meta and Bedrock Mantle; a local or private-network endpoint such as llama.cpp, vLLM or Ollama never expires on the clock), and right after a model or provider switch (`/model`, a fallback), because the new route has never seen this prompt. A window that may still be warm, or one the clock only assumes (such as a session saved before the clock existed), never counts as expired. Hermes then prunes old tool output: at the start of the next turn, after a tool round that outlasted the window, and, for a chat you type into, as soon as the window passes while the chat sits idle (so it is already small when you come back) or when the idle chat is released from memory. It runs without needing `proactive_prune_tokens`, and only when it frees at least 16,000 tokens (or `proactive_prune_min_reclaim_tokens`, if the warm prune is on with a lower value). It never touches the warm prune's regrowth wait, and what it frees counts toward the next warm gate. Turning `compression.enabled` off turns it off too. On the `local` backend each shortened result stays readable: before a result is shortened, its full original is written to `$HERMES_HOME/cache/pruned/___.json` (`.txt` when the output is not JSON), read back and compared byte-for-byte, and the one-line note ends with `— full result: ` so the model reads the exact output back with `read_file` (which pages a long one-line archive too) instead of re-running the tool. If that file cannot be written, or the terminal backend is anything other than `local` (docker, modal, ssh, daytona, vercel_sandbox, plugin backends: `read_file` there opens the sandbox's files, not the host's), the result is left whole. The protected recent tail never starts in the middle of one assistant step's parallel tool results (each older result is still judged on its own size), and the results of the turn in progress are never shortened; if Hermes cannot tell where that turn starts, it skips the prune for that round. The originals also stay in `state.db` (archived, still found by `session_search`). Archive files follow the spillover rules: a file is kept while any `state.db` row still points at it and is otherwise swept after `sessions.retention_days`. Rows keep pointing at it after a compaction (as inactive rows), so a referenced archive stays on disk until its session is deleted, and `sessions.auto_prune` is off by default: without it, archives are never removed. When more than 20,000 rows reference archives, the sweep cannot read the full reference set and keeps every file (spillover included). The prune never runs when `compression.checkpoint_required` is on (also after a live config change) or when the session has no store row to archive the originals into. This runs only under the built-in `compressor` engine; other context engines inherit a no-op. @@ -1546,19 +1546,19 @@ For Claude on **native Anthropic**, **OpenRouter**, and **Nous Portal**, Hermes The Qwen Cloud (Alibaba DashScope) upstream caps cache TTL at 5 minutes, so Hermes uses the 5-minute breakpoint TTL there instead. Other Claude-via-third-party paths (AWS Bedrock, Azure Foundry) fall back to the provider's own caching defaults. xAI Grok uses a separate session-pinned conversation-id mechanism — see [xAI prompt caching](../integrations/providers.md#xai-grok--responses-api--prompt-caching). -No knob exists to disable this — caching is always-on and saves money even on single-turn conversations because the system prompt alone is a meaningful fraction of the input token count. +Leave it on: it saves money even on single-turn conversations because the system prompt alone is a meaningful fraction of the input token count. `cache_ttl: "off"` (or `false`, `none`, `disabled`, `no`) turns it off entirely, for OAuth plans that bill cache writes as extra usage or proxies that inject their own `cache_control`. The one explicit knob is the cache TTL tier Hermes requests on Anthropic-style breakpoints: ```yaml prompt_caching: - cache_ttl: "auto" # default; or pin one of the Anthropic tiers, "5m" or "1h"; other values are ignored + cache_ttl: "auto" # default; or pin "5m" / "1h"; "off" disables caching; any other value means "5m" route_ttl_ceilings: {} # per-provider cap, e.g. {nous: "5m"}; {: "1h"} lifts a shipped cap ``` -`cache_ttl` selects the breakpoint TTL Hermes attaches for Claude via the native Anthropic API, OpenRouter, and Nous Portal. The two Anthropic tiers (`"5m"`, `"1h"`) are sent as-is; any other value is ignored. Providers with their own caps (e.g. Qwen Cloud, which maxes at 5 minutes) still clamp to what the upstream allows. +`cache_ttl` selects the breakpoint TTL Hermes attaches for Claude via the native Anthropic API, OpenRouter, and Nous Portal. The two Anthropic tiers (`"5m"`, `"1h"`) are sent as-is; an off value (`"off"`, `false`, `none`) disables prompt caching; any other value falls back to `"5m"`. Providers with their own caps (e.g. Qwen Cloud, which maxes at 5 minutes) still clamp to what the upstream allows. -The 1h tier writes at 2x the base input price (5m writes at 1.25x) and only pays off when your turns are more than five minutes apart — otherwise every tool result is written at the dearer rate for retention nobody uses. `"auto"` picks the tier per session from who paces it: `1h` for sessions a person types into (CLI, TUI, Desktop, Telegram/Discord/Slack and the other messaging platforms), `5m` for machine-paced ones (subagents, cron, `hermes -q` one-shots, webhooks, Kanban workers, the API server, the skill curator, tool-invoked and batch runs). `"auto"` is the default; set `"5m"` or `"1h"` to pin one tier for every session. A versioned config that still carries the old seeded `cache_ttl: "5m"` default has it removed by the v54 config migration, so it follows `"auto"` too (a config without `_config_version` keeps it). Under `"auto"`, side agents (`/background`, `/btw`) and background reviews routed to a different model stay on `5m`. On an install where interactive sessions are parked and resumed through the day, `auto` cut the interactive cache-write bill by roughly 40% while leaving fan-out subagent spend untouched. Delegated subagents are always clamped to `5m`, whatever the setting. +The 1h tier writes at 2x the base input price (5m writes at 1.25x) and only pays off when your turns are more than five minutes apart — otherwise every tool result is written at the dearer rate for retention nobody uses. `"auto"` picks the tier per session from who paces it: `1h` for sessions a person types into (CLI, TUI, Desktop, Telegram/Discord/Slack and the other messaging platforms), `5m` for machine-paced ones (subagents, cron, `hermes -q` one-shots, webhooks, Kanban workers, the API server, the skill curator, tool-invoked and batch runs). `"auto"` is the default; set `"5m"` or `"1h"` to pin one tier for every session. The v54 config migration removes `cache_ttl: "5m"` only when it is still the line the old config template wrote (with its `# use "1h" for long sessions with pauses between turns` comment), so that config follows `"auto"`; any other `"5m"` is kept as your pin, and the migration prints a notice saying so (a config without `_config_version` is not migrated). Under `"auto"`, side agents (`/background`, `/btw`) and background reviews routed to a different model stay on `5m`. On an install where interactive sessions are parked and resumed through the day, `auto` cut the interactive cache-write bill by roughly 40% while leaving fan-out subagent spend untouched. Delegated subagents are always clamped to `5m`, whatever the setting. `prompt_caching.route_ttl_ceilings` caps the tier per provider, so a route that silently keeps less than it is asked for is neither asked for (nor billed for) the dearer tier. A provider's profile can ship a ceiling too; the config entry overrides it. A ceiling clamps what the cache clock believes too: a route capped at `5m` is warmed and expired on the 5-minute schedule even when the request asked for `1h`. Nous Portal ships `1h` (it keeps, and bills, Claude's 1-hour tier). On Nous Portal the model catalog decides which models get cache markers: every model priced with a cache-write rate (`input_cache_write` in `/v1/models`: Claude, Qwen 3.7, Gemini, GPT-6 and others) gets them, at the tier asked for, and a 1-hour write is billed at the catalog's own rate (Claude 2x input, Qwen 1.25x); a model without one caches automatically. A `custom` endpoint pointed at a known provider's host gets that provider's ceiling. @@ -1578,10 +1578,12 @@ The clock is saved with the session (`sessions.prompt_cache_clock`), so a resume A cache entry expires a while after its last use; the next request after that re-sends the whole conversation at full price. One controller per chat watches the cache clock above and, shortly before the entry would expire, makes one decision. There is nothing to configure. -The cache rules follow the API spec. **Anthropic spec** (the request carries cache markers: Claude anywhere, including Nous Portal, OpenRouter and other gateways, and Anthropic-compatible endpoints) keeps the 5-minute or 1-hour lifetime the request asks for, capped by the route ceiling. **OpenAI spec** (no markers: OpenAI, DeepSeek, Qwen, GLM and other OpenAI-compatible endpoints, and non-Claude models on Nous Portal) caches automatically; Hermes assumes 30 minutes, renewed by each cache hit, and stops warming a session the first time that assumption fails. Both get the same loop. +The cache rules follow the API spec. **Anthropic spec** (the request carries cache markers: Claude anywhere, including Nous Portal, OpenRouter and other gateways, and Anthropic-compatible endpoints) keeps the 5-minute or 1-hour lifetime the request asks for, capped by the route ceiling. **OpenAI spec** (no markers: OpenAI, DeepSeek, Qwen, GLM and other OpenAI-compatible endpoints, and non-Claude models on Nous Portal) caches automatically; Hermes assumes 30 minutes, renewed by each cache hit, and stops warming a session the first time that assumption fails. Both get the same loop. Keep-alives go out only on the Anthropic Messages and Chat Completions wires; Responses API sessions (`openai-codex`, OpenAI Responses, xAI) are never warmed, so they are not compacted at the cache edge (a cold one is still offloaded). -1. **Warm.** At 5/6 of the lifetime (about 4 minutes 10 seconds on a 5-minute cache, 25 minutes on a 30-minute one, 50 minutes on a 1-hour one), Hermes re-sends the last request with the smallest output cap the model accepts (1 token; 16 for OpenAI's GPT-5 and o-series, which refuse less). One cache read keeps the entry alive for another lifetime. While the chat is idle it does this at most 3 times per message you send. While a tool call is running (the model is certain to send its next request when the tool returns), it keeps warming until the tool returns, with no count limit: only money bounds it, so the keep-alives of one tool wait together never cost more than the single full re-send they prevent (about 12 on a 5-minute Claude cache, about 20 on a 1-hour one, about 10 on a 30-minute automatic cache). Those do not use up the 3 idle keep-alives. A keep-alive that comes back without a cache hit means the entry was already gone: warming stops for the rest of the session (your next messages, and compactions, do not restart it; a `/model` or provider switch does, since that is a different cache), and the chat goes straight to step 2 or 3. -2. **Compact.** A set of idle keep-alives ends at whichever comes first: all 3 are used, one misses, or Hermes releases the idle chat from memory (the messaging gateway evicts a chat idle for an hour, or past its cache-size cap; the TUI/Desktop backend closes a session idle past its time limit or over its session cap). At that point Hermes compacts the chat (the normal compaction, with the normal compression model, or a summary already prepared by `compression.prepare_ahead`) if all of these hold: the cache window is at least 30 minutes (Claude at `1h`, or any 30-minute automatic cache), the context is at least half of the compaction threshold, and the chat grew since its last compaction. Whether the old cache is still warm does not matter: the summary is a separate request that never re-reads the cached chat. After a compaction nothing more is sent: there is no warm cache to keep, so the next keep-alives start with your next message. This happens at most once per idle stretch. +Two routes have no 30-minute expiry. A request that asks for 24-hour retention (`prompt_cache_retention: "24h"`, e.g. Meta's `api.meta.ai` and Bedrock Mantle GPT-5.x/4.1) is still warmed on the 30-minute schedule, but its cache is treated as alive for 24 hours, so it is not pruned as cold at minute 31 (ChatGPT Codex does not accept the field, Hermes strips it there, and 30 minutes stands). A local or self-hosted endpoint (loopback, private-network or `.local` host: llama.cpp, vLLM, Ollama) never expires on the clock; only a model or provider switch, or a compaction, offloads its history. + +1. **Warm.** At 5/6 of the lifetime (about 4 minutes 10 seconds on a 5-minute cache, 25 minutes on a 30-minute one, 50 minutes on a 1-hour one), Hermes re-sends the last request with the smallest output cap the model accepts (1 token; 16 for OpenAI's GPT-5 and o-series, which refuse less). One cache read keeps the entry alive for another lifetime. While the chat is idle it does this at most 3 times per message you send. While a tool call is running (the model is certain to send its next request when the tool returns), it keeps warming until the tool returns, with no count limit: only money bounds it, so the keep-alives of one tool wait together never cost more than the single full re-send they prevent (about 12 on a 5-minute Claude cache, about 20 on a 1-hour one, about 10 on a 30-minute automatic cache). Those do not use up the 3 idle keep-alives. A keep-alive that comes back without a cache hit means the entry was already gone: warming stops for the rest of the conversation (your next messages, a gateway reloading the chat, and compactions do not restart it; a `/model` or provider switch does, since that is a different cache, and `/new` starts fresh), and the chat goes straight to step 2 or 3. +2. **Compact.** A set of idle keep-alives ends at whichever comes first: all 3 are used, one misses, the next one would cost more than the re-send it prevents, or Hermes releases the idle chat from memory (the messaging gateway evicts a chat idle for an hour, or past its cache-size cap; the TUI/Desktop backend closes a session idle past its time limit or over its session cap). At that point Hermes compacts the chat (the normal compaction, with the normal compression model, or a summary already prepared by `compression.prepare_ahead`) if all of these hold: the cache window is at least 30 minutes (Claude at `1h`, or any 30-minute automatic cache), the context is at least half of the compaction threshold, and the chat grew since its last compaction. Whether the old cache is still warm does not matter: the summary is a separate request that never re-reads the cached chat. After a compaction nothing more is sent: there is no warm cache to keep, so the next keep-alives start with your next message. This happens at most once per idle stretch. 3. **Expire and offload.** Otherwise Hermes lets the entry expire, and once the window has passed it saves old tool results to disk and replaces them with a pointer right away (see the prune section above), so the chat is already small when you come back. A chat released from memory after its window is offloaded before it is released. 4. **Coming back cold.** If your next message goes out after the clock says the cache has expired and the chat was not offloaded yet, the same offload runs first. @@ -1591,9 +1593,9 @@ Who gets what: chats you type into (CLI, TUI, Desktop, messaging) get all four s A keep-alive that goes out while a tool call is still running is a separate side request with the same prompt: nothing is added to the conversation, the model never sees it, and the tool result lands normally. -If the clock cannot vouch for the window (`unknown` or `assumed` on `/context`), nothing is warmed and nothing is compacted at the cache edge. A keep-alive is sent only when its cache read costs less than the full re-send it prevents and the keep-alives of one idle set, or of one tool wait, together cost no more than that re-send; there is no fixed dollar minimum, so cheap models are warmed too. It also needs known cache prices (an unpriced or subscription-included route is never warmed) and a cached prompt of at least 1,024 tokens (no smaller prompt is cached). Route ceilings apply: a route capped at `5m` is warmed on the 5-minute schedule. Requests with budget-based extended thinking are never warmed because a 1-token cap would change the budget Anthropic keys the cache on. A keep-alive only goes out through a client that really sends the small cap: native Gemini raises any cap to 65,535 tokens when thinking is on, so those chats are not warmed (with thinking off they are, at 1 token), and a long-decode budget is dropped from the replay. Every stop is logged at INFO (`prompt cache warming stopped: ` in `agent.log`); keep-alives count toward the session totals (`session_cache_warm_calls`, `task=cache_warm` usage rows) but never enter the conversation, and a cache-edge compaction is recorded as a `cache_expiry` compression. +If the clock cannot vouch for the window (`unknown` or `assumed` on `/context`), nothing is warmed and nothing is compacted at the cache edge. A keep-alive is sent only when its cache read, plus the output it may generate (1 token, or 16 on GPT-5/o-series) and any per-request fee, costs less than the full re-send it prevents and the keep-alives of one idle set, or of one tool wait, together cost no more than that re-send; there is no fixed dollar minimum, so cheap models are warmed too. It also needs known cache prices (an unpriced or subscription-included route is never warmed) and a cached prompt of at least 1,024 tokens (no smaller prompt is cached). Route ceilings apply: a route capped at `5m` is warmed on the 5-minute schedule. Requests with budget-based extended thinking are never warmed because a 1-token cap would change the budget Anthropic keys the cache on. A keep-alive only goes out through a client that really sends the small cap: native Gemini raises any cap to 65,535 tokens when thinking is on, so those chats are not warmed (with thinking off they are, at 1 token), and a long-decode budget is dropped from the replay. Every stop is logged at INFO (`prompt cache warming stopped: ` in `agent.log`); keep-alives count toward the session totals (`session_cache_warm_calls`, `task=cache_warm` usage rows) but never enter the conversation, and a cache-edge compaction is recorded as a `cache_expiry` compression. -Every compaction (automatic, prepared, `/compress`, gateway hygiene and the cache-edge one) summarizes the original tool results first, so the summary reads what the tools actually returned. Only after the summary is written does it save each tool result it removed or shortened to `$HERMES_HOME/cache/pruned/`, byte for byte, once, and leave a `— full result: ` pointer in its place, so the model can still reach the exact output. Planning whether to compact writes nothing, and a summary that fails writes nothing. The committed summary ends with an `Archived tool output` list naming every saved file, so no file loses its last reference (the retention sweep keeps files a stored message names). A result whose file cannot be written stays whole. On non-`local` terminal backends the results are shortened as before, without a file. +Every compaction (automatic, prepared, `/compress`, gateway hygiene and the cache-edge one) summarizes the original tool results first, so the summary reads what the tools actually returned. Only after the summary is written does it save each tool result it removed or shortened to `$HERMES_HOME/cache/pruned/`, byte for byte, once, and leave a `— full result: ` pointer in its place, so the model can still reach the exact output; short and duplicate results the summary drops are saved too. Planning whether to compact writes nothing, and a summary that fails writes nothing. The committed summary ends with an `Archived tool output` list naming every saved file, so no file loses its last reference (the retention sweep keeps files a stored message names). A result whose file cannot be written stays whole. On sandboxed terminal backends (docker, ssh, modal, daytona and other non-`local` ones) the model cannot read host files, so tool output is summarized and shortened without an archive: the removed text is gone from the live context (the original rows stay in `state.db`). ## Auxiliary Models