Files
hermes-agent/tests/hermes_cli/test_context_switch_guard.py
yoniebans 58ab71a2b6 test(hermes_cli): CLI cases over the persistence interface, first half
The CLI tests seed and read the store through the storage fixtures.
2026-11-17 17:51:20 +01:00

262 lines
8.9 KiB
Python

"""Tests for hermes_cli.context_switch_guard."""
from __future__ import annotations
from types import SimpleNamespace
from hermes_cli.context_switch_guard import merge_preflight_compression_warning
from hermes_cli.model_switch import ModelSwitchResult
def _result(*, model: str = "small-model") -> ModelSwitchResult:
return ModelSwitchResult(
success=True,
new_model=model,
target_provider="openrouter",
provider_changed=False,
api_key="k",
base_url="https://example.com/v1",
api_mode="chat_completions",
provider_label="openrouter",
model_info={"context_length": 32_000},
)
def _compressor(
monkeypatch,
*,
context_length: int = 200_000,
threshold_tokens_cap: int | None = None,
):
from agent.context_compressor import ContextCompressor
monkeypatch.setattr(
"agent.context_compressor.get_model_context_length",
lambda *a, **k: context_length,
)
return ContextCompressor(
model="big-model",
threshold_percent=0.5,
protect_first_n=3,
protect_last_n=20,
quiet_mode=True,
config_context_length=context_length,
threshold_tokens_cap=threshold_tokens_cap,
)
def test_merge_appends_to_existing_warning(monkeypatch):
monkeypatch.setattr(
"hermes_cli.context_switch_guard._estimate_tokens",
lambda *a, **k: 90_000,
)
monkeypatch.setattr(
"hermes_cli.context_switch_guard.resolve_display_context_length",
lambda *a, **k: 32_000,
)
cc = _compressor(monkeypatch)
agent = SimpleNamespace(
context_compressor=cc,
compression_enabled=True,
base_url="",
api_key="",
)
result = _result()
result.warning_message = "expensive"
merge_preflight_compression_warning(result, agent=agent)
assert "expensive" in result.warning_message
assert "preflight compression" in result.warning_message
def test_cap_lowers_the_switch_warning_threshold_below_the_ratio(monkeypatch):
"""The warning quotes the trigger the compressor will install: on a 1M target the ratio alone says
500K (no warning at 300K in-flight), the cap says less — the guard must warn with the capped number."""
cap = 256_000
monkeypatch.setattr(
"hermes_cli.context_switch_guard._estimate_tokens",
lambda *a, **k: 300_000,
)
monkeypatch.setattr(
"hermes_cli.context_switch_guard.resolve_display_context_length",
lambda *a, **k: 1_000_000,
)
cc = _compressor(
monkeypatch,
context_length=200_000,
threshold_tokens_cap=cap,
)
agent = SimpleNamespace(
context_compressor=cc,
compression_enabled=True,
base_url="",
api_key="",
)
result = _result(model="large-model")
merge_preflight_compression_warning(result, agent=agent)
assert "preflight compression" in result.warning_message
assert f"auto-compress at ~{cap:,}" in result.warning_message
def test_custom_provider_context_avoids_false_shrink_warning(monkeypatch):
"""Classic CLI used to omit custom_providers from the shrink warning.
Repro: switch onto a custom endpoint with models.<id>.context_length=1M
while session ~147k. Probe fails → hardcoded catalog match on "qwen"
(131072) → false "Context window shrinks (... → 131,072)" warning, even
though /model confirmation and the status bar correctly show 1M.
"""
custom_provs = [
{
"name": "qwen-token-plan",
"base_url": "https://token-plan.example/compatible-mode/v1",
"models": {
"qwen3.9-max-preview": {"context_length": 1_048_576},
},
}
]
# Force the probe-down path that hit the "qwen" → 131072 catalog match
# when custom_providers was not threaded through.
monkeypatch.setattr(
"agent.model_metadata._resolve_endpoint_context_length",
lambda *a, **k: None,
)
monkeypatch.setattr(
"agent.model_metadata._query_ollama_api_show",
lambda *a, **k: None,
)
monkeypatch.setattr(
"hermes_cli.context_switch_guard._estimate_tokens",
lambda *a, **k: 147_053,
)
cc = _compressor(monkeypatch, context_length=1_000_000)
agent = SimpleNamespace(
model="MiniMax-M3",
provider="minimax",
context_compressor=cc,
compression_enabled=True,
conversation_history=[],
base_url="https://api.minimax.example/v1",
api_key="",
_custom_providers=custom_provs,
)
result = ModelSwitchResult(
success=True,
new_model="qwen3.9-max-preview",
target_provider="qwen-token-plan",
provider_changed=True,
api_key="k",
base_url="https://token-plan.example/compatible-mode/v1",
api_mode="chat_completions",
provider_label="qwen-token-plan",
model_info=None,
)
# Explicit custom_providers — no false shrink warning (1M > 147k*2).
merge_preflight_compression_warning(
result,
agent=agent,
custom_providers=custom_provs,
)
assert not result.warning_message
# Agent snapshot alone (classic CLI historically forgot to pass the kwarg).
result2 = ModelSwitchResult(
success=True,
new_model="qwen3.9-max-preview",
target_provider="qwen-token-plan",
provider_changed=True,
api_key="k",
base_url="https://token-plan.example/compatible-mode/v1",
api_mode="chat_completions",
provider_label="qwen-token-plan",
model_info=None,
)
merge_preflight_compression_warning(result2, agent=agent)
assert not result2.warning_message
# Without any custom_providers source, catalog match still warns (131k).
agent_no_cp = SimpleNamespace(
model="MiniMax-M3",
provider="minimax",
context_compressor=cc,
compression_enabled=True,
conversation_history=[],
base_url="https://api.minimax.example/v1",
api_key="",
_custom_providers=None,
)
result3 = ModelSwitchResult(
success=True,
new_model="qwen3.9-max-preview",
target_provider="qwen-token-plan",
provider_changed=True,
api_key="k",
base_url="https://token-plan.example/compatible-mode/v1",
api_mode="chat_completions",
provider_label="qwen-token-plan",
model_info=None,
)
merge_preflight_compression_warning(result3, agent=agent_no_cp)
assert result3.warning_message
assert "preflight compression" in result3.warning_message
assert "shrinks" in result3.warning_message
# Must not honor the unused 1M custom override when no providers were passed.
assert "1,048,576" not in result3.warning_message
def test_lifetime_prompt_counter_is_not_a_size_estimate():
"""``session_prompt_tokens`` sums every call; without a measured size the guard stays silent (#126343)."""
from hermes_cli.context_switch_guard import _estimate_tokens
agent = SimpleNamespace(context_compressor=SimpleNamespace(last_prompt_tokens=0),
session_prompt_tokens=16_282_033)
assert _estimate_tokens(agent, None) is None
def test_gateway_warning_estimates_from_the_stored_transcript(tmp_path, monkeypatch):
"""A messaging chat's agent has no measured prompt size before its first turn on the new model; the warning
comes from the stored transcript, as the CLI's comes from its in-memory history."""
import threading
from hermes_cli.context_switch_guard import enrich_model_switch_warnings_for_gateway
from persistence_assembly import open_persistence, profile_scope
from tests.fixtures.storage_seed import seed_messages, seed_session
monkeypatch.setattr(
"hermes_cli.context_switch_guard.resolve_display_context_length",
lambda *a, **k: 32_000,
)
cc = _compressor(monkeypatch)
assert not cc.last_prompt_tokens
agent = SimpleNamespace(
context_compressor=cc,
compression_enabled=True,
base_url="",
api_key="",
)
result = _result()
store = open_persistence(profile_scope(home=tmp_path), eager=True)
try:
seed_session(store, "s-chat", "telegram")
seed_messages(store, "s-chat", *(
{"role": ("user", "assistant")[i % 2], "content": f"turn {i} " + "words " * 4_000}
for i in range(40)))
runner = SimpleNamespace(
_agent_cache_lock=threading.Lock(),
_agent_cache={"chat-key": (agent, None)},
_session_db=store,
session_store=SimpleNamespace(
get_or_create_session=lambda source: SimpleNamespace(session_id="s-chat")),
)
enrich_model_switch_warnings_for_gateway(result, runner, session_key="chat-key", source=None)
finally:
store.close()
assert "preflight compression" in (result.warning_message or "")
assert "auto-compress at ~" in result.warning_message