Files
hermes-agent/tests/agent/test_bedrock_request_timeout.py
Teknium e286e63e0e fix(bedrock): a hung Bedrock call fails after one read timeout, not five
botocore's default retry policy (legacy mode, 5 total attempts) re-sent a
Converse/ConverseStream read timeout, so a hung call surfaced after ~5x the
configured request timeout (5x botocore's 60s default when unset), and
throttles were retried under Hermes's own adaptive rate-limit backoff.

The bedrock-runtime client now makes one attempt per call
(retries.total_max_attempts=1; botocore's max_attempts counts retries, not
attempts). Hermes's retry loop already classifies ReadTimeoutError as a
retryable timeout and ThrottlingException as rate_limit with backoff; the
AnthropicBedrock client already runs with max_retries=0.
2026-10-28 21:22:10 -07:00

262 lines
12 KiB
Python

"""Bedrock request timeouts reach the clients that serve the call.
``providers.bedrock.request_timeout_seconds`` and ``providers.bedrock.models.<id>.timeout_seconds``
resolved correctly but never reached a client: the boto3 Converse client kept botocore's 60s read
timeout and the AnthropicBedrock SDK client its 900s default. These tests read the real botocore /
httpx client config (dummy credentials, IMDS off) and intercept the botocore API call itself, so no
request leaves the process.
"""
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
pytest.importorskip("boto3")
pytest.importorskip("botocore.client")
import boto3
import botocore.client
from agent import bedrock_adapter
from agent.bedrock_adapter import _get_bedrock_runtime_client, invalidate_runtime_client
REGION = "us-east-1"
RUNTIME_URL = f"https://bedrock-runtime.{REGION}.amazonaws.com"
CLAUDE = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" # model override 300
NOVA = "us.amazon.nova-pro-v1:0" # no model override -> provider 600
LLAMA = "us.meta.llama4-maverick-17b-instruct-v1:0" # model override 120
OVERRIDES = {"providers": {"bedrock": {
"request_timeout_seconds": 600,
"models": {CLAUDE: {"timeout_seconds": 300}, LLAMA: {"timeout_seconds": 120}},
}}}
BOTOCORE_DEFAULT_TIMEOUT = 60
@pytest.fixture(autouse=True)
def aws_env(monkeypatch):
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIA-AMBIENT")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "ambient-secret")
monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true")
for var in ("AWS_PROFILE", "AWS_SESSION_TOKEN", "AWS_BEARER_TOKEN_BEDROCK"):
monkeypatch.delenv(var, raising=False)
bedrock_adapter.reset_client_cache()
yield
bedrock_adapter.reset_client_cache()
@pytest.fixture
def config(monkeypatch):
"""Install ``cfg`` as the loaded config (the operator report's offline technique)."""
def install(cfg):
monkeypatch.setattr("hermes_cli.config.load_config_readonly", lambda: cfg)
install({})
return install
@pytest.fixture
def botocore_calls(monkeypatch):
"""Intercept every botocore API call: record (operation, serving client, its read timeout)."""
calls = []
def fake_make_api_call(client, operation_name, params):
calls.append((operation_name, client, client.meta.config.read_timeout, params.get("modelId")))
if operation_name == "ConverseStream":
return {"stream": []}
return {"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
"stopReason": "end_turn", "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}}
monkeypatch.setattr(botocore.client.BaseClient, "_make_api_call", fake_make_api_call)
return calls
def _converse_kwargs(model):
return {"__bedrock_converse__": True, "__bedrock_region__": REGION, "modelId": model,
"messages": [{"role": "user", "content": [{"text": "hi"}]}]}
# --- client construction ----------------------------------------------------------------------
def test_unset_config_builds_exactly_the_botocore_default_client(config):
baseline = boto3.client("bedrock-runtime", region_name=REGION).meta.config
built = _get_bedrock_runtime_client(REGION, CLAUDE).meta.config
assert (built.read_timeout, built.connect_timeout) == (BOTOCORE_DEFAULT_TIMEOUT, BOTOCORE_DEFAULT_TIMEOUT)
assert (built.read_timeout, built.connect_timeout, built.max_pool_connections) == (
baseline.read_timeout, baseline.connect_timeout, baseline.max_pool_connections)
@pytest.mark.parametrize("model, expected", [(NOVA, 600), (CLAUDE, 300)], ids=["provider", "model-wins"])
def test_configured_timeout_is_the_client_read_timeout_and_connect_stays_bounded(config, model, expected):
config(OVERRIDES)
client_config = _get_bedrock_runtime_client(REGION, model).meta.config
assert client_config.read_timeout == expected
assert client_config.connect_timeout == BOTOCORE_DEFAULT_TIMEOUT, "a long read budget must not stretch connect"
@pytest.mark.parametrize("provider_value, model_value", [("soon", -5), (0, "later"), ("", None)])
def test_invalid_values_behave_like_unset(config, provider_value, model_value):
config({"providers": {"bedrock": {"request_timeout_seconds": provider_value,
"models": {CLAUDE: {"timeout_seconds": model_value}}}}})
client = _get_bedrock_runtime_client(REGION, CLAUDE)
assert client.meta.config.read_timeout == BOTOCORE_DEFAULT_TIMEOUT
assert client is _get_bedrock_runtime_client(REGION), "invalid must share the unset client, not a new one"
def test_models_with_different_overrides_never_share_a_client_in_one_region(config):
config(OVERRIDES)
claude, nova, llama = (_get_bedrock_runtime_client(REGION, m) for m in (CLAUDE, NOVA, LLAMA))
assert [c.meta.config.read_timeout for c in (claude, nova, llama)] == [300, 600, 120]
assert len({id(claude), id(nova), id(llama)}) == 3
# Bounded: same region + same effective timeout reuses one client.
assert _get_bedrock_runtime_client(REGION, CLAUDE) is claude
assert _get_bedrock_runtime_client(REGION, "us.amazon.nova-lite-v1:0") is nova # provider-level 600 too
config({}) # unset must not collide with any configured slot
unset = _get_bedrock_runtime_client(REGION, CLAUDE)
assert unset not in (claude, nova, llama)
assert unset.meta.config.read_timeout == BOTOCORE_DEFAULT_TIMEOUT
def test_profile_scoped_clients_carry_their_own_profile_timeouts(tmp_path, monkeypatch):
"""A routed profile's clients come from its own config.yaml and own cache slot, per timeout."""
from agent import secret_scope
from hermes_constants import reset_hermes_home_override, set_hermes_home_override
home = tmp_path / "profile-home"
home.mkdir()
(home / ".env").write_text("AWS_ACCESS_KEY_ID=AKIA-PROFILE\nAWS_SECRET_ACCESS_KEY=profile-secret\n")
(home / "config.yaml").write_text(
"providers:\n bedrock:\n request_timeout_seconds: 600\n"
f" models:\n {CLAUDE}:\n timeout_seconds: 300\n")
monkeypatch.setattr(secret_scope, "_MULTIPLEX_ACTIVE", True)
def in_profile(fn):
h_tok = set_hermes_home_override(str(home))
s_tok = secret_scope.set_secret_scope(secret_scope.build_profile_secret_scope(home))
try:
return fn()
finally:
secret_scope.reset_secret_scope(s_tok)
reset_hermes_home_override(h_tok)
claude, nova, claude_again = in_profile(lambda: (
_get_bedrock_runtime_client(REGION, CLAUDE), _get_bedrock_runtime_client(REGION, NOVA),
_get_bedrock_runtime_client(REGION, CLAUDE)))
assert (claude.meta.config.read_timeout, nova.meta.config.read_timeout) == (300, 600)
assert claude.meta.config.connect_timeout == nova.meta.config.connect_timeout == BOTOCORE_DEFAULT_TIMEOUT
assert claude is claude_again and claude is not nova
assert claude._request_signer._credentials.access_key == "AKIA-PROFILE"
# The launch context (no override, its own unset config) keeps botocore's default client.
ambient = _get_bedrock_runtime_client(REGION, CLAUDE)
assert ambient is not claude and ambient.meta.config.read_timeout == BOTOCORE_DEFAULT_TIMEOUT
# Stale-pool eviction drops every timeout slot of the profile's region, never the launch context's.
assert in_profile(lambda: invalidate_runtime_client(REGION)) is True
assert in_profile(lambda: _get_bedrock_runtime_client(REGION, CLAUDE)) is not claude
assert _get_bedrock_runtime_client(REGION, CLAUDE) is ambient
# --- dispatch ---------------------------------------------------------------------------------
def test_converse_and_converse_stream_are_served_with_the_requested_models_timeout(config, botocore_calls):
from agent.chat_completion_helpers import _BedrockStream, _dispatch_nonstreaming_api_request
config(OVERRIDES)
agent = SimpleNamespace(api_mode="bedrock_converse", provider="bedrock", model=CLAUDE,
_disable_streaming=False, _safe_print=MagicMock())
_dispatch_nonstreaming_api_request(agent, _converse_kwargs(CLAUDE), make_client=None)
_BedrockStream(agent, _converse_kwargs(NOVA), on_first_delta=None)._open_stream(_converse_kwargs(NOVA))
assert [(op, timeout, model) for op, _client, timeout, model in botocore_calls] == [
("Converse", 300, CLAUDE), ("ConverseStream", 600, NOVA)]
def test_auxiliary_call_converse_uses_the_models_timeout(config, botocore_calls):
from agent.bedrock_adapter import call_converse
config(OVERRIDES)
call_converse(region=REGION, model=LLAMA, messages=[{"role": "user", "content": "hi"}])
assert [(op, timeout) for op, _c, timeout, _m in botocore_calls] == [("Converse", 120)]
def _agent():
from agent.context_compressor import ContextCompressor
from run_agent import AIAgent
agent = AIAgent.__new__(AIAgent)
agent.model, agent.provider = "claude-opus-4.8", "anthropic"
agent.base_url, agent.api_key, agent.api_mode = "https://api.anthropic.com", "sk-ant", "anthropic_messages"
agent.client, agent._anthropic_client, agent._client_kwargs = None, MagicMock(), {}
agent.quiet_mode, agent._config_context_length, agent._primary_runtime = True, None, {}
agent.max_tokens = None
agent.context_compressor = ContextCompressor(
model=agent.model, threshold_percent=0.5, base_url=agent.base_url, api_key="sk-ant",
provider="anthropic", quiet_mode=True, config_context_length=None,
)
return agent
def _dispatch_turn(agent):
from agent.chat_completion_helpers import _build_bedrock_kwargs, _dispatch_nonstreaming_api_request
kwargs = _build_bedrock_kwargs(agent, [{"role": "user", "content": "hi"}], None, None)
_dispatch_nonstreaming_api_request(agent, kwargs, make_client=None)
def test_model_switch_and_fallback_dispatch_with_the_new_models_timeout(config, botocore_calls, monkeypatch):
from agent.client_lifecycle import _swap_fallback_clients
monkeypatch.setattr("agent.model_metadata.get_model_context_length", lambda *a, **k: 200_000)
config(OVERRIDES)
agent = _agent()
agent.switch_model(LLAMA, "bedrock", api_key="aws-sdk", base_url=RUNTIME_URL, api_mode="bedrock_converse")
_dispatch_turn(agent)
# Fallback onto another Bedrock model, bound the way route_binding.bind_route_entry binds it.
agent.model, agent.provider, agent.base_url, agent.api_mode = NOVA, "bedrock", RUNTIME_URL, "bedrock_converse"
_swap_fallback_clients(agent, MagicMock(base_url=RUNTIME_URL, api_key="aws-sdk"), "bedrock", NOVA,
RUNTIME_URL, "bedrock_converse")
_dispatch_turn(agent)
agent.switch_model(LLAMA, "bedrock", api_key="aws-sdk", base_url=RUNTIME_URL, api_mode="bedrock_converse")
_dispatch_turn(agent)
assert [(timeout, model) for _op, _c, timeout, model in botocore_calls] == [
(120, LLAMA), (600, NOVA), (120, LLAMA)]
first, second, third = (client for _op, client, _t, _m in botocore_calls)
assert first is third and first is not second
# --- AnthropicBedrock (Claude via the anthropic SDK) ------------------------------------------
def test_anthropic_bedrock_client_takes_the_timeout_and_keeps_default_when_unset():
pytest.importorskip("anthropic")
from agent.anthropic_adapter import build_anthropic_bedrock_client
configured = build_anthropic_bedrock_client(REGION, timeout=300.0).timeout
unset = build_anthropic_bedrock_client(REGION).timeout
assert (configured.read, configured.connect) == (300.0, 10.0)
assert (unset.read, unset.connect) == (900.0, 10.0)
def test_anthropic_bedrock_request_client_follows_the_active_model(config, monkeypatch):
pytest.importorskip("anthropic")
monkeypatch.setattr("agent.model_metadata.get_model_context_length", lambda *a, **k: 200_000)
other_claude = "us.anthropic.claude-haiku-4-5-20251001-v1:0" # provider-level 600
config(OVERRIDES)
agent = _agent()
agent.switch_model(CLAUDE, "bedrock", api_key="aws-sdk", base_url=RUNTIME_URL, api_mode="anthropic_messages")
shared = agent._anthropic_client.timeout
first = agent._create_request_anthropic_client(reason="test").timeout
agent.model = other_claude # e.g. a fallback onto another Bedrock Claude in the same region
second = agent._create_request_anthropic_client(reason="test").timeout
assert (shared.read, first.read, second.read) == (300.0, 300.0, 600.0)
assert first.connect == second.connect == 10.0