mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 02:46:38 +03:00
f3a5459eee
Final gate wave on #152477 (6b78fd266d): - OpenRouter relays an upstream failure on the choice (``finish_reason: "error"``, a ``choices[0].error``), possibly beside partial text: both now fail, the error classified like any provider error (401 -> live_auth); - a malformed Responses-style body (``output[].content`` not a list) is live_response, as the docs say, instead of live_unknown; a failed/cancelled status is handled in its own branch; - tui_gateway/methods_config.py's setup readiness still hand-rolled the credential predicate this PR extracted; it now calls runtime_readiness.runtime_has_credentials (identical rules); - ``hermes --help`` lists ``hermes fallback check``; docs name the failing and answering statuses. Second-model findings deliberately left to #152505 (wrong verdicts inside the new command, no existing-behaviour change): final-wire re-detection, swap-policy credentials, stream failure state.
334 lines
18 KiB
Python
334 lines
18 KiB
Python
"""``hermes fallback check`` — readiness of every fallback-chain entry without an agent session (#63852).
|
|
|
|
Static mode resolves each entry with the switch's own kwargs; ``--live`` sends one request on the
|
|
client the swap builds. The live request never goes through ``call_llm``: its recovery ladder would
|
|
retry or answer from another provider and hide the dead entry. Exit status is 1 when any entry is
|
|
not ready, so cron and CI can gate on it. User-facing contract: docs ``fallback-providers.md``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import functools
|
|
import json
|
|
import logging
|
|
import re
|
|
import time
|
|
from datetime import UTC, datetime
|
|
from typing import Any, Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Room for reasoning models, which spend their first tokens thinking (same rationale as the local
|
|
# runtime's touch generation); providers that need no cap never receive it (``_forwards_max_tokens``).
|
|
_LIVE_MAX_TOKENS = 512
|
|
_LIVE_TIMEOUT_S = 30.0
|
|
_LIVE_PROMPT = [{"role": "user", "content": "Reply with exactly: OK"}]
|
|
_NOT_PROBED = frozenset({"auto", "moa"})
|
|
# Router meta-models (``openrouter/auto``, ``[<model>]@preset/<slug>``) answer under the model they
|
|
# picked, so the reported id proves nothing.
|
|
_ROUTER_AUTO_MODEL = "openrouter/auto"
|
|
_ROUTER_PRESET_MARKER = "@preset/"
|
|
# Vertex's OpenAI-compatible surface may name a model by its (project-scoped) publisher path.
|
|
_VERTEX_PUBLISHER_PATH = re.compile(r"^(?:projects/[^/]+/locations/[^/]+/)?publishers/([^/]+)/models/")
|
|
# Responses-style terminal states that mean no answer, even when partial text came back.
|
|
_FAILED_STATUSES = frozenset({"failed", "cancelled"}) # not "incomplete": a token cap still answered
|
|
# Azure reports the base model, not the deployment name; hosts padded like
|
|
# ``anthropic_endpoints._is_azure_anthropic_endpoint`` so sovereign clouds match too.
|
|
_AZURE_HOST_LABELS = (".openai.azure.", ".services.ai.azure.", ".cognitiveservices.azure.")
|
|
# A provider may answer with a dated snapshot of the requested alias (gpt-4o → gpt-4o-2024-08-06,
|
|
# claude-sonnet-4 → claude-sonnet-4-20250514) or a ``-latest`` pointer; anything else is another model.
|
|
_SNAPSHOT_SUFFIX = r"(?:-\d{4}-?\d{2}-?\d{2}|-\d{4}|-latest)?"
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class EntryCheck:
|
|
"""One fallback entry's verdict; ``inference_ready`` stays None until ``--live`` probes it."""
|
|
|
|
provider: str
|
|
model: str
|
|
base_url: str = ""
|
|
api_mode: str = ""
|
|
auth_source: str = ""
|
|
configured: bool = False
|
|
auth_ready: bool = False
|
|
inference_ready: Optional[bool] = None
|
|
latency_ms: Optional[int] = None
|
|
model_returned: str = ""
|
|
error_class: str = ""
|
|
error: str = ""
|
|
|
|
@property
|
|
def ready(self) -> bool:
|
|
return self.configured and self.auth_ready and self.inference_ready is not False
|
|
|
|
@property
|
|
def status(self) -> str:
|
|
"""``ready`` / ``ready (static)``, else the ``error_class`` every not-ready path sets."""
|
|
if self.ready:
|
|
return "ready" if self.inference_ready else "ready (static)"
|
|
return self.error_class
|
|
|
|
|
|
def _redacted(text: Any, secrets: tuple[str, ...] = (), max_chars: Optional[int] = None) -> str:
|
|
"""Known credential values removed first (pattern redaction misses keys without a known
|
|
prefix), then the support redactor."""
|
|
from hermes_cli.debug_redaction import redact_debug_support_text
|
|
text = str(text or "")
|
|
for secret in secrets:
|
|
text = text.replace(secret, "***")
|
|
return redact_debug_support_text(text, max_chars=max_chars)
|
|
|
|
|
|
def _secret_values(*candidates: Any) -> tuple[str, ...]:
|
|
return tuple(c for c in candidates if isinstance(c, str) and len(c) >= 8)
|
|
|
|
|
|
def _reported_model_is_evidence(provider: str, model: str, client: Any) -> bool:
|
|
"""Whether a response's ``model`` field can prove which model answered.
|
|
|
|
Only a real OpenAI SDK client hands back the provider's own field: the Anthropic Messages and
|
|
Codex Responses adapters rebuild the response with the REQUESTED model. Local or private-network
|
|
servers (llama.cpp, LM Studio, Ollama, LAN boxes) often report a placeholder or a weights path,
|
|
and a Bedrock ARN is not what the endpoint reports back.
|
|
"""
|
|
from openai import OpenAI
|
|
from agent.model_metadata import is_local_endpoint
|
|
from providers import get_provider_profile
|
|
from utils import base_url_hostname
|
|
base_url = str(getattr(client, "base_url", "") or "")
|
|
host = f".{base_url_hostname(base_url)}."
|
|
lowered = model.strip().lower()
|
|
is_azure = (getattr(get_provider_profile(provider), "name", provider) == "azure-foundry" # and its aliases
|
|
or any(label in host for label in _AZURE_HOST_LABELS))
|
|
return (isinstance(client, OpenAI) and not is_local_endpoint(base_url) and not is_azure
|
|
and not lowered.startswith(("arn:", _ROUTER_AUTO_MODEL)) and _ROUTER_PRESET_MARKER not in lowered)
|
|
|
|
|
|
def _canonical_model_id(model_id: str) -> str:
|
|
return _VERTEX_PUBLISHER_PATH.sub(r"\1/", model_id.strip().lower())
|
|
|
|
|
|
def answered_by_requested_model(requested: str, returned: str) -> bool:
|
|
"""True when ``returned`` (the provider-reported model) is ``requested`` or a dated snapshot of it.
|
|
|
|
A namespace is compared when both ids carry one (``org-a/m`` is not ``org-b/m``); a router may
|
|
drop the requested one (``anthropic/claude-x`` answered as ``claude-x``). A ``:tag`` the provider
|
|
reports must equal the requested tag (``qwen3:8b`` is not ``qwen3:32b``); a router may drop it
|
|
(``x/llama-3:free`` answered as ``llama-3``). An empty ``returned`` counts as a match: some
|
|
providers omit the field and adapter wires hide it (:func:`_reported_model_is_evidence`).
|
|
"""
|
|
if not returned:
|
|
return True
|
|
want, _, want_tag = _canonical_model_id(requested).partition(":")
|
|
got, _, got_tag = _canonical_model_id(returned).partition(":")
|
|
if got_tag and got_tag != want_tag:
|
|
return False
|
|
if "/" not in want or "/" not in got:
|
|
want, got = want.rsplit("/", 1)[-1], got.rsplit("/", 1)[-1]
|
|
return bool(re.fullmatch(re.escape(want) + _SNAPSHOT_SUFFIX, got))
|
|
|
|
|
|
def _without_sdk_retries(client: Any) -> Any:
|
|
"""``client`` with OpenAI SDK retries off, so one probe is one request. Some builders keep the
|
|
SDK default (Vertex, Bedrock Mantle), including inside the Codex Responses wrapper."""
|
|
from openai import OpenAI
|
|
from agent.auxiliary_client import CodexAuxiliaryClient
|
|
if isinstance(client, OpenAI):
|
|
return client.with_options(max_retries=0)
|
|
if isinstance(client, CodexAuxiliaryClient) and isinstance(client._real_client, OpenAI):
|
|
return CodexAuxiliaryClient(client._real_client.with_options(max_retries=0), client.chat.completions._model)
|
|
return client
|
|
|
|
|
|
def check_entry_static(entry: dict[str, Any]) -> EntryCheck:
|
|
"""Resolve ``entry`` the way the fallback switch does; never sends an inference request."""
|
|
from hermes_cli.auth import AuthError
|
|
from hermes_cli.fallback_config import fallback_entry_runtime_kwargs
|
|
from hermes_cli.runtime_provider import resolve_runtime_provider
|
|
from hermes_cli.runtime_readiness import missing_credentials_suffix, runtime_has_credentials
|
|
|
|
# Lowercased as the swap's callers pass it, so ``provider: MoA`` is ``moa`` everywhere below.
|
|
result = EntryCheck(provider=str(entry.get("provider") or "").strip().lower(), model=str(entry.get("model") or ""))
|
|
try:
|
|
runtime = resolve_runtime_provider(**fallback_entry_runtime_kwargs(entry))
|
|
except AuthError as exc:
|
|
# An unknown provider is a config error; every other AuthError is a credential problem,
|
|
# which is also what makes the real resolution-time walker skip the entry.
|
|
result.configured = exc.code != "invalid_provider"
|
|
result.error_class, result.error = ("auth" if result.configured else "config"), _redacted(exc)
|
|
return result
|
|
except Exception as exc: # health: allow BLE001 -- per-entry diagnostic boundary; logged with traceback
|
|
logger.debug("fallback check: %s/%s did not resolve", result.provider, result.model, exc_info=True)
|
|
result.error_class, result.error = "config", _redacted(exc) or type(exc).__name__
|
|
return result
|
|
result.configured = True
|
|
from agent.chat_completion_helpers import _fallback_api_mode_hint
|
|
result.base_url = _redacted(runtime.get("base_url"))
|
|
explicit, hinted = _fallback_api_mode_hint(entry, result.provider, str(entry.get("base_url") or "").strip())
|
|
# bind_route_entry keeps an explicit or non-chat hint and re-detects only a default chat one.
|
|
result.api_mode = hinted if explicit or hinted != "chat_completions" else str(runtime.get("api_mode") or hinted)
|
|
result.auth_source = str(runtime.get("source") or "")
|
|
result.auth_ready = runtime_has_credentials(runtime)
|
|
if not result.auth_ready:
|
|
result.error_class = "auth"
|
|
result.error = f"No usable credentials found for {result.provider}.{missing_credentials_suffix(result.provider)}"
|
|
return result
|
|
|
|
|
|
def check_entry_live(entry: dict[str, Any], result: EntryCheck) -> EntryCheck:
|
|
"""Send one minimal completion for ``entry`` (already statically ready) and record the verdict."""
|
|
from agent.auxiliary_client import _build_call_kwargs, _create_with_progress_once, _elapsed_ms, _provider_requires_stream
|
|
from hermes_cli.fallback_config import resolve_entry_api_key
|
|
from hermes_cli.timeouts import get_provider_request_timeout
|
|
from agent.error_classifier import classify_api_error
|
|
from agent.route_binding import resolve_route_entry_client
|
|
|
|
provider, secrets, started = result.provider, _secret_values(resolve_entry_api_key(entry)), None
|
|
try:
|
|
# raw_codex=False: the chat-completions adapter, so one create() call fits every wire.
|
|
route = resolve_route_entry_client(entry, provider, result.model, raw_codex=False)
|
|
client, model = route.client, route.model
|
|
if client is None:
|
|
result.inference_ready, result.error_class = False, "auth"
|
|
result.error = "the provider router built no client for this entry (no usable credential)"
|
|
return result
|
|
client = _without_sdk_retries(client)
|
|
secrets += _secret_values(getattr(client, "api_key", None))
|
|
base_url = str(getattr(client, "base_url", "") or "")
|
|
timeout = get_provider_request_timeout(provider, model) or _LIVE_TIMEOUT_S
|
|
kwargs = _build_call_kwargs(provider, model, list(_LIVE_PROMPT),
|
|
max_tokens=_LIVE_MAX_TOKENS, timeout=timeout, base_url=base_url)
|
|
started = time.monotonic() # provider latency, not SDK import / client construction
|
|
raw = _create_with_progress_once(client, kwargs, None, provider=provider,
|
|
force_stream=_provider_requires_stream(provider, base_url))
|
|
result.latency_ms = _elapsed_ms(started)
|
|
# Inside the boundary: an error envelope without an error object raises and is classified
|
|
# here, and nothing in an odd body can fail the whole command.
|
|
_judge_answer(result, raw, model, client, secrets)
|
|
except Exception as exc: # health: allow BLE001 -- the probe reports every provider failure as its verdict
|
|
if started is not None and result.latency_ms is None: # none for an entry that sent no request
|
|
result.latency_ms = _elapsed_ms(started)
|
|
verdict = classify_api_error(exc, provider=provider, model=result.model)
|
|
result.inference_ready, result.error_class = False, f"live_{verdict.reason.value}"
|
|
result.error = _redacted(exc, secrets, max_chars=300) or type(exc).__name__
|
|
# No traceback: exception text can carry the key, and log redaction is pattern-based too.
|
|
logger.debug("fallback check: live probe of %s/%s failed: %s", result.provider, result.model, result.error)
|
|
return result
|
|
|
|
|
|
def _body_failure(raw: Any) -> tuple[Optional[dict[str, Any]], bool]:
|
|
"""``(provider error, failed status)`` from the body AND a gateway ``{"data": ...}`` envelope,
|
|
read before any normalization: unwrapping rebuilds the answer without ``status`` / ``error``."""
|
|
from agent import auxiliary_call_outcome as outcome
|
|
from agent.auxiliary_client import _field
|
|
data = _field(raw, "data")
|
|
layers = [raw, data] if isinstance(data, dict) else [raw]
|
|
choices = _field(raw, "choices")
|
|
if isinstance(choices, list) and choices: # OpenRouter puts an upstream failure on the choice
|
|
layers.append(choices[0])
|
|
# Some providers always send an ``error`` object; one whose fields are all empty is not a failure.
|
|
embedded = next((e for e in map(outcome.embedded_provider_error, layers) if e and any(e.values())), None)
|
|
failed = any(str(_field(layer, "status") or "").lower() in _FAILED_STATUSES for layer in layers)
|
|
return embedded, failed
|
|
|
|
|
|
def _judge_answer(result: EntryCheck, raw: Any, model: str, client: Any, secrets: tuple[str, ...]) -> None:
|
|
"""Verdict for an answered request: a provider error in the body, no usable completion, another
|
|
model, or ready."""
|
|
from agent import auxiliary_call_outcome as outcome
|
|
from agent.auxiliary_client import _recover_aux_response_message, _unwrap_data_envelope, extract_content_or_reasoning
|
|
from agent.errors import InvalidProviderResponseError
|
|
from agent.transports.chat_completions import is_router_timeout_shim
|
|
provider = result.provider # already lowercased by check_entry_static
|
|
embedded, failed = _body_failure(raw)
|
|
response = None
|
|
if embedded is None and not failed:
|
|
response = _unwrap_data_envelope(raw)
|
|
# Responses-style text (``output_text`` / ``output[]``) outside ``choices``, as the aux path accepts.
|
|
if not getattr(response, "choices", None):
|
|
try:
|
|
response = _recover_aux_response_message(response) or response
|
|
except (TypeError, ValueError, AttributeError): # a malformed output[] is no answer
|
|
response = None
|
|
returned = str(getattr(response, "model", "") or "") if _reported_model_is_evidence(provider, model, client) else ""
|
|
result.model_returned = _redacted(returned, secrets)
|
|
choices = getattr(response, "choices", None)
|
|
if embedded is not None: # an HTTP-200 relay of an upstream failure: classify it like the aux path
|
|
error = InvalidProviderResponseError(str(embedded.get("message") or embedded))
|
|
setattr(error, outcome.PROVIDER_ERROR_ATTR, embedded)
|
|
result.inference_ready = False
|
|
result.error_class = f"live_{outcome.error_class(error, provider=provider, model=result.model)}"
|
|
result.error = _redacted(error, secrets, max_chars=300)
|
|
# Partial text under a failed status, or beside a choice-level error (OpenRouter's
|
|
# ``finish_reason: "error"``), is not an answer. An empty stream is re-aggregated into a choice
|
|
# with an empty message: content (or reasoning) is the evidence a model ran, not the shape.
|
|
elif (failed or not choices or getattr(choices[0], "message", None) is None
|
|
or str(getattr(choices[0], "finish_reason", "") or "").lower() == "error"
|
|
or is_router_timeout_shim(response) or not extract_content_or_reasoning(response)):
|
|
result.inference_ready, result.error_class = False, "live_response"
|
|
result.error = "the provider answered without a usable completion"
|
|
elif not answered_by_requested_model(model, returned):
|
|
result.inference_ready, result.error_class = False, "model_mismatch"
|
|
result.error = _redacted(f"requested {model!r} but {returned!r} answered", secrets)
|
|
else:
|
|
result.inference_ready = True
|
|
|
|
|
|
def run_checks(chain: list[dict[str, Any]], *, live: bool) -> list[EntryCheck]:
|
|
"""Static checks in chain order; with ``live``, probe the statically ready entries in parallel."""
|
|
from hermes_cli.left_core_migration import installs_held
|
|
with installs_held(): # a diagnostic: resolving a provider whose plugin left core must not install it
|
|
results = [check_entry_static(entry) for entry in chain]
|
|
if live:
|
|
from hermes_cli.doctor_connectivity import run_probes
|
|
# Not one fixed route, so one request proves nothing: ``auto`` picks its backend per call (and can
|
|
# walk the chain); ``moa`` is a fan-out preset whose facade only an agent binds. Both stay static.
|
|
# IMDS stays on: an EC2 instance role is a real Bedrock credential for an inference request.
|
|
run_probes([(r.provider, functools.partial(check_entry_live, entry, r))
|
|
for entry, r in zip(chain, results) if r.ready and r.provider not in _NOT_PROBED],
|
|
aws_imds=True)
|
|
return results
|
|
|
|
|
|
def _print_table(results: list[EntryCheck], *, live: bool) -> None:
|
|
def mark(value: Optional[bool]) -> str:
|
|
return "—" if value is None else "ok" if value else "FAIL"
|
|
|
|
print()
|
|
print(f" {'#':<3}{'Provider / model':<44}{'Config':<8}{'Auth':<8}{'Live':<8}{'Latency':<10}Status")
|
|
for index, r in enumerate(results, 1):
|
|
latency = f"{r.latency_ms} ms" if r.latency_ms is not None else "—"
|
|
label = f"{r.provider} / {r.model}"
|
|
print(f" {index:<3}{label[:43]:<44}{mark(r.configured):<8}{mark(r.auth_ready if r.configured else None):<8}"
|
|
f"{mark(r.inference_ready):<8}{latency:<10}{r.status}")
|
|
if r.error:
|
|
print(f" {r.error}")
|
|
print()
|
|
if not live:
|
|
print(" Static check only (no inference request sent). Use --live to send one minimal request per entry.")
|
|
print()
|
|
|
|
|
|
def cmd_fallback_check(args) -> None:
|
|
"""``hermes fallback check [--live] [--json]``; exits 1 when any entry is not ready."""
|
|
from hermes_cli.config import load_config_readonly
|
|
from hermes_cli.fallback_config import get_fallback_chain
|
|
from hermes_cli.profiles import get_active_profile_name
|
|
|
|
chain = get_fallback_chain(load_config_readonly())
|
|
results = run_checks(chain, live=args.live)
|
|
ready = all(r.ready for r in results)
|
|
if args.json:
|
|
print(json.dumps({
|
|
"profile": get_active_profile_name(), "live": bool(args.live), "ready": ready,
|
|
"checked_at": datetime.now(UTC).isoformat(),
|
|
"results": [{**dataclasses.asdict(r), "ready": r.ready, "status": r.status} for r in results],
|
|
}, indent=2))
|
|
elif not results:
|
|
print("\n No fallback providers configured. Add one with: hermes fallback add\n")
|
|
else:
|
|
_print_table(results, live=args.live)
|
|
if not ready:
|
|
raise SystemExit(1)
|