mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 02:46:38 +03:00
b6e475170e
The endpoint each repair was spent on is not enough: after a move away and back to the same endpoint (a Nous 401 refresh to the welcome host, then the welcome heal back to the paid host), that endpoint's verdicts were reset and the rejected effort was sent again, but its repair still counted as spent, so the turn went to the fallback chain. reset_route_verdicts now starts a new route visit (route_visit), and the reasoning and encrypted-content repairs record the visit they were spent on. Every route change re-arms them; a recovery that stays on the route keeps them spent, so the bound is still one repair per route visit. The Nous 401 refresh test fixture gains the pool refresh counter main reads, so it runs against current main too.
441 lines
24 KiB
Python
441 lines
24 KiB
Python
"""API-call exception handler for the conversation turn's retry loop: pre-/post-classification
|
|
recovery, interpreter-shutdown abandon, classified-error routing, overflow recovery, the
|
|
non-retryable client-error exit, max-retries exhaustion (primary transport recovery →
|
|
fallback → terminal result) and the interruptible backoff. Nothing here imports
|
|
``agent.conversation_loop`` at module level (cycle) — loop-internal helpers resolve lazily so
|
|
``patch("agent.conversation_loop.X")`` sites keep intercepting.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import json
|
|
import logging
|
|
import ssl
|
|
import time
|
|
from typing import Any, Dict, Optional
|
|
|
|
from agent.api_error_summary import is_provider_stream_parse_error
|
|
from agent.error_classifier import RETRYABLE_CLIENT_REASONS, FailoverReason, classify_api_error
|
|
from agent.route_verdicts import route_visit
|
|
from agent.turn_overflow import recover_from_overflow
|
|
from agent.turn_recovery import (
|
|
_NONRETRYABLE_LABELS, abort_turn_on_interrupt, compute_error_backoff, free_tier_cooldown_ends_turn,
|
|
interruptible_backoff_sleep, log_api_error_attempt,
|
|
max_retries_exhausted_result, nonretryable_client_error_result, recover_after_classification,
|
|
recover_before_classification, route_classified_error, settle_delivered_partial,
|
|
)
|
|
|
|
logger = logging.getLogger("agent.conversation_loop")
|
|
|
|
|
|
@dataclass
|
|
class ApiErrorVerdict:
|
|
"""``action``: ``"continue"`` (retry the API call), ``"break"`` (leave the retry loop:
|
|
fallback armed / redirect pending) or ``"return"`` (``result`` is the turn's result dict);
|
|
``"fallthrough"`` never happens — the handler always ends in an exit. The other fields
|
|
are the retry-loop locals the handler rebinds; ``_provider_overflow_recovery_pending`` is
|
|
merge-only (caller sets True when set)."""
|
|
|
|
action: str
|
|
thinking_spinner: Any
|
|
messages: Any
|
|
active_system_prompt: Any
|
|
conversation_history: Any
|
|
approx_tokens: Any
|
|
retry_count: Any
|
|
max_retries: Any
|
|
compression_attempts: Any
|
|
_provider_overflow_recovery_pending: Any
|
|
result: Optional[dict[str, Any]] = None
|
|
|
|
|
|
def handle_api_error(
|
|
agent: Any, *, api_error: Any, _retry: Any, thinking_spinner: Any, messages: Any,
|
|
api_messages: Any, api_kwargs: Any, system_message: Any, active_system_prompt: Any,
|
|
conversation_history: Any, approx_tokens: Any, retry_count: Any, max_retries: Any,
|
|
compression_attempts: Any, max_compression_attempts: Any, api_call_count: Any,
|
|
api_request_id: Any, api_start_time: Any, effective_task_id: Any, turn_id: Any,
|
|
current_turn_user_idx: Any = None,
|
|
) -> ApiErrorVerdict:
|
|
"""Recover from ``api_error`` in the original order. Every fallback activation must leave
|
|
the retry loop with ``restart_with_rebuilt_messages`` armed (``"break"``) so the pre-API
|
|
preflight re-runs against the fallback's context window (#84733)."""
|
|
_provider_overflow_recovery_pending = False
|
|
|
|
def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> ApiErrorVerdict:
|
|
return ApiErrorVerdict(
|
|
action=action, thinking_spinner=thinking_spinner, messages=messages,
|
|
active_system_prompt=active_system_prompt, conversation_history=conversation_history,
|
|
approx_tokens=approx_tokens, retry_count=retry_count, max_retries=max_retries,
|
|
compression_attempts=compression_attempts,
|
|
_provider_overflow_recovery_pending=_provider_overflow_recovery_pending, result=result,
|
|
)
|
|
|
|
# Stop spinner silently — retry status is buffered and only flushed when every
|
|
# retry+fallback is exhausted.
|
|
if thinking_spinner:
|
|
thinking_spinner.stop("")
|
|
thinking_spinner = None
|
|
if agent.thinking_callback:
|
|
agent.thinking_callback("")
|
|
|
|
_recovered, active_system_prompt = recover_before_classification(
|
|
agent, api_error, messages=messages, api_messages=api_messages, api_kwargs=api_kwargs,
|
|
active_system_prompt=active_system_prompt,
|
|
)
|
|
if _recovered:
|
|
return _verdict("continue")
|
|
|
|
status_code = getattr(api_error, "status_code", None)
|
|
error_context = agent._extract_api_error_context(api_error)
|
|
|
|
# A Nous 402/403 carries the same x-nous-credits-* headers as a success, and a usage-window plan's
|
|
# "ran out for this model" / "payment paused" state is only visible on the blocking response itself
|
|
# (the stream path captures headers after the stream opens, which a 402 never does). Fail-open.
|
|
# Usage-window responses only: every other account keeps main's behaviour (no capture here).
|
|
if status_code in (402, 403) and getattr(agent, "provider", "") == "nous":
|
|
try:
|
|
from agent.credits_tracker import response_is_usage_window
|
|
response = getattr(api_error, "response", None)
|
|
if response_is_usage_window(response):
|
|
agent._capture_credits(response, refused=True)
|
|
except Exception:
|
|
logger.debug("credits ▸ header capture on %s failed (fail-open)", status_code, exc_info=True)
|
|
|
|
# Process is exiting mid-flight: retries/rotation/fallbacks are futile and the
|
|
# retry trace spams the shell. One log line.
|
|
from tools.interpreter_shutdown import interpreter_shutting_down
|
|
|
|
if interpreter_shutting_down(api_error):
|
|
logger.warning(
|
|
"%sInterpreter is shutting down — abandoning turn "
|
|
"during API call #%d (%s)",
|
|
agent.log_prefix, api_call_count, api_error,
|
|
)
|
|
_shutdown_summary = "Turn abandoned: the process was shutting down before the model call could complete."
|
|
return _verdict("return", {
|
|
"final_response": _shutdown_summary, "messages": messages, "api_calls": api_call_count,
|
|
"completed": False, "failed": True, "error": _shutdown_summary,
|
|
"failure_reason": "interpreter_shutdown", "failure_retryable": False,
|
|
})
|
|
|
|
_compressor = getattr(agent, "context_compressor", None)
|
|
_ctx_len = getattr(_compressor, "context_length", 200000) if _compressor else 200000
|
|
classified = classify_api_error(
|
|
api_error, provider=getattr(agent, "provider", "") or "",
|
|
model=getattr(agent, "model", "") or "", approx_tokens=approx_tokens,
|
|
context_length=_ctx_len, num_messages=len(api_messages) if api_messages else 0,
|
|
base_url=str(getattr(agent, "base_url", "") or ""),
|
|
api_key=getattr(agent, "api_key", None),
|
|
)
|
|
logger.debug(
|
|
"Error classified: reason=%s status=%s retryable=%s compress=%s rotate=%s fallback=%s",
|
|
classified.reason.value, classified.status_code,
|
|
classified.retryable, classified.should_compress,
|
|
classified.should_rotate_credential, classified.should_fallback,
|
|
)
|
|
agent._invoke_api_request_error_hook(
|
|
task_id=effective_task_id, turn_id=turn_id, api_request_id=api_request_id,
|
|
api_call_count=api_call_count, api_start_time=api_start_time, api_kwargs=api_kwargs,
|
|
error_type=type(api_error).__name__, error_message=str(api_error), status_code=status_code,
|
|
retry_count=retry_count, max_retries=max_retries, retryable=classified.retryable,
|
|
reason=classified.reason.value,
|
|
)
|
|
|
|
_recovered, recovered_with_pool = recover_after_classification(
|
|
agent, api_error, classified, _retry, status_code=status_code, error_context=error_context,
|
|
messages=messages, api_messages=api_messages,
|
|
)
|
|
if _recovered:
|
|
return _verdict("continue")
|
|
|
|
retry_count += 1
|
|
elapsed_time = time.time() - api_start_time
|
|
# Liveness/watchdog label only (never shown in chat), so the classifier's
|
|
# "not retryable" verdict is named on the logged attempt line below instead.
|
|
agent._touch_activity(f"API error recovery (attempt {retry_count}/{max_retries})")
|
|
|
|
error_type, error_msg, _provider, _base, _model = log_api_error_attempt(
|
|
agent, api_error, retry_count=retry_count, max_retries=max_retries, status_code=status_code,
|
|
elapsed_time=elapsed_time, api_messages=api_messages, approx_tokens=approx_tokens,
|
|
retryable=bool(classified.retryable),
|
|
)
|
|
|
|
if agent._interrupt_requested:
|
|
# Preserve a pending redirect: the user is steering, not stopping — rebuild the
|
|
# turn from the correction instead of aborting.
|
|
if agent.clear_interrupt(preserve_redirect=True):
|
|
_retry.restart_with_redirected_messages = True
|
|
return _verdict("break")
|
|
return _verdict("return", abort_turn_on_interrupt(
|
|
agent, messages, conversation_history, api_call_count,
|
|
abort_message="Interrupt detected during error handling, aborting retries.",
|
|
interrupt_text=f"Operation interrupted: handling API error ({error_type}: {agent._clean_error_message(str(api_error))}).",
|
|
))
|
|
|
|
_ce = route_classified_error(
|
|
agent, api_error, classified, _retry, error_msg=error_msg, error_context=error_context,
|
|
recovered_with_pool=recovered_with_pool, base_url=_base, model=_model, messages=messages,
|
|
api_messages=api_messages, system_message=system_message,
|
|
active_system_prompt=active_system_prompt, conversation_history=conversation_history,
|
|
retry_count=retry_count, max_retries=max_retries, compression_attempts=compression_attempts,
|
|
max_compression_attempts=max_compression_attempts, api_call_count=api_call_count,
|
|
effective_task_id=effective_task_id,
|
|
)
|
|
status_code = _ce.status_code
|
|
messages = _ce.messages
|
|
active_system_prompt = _ce.active_system_prompt
|
|
conversation_history = _ce.conversation_history
|
|
retry_count = _ce.retry_count
|
|
max_retries = _ce.max_retries
|
|
compression_attempts = _ce.compression_attempts
|
|
is_rate_limited = _ce.is_rate_limited
|
|
_wrapped_output_cap_budget = _ce.wrapped_output_cap_budget
|
|
_is_zai_coding_overload = _ce.is_zai_coding_overload
|
|
if _ce.provider_overflow_recovery_pending:
|
|
_provider_overflow_recovery_pending = True
|
|
if _ce.action != "fallthrough":
|
|
return _verdict(_ce.action, _ce.result)
|
|
|
|
_ov = recover_from_overflow(
|
|
agent, api_error, classified, _retry, status_code=status_code, error_msg=error_msg,
|
|
wrapped_output_cap_budget=_wrapped_output_cap_budget, messages=messages,
|
|
api_messages=api_messages, system_message=system_message,
|
|
active_system_prompt=active_system_prompt, conversation_history=conversation_history,
|
|
approx_tokens=approx_tokens, compression_attempts=compression_attempts,
|
|
max_compression_attempts=max_compression_attempts, api_call_count=api_call_count,
|
|
effective_task_id=effective_task_id,
|
|
)
|
|
messages = _ov.messages
|
|
active_system_prompt = _ov.active_system_prompt
|
|
conversation_history = _ov.conversation_history
|
|
approx_tokens = _ov.approx_tokens
|
|
compression_attempts = _ov.compression_attempts
|
|
is_context_length_error = _ov.is_context_length_error
|
|
if _ov.provider_overflow_recovery_pending:
|
|
_provider_overflow_recovery_pending = True
|
|
if _ov.action != "fallthrough":
|
|
return _verdict(_ov.action, _ov.result)
|
|
|
|
_ue = settle_unrecovered_error(
|
|
agent, api_error=api_error, classified=classified, _retry=_retry, status_code=status_code,
|
|
error_msg=error_msg, error_context=error_context,
|
|
is_context_length_error=is_context_length_error,
|
|
is_rate_limited=is_rate_limited, _is_zai_coding_overload=_is_zai_coding_overload,
|
|
_provider=_provider, _base=_base, _model=_model, messages=messages,
|
|
api_messages=api_messages, api_kwargs=api_kwargs, active_system_prompt=active_system_prompt,
|
|
conversation_history=conversation_history, approx_tokens=approx_tokens,
|
|
retry_count=retry_count, max_retries=max_retries, compression_attempts=compression_attempts,
|
|
api_call_count=api_call_count, current_turn_user_idx=current_turn_user_idx,
|
|
)
|
|
active_system_prompt = _ue.active_system_prompt
|
|
retry_count = _ue.retry_count
|
|
compression_attempts = _ue.compression_attempts
|
|
return _verdict(_ue.action, _ue.result)
|
|
|
|
|
|
def _is_local_validation_error(api_error: Any) -> bool:
|
|
"""ValueError/TypeError are local bugs, except: UnicodeEncodeError (surrogate recovery
|
|
path), json.JSONDecodeError / provider stream-parse ValueErrors (transient provider/network failure, must retry),
|
|
ssl.SSLError (inherits OSError *and* ValueError — a TLS failure is not a local bug)
|
|
and "NoneType is not iterable" TypeErrors (upstream shape mismatches, e.g. Codex
|
|
response.completed.output=null — retryable so the fallback path runs)."""
|
|
if not isinstance(api_error, (ValueError, TypeError)):
|
|
return False
|
|
if isinstance(api_error, (UnicodeEncodeError, json.JSONDecodeError, ssl.SSLError)):
|
|
return False
|
|
_text = str(api_error).lower()
|
|
if is_provider_stream_parse_error(api_error):
|
|
return False # corrupted provider SSE chunk (jiter), transient — not a local bug (#65147)
|
|
return not (isinstance(api_error, TypeError) and "nonetype" in _text and "not iterable" in _text)
|
|
|
|
|
|
@dataclass
|
|
class UnrecoveredErrorVerdict:
|
|
"""``action``: ``"continue"`` (retry), ``"break"`` (fallback armed / redirect pending) or
|
|
``"return"`` (``result`` is the terminal result dict). Rebinds ``active_system_prompt``,
|
|
``retry_count`` and ``compression_attempts``."""
|
|
|
|
action: str
|
|
active_system_prompt: Any
|
|
retry_count: Any
|
|
compression_attempts: Any
|
|
result: Optional[dict[str, Any]] = None
|
|
|
|
|
|
def settle_unrecovered_error(
|
|
agent: Any, *, api_error: Any, classified: Any, _retry: Any, status_code: Any, error_msg: Any,
|
|
is_context_length_error: Any, is_rate_limited: Any, _is_zai_coding_overload: Any,
|
|
_provider: Any, _base: Any, _model: Any, messages: Any, api_messages: Any, api_kwargs: Any,
|
|
active_system_prompt: Any, conversation_history: Any, approx_tokens: Any, retry_count: Any,
|
|
max_retries: Any, compression_attempts: Any, api_call_count: Any, error_context: Any = None,
|
|
current_turn_user_idx: Any = None,
|
|
) -> UnrecoveredErrorVerdict:
|
|
"""Decide the fate of an API error that every recovery chain declined: local validation /
|
|
non-retryable client errors (provider stale-credential self-heal first, then fallback, then a
|
|
terminal result), max-retries exhaustion (primary transport recovery -> fallback -> terminal
|
|
result), else the interruptible error backoff. ``FailoverReason.billing`` (402) is deliberately
|
|
treated as non-retryable (#31273)."""
|
|
from agent.conversation_loop import _arm_fallback_restart, _credential_refresh_profile
|
|
from agent.provider_route_hooks import ask_on_error_path
|
|
|
|
def _verdict(action: str, result: Optional[dict[str, Any]] = None) -> UnrecoveredErrorVerdict:
|
|
return UnrecoveredErrorVerdict(
|
|
action=action, active_system_prompt=active_system_prompt, retry_count=retry_count,
|
|
compression_attempts=compression_attempts, result=result,
|
|
)
|
|
|
|
# ``FailoverReason.billing`` (402) is deliberately NOT excluded: pool rotation and
|
|
# eager fallback already gave up, so retrying only burns paid requests on a depleted
|
|
# balance. Mirrors 401/403.
|
|
is_local_validation_error = _is_local_validation_error(api_error)
|
|
# ``recover_after_classification`` sets ``image_shrink_retry_attempted`` BEFORE it runs the shrink,
|
|
# so an image-size rejection reaching this point with the flag set had nothing left to shrink (the
|
|
# excess is text on a host whose cap is payload-scoped, or an unshrinkable image). Re-sending the
|
|
# byte-identical body ``max_retries`` times changes nothing: treat it as a client error and try
|
|
# the fallback chain now, as the format_error verdict these 400s carried before did (#112473).
|
|
shrink_spent = classified.reason == FailoverReason.image_too_large and bool(
|
|
getattr(_retry, "image_shrink_retry_attempted", False)
|
|
)
|
|
# Same shape for the reasoning-disable rung: the retry already went out without the disable,
|
|
# so a second reasoning-field rejection means the route refuses the configured reasoning
|
|
# controls themselves — nothing left to drop, so take the fallback chain now instead of
|
|
# replaying the identical request ``max_retries`` times (#114460).
|
|
reasoning_spent = (
|
|
classified.reason == FailoverReason.reasoning_mandatory
|
|
and getattr(_retry, "reasoning_mandatory_retry_visit", None) == route_visit(agent)
|
|
)
|
|
is_client_error = (
|
|
is_local_validation_error
|
|
or shrink_spent
|
|
or reasoning_spent
|
|
or (
|
|
not classified.retryable
|
|
and not classified.should_compress
|
|
and classified.reason not in RETRYABLE_CLIENT_REASONS
|
|
)
|
|
) and not is_context_length_error
|
|
|
|
if is_client_error:
|
|
# A Codex ChatGPT-account entitlement 400 names the model: with nothing to rotate the
|
|
# slug is dead for this account, so record it before the fallback walk runs (#106475).
|
|
from agent.fallback_cooldown import _mark_entitlement_rejected_model
|
|
_mark_entitlement_rejected_model(agent, api_error)
|
|
# Provider self-heal BEFORE fallback: some providers surface a stale credential as a
|
|
# non-auth error (the profile's ``is_stale_credential_error``), not a 401. Fresh
|
|
# credential + client rebuild, one retry, SAME provider.
|
|
if (
|
|
not _retry.provider_stale_refresh_attempted
|
|
and (profile := _credential_refresh_profile(agent)) is not None
|
|
and ask_on_error_path(
|
|
profile, "is_stale_credential_error", False,
|
|
status_code=status_code, message=str(getattr(api_error, "message", "") or api_error),
|
|
)
|
|
):
|
|
_retry.provider_stale_refresh_attempted = True
|
|
if agent._try_refresh_provider_runtime_credentials(reason="stale"):
|
|
agent._buffer_vprint("🔐 " + (ask_on_error_path(profile, "credential_refresh_notice", "", reason="stale") or (
|
|
f"{profile.display_name or profile.name} credential renewed after a "
|
|
"stale-credential error. Retrying request...")))
|
|
retry_count = 0
|
|
return _verdict("continue")
|
|
# ``should_fallback=False`` marks a deterministic failure no other provider can fix (the
|
|
# model's own malformed tool-call JSON, #12770; MoA preset/adapter faults, #55933): skip
|
|
# the cascade. An UNCLASSIFIED local ValueError/TypeError keeps its historical fallback;
|
|
# a recognised verdict that opts out wins even when the exception is a ValueError subclass.
|
|
_unclassified_local = is_local_validation_error and classified.reason == FailoverReason.unknown
|
|
if classified.should_fallback or _unclassified_local or shrink_spent or reasoning_spent:
|
|
# Announce the fallback only when a chain exists, else "trying fallback..." lies
|
|
# before a silent abort.
|
|
if agent._has_pending_fallback():
|
|
_label = _NONRETRYABLE_LABELS.get(classified.reason, f"Non-retryable error (HTTP {status_code})")
|
|
agent._buffer_diagnostic_status(f"⚠️ {_label} — trying fallback...")
|
|
reset_at = error_context.get("reset_at") if isinstance(error_context, dict) else None
|
|
if agent._try_activate_fallback(reason=classified.reason, reset_at=reset_at):
|
|
# Direct ``return _verdict("break")`` is load-bearing: the restart handler
|
|
# re-runs the pre-API preflight against the fallback's context window.
|
|
active_system_prompt = _arm_fallback_restart(agent, api_messages, active_system_prompt, _retry)
|
|
retry_count = compression_attempts = 0
|
|
return _verdict("break")
|
|
# Terminal from here: collapse the continuation trail once, before the persist.
|
|
_delivered = settle_delivered_partial(agent, messages, current_turn_user_idx)
|
|
return _verdict("return", nonretryable_client_error_result(
|
|
agent, api_error, classified, status_code=status_code, api_kwargs=api_kwargs,
|
|
api_messages=api_messages, messages=messages, conversation_history=conversation_history,
|
|
api_call_count=api_call_count, approx_tokens=approx_tokens, provider=_provider,
|
|
base_url=_base, model=_model, delivered=_delivered, current_turn_user_idx=current_turn_user_idx,
|
|
))
|
|
|
|
# An attended session on the free model does not sit through a long cooldown: end the attempt
|
|
# cycle now (fallback, else the reset time and the ways forward); the copy counts attempts made.
|
|
attempts_made = min(retry_count, max_retries)
|
|
if is_rate_limited and retry_count < max_retries and free_tier_cooldown_ends_turn(agent, api_error, _base):
|
|
logger.info("%sFree-tier cooldown outlasts the attended wait — ending retries after attempt %s",
|
|
agent.log_prefix, retry_count)
|
|
retry_count = max_retries
|
|
if retry_count >= max_retries:
|
|
# Before fallback, rebuild the primary client once per API call block for
|
|
# transient transport errors (stale pool, TCP reset).
|
|
if not _retry.primary_recovery_attempted and agent._try_recover_primary_transport(
|
|
api_error, retry_count=retry_count, max_retries=max_retries,
|
|
):
|
|
_retry.primary_recovery_attempted = True
|
|
retry_count = 0
|
|
# Fresh attempt cycle: re-open fallback state so a follow-on 429 can still
|
|
# activate fallback_providers.
|
|
_retry.has_retried_429 = False
|
|
agent._fallback_index = 0
|
|
agent._fallback_activated = False
|
|
return _verdict("continue")
|
|
if agent._has_pending_fallback():
|
|
agent._buffer_diagnostic_status(f"⚠️ Max retries ({max_retries}) exhausted — trying fallback...")
|
|
reset_at = error_context.get("reset_at") if isinstance(error_context, dict) else None
|
|
if agent._try_activate_fallback(reason=classified.reason, reset_at=reset_at):
|
|
# Direct ``return _verdict("break")`` is load-bearing: the restart handler
|
|
# re-runs the pre-API preflight against the fallback's context window.
|
|
active_system_prompt = _arm_fallback_restart(agent, api_messages, active_system_prompt, _retry)
|
|
retry_count = compression_attempts = 0
|
|
return _verdict("break")
|
|
# Fallback first (above); only with nothing left to move to does the bounded auto-recovery
|
|
# ladder park the turn on a transient outage instead of ending it (#85426, #107307).
|
|
from agent.turn_recovery_autorecover import auto_recover_after_exhaustion
|
|
_ladder = auto_recover_after_exhaustion(
|
|
agent, api_error, classified, _retry, messages=messages,
|
|
conversation_history=conversation_history, api_call_count=api_call_count,
|
|
)
|
|
if _ladder is not None:
|
|
if _ladder["action"] == "continue":
|
|
retry_count = 0
|
|
return _verdict(_ladder["action"], _ladder.get("result"))
|
|
_delivered = settle_delivered_partial(agent, messages, current_turn_user_idx)
|
|
return _verdict("return", max_retries_exhausted_result(
|
|
agent, api_error, classified, attempts=attempts_made, is_rate_limited=is_rate_limited,
|
|
error_msg=error_msg, api_kwargs=api_kwargs, api_messages=api_messages,
|
|
messages=messages, conversation_history=conversation_history,
|
|
api_call_count=api_call_count, approx_tokens=approx_tokens, provider=_provider,
|
|
base_url=_base, model=_model, delivered=_delivered,
|
|
))
|
|
|
|
wait_time = compute_error_backoff(
|
|
agent, api_error, retry_count=retry_count, max_retries=max_retries,
|
|
is_rate_limited=is_rate_limited, is_zai_coding_overload=_is_zai_coding_overload,
|
|
base_url=_base, model=_model,
|
|
)
|
|
# Same preserve-redirect rule as the invalid-response wait: a steering correction
|
|
# must survive backoff, not die as "Operation interrupted".
|
|
_interrupted = interruptible_backoff_sleep(
|
|
agent, wait_time, _retry, messages=messages, conversation_history=conversation_history,
|
|
api_call_count=api_call_count,
|
|
abort_message="Interrupt detected during retry wait, aborting.",
|
|
interrupt_text=f"Operation interrupted: retrying API call after error (retry {retry_count}/{max_retries}).",
|
|
activity_label=f"error retry backoff ({retry_count}/{max_retries})",
|
|
)
|
|
if _interrupted is not None:
|
|
return _verdict("return", _interrupted)
|
|
if _retry.restart_with_redirected_messages:
|
|
# Leave the retry loop — the caller rebuilds this iteration from the correction
|
|
# instead of re-firing the stale request.
|
|
return _verdict("break")
|
|
return _verdict("fallthrough")
|