Files
hermes-agent/tests/hermes_cli/test_model_cache_fallback_rows.py
Teknium 1da2eb42dc refactor!: remove GitHub Copilot from core (now the github-copilot labs plugin)
Delete the in-tree model-provider plugins plugins/model-providers/copilot/
(provider `copilot`: Copilot API via the GitHub->Copilot token exchange) and
plugins/model-providers/copilot-acp/ (provider `copilot-acp`: Copilot CLI over
ACP). Both ship as ONE plugin, `github-copilot`, in
NousResearch/hermes-labs-plugins; the LEFT_CORE row added before this commit
installs it for every profile that routes through copilot/copilot-acp, and
every "unknown provider" surface names `hermes plugins install
github-copilot`. Core keeps no Copilot code: the generic seams the plugins
fill stay, and agent/copilot_acp_client.py stays as the deprecated alias the
kimchi-acp catalog plugin subclasses (agent.acp_client.ACPClient is the
public base). This commit is revertible on its own: reverting it restores the
bundled providers (the labs plugin stays inert on a core that bundles them).

Test files that only tested the plugins move to the labs plugin's tests:
  tests/plugins/model_providers/test_copilot_acp_client.py
  tests/plugins/model_providers/test_copilot_acp_deprecation.py
  tests/plugins/model_providers/test_copilot_auth.py
  tests/plugins/model_providers/test_copilot_catalog_oauth_fallback.py
  tests/plugins/model_providers/test_copilot_context.py
  tests/plugins/model_providers/test_copilot_custom_model_ids.py
  tests/plugins/model_providers/test_copilot_hosts.py
  tests/plugins/model_providers/test_copilot_pool_exchange.py
  tests/plugins/model_providers/test_copilot_profile.py
  tests/plugins/model_providers/test_copilot_refresh_notice.py
  tests/plugins/model_providers/test_copilot_setup.py
  tests/plugins/model_providers/test_copilot_token_exchange.py
  tests/agent/test_copilot_initiator.py
  tests/agent/test_copilot_native_vision_headers.py
  tests/hermes_cli/test_copilot_in_model_list.py
  tests/hermes_cli/test_copilot_model_api_mode.py
  tests/hermes_cli/test_copilot_runtime_api_mode.py
  tests/hermes_cli/test_copilot_runtime_api_mode_stale.py
  tests/hermes_cli/test_model_switch_copilot_api_mode.py
  tests/e2e/core/providers/test_native_copilot_acp.py
  tests/e2e/core/providers/test_native_copilot_acp_errors.py
  tests/e2e/core/providers/test_native_copilot_acp_streaming.py
  tests/fakes/providers/copilot_acp.py

Copilot-behaviour cases extracted from generic test files (same move):
  test_credential_pool (Copilot pool seeding / gh / env suppression),
  test_reasoning_carriers (Copilot carriers), test_run_agent (Copilot gpt-5
  reasoning, xhigh), test_run_agent_codex_responses (Copilot 401 refresh),
  test_codex_transport (Copilot reasoning), test_auxiliary_main_first (vision
  headers), test_private_credential_writers (JWT store row),
  test_provider_client_seam (copilot-acp client), test_provider_wire_snapshot
  + its fixture (copilot / github-models cells), test_api_key_providers,
  test_auth_commands, test_credential_pool_off_loop (exchange single-flight),
  test_external_process_auth_status (Copilot CLI token stores),
  test_external_process_provider_seam, test_model_cache_fallback_rows,
  test_model_validation, test_models_relay_base_url (copilot row),
  test_multiplex_cli_cache_scope, test_provider_catalog, test_cold_start_gil_stall.

Tests that used Copilot only as an example provider now use a fake
ProviderProfile (or another in-tree provider) with the same assertions:
  test_auxiliary_client, test_auxiliary_named_custom_providers,
  test_init_fallback_on_exhausted_pool, test_intent_ack_continuation,
  test_models_dev, test_turn_retry_state, test_one_turn_route_publish,
  test_mcp_catalog_env_boundary, test_plugin_provider_picker_residue,
  test_delegate_external_provider, test_delegation_profile_route,
  test_overlay_slug_resolution, test_installed_plugin_discovery, plus the
  generic halves of test_run_agent_codex_responses, test_codex_transport,
  test_api_key_providers, test_external_process_auth_status and
  test_model_cache_fallback_rows.
2026-10-25 20:57:22 -07:00

105 lines
5.0 KiB
Python

"""A curated fallback served because the live catalog fetch failed must never be pinned in the
disk cache as if it were the account's real catalog (#107391): a transient Copilot outage wrote
the 17-model static list over the account's 10 live models with a fresh 1h TTL, and every picker
surface served the wrong list until the TTL lapsed.
"""
from __future__ import annotations
import time
from unittest.mock import patch
import pytest
import hermes_cli.models as mod
@pytest.fixture(autouse=True)
def _reset_swr_state():
with mod._swr_refresh_lock:
mod._swr_refresh_inflight.clear()
yield
with mod._swr_refresh_lock:
mod._swr_refresh_inflight.clear()
def _row(models, age_seconds, **extra):
return {"fp": "fp", "at": time.time() - age_seconds, "models": list(models), **extra}
LIVE = ["claude-opus-5", "kimi-k3", "gpt-5.6-sol"]
STATIC = ["gpt-5.4", "gpt-4o", "claude-sonnet-4.6"]
def test_fallback_does_not_overwrite_the_same_credentials_live_row():
# 2h-old live row (past the 1h TTL, inside the SWR window) + a live fetch that degrades to the
# curated list: the account's real catalog must survive, in the cache AND in the return value.
cache = {"deepinfra": _row(LIVE, age_seconds=7200)}
with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
patch.object(mod, "_credential_fingerprint", return_value="fp"), \
patch.object(mod, "_save_provider_models_cache") as save, \
patch.object(mod, "provider_model_ids", return_value=mod.CuratedFallbackModels(STATIC)):
out = mod.cached_provider_model_ids("deepinfra", force_refresh=True)
assert out == LIVE
assert cache["deepinfra"]["models"] == LIVE
save.assert_not_called()
def test_fallback_row_is_recorded_as_fallback_and_never_served_stale():
# Cold cache: the curated list is served (the picker must not be empty) but recorded as a
# fallback row, which the SWR stale window must not resurrect — a stale fallback re-probes.
cache: dict = {}
with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
patch.object(mod, "_credential_fingerprint", return_value="fp"), \
patch.object(mod, "_save_provider_models_cache"), \
patch.object(mod, "provider_model_ids", return_value=mod.CuratedFallbackModels(STATIC)):
assert mod.cached_provider_model_ids("deepinfra") == STATIC
assert cache["deepinfra"]["fallback"] is True
# Same row, now past the fallback TTL: the live fetch runs instead of the stale-serve path,
# and a real catalog replaces the fallback row.
cache["deepinfra"]["at"] = time.time() - mod._PROVIDER_MODELS_FALLBACK_TTL - 1
with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
patch.object(mod, "_credential_fingerprint", return_value="fp"), \
patch.object(mod, "_save_provider_models_cache"), \
patch.object(mod, "_spawn_swr_refresh") as spawn, \
patch.object(mod, "provider_model_ids", return_value=list(LIVE)) as live:
assert mod.cached_provider_model_ids("deepinfra") == LIVE
live.assert_called_once()
spawn.assert_not_called()
assert cache["deepinfra"]["models"] == LIVE
assert "fallback" not in cache["deepinfra"]
def test_codex_cold_fallback_retries_after_short_ttl(monkeypatch, tmp_path):
from hermes_cli import auth, codex_models
monkeypatch.setenv("CODEX_HOME", str(tmp_path))
monkeypatch.setattr(auth, "resolve_codex_runtime_credentials", lambda **_: {"api_key": "test-token"})
monkeypatch.setattr(auth, "_codex_access_token_is_expiring", lambda *_: False)
responses = iter([[], ["gpt-6-astra"]])
monkeypatch.setattr(codex_models, "_fetch_models_from_api", lambda *_args, **_kw: next(responses))
cache = {}
with patch.object(mod, "_load_provider_models_cache", return_value=cache), \
patch.object(mod, "_credential_fingerprint", return_value="fp"), \
patch.object(mod, "_save_provider_models_cache"), \
patch.object(mod, "_spawn_swr_refresh") as spawn:
mod.cached_provider_model_ids("openai-codex")
assert cache["openai-codex"]["fallback"] is True
cache["openai-codex"]["at"] = time.time() - mod._PROVIDER_MODELS_FALLBACK_TTL - 1
assert "gpt-6-astra" in mod.cached_provider_model_ids("openai-codex")
assert "fallback" not in cache["openai-codex"]
spawn.assert_not_called()
def test_anthropic_catalog_marks_the_curated_list_as_fallback_on_a_failed_live_fetch():
# _anthropic_catalog used to `return curated` (a plain list) on a failed live fetch, so a
# transient outage on a proxy base_url cached the generic curated list over the account's/
# proxy's real catalog for a full hour (same class of bug as #107391, just missed by 3fb8b3f6).
with patch.object(mod, "_get_model_config_dict", return_value={}), \
patch.object(mod, "_fetch_anthropic_models", return_value=None):
rows = mod.provider_model_ids("anthropic")
assert isinstance(rows, mod.CuratedFallbackModels)
assert rows == list(mod._PROVIDER_MODELS["anthropic"])