mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:04:08 +03:00
9ad5a62708
test_inventory.py::test_include_unconfigured_appends_canonical_skeletons
passes alone and fails when it runs in one process after
test_models_dev_provider_capabilities.py (with any earlier file that
already ran provider discovery, e.g. test_model_metadata.py):
AssertionError: assert 'solstice' not in {...}
_isolated_registry() swaps providers._REGISTRY for {} but leaves
providers._PROVIDER_LIST_CACHE alone. register_provider() clears the cache,
the next list_providers() refills it from the fixture registry, and the
monkeypatch undo restores _REGISTRY but not the cache. Every later
list_providers() then returns only the fixture profile, so
unlisted_provider_names() never sees the hidden pre-release "solstice"
profile and the picker lists it.
The same fixture also left its fixture-provider/fixture-relay rows in
hermes_cli.models.CANONICAL_PROVIDERS: a post-discovery registration
mirrors into the picker catalog, and a string setattr on
"hermes_cli.models..." imports that module inside the patch window, so
the undo "restores" the patched copy.
Fix at the polluter: patch _PROVIDER_LIST_CACHE and _SOURCES alongside
_REGISTRY, and scope the catalog tables (both modules imported before any
patch). test_account_usage_fetch.py's _register_profile had the same
missing list-cache patch.
Repro (test venv python, -p no:randomly, one process; run_tests.sh runs
each file in its own process and cannot hold the order):
test_model_metadata.py test_models_dev_provider_capabilities.py <victim>
main: 1 failed, 120 passed fix: 121 passed
48 models_dev/model_metadata_http files + victim
main: 1 failed, 774 passed fix: 775 passed
107 lines
6.0 KiB
Python
107 lines
6.0 KiB
Python
"""Plugin-declared per-model capabilities (#102115): one declaration on ``ProviderProfile`` feeds every
|
|
models.dev consumer — capability lookup, context lookup, the picker's reasoning badge and image routing."""
|
|
|
|
from copy import deepcopy
|
|
|
|
import providers
|
|
from providers.base import ProviderProfile
|
|
from agent import models_dev
|
|
|
|
|
|
def _isolated_registry(monkeypatch, catalog=None):
|
|
# A post-discovery registration mirrors the profile into the picker catalog; keep that scoped too.
|
|
# Both imported before any patch: the catalog's import-time sync must see the real registry, and
|
|
# hermes_cli.models must bind the real list so the undo restores it.
|
|
import hermes_cli.models as hm
|
|
import hermes_cli.models_catalog_static as static
|
|
monkeypatch.setattr(static, "CANONICAL_PROVIDERS", list(static.CANONICAL_PROVIDERS))
|
|
monkeypatch.setattr(static, "_canonical_slugs", set(static._canonical_slugs))
|
|
monkeypatch.setattr(static, "_PROVIDER_LABELS", dict(static._PROVIDER_LABELS))
|
|
monkeypatch.setattr(hm, "CANONICAL_PROVIDERS", static.CANONICAL_PROVIDERS)
|
|
monkeypatch.setattr(hm, "_PROVIDER_LABELS", static._PROVIDER_LABELS)
|
|
monkeypatch.setattr(providers, "_REGISTRY", {})
|
|
monkeypatch.setattr(providers, "_ALIASES", {})
|
|
# The list cache is filled from the swapped-in registry; without its own patch it outlives the
|
|
# test and later list_providers() calls see only the fixture profile (hidden ones go unfiltered).
|
|
monkeypatch.setattr(providers, "_PROVIDER_LIST_CACHE", None)
|
|
monkeypatch.setattr(providers, "_SOURCES", {})
|
|
monkeypatch.setattr(providers, "_discovered", True)
|
|
monkeypatch.setattr(models_dev, "_registry_models", lambda *a, **k: catalog)
|
|
|
|
|
|
def test_declared_capabilities_reach_every_consumer_and_user_override_wins(monkeypatch):
|
|
from agent.image_routing import decide_image_input_mode
|
|
from hermes_cli.inventory import _apply_capabilities
|
|
|
|
_isolated_registry(monkeypatch)
|
|
overrides: dict = {}
|
|
monkeypatch.setattr(models_dev, "_load_model_overrides", lambda: overrides)
|
|
monkeypatch.setattr("hermes_cli.inventory._reasoning_catalog_reader", lambda slug: None)
|
|
declaration = {
|
|
"tier-high": {"supports_reasoning": False, "supports_vision": True, "context_window": 64000},
|
|
}
|
|
original = deepcopy(declaration)
|
|
providers.register_provider(ProviderProfile(
|
|
name="fixture-provider", aliases=("fixture-alias",), model_capabilities=declaration))
|
|
|
|
for name in ("fixture-provider", "fixture-alias"):
|
|
caps = models_dev.get_model_capabilities(name, "tier-high")
|
|
assert (caps.supports_reasoning, caps.supports_vision, caps.context_window) == (False, True, 64000)
|
|
assert models_dev.lookup_models_dev_context(name, "tier-high") == 64000
|
|
# Negative: an undeclared model keeps the catalog/heuristic path (catalog miss → None).
|
|
assert models_dev.get_model_capabilities(name, "undeclared") is None
|
|
|
|
cfg = {"model": {"provider": "fixture-provider", "default": "tier-high"}}
|
|
assert decide_image_input_mode("fixture-provider", "tier-high", cfg) == "native"
|
|
assert decide_image_input_mode("fixture-provider", "undeclared", cfg) == "text"
|
|
|
|
rows = [{"slug": "fixture-provider", "models": ["tier-high", "undeclared"]}]
|
|
_apply_capabilities(rows)
|
|
assert rows[0]["capabilities"]["tier-high"]["reasoning"] is False
|
|
assert rows[0]["capabilities"]["undeclared"]["reasoning"] is True
|
|
|
|
overrides["fixture-provider"] = {"tier-high": {"supports_reasoning": True, "context_window": 96000}}
|
|
caps = models_dev.get_model_capabilities("fixture-provider", "tier-high")
|
|
assert (caps.supports_reasoning, caps.supports_vision, caps.context_window) == (True, True, 96000)
|
|
assert models_dev.lookup_models_dev_context("fixture-provider", "tier-high") == 96000
|
|
assert declaration == original
|
|
|
|
|
|
def test_partial_plugin_metadata_preserves_unknowns_and_catalog_fields(monkeypatch):
|
|
catalog = {"known": {"reasoning": True, "tool_call": True,
|
|
"limit": {"context": 32000, "output": 4000},
|
|
"modalities": {"input": ["text", "image"]}}}
|
|
original = deepcopy(catalog)
|
|
_isolated_registry(monkeypatch, catalog)
|
|
monkeypatch.setitem(models_dev.PROVIDER_TO_MODELS_DEV, "fixture-provider", "fixture-provider")
|
|
monkeypatch.setattr(models_dev, "_load_model_overrides", dict)
|
|
providers.register_provider(ProviderProfile(name="fixture-provider", model_capabilities={
|
|
"known": {"context_window": 64000},
|
|
"unknown": {"context_window": 48000},
|
|
}))
|
|
known = models_dev.get_model_capabilities("fixture-provider", "known")
|
|
assert (known.context_window, known.max_output_tokens) == (64000, 4000)
|
|
assert known.supports_reasoning is True and known.supports_vision is True
|
|
unknown = models_dev.get_model_capabilities("fixture-provider", "unknown")
|
|
assert unknown.context_window == 48000
|
|
assert unknown.supports_reasoning is None and unknown.supports_vision is None
|
|
assert catalog == original
|
|
|
|
|
|
def test_provider_wide_vision_flag_gates_tool_results_not_attachments(monkeypatch):
|
|
"""``ProviderProfile.supports_vision`` is a provider-wide tool-result wire capability; per-model attachment
|
|
routing comes only from ``model_capabilities``. Relays (``router``) set the flag over mixed catalogs, so
|
|
honouring it for attachments sent images to text-only models (reverted in 08feaa98b5b)."""
|
|
from agent.image_routing import decide_image_input_mode
|
|
from tools.vision_tools import _supports_media_in_tool_results
|
|
|
|
_isolated_registry(monkeypatch)
|
|
monkeypatch.setattr(models_dev, "_load_model_overrides", dict)
|
|
providers.register_provider(ProviderProfile(
|
|
name="fixture-relay", supports_vision=True,
|
|
model_capabilities={"seeing": {"supports_vision": True}}))
|
|
|
|
assert _supports_media_in_tool_results("fixture-relay", "undeclared") is True
|
|
assert decide_image_input_mode("fixture-relay", "undeclared", {}) == "text"
|
|
assert decide_image_input_mode("fixture-relay", "seeing", {}) == "native"
|