mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 11:22:02 +03:00
7e8c8a5856
A hosted load abandoned at its load deadline can fail after a reload of the same plugin succeeded. Its failure cleanup unloaded the plugin key, which tore down the live load: its tools failed with "plugin host callable was released" and its on_unload had already run. Claim the plugin key before the host start can block, and run the failure cleanup only while this load still owns the key.
451 lines
20 KiB
Python
451 lines
20 KiB
Python
"""plugins.isolation: host — third-party plugins run in a per-profile host process, reached only via ctx."""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
import hermes_yaml as yaml
|
|
import pytest
|
|
|
|
from hermes_cli import plugins as plugins_mod
|
|
from hermes_cli.plugins import PluginManager
|
|
|
|
PROBE_PLUGIN = '''
|
|
import json, os
|
|
from agent.image_gen_provider import ImageGenProvider
|
|
|
|
class Painter(ImageGenProvider):
|
|
@property
|
|
def name(self):
|
|
return "hostpainter"
|
|
def generate(self, prompt, aspect_ratio="landscape", **kw):
|
|
return {"success": True, "image": f"{prompt}@{os.getpid()}"}
|
|
|
|
SEEN = []
|
|
|
|
def register(ctx):
|
|
ctx.register_tool(name="hostprobe_pid", toolset="hostprobe", schema={"name": "hostprobe_pid",
|
|
"description": "d", "parameters": {"type": "object", "properties": {}}},
|
|
handler=lambda args, **kw: json.dumps({"pid": os.getpid(), "seen": SEEN, "x": args.get("x")}))
|
|
ctx.register_tool(name="hostprobe_nested", toolset="hostprobe", schema={"name": "hostprobe_nested",
|
|
"description": "d", "parameters": {"type": "object", "properties": {}}},
|
|
handler=lambda args, **kw: ctx.dispatch_tool("hostprobe_pid", {"x": "nested"}))
|
|
ctx.register_tool(name="hostprobe_crash", toolset="hostprobe", schema={"name": "hostprobe_crash",
|
|
"description": "d", "parameters": {"type": "object", "properties": {}}},
|
|
handler=lambda args, **kw: os._exit(3))
|
|
ctx.register_hook("post_tool_call", lambda tool_name=None, **kw: SEEN.append(tool_name))
|
|
ctx.register_image_gen_provider(Painter())
|
|
'''
|
|
|
|
PLATFORM_PLUGIN = '''
|
|
def register(ctx):
|
|
ctx.register_platform("x", "X", adapter_factory=lambda cfg: None, check_fn=lambda: True)
|
|
'''
|
|
|
|
|
|
def _home_with_plugins(tmp_path, monkeypatch, plugins: dict, isolation="host"):
|
|
home = tmp_path / ".hermes"
|
|
home.mkdir(parents=True, exist_ok=True)
|
|
for name, body in plugins.items():
|
|
plugin_dir = home / "plugins" / name
|
|
plugin_dir.mkdir(parents=True)
|
|
(plugin_dir / "plugin.yaml").write_text(yaml.safe_dump({"name": name, "version": "1.0"}), encoding="utf-8")
|
|
(plugin_dir / "__init__.py").write_text(body, encoding="utf-8")
|
|
(home / "config.yaml").write_text(yaml.safe_dump(
|
|
{"plugins": {"enabled": list(plugins), "isolation": isolation}}), encoding="utf-8")
|
|
empty_bundled = tmp_path / "bundled"
|
|
empty_bundled.mkdir()
|
|
monkeypatch.setenv("HOME", str(tmp_path / "os-home"))
|
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
|
monkeypatch.setattr(plugins_mod, "get_bundled_plugins_dir", lambda: empty_bundled)
|
|
return home
|
|
|
|
|
|
@pytest.mark.platforms("posix") # os._exit / SIGKILL process semantics
|
|
def test_plugin_runs_out_of_process_with_ctx_round_trips_and_survives_host_crash(tmp_path, monkeypatch):
|
|
from agent import image_gen_registry
|
|
from agent.image_gen_provider import ImageGenProvider
|
|
from tools.registry import registry
|
|
|
|
_home_with_plugins(tmp_path, monkeypatch, {"hostprobe": PROBE_PLUGIN, "platformprobe": PLATFORM_PLUGIN})
|
|
manager = PluginManager()
|
|
manager.discover_and_load()
|
|
try:
|
|
# The plugin's code never entered this interpreter, yet its registrations are ordinary.
|
|
assert not any("hostprobe" in name for name in sys.modules)
|
|
assert manager._plugins["hostprobe"].error is None
|
|
first = json.loads(registry.dispatch("hostprobe_pid", {"x": 1}, scope=manager.scope_key))
|
|
host_pid = first["pid"]
|
|
assert host_pid != os.getpid()
|
|
# A handler calling back into Hermes while Hermes waits on it (nested ctx.dispatch_tool).
|
|
nested = json.loads(registry.dispatch("hostprobe_nested", {}, scope=manager.scope_key))
|
|
assert nested["pid"] == host_pid and nested["x"] == "nested"
|
|
# Hooks fire in the host; provider objects arrive as instances of the ABC Hermes checks.
|
|
manager.invoke_hook("post_tool_call", tool_name="read_file", args={}, result="ok")
|
|
assert "read_file" in json.loads(registry.dispatch("hostprobe_pid", {}, scope=manager.scope_key))["seen"]
|
|
painter = image_gen_registry.get_provider("hostpainter")
|
|
assert isinstance(painter, ImageGenProvider)
|
|
assert painter.generate("cat") == {"success": True, "image": f"cat@{host_pid}"}
|
|
# Live-object ctx surfaces fail the plugin with the boundary's reason, not a crash.
|
|
assert "register_platform" in str(manager._plugins["platformprobe"].error)
|
|
|
|
# A plugin killing its host costs one tool error; Hermes keeps running and the host restarts.
|
|
crashed = registry.dispatch("hostprobe_crash", {}, scope=manager.scope_key)
|
|
assert "plugin host" in crashed
|
|
deadline, result = time.monotonic() + 15, ""
|
|
while time.monotonic() < deadline:
|
|
result = registry.dispatch("hostprobe_pid", {}, scope=manager.scope_key)
|
|
if '"pid"' in result:
|
|
break
|
|
time.sleep(0.2)
|
|
assert json.loads(result)["pid"] not in {host_pid, os.getpid()}
|
|
finally:
|
|
manager.unload()
|
|
manager._plugin_host().shutdown()
|
|
|
|
|
|
@pytest.mark.platforms("any") # the host is a child process: its env/home resolution is per-OS
|
|
def test_isolation_host_keeps_every_user_import_path_out_of_process(tmp_path, monkeypatch):
|
|
from hermes_cli.plugin_isolation_audit import audit_plugin_dir
|
|
from plugins import plugin_loader
|
|
|
|
home = _home_with_plugins(tmp_path, monkeypatch, {"platformprobe": PLATFORM_PLUGIN, "fine": PROBE_PLUGIN})
|
|
# Category loaders' in-process import refuses user code under host isolation (the backstop
|
|
# behind the memory / context-engine / cron host routes).
|
|
import logging
|
|
assert plugin_loader.load_plugin_module(
|
|
"_hermes_user_x.fine", home / "plugins" / "fine", parents=("_hermes_user_x",),
|
|
logger=logging.getLogger("t"), synthetic_namespace="_hermes_user_x") is None
|
|
assert "_hermes_user_x.fine" not in sys.modules
|
|
# The static audit reads the same boundary table the host enforces.
|
|
assert audit_plugin_dir(home / "plugins" / "fine").verdict == "host"
|
|
blocked = audit_plugin_dir(home / "plugins" / "platformprobe")
|
|
assert blocked.verdict == "in_process" and "register_platform" in blocked.reasons[0]
|
|
|
|
|
|
|
|
def test_managed_scope_pins_host_isolation_over_the_profiles_own_config(tmp_path, monkeypatch):
|
|
"""The isolated party must not be able to opt out: an operator pin in the managed scope
|
|
(/etc/hermes/config.yaml) wins over a profile config that says in_process."""
|
|
from hermes_cli import managed_scope
|
|
from tools.registry import registry
|
|
|
|
_home_with_plugins(tmp_path, monkeypatch, {"hostprobe": PROBE_PLUGIN}, isolation="in_process")
|
|
managed = tmp_path / "managed"
|
|
managed.mkdir()
|
|
(managed / "config.yaml").write_text(yaml.safe_dump({"plugins": {"isolation": "host"}}), encoding="utf-8")
|
|
monkeypatch.setenv("HERMES_MANAGED_DIR", str(managed))
|
|
managed_scope.invalidate_managed_cache()
|
|
manager = PluginManager()
|
|
manager.discover_and_load()
|
|
try:
|
|
assert not any("hostprobe" in name for name in sys.modules)
|
|
result = json.loads(registry.dispatch("hostprobe_pid", {}, scope=manager.scope_key))
|
|
assert result["pid"] != os.getpid()
|
|
finally:
|
|
host = getattr(manager, "_plugin_host_instance", None)
|
|
if host is not None:
|
|
host.shutdown()
|
|
managed_scope.invalidate_managed_cache()
|
|
|
|
MODEL_PROVIDER_PLUGIN = '''
|
|
import os
|
|
from providers import register_provider
|
|
from providers.base import ProviderProfile
|
|
|
|
class HostModel(ProviderProfile):
|
|
def build_extra_body(self, *, session_id=None, **context):
|
|
return {"pid": os.getpid(), "session_id": session_id}
|
|
|
|
register_provider(HostModel(name="hostmodel", base_url="https://hostmodel.example/v1", env_vars=("HOSTMODEL_KEY",)))
|
|
'''
|
|
|
|
|
|
@pytest.mark.platforms("any") # the host is a child process: its env/home resolution is per-OS
|
|
def test_model_provider_profile_data_is_local_and_overrides_run_in_the_host(tmp_path, monkeypatch):
|
|
import providers
|
|
|
|
home = _home_with_plugins(tmp_path, monkeypatch, {})
|
|
plugin_dir = home / "plugins" / "model-providers" / "hostmodel"
|
|
plugin_dir.mkdir(parents=True)
|
|
(plugin_dir / "plugin.yaml").write_text("name: hostmodel\nkind: model-provider\n", encoding="utf-8")
|
|
(plugin_dir / "__init__.py").write_text(MODEL_PROVIDER_PLUGIN, encoding="utf-8")
|
|
profile = providers.get_provider_profile("hostmodel")
|
|
try:
|
|
assert isinstance(profile, providers.ProviderProfile)
|
|
# Discovery never started a host: data came from the cached, credential-free extraction.
|
|
assert profile.base_url == "https://hostmodel.example/v1" and tuple(profile.env_vars) == ("HOSTMODEL_KEY",)
|
|
assert list((home / "cache" / "plugin_host" / "model-providers").glob("hostmodel-*.json"))
|
|
assert not any("hostmodel" in name for name in sys.modules)
|
|
body = profile.build_extra_body(session_id="s1")
|
|
assert body["session_id"] == "s1" and body["pid"] != os.getpid()
|
|
finally:
|
|
host = getattr(plugins_mod.get_plugin_manager(), "_plugin_host_instance", None)
|
|
if host is not None:
|
|
host.shutdown()
|
|
|
|
|
|
ASYNC_PLUGIN = '''
|
|
import asyncio, json
|
|
S = {"type": "object", "properties": {}}
|
|
|
|
def register(ctx):
|
|
async def whoami(args, **kw): # async plugin code calling back into Hermes
|
|
return json.dumps({"inner": json.loads(ctx.dispatch_tool("session_probe", {})),
|
|
"acomplete_awaitable": asyncio.iscoroutinefunction(ctx.llm.acomplete)})
|
|
ctx.register_tool(name="async_whoami", toolset="asyncprobe", schema={"name": "async_whoami",
|
|
"description": "d", "parameters": S}, handler=whoami, is_async=True)
|
|
'''
|
|
|
|
|
|
@pytest.mark.platforms("any") # the host is a child process: its env/home resolution is per-OS
|
|
def test_async_plugin_code_calls_back_in_the_callers_session(tmp_path, monkeypatch):
|
|
import contextvars
|
|
from tools.registry import registry
|
|
|
|
session = contextvars.ContextVar("session", default="<unset>")
|
|
session.set("host-creator") # whoever first touches the host must not leak into later calls
|
|
registry.register(name="session_probe", toolset="asyncprobe", schema={"name": "session_probe",
|
|
"description": "d", "parameters": {"type": "object", "properties": {}}},
|
|
handler=lambda args, **kw: json.dumps({"session": session.get()}))
|
|
_home_with_plugins(tmp_path, monkeypatch, {"asyncprobe": ASYNC_PLUGIN})
|
|
manager = PluginManager()
|
|
manager.discover_and_load()
|
|
try:
|
|
def call_as(name):
|
|
def run():
|
|
session.set(name)
|
|
return json.loads(registry.dispatch("async_whoami", {}, scope=manager.scope_key))
|
|
return contextvars.copy_context().run(run)
|
|
|
|
assert call_as("session-B") == {"inner": {"session": "session-B"}, "acomplete_awaitable": True}
|
|
assert call_as("session-C")["inner"] == {"session": "session-C"}
|
|
finally:
|
|
registry.deregister("session_probe")
|
|
manager.unload()
|
|
manager._plugin_host().shutdown()
|
|
|
|
|
|
MEMORY_PLUGIN = '''
|
|
import os
|
|
from agent.memory_provider import MemoryProvider
|
|
|
|
class Probe(MemoryProvider):
|
|
level = 0 # a class-level default the provider changes later
|
|
@property
|
|
def name(self): return "memprobe"
|
|
def is_available(self): return True
|
|
def initialize(self, session_id, **kw): pass
|
|
def get_tool_schemas(self): return []
|
|
def whoami(self):
|
|
self.level += 1
|
|
return os.getpid()
|
|
|
|
def register(ctx):
|
|
ctx.register_memory_provider(Probe())
|
|
'''
|
|
|
|
|
|
@pytest.mark.platforms("posix") # SIGKILL
|
|
def test_hosted_memory_provider_stays_live_across_a_host_crash(tmp_path, monkeypatch):
|
|
import signal
|
|
from plugins.memory import load_memory_provider
|
|
from plugins.memory.config_schema import get_provider_config_schema
|
|
|
|
home = _home_with_plugins(tmp_path, monkeypatch, {})
|
|
provider_dir = home / "plugins" / "memprobe"
|
|
provider_dir.mkdir(parents=True)
|
|
(provider_dir / "__init__.py").write_text(MEMORY_PLUGIN, encoding="utf-8")
|
|
marker = tmp_path / "schema_pid"
|
|
(provider_dir / "config_schema.py").write_text(
|
|
f"import os, pathlib\npathlib.Path({str(marker)!r}).write_text(str(os.getpid()))\nCONFIG_SCHEMA = None\n",
|
|
encoding="utf-8")
|
|
host = plugins_mod.get_plugin_manager()._plugin_host()
|
|
try:
|
|
provider = load_memory_provider("memprobe")
|
|
first_pid = provider.whoami()
|
|
assert first_pid == host.pid != os.getpid()
|
|
assert provider.level == 1 # read live from the plugin's object, not the class default
|
|
# The provider's schema file is user code too: it runs in the host, never here.
|
|
get_provider_config_schema("memprobe")
|
|
assert int(marker.read_text(encoding="utf-8-sig")) == host.pid
|
|
|
|
os.kill(first_pid, signal.SIGKILL) # windows-footgun: ok — posix-only test (platforms marker)
|
|
deadline = time.monotonic() + 10
|
|
while host.alive and time.monotonic() < deadline:
|
|
time.sleep(0.1)
|
|
# The proxy Hermes already holds reloads the provider into the new host on next use.
|
|
assert provider.whoami() not in {first_pid, os.getpid()}
|
|
finally:
|
|
host.shutdown()
|
|
|
|
|
|
@pytest.mark.platforms("any")
|
|
def test_host_plugin_can_launch_status_and_steer_subagent(tmp_path, monkeypatch):
|
|
import threading
|
|
from types import SimpleNamespace
|
|
from agent.subagent_lifecycle import bind_subagent_parent, SubagentLifecycleService
|
|
from tools.delegate_tool_registry import _register_subagent, _unregister_subagent
|
|
from tools.registry import registry
|
|
|
|
body = '''
|
|
import json, os
|
|
from agent.subagent_lifecycle import SubagentLaunchRequest
|
|
def register(ctx):
|
|
def run(args, **kw):
|
|
service = ctx.subagent_lifecycle
|
|
handle = service.launch(SubagentLaunchRequest(goal="review from host"))
|
|
return json.dumps({"pid": os.getpid(), "state": service.status(handle).state,
|
|
"accepted": service.steer(handle, "check correctness"), "handle": handle})
|
|
ctx.register_tool(name="host_lifecycle", toolset="hostlife",
|
|
schema={"name": "host_lifecycle", "description": "d",
|
|
"parameters": {"type": "object", "properties": {}}}, handler=run)
|
|
'''
|
|
finish, steering = threading.Event(), []
|
|
child = SimpleNamespace(_subagent_id="sa-host-lifecycle", provider="test", model="test-model",
|
|
steer=lambda text: steering.append(text) is None)
|
|
parent = SimpleNamespace(session_id="parent-host-lifecycle")
|
|
|
|
def build(**kwargs):
|
|
assert kwargs["parent_agent"] is parent
|
|
assert kwargs["goal"] == "review from host"
|
|
_register_subagent({"subagent_id": child._subagent_id, "agent": child})
|
|
return child
|
|
|
|
def run(*_args):
|
|
assert finish.wait(5)
|
|
return {"status": "completed", "summary": "host child done"}
|
|
|
|
monkeypatch.setattr("tools.delegate_tool._build_child_agent", build)
|
|
monkeypatch.setattr("tools.delegate_tool._run_single_child", run)
|
|
_home_with_plugins(tmp_path, monkeypatch, {"hostlife": body})
|
|
manager = PluginManager()
|
|
manager.discover_and_load()
|
|
service = SubagentLifecycleService(lambda: parent)
|
|
result = None
|
|
try:
|
|
with bind_subagent_parent(parent):
|
|
result = json.loads(registry.dispatch("host_lifecycle", {}, scope=manager.scope_key))
|
|
assert result["pid"] != os.getpid()
|
|
assert result["state"] in {"PENDING", "RUNNING"}
|
|
assert result["accepted"] is True
|
|
assert steering == ["check correctness"]
|
|
finally:
|
|
finish.set()
|
|
from agent.subagent_lifecycle import _REGISTRY
|
|
record = _REGISTRY.records.get(child._subagent_id)
|
|
if record is not None:
|
|
record.future.result(timeout=5)
|
|
_unregister_subagent(child._subagent_id, agent=child)
|
|
manager.unload()
|
|
manager._plugin_host().shutdown()
|
|
assert service.result(result["handle"]).summary == "host child done"
|
|
|
|
|
|
@pytest.mark.platforms("any")
|
|
@pytest.mark.parametrize("isolation", ["host", "in_process"])
|
|
def test_failed_register_runs_its_on_unload_cleanup(tmp_path, monkeypatch, isolation):
|
|
# A register() that queued cleanup and then raised must have that cleanup run, in either mode.
|
|
marker = tmp_path / "cleaned"
|
|
body = f'''
|
|
from pathlib import Path
|
|
def register(ctx):
|
|
ctx.on_unload(lambda: Path({str(marker)!r}).write_text("cleaned"))
|
|
raise RuntimeError("register failed")
|
|
'''
|
|
_home_with_plugins(tmp_path, monkeypatch, {"halfload": body}, isolation=isolation)
|
|
manager = PluginManager()
|
|
manager.discover_and_load()
|
|
try:
|
|
assert "register failed" in str(manager._plugins["halfload"].error)
|
|
assert marker.is_file(), "the failed load's on_unload cleanup never ran"
|
|
finally:
|
|
manager.unload()
|
|
if isolation == "host":
|
|
manager._plugin_host().shutdown()
|
|
|
|
|
|
@pytest.mark.platforms("any")
|
|
@pytest.mark.parametrize("cleanup", ['raise SystemExit("cleanup exit")', "import os; os._exit(7)"])
|
|
def test_failed_hosted_load_keeps_its_error_when_cleanup_fails(tmp_path, monkeypatch, cleanup):
|
|
# Cleanup that raises or kills the host must not replace the register() error that is recorded.
|
|
body = f'''
|
|
def register(ctx):
|
|
def cleanup():
|
|
{cleanup}
|
|
ctx.on_unload(cleanup)
|
|
raise RuntimeError("register failed")
|
|
'''
|
|
_home_with_plugins(tmp_path, monkeypatch, {"halfload": body})
|
|
manager = PluginManager()
|
|
manager.discover_and_load()
|
|
try:
|
|
assert "register failed" in str(manager._plugins["halfload"].error)
|
|
finally:
|
|
manager.unload()
|
|
manager._plugin_host().shutdown()
|
|
|
|
|
|
@pytest.mark.platforms("any")
|
|
@pytest.mark.parametrize("held_in", ["register", "host_start"])
|
|
def test_late_failure_of_an_abandoned_load_leaves_the_reloaded_plugin_alone(tmp_path, monkeypatch, held_in):
|
|
# A hosted load abandoned at its deadline (inside register(), or still waiting for the host to start)
|
|
# can fail after a reload of the same plugin succeeded. Host state is keyed by plugin, not by load,
|
|
# so that late failure must not unload the live load.
|
|
from hermes_cli import plugin_host, plugins_loader
|
|
from tests.waiting import wait_until
|
|
from tools.registry import registry
|
|
|
|
state = tmp_path / "state"
|
|
state.mkdir()
|
|
# The abandoned load's register() is the first to run when it is held inside register(), the second
|
|
# when it is held before the host start returns (the reload reaches the host first).
|
|
failing_attempt = 1 if held_in == "register" else 2
|
|
body = f'''
|
|
import json, time
|
|
from pathlib import Path
|
|
STATE = Path({str(state)!r})
|
|
def register(ctx):
|
|
attempt = len(list(STATE.glob("attempt-*"))) + 1
|
|
(STATE / f"attempt-{{attempt}}").write_text("x")
|
|
if attempt == {failing_attempt}:
|
|
while not (STATE / "go").exists():
|
|
time.sleep(0.05)
|
|
raise RuntimeError("abandoned load failed late")
|
|
ctx.register_tool(name="lateprobe_ping", toolset="lateprobe", schema={{"name": "lateprobe_ping",
|
|
"description": "d", "parameters": {{"type": "object", "properties": {{}}}}}},
|
|
handler=lambda args, **kw: json.dumps({{"pong": True}}))
|
|
'''
|
|
home = _home_with_plugins(tmp_path, monkeypatch, {"lateprobe": body})
|
|
(home / "config.yaml").write_text(yaml.safe_dump({"plugins": {
|
|
"enabled": ["lateprobe"], "isolation": "host", "load_timeout_seconds": 2}}), encoding="utf-8")
|
|
manager = PluginManager()
|
|
manager._plugin_host().ensure_started() # outside any load deadline: a slow host start can't time out the reload
|
|
if held_in == "host_start":
|
|
start, callers = plugin_host.PluginHost.ensure_started, []
|
|
|
|
def slow_first_start(self):
|
|
channel = start(self)
|
|
callers.append(self)
|
|
if len(callers) == 1:
|
|
wait_until(lambda: (state / "go").exists(), timeout=30, what="the test to release the host start")
|
|
return channel
|
|
|
|
monkeypatch.setattr(plugin_host.PluginHost, "ensure_started", slow_first_start)
|
|
manager.discover_and_load()
|
|
try:
|
|
assert "timed out" in str(manager._plugins["lateprobe"].error)
|
|
manager.discover_and_load(force=True)
|
|
assert manager._plugins["lateprobe"].enabled
|
|
(state / "go").write_text("go")
|
|
loader = f"{plugins_loader._LOADER_THREAD_PREFIX}lateprobe"
|
|
wait_until(lambda: not any(t.is_alive() and t.name == loader for t in plugins_loader._ABANDONED_LOADERS),
|
|
timeout=15, what="the abandoned first load to finish")
|
|
assert len(list(state.glob("attempt-*"))) == 2
|
|
assert json.loads(registry.dispatch("lateprobe_ping", {}, scope=manager.scope_key)) == {"pong": True}
|
|
finally:
|
|
(state / "go").write_text("go")
|
|
manager.unload()
|
|
manager._plugin_host().shutdown()
|