Files
Maciej Lisiewski eb1b37dbc2 fix(code-execution): read the cell spill as utf-8-sig; gate POSIX spill tests with platforms
The footgun lint requires BOM-tolerant reads. The runner writes the spill
without a BOM, so the only change is that output starting with U+FEFF loses
that character in the spill. The bare skipif decorators hid the POSIX-only
spill tests from the macOS lane; platforms("posix") lets it import them.

Refs #122024

Co-authored-by: BearHuddleston <196404565+BearHuddleston@users.noreply.github.com>
2026-11-14 21:02:31 +01:00

873 lines
41 KiB
Python

#!/usr/bin/env python3
"""Tests for execute_code's session kernel.
Session kernels are always on (the ``code_execution.kernel_mode`` key is
retired): each (task, mode, interpreter, cwd, tool-set) owner keeps one
Python child alive so state survives across calls. These tests pin the
contract:
- state persists across cells and reset=true discards it
- a raised exception keeps the kernel (and its state) alive
- a timeout kills the kernel; the next call gets a fresh one
- fd-level output from user-spawned subprocesses reaches the result
- sys.exit() inside a cell ends the kernel deliberately
Mode is sourced from ``code_execution.mode`` in config.yaml only;
tests patch ``_load_config`` directly, mirroring test_code_execution_modes.
"""
import json
import os
import subprocess
import sys
import tempfile
import textwrap
import time
import unittest
from contextlib import contextmanager
from pathlib import Path
from unittest.mock import patch
import pytest
os.environ["TERMINAL_ENV"] = "local"
@pytest.fixture(autouse=True)
def _force_local_terminal(monkeypatch):
"""Mirror test_code_execution.py — guarantee local backend."""
monkeypatch.setenv("TERMINAL_ENV", "local")
from tools.code_execution_tool import execute_code
from tools.code_kernel import _KERNELS, shutdown_all_kernels
@contextmanager
def _kernel_config(**overrides):
"""Pin code_execution config; strict mode keeps the test hermetic.
``mode`` (strict/project) is the only config knob — session kernels are
always on; the retired ``kernel_mode`` key is ignored by the tool."""
config = {"mode": "strict", "timeout": 30}
config.update(overrides)
with patch("tools.code_execution_tool._load_config", return_value=config):
yield
@pytest.fixture(autouse=True)
def _fresh_kernel_registry():
shutdown_all_kernels()
yield
shutdown_all_kernels()
def _run(code, **kwargs):
return json.loads(execute_code(code, task_id="kernel-test", **kwargs))
class TestSessionStatePersistence(unittest.TestCase):
def test_state_persists_across_cells(self):
with _kernel_config():
first = _run("x = 41")
self.assertEqual(first["status"], "success", first)
self.assertEqual(first["kernel"]["reused"], False)
second = _run("print(x + 1)")
self.assertEqual(second["status"], "success", second)
self.assertIn("42", second["output"])
self.assertEqual(second["kernel"]["reused"], True)
self.assertEqual(second["kernel"]["execution_count"], 2)
def test_reset_discards_state(self):
with _kernel_config():
_run("x = 41")
second = _run("print(x + 1)", reset=True)
self.assertEqual(second["status"], "error", second)
self.assertIn("NameError", second.get("error", ""))
self.assertEqual(second["kernel"]["state_reset"], True)
def test_exception_keeps_the_kernel_alive(self):
with _kernel_config():
_run("a = 7")
boom = _run("1 / 0")
self.assertEqual(boom["status"], "error")
self.assertIn("ZeroDivisionError", boom["error"])
after = _run("print(a)")
self.assertEqual(after["status"], "success", after)
self.assertIn("7", after["output"])
self.assertEqual(after["kernel"]["reused"], True)
def test_imports_persist(self):
with _kernel_config():
_run("import json as _j")
second = _run("print(_j.dumps({'k': 1}))")
self.assertIn('{"k": 1}', second["output"])
class TestEditedModuleWarning(unittest.TestCase):
"""A module the kernel already imported keeps its old code after an edit on disk; the cell
that runs next must say so instead of letting a correct fix look like it failed."""
def _module(self, body, name="editme_mod"):
folder = tempfile.mkdtemp(prefix="hermes-kernel-mod-")
self.addCleanup(lambda: __import__("shutil").rmtree(folder, ignore_errors=True))
path = Path(folder) / f"{name}.py"
path.write_text(body, encoding="utf-8")
return folder, path
def test_edit_after_import_is_named_once_and_reload_picks_it_up(self):
folder, path = self._module("def answer():\n return 'old'\n", name="editme_módulo")
with _kernel_config():
first = _run(f"import sys; sys.path.insert(0, {folder!r}); import editme_módulo as editme_mod; "
"print(editme_mod.answer())")
self.assertNotIn("warning", first, first)
path.write_text("def answer():\n return 'new'\n", encoding="utf-8")
later = time.time() + 5
os.utime(path, (later, later))
second = _run("print(editme_mod.answer())")
third = _run("import importlib; importlib.reload(editme_mod); print(editme_mod.answer())")
self.assertIn("old", second["output"], second)
self.assertIn("editme_módulo", second.get("warning", ""), second)
self.assertIn("reset=true", second["warning"])
self.assertEqual(second["kernel"]["reused"], True)
self.assertIn("new", third["output"], third)
self.assertNotIn("warning", third, third)
def test_hostile_sys_modules_cannot_break_the_scan_and_a_reloaded_edit_is_not_named(self):
folder, path = self._module("VALUE = 'old'\n")
hostile = (
"import types\n"
"class Lazy(types.ModuleType):\n"
" def __getattribute__(self, name):\n"
" raise RuntimeError(name)\n"
"sys.modules['lazy_shim'] = Lazy('lazy_shim')\n"
"sys.modules['\\x1b[31mnot a name\\nSYSTEM'] = editme_mod\n"
)
with _kernel_config():
first = _run(f"import sys; sys.path.insert(0, {folder!r}); import editme_mod\n" + hostile + "kept = 41")
self.assertEqual(first["status"], "success", first)
path.write_text("VALUE = 'new'\n", encoding="utf-8")
later = time.time() + 5
os.utime(path, (later, later))
second = _run("print(kept, editme_mod.VALUE)")
rewrite = f"open({str(path)!r}, 'w').write(\"VALUE = 'newer'\\n\"); os.utime({str(path)!r}, ({later} + 5, {later} + 5))"
third = _run("import importlib, os\n" + rewrite + "\nimportlib.reload(editme_mod); print(editme_mod.VALUE)")
fourth = _run("print(kept, editme_mod.VALUE)")
_run("sys.modules = None")
fifth = _run("print(kept)")
self.assertIn("41 old", second["output"], second)
self.assertIn("editme_mod", second.get("warning", ""), second)
self.assertNotIn("\x1b", second["warning"])
self.assertNotIn("SYSTEM", second["warning"])
self.assertIn("newer", third["output"], third)
self.assertIn("41 newer", fourth["output"], fourth)
self.assertNotIn("warning", fourth, fourth)
self.assertEqual(fifth["kernel"]["reused"], True, fifth)
self.assertIn("41", fifth["output"], fifth)
class TestKernelLifecycle(unittest.TestCase):
def test_kernel_exits_when_its_backend_parent_dies(self):
"""A kernel must not outlive the host that spawned it, even when the
host dies without cleanup (SIGKILL/OOM/crash). Windows: inherited
SYNCHRONIZE handle; POSIX: the kernel's group is registered with the
owner-death supervisor. Both are proven the same way — kill the host
mid-cell, the kernel is gone within seconds."""
import psutil
repo_root = str(Path(__file__).resolve().parents[2])
host_src = textwrap.dedent(f"""
import json, os, sys, time
os.environ["HERMES_HOME"] = sys.argv[1]
sys.path.insert(0, {repo_root!r})
from tools.code_kernel import SessionKernel, _spawn
k = SessionKernel(("parent-death",))
_spawn(k, task_id="parent-death", child_python=sys.executable,
child_cwd="", sandbox_tools=frozenset(), max_tool_calls=1)
cell = json.dumps({{"id": "x", "code": "import os, time\\n"
"assert 'HERMES_KERNEL_PARENT_PROCESS_HANDLE' not in os.environ\\n"
"assert 'HERMES_KERNEL_PARENT_DEATH_FD' not in os.environ\\n"
"time.sleep(300)"}}) + "\\n"
k.proc.stdin.write(cell.encode()); k.proc.stdin.flush()
print(k.proc.pid, flush=True)
time.sleep(600)
""")
with tempfile.TemporaryDirectory() as home:
host = subprocess.Popen(
[sys.executable, "-c", host_src, home],
stdout=subprocess.PIPE, text=True,
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
)
try:
kernel = psutil.Process(int(host.stdout.readline()))
time.sleep(0.5)
self.assertTrue(kernel.is_running(), "kernel never came up")
host.kill()
host.wait(timeout=10)
try:
kernel.wait(timeout=10)
except psutil.TimeoutExpired:
kernel.kill()
self.fail("session kernel survived its backend parent")
finally:
if host.poll() is None:
host.kill()
def test_timeout_kills_the_kernel_and_reports_state_loss(self):
with _kernel_config(timeout=1):
slow = _run("import time\ntime.sleep(30)")
self.assertEqual(slow["status"], "timeout", slow)
self.assertIn("state was lost", slow["error"])
self.assertEqual(len(_KERNELS), 0)
with _kernel_config():
fresh = _run("print('alive')")
self.assertEqual(fresh["status"], "success", fresh)
self.assertEqual(fresh["kernel"]["reused"], False)
self.assertIn("alive", fresh["output"])
def test_sys_exit_ends_the_kernel(self):
with _kernel_config():
done = _run("import sys\nsys.exit(0)")
self.assertEqual(done["kernel"].get("ended"), True, done)
self.assertEqual(len(_KERNELS), 0)
fresh = _run("print('respawned')")
self.assertEqual(fresh["kernel"]["reused"], False)
self.assertIn("respawned", fresh["output"])
def test_subprocess_fd_output_reaches_the_result(self):
code = (
"import subprocess, sys\n"
"subprocess.run([sys.executable, '-c', \"print('raw-passthrough')\"])\n"
)
with _kernel_config():
result = _run(code)
self.assertEqual(result["status"], "success", result)
self.assertIn("raw-passthrough", result["output"])
class TestModelFacingReset(unittest.TestCase):
def test_reset_is_reachable_from_a_model_call_despite_stale_kernel_mode(self):
"""Session kernels are always on (#96787), so ``reset`` is the model's only
way out of poisoned state. A stale ``kernel_mode: per-call`` key must not
drop it from the schema, and a model-shaped call routed through the
registered handler must actually discard the kernel's state."""
from tools.code_execution_tool import _execute_code_handler, build_execute_code_schema
with _kernel_config(kernel_mode="per-call"):
schema = build_execute_code_schema(mode="strict")
self.assertEqual(schema["parameters"]["properties"]["reset"]["type"], "boolean")
_execute_code_handler({"code": "x = 41"}, task_id="kernel-test")
kept = json.loads(_execute_code_handler({"code": "print(x + 1)"}, task_id="kernel-test"))
self.assertIn("42", kept["output"], kept)
reset = json.loads(_execute_code_handler(
{"code": "print(x + 1)", "reset": True}, task_id="kernel-test"))
self.assertEqual(reset["status"], "error", reset)
self.assertIn("NameError", reset.get("error", ""))
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
class TestKernelOwnershipAndLifecycle(unittest.TestCase):
"""The kernel belongs to the conversation, and its lifetime is bounded.
run_agent mints a fresh task id per top-level turn, so a task-keyed
kernel would neither survive the next user turn nor ever be disposed
with anything. The owner is the approval session key; disposal rides
the same session boundary that clears approval/yolo state, idle
kernels are reaped, and the process-wide live count is capped (the
lifecycle shape carried forward from hermes-agent#88637).
"""
def _run_as(self, session_key, code, task_id, **kwargs):
from tools.approval_context import reset_current_session_key, set_current_session_key
token = set_current_session_key(session_key)
try:
return json.loads(execute_code(code, task_id=task_id, **kwargs))
finally:
reset_current_session_key(token)
def test_state_survives_across_turns_of_one_conversation(self):
# Two top-level turns: same session, different per-turn task ids.
with _kernel_config():
first = self._run_as("conv-a", "x = 41", task_id="turn-1")
self.assertEqual(first["status"], "success", first)
second = self._run_as("conv-a", "print(x + 1)", task_id="turn-2")
self.assertEqual(second["status"], "success", second)
self.assertIn("42", second["output"])
self.assertEqual(second["kernel"]["reused"], True)
def test_sessions_are_isolated_from_each_other(self):
# Same task id, different sessions: no state may cross.
with _kernel_config():
self._run_as("conv-a", "x = 41", task_id="turn-1")
other = self._run_as("conv-b", "print(x + 1)", task_id="turn-1")
self.assertEqual(other["status"], "error", other)
self.assertIn("NameError", other.get("error", ""))
def test_delegated_children_get_their_own_kernels(self):
"""A delegated child runs in a COPY of the parent's context and
inherits the parent's approval session key — the naive owner
resolution attached the child to the parent's kernel and leaked
in-memory state across the delegation boundary (both directions,
verified live). The owner must be qualified for child contexts."""
from agent.delegation_context import delegated_child_context
with _kernel_config():
self._run_as("conv-a", "parent_secret = 'p'", task_id="turn-1")
with delegated_child_context("child-1"):
leak = self._run_as(
"conv-a",
"print(globals().get('parent_secret', 'ISOLATED'))",
task_id="child-task",
)
self._run_as("conv-a", "child_secret = 'c'", task_id="child-task")
back = self._run_as(
"conv-a",
"print(globals().get('child_secret', 'ISOLATED'))",
task_id="turn-2",
)
self.assertIn("ISOLATED", leak.get("output", ""), leak)
self.assertIn("ISOLATED", back.get("output", ""), back)
def test_two_delegated_children_are_isolated_from_each_other(self):
"""Sibling children in one batch must not share a kernel either —
each child context carries its own delegation session id."""
from agent.delegation_context import delegated_child_context
with _kernel_config():
with delegated_child_context("child-A"):
self._run_as("conv-a", "sibling_secret = 'A'", task_id="t")
with delegated_child_context("child-B"):
peek = self._run_as(
"conv-a",
"print(globals().get('sibling_secret', 'ISOLATED'))",
task_id="t",
)
self.assertIn("ISOLATED", peek.get("output", ""), peek)
def test_live_children_keep_their_kernels_past_the_lru_cap(self):
"""A fan-out wider than max_session_kernels used to evict LIVE children's kernels (each
child's execute_code spawned a kernel, the cap reaped the oldest sibling's), so a child's
second call hit NameError on state its first call had set — 48 NameErrors across 28 lanes,
while the schema promised persistence. A live child's kernel is pinned for the child's life."""
import contextvars
from agent.delegation_context import delegated_child_context
with _kernel_config(max_session_kernels=2):
contexts = []
for index in range(5):
def _set(index=index):
with delegated_child_context(f"child-{index}"):
self._run_as("conv", f"v = {index}", task_id=f"child-{index}")
ctx = contextvars.copy_context()
ctx.run(_set)
contexts.append(ctx)
outcomes = {}
for index, ctx in enumerate(contexts):
def _read(index=index):
with delegated_child_context(f"child-{index}"):
outcomes[index] = self._run_as("conv", "print(v)", task_id=f"child-{index}")
ctx.run(_read)
for index, outcome in outcomes.items():
self.assertEqual(outcome["status"], "success", outcome)
self.assertTrue(outcome["kernel"]["reused"], outcome)
self.assertIn(str(index), outcome["output"])
def test_finished_children_release_their_kernels(self):
"""The pin is not a leak: when the child is torn down (the delegate_task cleanup path calls
``shutdown_kernels_for_delegated_child``) its kernels die and stop counting."""
from agent.delegation_context import delegated_child_context
from tools.code_kernel import shutdown_kernels_for_delegated_child
with _kernel_config():
with delegated_child_context("child-done"):
self._run_as("conv", "v = 1", task_id="child-done")
with delegated_child_context("child-live"):
self._run_as("conv", "v = 2", task_id="child-live")
doomed = [k for k in _KERNELS.values() if k.owner.endswith("::child::child-done")]
self.assertEqual(len(doomed), 1)
shutdown_kernels_for_delegated_child("child-done")
self.assertEqual([k for k in _KERNELS.values() if k.owner.endswith("::child::child-done")], [])
doomed[0].proc.wait(timeout=10)
self.assertFalse(doomed[0].alive())
# The sibling's kernel is untouched.
with delegated_child_context("child-live"):
still = self._run_as("conv", "print(v)", task_id="child-live")
self.assertIn("2", still["output"])
def test_session_clear_disposes_the_owners_kernels(self):
from tools.approval import clear_session
with _kernel_config():
self._run_as("conv-a", "x = 41", task_id="turn-1")
self.assertEqual(len(_KERNELS), 1)
kernel = next(iter(_KERNELS.values()))
self.assertTrue(kernel.alive())
clear_session("conv-a")
self.assertEqual(len(_KERNELS), 0)
kernel.proc.wait(timeout=10)
self.assertFalse(kernel.alive())
# The next turn in a cleared session starts fresh.
after = self._run_as("conv-a", "print('x' in dir())", task_id="turn-2")
self.assertEqual(after["status"], "success", after)
self.assertIn("False", after["output"])
def test_live_kernels_are_capped_lru_across_owners(self):
with _kernel_config(max_session_kernels=2):
kernels = []
for index in range(4):
self._run_as(f"conv-{index}", "x = 1", task_id=f"turn-{index}")
kernels.append(list(_KERNELS.values()))
self.assertLessEqual(len(_KERNELS), 2)
live_owners = {key[0] for key in _KERNELS}
# The two most recently used owners survive.
self.assertEqual(live_owners, {"conv-2", "conv-3"})
# Evicted kernels are actually dead, not orphaned.
evicted = [
kernel
for snapshot in kernels
for kernel in snapshot
if kernel.key not in _KERNELS
]
for kernel in evicted:
kernel.proc.wait(timeout=10)
self.assertFalse(kernel.alive())
def test_idle_kernels_are_reaped(self):
import time as time_module
with _kernel_config(kernel_idle_timeout=1):
self._run_as("conv-a", "x = 41", task_id="turn-1")
stale = next(iter(_KERNELS.values()))
time_module.sleep(1.2)
# Any owner's next call sweeps expired kernels process-wide.
self._run_as("conv-b", "y = 1", task_id="turn-2")
self.assertNotIn(stale.key, _KERNELS)
stale.proc.wait(timeout=10)
self.assertFalse(stale.alive())
def test_parallel_cells_share_one_kernel_process(self):
"""Parallel cells for one owner race the first spawn. Each racer
used to see proc=None as 'dead', replace the registry entry, and
orphan the winner's process — 110 live kernels under a 4-capped
process (Sep 2026). Every kernel process must stay registry-owned.
Capture actual children so an unregistered spawn cannot hide behind
the registry count. The owned process must exit on teardown."""
import threading
spawned = []
real_popen = subprocess.Popen
def _capturing_popen(args, **kwargs):
proc = real_popen(args, **kwargs)
spawned.append((proc, list(args)))
return proc
results = []
with patch("tools.code_kernel.subprocess.Popen", side_effect=_capturing_popen):
with _kernel_config():
def _cell():
results.append(self._run_as("conv-a", "import time; time.sleep(0.3)", task_id="t"))
threads = [threading.Thread(target=_cell) for _ in range(6)]
for t in threads:
t.start()
for t in threads:
t.join()
self.assertEqual([r["status"] for r in results], ["success"] * 6)
self.assertEqual(len(_KERNELS), 1)
runners = [proc for proc, args in spawned
if len(args) == 2 and Path(args[1]).name == "hermes_kernel_runner.py"]
self.assertEqual(len(runners), 1, "parallel cells spawned an unowned kernel")
kernel = next(iter(_KERNELS.values()))
self.assertIs(kernel.proc, runners[0])
self.assertIsNone(kernel.proc.poll())
shutdown_all_kernels()
for proc in runners:
proc.wait(timeout=10)
self.assertIsNotNone(proc.returncode)
class TestInScriptToolErrors(unittest.TestCase):
def test_ignored_helper_error_is_reported_for_that_cell_only(self):
"""A script that drops a helper's {"error": ...} return must not read as a clean success."""
def _handle(tool_name, tool_args, task_id=None):
if tool_name == "write_file":
return json.dumps({"error": "Refusing to overwrite a.py: never read"})
return json.dumps({"ok": True})
with _kernel_config(), patch("model_tools.handle_function_call", new=_handle):
failed = _run("import hermes_tools\nhermes_tools.write_file('a.py', 'x')\nprint('done')\n")
clean = _run("import hermes_tools\nhermes_tools.web_search(query='q')\n")
self.assertEqual(failed["status"], "success", failed)
self.assertEqual(failed["tool_errors"], [{"tool": "write_file", "error": "Refusing to overwrite a.py: never read"}])
self.assertNotIn("tool_errors", clean)
class TestPerCellRpcAuthority(unittest.TestCase):
"""Interpreter state persists across cells; RPC authority must not."""
def _recorder(self, seen):
def _handle(tool_name, tool_args, task_id=None):
from tools.thread_context import _callback_api
(get_approval, _set_a), *_rest = _callback_api()
seen.append(
{
"tool": tool_name,
"task_id": task_id,
"approval_cb": get_approval(),
}
)
return json.dumps({"ok": True})
return _handle
def test_a_later_cells_rpc_runs_under_that_cells_authority(self):
from tools.terminal_tool import set_approval_callback
seen = []
cell = "import hermes_tools\nhermes_tools.web_search(query='q')\n"
with _kernel_config(), patch(
"model_tools.handle_function_call", new=self._recorder(seen)
):
def cb_one():
return "one"
def cb_two():
return "two"
set_approval_callback(cb_one)
try:
first = _run(cell)
set_approval_callback(cb_two)
second = _run(cell)
finally:
set_approval_callback(None)
self.assertEqual(first["status"], "success", first)
self.assertEqual(second["status"], "success", second)
self.assertEqual(len(seen), 2)
self.assertIs(seen[0]["approval_cb"], cb_one)
self.assertIs(seen[1]["approval_cb"], cb_two)
self.assertEqual(seen[0]["task_id"], "kernel-test")
def test_cross_cell_alias_dispatches_under_the_current_cell(self):
# Adversarial cross-cell dataflow: a callable captured in cell 1 and
# invoked by an opaque global name in cell 2 still crosses the RPC
# boundary — under cell 2's authority, allow-list, and budget — the
# operative enforcement a per-script static scan cannot provide once
# state persists (composition contract with the execute-code guard).
from tools.terminal_tool import set_approval_callback
seen = []
with _kernel_config(), patch(
"model_tools.handle_function_call", new=self._recorder(seen)
):
def cb_one():
return "one"
def cb_two():
return "two"
set_approval_callback(cb_one)
try:
first = _run("import hermes_tools\nalias = hermes_tools.web_search\n")
set_approval_callback(cb_two)
second = _run("alias(query='q')\n")
finally:
set_approval_callback(None)
self.assertEqual(first["status"], "success", first)
self.assertEqual(second["status"], "success", second)
self.assertEqual(len(seen), 1)
self.assertIs(seen[0]["approval_cb"], cb_two)
def test_cell_authority_forwards_session_identity(self):
from tools.code_kernel import CellAuthority
seen = {}
def _handle(tool_name, tool_args, task_id=None, session_id=None, turn_id=None):
seen.update(tool=tool_name, task_id=task_id, session_id=session_id, turn_id=turn_id)
return json.dumps({"ok": True})
authority = CellAuthority("turn-1", session_id="sess-A", turn_id="turn-A")
with patch("model_tools.handle_function_call", new=_handle):
assert json.loads(authority.dispatch("web_search", {"query": "q"})) == {"ok": True}
self.assertEqual(
seen,
{"tool": "web_search", "task_id": "turn-1", "session_id": "sess-A", "turn_id": "turn-A"},
)
def test_a_settled_cells_authority_refuses_dispatch(self):
from tools.code_kernel import CellAuthority
authority = CellAuthority("turn-1")
authority.retire()
result = authority.dispatch("web_search", {"query": "q"})
self.assertIn("No active execute_code cell", result)
def test_each_cell_installs_a_fresh_authority(self):
with _kernel_config():
_run("x = 1")
kernel = next(iter(_KERNELS.values()))
first_authority = kernel.cell_authority
self.assertFalse(first_authority.active)
_run("y = 2")
self.assertIsNot(kernel.cell_authority, first_authority)
self.assertFalse(kernel.cell_authority.active)
class TestBackgroundIdleReaper(unittest.TestCase):
"""#117169: the idle sweep must not depend on the next kernel acquire — a host
that stays alive but wedged (e.g. pids exhaustion fail-closing every tool call)
never acquires again, so a background reaper reapplies the acquire-path criteria
on its own schedule, and staging dirs that outlived a dead host are swept by age."""
def _run_as(self, session_key, code, task_id, **kwargs):
from tools.approval_context import reset_current_session_key, set_current_session_key
token = set_current_session_key(session_key)
try:
return json.loads(execute_code(code, task_id=task_id, **kwargs))
finally:
reset_current_session_key(token)
def test_reap_once_sweeps_idle_kernels_without_a_new_acquire(self):
import time as time_module
from tools.code_kernel import _reap_once
with _kernel_config(kernel_idle_timeout=1):
self._run_as("conv-a", "x = 41", task_id="turn-1")
stale = next(iter(_KERNELS.values()))
time_module.sleep(1.2)
# No conv-b acquire here: the reaper pass alone must retire the kernel.
_reap_once()
self.assertNotIn(stale.key, _KERNELS)
stale.proc.wait(timeout=10)
self.assertFalse(stale.alive())
def test_reap_once_spares_attached_and_fresh_kernels(self):
from tools.code_kernel import _reap_once
with _kernel_config(kernel_idle_timeout=1):
fresh = self._run_as("conv-fresh", "x = 1", task_id="turn-1")
self.assertEqual(fresh["status"], "success", fresh)
kernel = next(iter(_KERNELS.values()))
kernel.attached += 1 # a cell is mid-flight: reaping must skip it
try:
_reap_once()
self.assertIn(kernel.key, _KERNELS)
self.assertTrue(kernel.alive())
finally:
kernel.attached -= 1
class TestStaleStagingDirSweep(unittest.TestCase):
def test_week_old_kernel_dirs_go_and_fresh_ones_stay(self):
import time as time_module
from tools.code_kernel import _sweep_stale_staging_dirs
with tempfile.TemporaryDirectory() as tmp:
with patch("tools.code_kernel.tempfile.gettempdir", return_value=tmp):
old = Path(tmp, "hermes_kernel_old")
young = Path(tmp, "hermes_kernel_young")
bystander = Path(tmp, "unrelated_dir")
for path in (old, young, bystander):
path.mkdir()
week_and_a_bit = time_module.time() - 8 * 86400
os.utime(old, (week_and_a_bit, week_and_a_bit))
removed = _sweep_stale_staging_dirs()
# Asserted inside the TemporaryDirectory: cleanup would flatten everything.
self.assertEqual(removed, 1)
self.assertFalse(old.exists())
self.assertTrue(young.exists())
self.assertTrue(bystander.exists())
class TestCellStdoutSpill(unittest.TestCase):
"""The runner-side full-stdout spill is a second emitter of cell output: it must
carry the same secret redaction the inline result gets, and it must never write
through a name the cell could have replanted as a symlink."""
def test_cell_spill_on_disk_is_sanitized(self):
import stat as stat_mod
from agent.redact import redact_sensitive_text
from hermes_constants import get_hermes_dir
secret = "ghp_" + "0123456789abcdef"
masked = redact_sensitive_text(secret, code_file=True)
with _kernel_config():
probe = _run("import os\nprint(os.environ['HERMES_KERNEL_SPILL_DIR'])")
kernel_tmp = probe["output"].strip()
# The secret lands PAST the runner's 1MB clip: the host-side spill of the
# clipped head cannot contain it, so a masked remnant in the published file
# proves the runner spill (not the fallback) is what was published.
result = _run("print('x' * 1_050_000)\n"
"print('\\x1b[31m" + secret + "\\x1b[0m')")
self.assertEqual(result["status"], "success", result)
self.assertNotIn(secret, result["output"]) # inline copy is masked
spill = result.get("stdout_spill_path", "")
self.assertTrue(spill, result)
# Published under the host spill dir, not the cell-writable kernel tmpdir:
# survives kernel teardown and post-publish mutation by later cells.
self.assertEqual(os.path.dirname(spill),
str(get_hermes_dir("cache/exec", "exec_spill")))
self.assertIn("FULL output saved", result.get("warning", ""))
# The raw tmpdir copy is dropped once republished.
self.assertFalse(os.path.exists(
os.path.join(kernel_tmp, "cell_000002_stdout.txt")))
body = Path(spill).read_text(encoding="utf-8")
self.assertNotIn(secret, body)
self.assertIn(masked, body) # masked remnant proves the pipeline ran
self.assertNotIn("\x1b[", body) # ANSI-stripped like the inline copy
self.assertGreater(len(body), 1_000_000) # full spill, not the 1MB clip
if os.name == "posix":
self.assertEqual(stat_mod.S_IMODE(os.stat(spill).st_mode), 0o600)
def test_cell_spill_keeps_full_non_ascii_output(self):
"""The runner caps its spill in characters written as UTF-8, so a multibyte
spill under that cap is several times larger on disk; it is still our spill
and must be published in full, not dropped to the clipped-head fallback."""
with _kernel_config():
result = _run("print('\\u6f22' * 2_000_000)") # ~6MB of UTF-8 on disk
self.assertEqual(result["status"], "success", result)
self.assertIn("FULL output saved", result.get("warning", ""))
body = Path(result["stdout_spill_path"]).read_text(encoding="utf-8")
self.assertEqual(body.count("\u6f22"), 2_000_000)
def test_cell_spill_path_survives_sys_exit_teardown(self):
"""A cell that spills and then sys.exit()s kills its kernel, which rmtree()s the
tmpdir the runner wrote to. The advertised spill path must still resolve."""
import stat as stat_mod
with _kernel_config():
result = _run("print('y' * 1_100_000)\nimport sys\nsys.exit(0)")
# sys.exit() ends the kernel: tmpdir is rmtree'd before the caller pages the
# spill, so the published path must live outside it.
self.assertTrue(result["kernel"]["ended"], result)
spill = result.get("stdout_spill_path", "")
self.assertTrue(spill, result)
self.assertTrue(os.path.isfile(spill))
self.assertIn("y" * 100, Path(spill).read_text(encoding="utf-8"))
if os.name == "posix":
self.assertEqual(stat_mod.S_IMODE(os.stat(spill).st_mode), 0o600)
@pytest.mark.platforms("posix")
def test_cell_spill_refuses_preplanted_symlink(self):
"""A cell sharing the kernel tmpdir can squat cell_NNNNNN_stdout.txt with a
symlink; the runner's spill write must not follow it onto a host file."""
with tempfile.TemporaryDirectory() as tmp:
target = Path(tmp, "victim.txt")
target.write_text("keep me", encoding="utf-8")
with _kernel_config():
result = _run(
"import os\n"
"d = os.environ['HERMES_KERNEL_SPILL_DIR']\n"
"os.symlink(" + repr(str(target)) + ", os.path.join(d, 'cell_000001_stdout.txt'))\n"
"print('x' * 1_100_000)")
self.assertEqual(result["status"], "success", result)
self.assertEqual(target.read_text(encoding="utf-8"), "keep me")
spill = result.get("stdout_spill_path", "")
if spill:
# If a spill was still produced it must be a real file, not the link.
self.assertFalse(os.path.islink(spill))
def test_sanitize_cell_spill_drops_paths_outside_tmpdir(self):
"""A spill path outside the kernel tmpdir is never read or rewritten."""
from types import SimpleNamespace
from tools.code_kernel import _sanitize_cell_spill
with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b:
foreign = Path(b, "cell_000001_stdout.txt")
foreign.write_text("raw ghp_0123456789abcdef", encoding="utf-8")
kernel = SimpleNamespace(tmpdir=a)
self.assertEqual(_sanitize_cell_spill(kernel, str(foreign)), "")
self.assertIn("ghp_0123456789abcdef",
foreign.read_text(encoding="utf-8"))
@pytest.mark.platforms("posix")
def test_sanitize_cell_spill_refuses_replanted_symlink(self):
"""A link swapped in after the runner wrote must not be read through."""
from types import SimpleNamespace
from tools.code_kernel import _sanitize_cell_spill
with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b:
foreign = Path(b, "victim.txt")
foreign.write_text("sentinel", encoding="utf-8")
link = Path(a, "cell_000001_stdout.txt")
os.symlink(foreign, link)
kernel = SimpleNamespace(tmpdir=a)
self.assertEqual(_sanitize_cell_spill(kernel, str(link)), "")
self.assertEqual(foreign.read_text(encoding="utf-8"), "sentinel")
@pytest.mark.platforms("posix")
def test_sanitize_cell_spill_refuses_fifo_and_dotdot(self):
"""A cell can forge or swap the published name; a FIFO there would park the
host thread at open() (under kernel.lock), and a 'tmpdir/link/../victim'
spelling escapes the lexical parent check through a planted intermediate
link. Both must fail closed without touching the foreign file."""
import threading
from types import SimpleNamespace
from tools.code_kernel import _sanitize_cell_spill
with tempfile.TemporaryDirectory() as outer:
a = os.path.join(outer, "kertmp") # kernel tmpdir
b = os.path.join(outer, "b") # symlink target dir
os.mkdir(a)
os.mkdir(b)
kernel = SimpleNamespace(tmpdir=a)
fifo = Path(a, "cell_000001_stdout.txt")
os.mkfifo(fifo)
done = []
t = threading.Thread(
target=lambda: done.append(_sanitize_cell_spill(kernel, str(fifo))))
t.start()
t.join(timeout=15)
self.assertEqual(done, [""]) # returned, did not block on the FIFO
self.assertFalse(fifo.exists()) # the name is cleaned up either way
# a/link/../victim.txt resolves through the planted link to
# outer/victim.txt: a lexical abspath() parent check sees 'a', the
# filesystem sees outer/. The read and the cleanup unlink must both
# stay confined.
victim = Path(outer, "victim.txt")
victim.write_text("sentinel", encoding="utf-8")
os.symlink(b, os.path.join(a, "link"))
spell = os.path.join(a, "link", "..", "victim.txt")
self.assertEqual(_sanitize_cell_spill(kernel, spell), "")
self.assertEqual(victim.read_text(encoding="utf-8"), "sentinel")
def test_sanitize_cell_spill_refuses_oversized_swap(self):
"""A swapped-in file bigger than the runner's spill cap is not a spill."""
from types import SimpleNamespace
from tools.code_kernel import _CELL_SPILL_READ_CAP, _sanitize_cell_spill
with tempfile.TemporaryDirectory() as a:
big = Path(a, "cell_000001_stdout.txt")
with open(big, "wb") as f:
f.truncate(_CELL_SPILL_READ_CAP + 1)
kernel = SimpleNamespace(tmpdir=a)
self.assertEqual(_sanitize_cell_spill(kernel, str(big)), "")
self.assertFalse(big.exists())
def test_cell_result_failclosed_keeps_partial_host_spill(self):
"""When the cell spill cannot be republished, the result must not leave a
spill path the warning disclaims: the host spill of the clipped head is
sanitized and pageable, so the warning names it as the partial file."""
from types import SimpleNamespace
from tools.code_kernel import _BoundedBuffer, _cell_result
with tempfile.TemporaryDirectory() as a, tempfile.TemporaryDirectory() as b:
foreign = Path(a, "cell_000001_stdout.txt")
foreign.write_text("raw ghp_0123456789abcdef", encoding="utf-8")
kernel = SimpleNamespace(tmpdir=b, raw=_BoundedBuffer(),
stderr=_BoundedBuffer(), execution_count=1,
tool_call_counter=[0], tool_call_log=[],
cell_log_start=0)
payload = {"status": "ok", "stdout": "x" * 60_000,
"stdout_clipped": True,
"stdout_spill_path": str(foreign), "execution_count": 1}
result = _cell_result(kernel, ("k",), "success", payload, timeout=30,
sandbox_tools=frozenset(), reused=False,
state_reset=False, exec_start=time.monotonic())
self.assertTrue(foreign.exists()) # foreign file never touched
self.assertIn("stdout_spill_path", result) # host spill of the clipped head
self.assertIn("Only the first part", result["warning"])
self.assertIn(result["stdout_spill_path"], result["warning"])