Files
Teknium 065f5abbb3 fix(terminal): parse cwd markers without publishing them; keep the fallback notice out of truncation
Background polls run from '/' on the shared environment: _update_cwd wrote
self.cwd from their marker and execute() restored it afterwards, so a
concurrent file op could resolve a relative path against '/' (and the restore
could clobber a concurrent cd). Marker parsing now only fills the result;
self.cwd is assigned only when the command may move the session.

The cwd-fallback line leads the stream, so a small tool_output.max_bytes or a
timeout could cut or drop it. The output collector now strips it before the
head/tail window applies.
2026-11-17 10:29:07 -08:00

904 lines
46 KiB
Python

"""Base class for all Hermes execution environment backends.
Unified spawn-per-call model: every command spawns a fresh ``bash -c`` process.
A session snapshot (env vars, functions, aliases) is captured once at init and
re-sourced before each command. CWD persists via in-band stdout markers (remote)
or a temp file (local). Cohesive pieces live in sibling modules (``base_output``,
``base_session_env``, ``base_wait``, ``path_utils``).
"""
import contextvars
import hashlib
import logging
import os
import shlex
import subprocess
import threading
import time
import uuid
from abc import ABC, abstractmethod
from contextlib import contextmanager
from pathlib import Path
from typing import Callable, Iterable
from hermes_constants import get_hermes_home
from tools.interrupt import consume_yield, is_interrupted, is_thread_interrupted
from tools.environments.base_output import (
ProcessHandle, _finalize_wait_result, _new_output_collector, _start_drain_thread,
)
from tools.environments.base_session_env import (
_SHELL_ENV_NAME_RE, _SNAP_TMP_SUFFIX, _cwd_fallback_marker, _cwd_marker, _pipe_marker, _snapshot_bootstrap_script, _split_cwd_marker,
_wrap_command_script,
)
from tools.environments.base_processes import LongLivedProcessMixin
from tools.environments.base_wait import _WaitTrace
from tools.terminal_tool_sudo_askpass import fill_timeout_token, sudo_askpass_supported_once
from utils import atomic_json_write, env_var_enabled, file_signature, read_json_or_empty
logger = logging.getLogger(__name__)
# Opt-in debug tracing for the interrupt/activity/poll machinery
# (HERMES_DEBUG_INTERRUPT=1). Off by default to avoid flooding gateway logs.
_DEBUG_INTERRUPT = env_var_enabled("HERMES_DEBUG_INTERRUPT")
# Extra seconds the ``run_bounded_sync`` backstop waits past the inner ``_wait_for_process``
# deadline: the inner loop returns partial output + 124; the outer bound only fires when that
# loop never returns. Keep small so a healthy timeout still comes from the inner path.
# The inner poll loop is what returns partial output + returncode 124; this outer bound only exists for when
# that loop itself never returns (family A of #94285: a blocked wait that silently disables asyncio timers).
_EXECUTE_WAIT_BOUND_GRACE_S = 2.0
if _DEBUG_INTERRUPT:
# quiet_mode forces the `tools` logger to ERROR on CLI startup, which would
# swallow every trace; force this logger back to INFO in the opt-in case.
logger.setLevel(logging.INFO)
# Thread-local activity callback: the agent sets it before a tool call so
# long-running _wait_for_process loops can report liveness to the gateway.
_activity_callback_local = threading.local()
# Foreground commands in flight in THIS process, across every environment. Each runs in its
# own session/process group, so a host that exits mid-command (TUI client gone, SIGTERM) would
# orphan the whole tree; the process-exit funnel ``cleanup_all_environments`` kills them.
_live_foreground: "dict[int, tuple[BaseEnvironment, ProcessHandle]]" = {}
# Reentrant, and the hard-exit path only ever takes it with a timeout: a signal handler can run
# on a thread that already holds it.
_live_foreground_cond = threading.Condition(threading.RLock())
_exit_fenced = False # one-way, set by the hard-exit kill: no foreground command spawns after it
_spawns_in_flight = 0 # past the fence check, child maybe alive, not yet in _live_foreground
_HARD_KILL_BUDGET_S = 0.5
# Task ids whose shell state a command reads and writes, own id first, then its delegation
# ancestors; empty means the environment's own snapshot. One environment serves a parent and
# its delegate_task children (shared sandbox, #16177), so without this a child's ``export``
# became the parent's shell state (#84536).
_shell_state_chain: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar(
"terminal_shell_state_chain", default=())
@contextmanager
def shell_state_scope(chain: tuple[str, ...]):
"""Run commands against the per-task snapshot of ``chain[0]`` (seeded from its ancestors)."""
token = _shell_state_chain.set(tuple(chain))
try:
yield
finally:
_shell_state_chain.reset(token)
def _enter_foreground_spawn() -> bool:
global _spawns_in_flight
with _live_foreground_cond:
if _exit_fenced:
return False
_spawns_in_flight += 1
return True
def _leave_foreground_spawn(env: "BaseEnvironment", spawned) -> bool:
"""Publish ``spawned`` (None: the spawn failed); True when the exit fence went up meanwhile."""
global _spawns_in_flight
with _live_foreground_cond:
_spawns_in_flight -= 1
if spawned is not None:
_live_foreground[id(spawned)] = (env, spawned)
_live_foreground_cond.notify_all()
return _exit_fenced
def _quiet_kill(kill: Callable, proc) -> None:
try:
kill(proc)
except Exception:
logger.debug("exit-time kill of a foreground command failed", exc_info=True)
def kill_live_foreground_processes(*, now: bool = False) -> int:
"""Kill every in-flight foreground command's process tree; returns how many were signalled.
``now=True`` is for a caller about to ``os._exit``: the graceful kill TERMs, waits and only then
KILLs, so a SIGTERM-ignoring command outlives a hard exit that lands inside that window. It also
raises the exit fence and waits for spawns already past it to register, so no command started
around the snapshot survives, and it never blocks past ``_HARD_KILL_BUDGET_S``: SDK cancels
(Modal, Daytona, Vercel) run on daemon threads under that one deadline."""
global _exit_fenced
if not now:
with _live_foreground_cond:
live = list(_live_foreground.values())
for env, proc in live:
_quiet_kill(env._kill_process, proc)
return len(live)
deadline = time.monotonic() + _HARD_KILL_BUDGET_S
_exit_fenced = True
if _live_foreground_cond.acquire(timeout=_HARD_KILL_BUDGET_S):
try:
_live_foreground_cond.wait_for(lambda: _spawns_in_flight == 0, max(0.0, deadline - time.monotonic()))
live = list(_live_foreground.values())
finally:
_live_foreground_cond.release()
else: # the holder is stuck under our signal: a lock-free copy beats hanging the exit
live = list(_live_foreground.values())
remote = []
for env, proc in live:
if isinstance(proc, subprocess.Popen): # killpg/kill: never blocks
_quiet_kill(env._force_kill_process, proc)
else:
remote.append(threading.Thread(target=_quiet_kill, args=(env._force_kill_process, proc), daemon=True))
remote[-1].start()
for t in remote:
t.join(max(0.0, deadline - time.monotonic()))
return len(live)
class FileFetchError(RuntimeError):
"""A file could not be extracted from the backend filesystem."""
# Pulling a large artifact over a slow exec channel can outlast the per-command terminal timeout.
_FETCH_TIMEOUT_SECONDS = 300
class EnvironmentConnectionError(RuntimeError):
"""Infrastructure/connection-class failure of a terminal backend (SSH host down, Docker
daemon not running, remote sync on a dead link) — never a command that merely exited
nonzero. Subclassing RuntimeError keeps every ``except RuntimeError`` catcher working.
``terminal_tool`` turns this into a structured ``status: "degraded"`` result; the failed
backend is never cached, so a later call retries from scratch."""
def __init__(self, reason: str, *, retry_hint: str = ""):
super().__init__(reason)
self.reason = reason
self.retry_hint = retry_hint or (
"This is an infrastructure failure, not a command failure. "
"Verify the backend is reachable (network, service running, "
"credentials), then retry the same command — recovery is "
"automatic once the backend is back.")
def set_activity_callback(cb: Callable[[str], None] | None) -> None:
"""Register a callback that _wait_for_process fires periodically."""
_activity_callback_local.callback = cb
def get_activity_callback() -> Callable[[str], None] | None:
"""Thread-local activity callback; capture it before handing work to another thread.
Public accessor for callers outside this module that need to capture the calling thread's callback
before handing work to another thread (the callback is thread-local, so a freshly spawned thread cannot
read it back) — e.g. the manual cron-run heartbeat (#76502).
"""
return getattr(_activity_callback_local, "callback", None)
def touch_activity_if_due(state: dict, label: str) -> None:
"""Fire the activity callback at most once every ``state['interval']`` (default 10 s).
*state* holds ``last_touch``/``start`` monotonic timestamps. Swallows all exceptions."""
now = time.monotonic()
if now - state["last_touch"] < state.get("interval", 10.0):
return
state["last_touch"] = now
try:
cb = get_activity_callback()
if cb:
cb(f"{label} ({int(now - state['start'])}s elapsed)")
except Exception:
logger.debug("activity callback failed during a long-running command", exc_info=True)
def get_sandbox_dir() -> Path:
"""Host-side root for all sandbox storage (Docker workspaces, plugin backends'
host-side state). ``TERMINAL_SANDBOX_DIR`` overrides ``{HERMES_HOME}/sandboxes``."""
custom = os.getenv("TERMINAL_SANDBOX_DIR")
p = Path(custom) if custom else get_hermes_home() / "sandboxes"
p.mkdir(parents=True, exist_ok=True)
return p
def _save_json_store(path: Path, data: dict) -> None:
"""Write *data* as pretty-printed JSON to *path* atomically (readers never see a torn file)."""
path.parent.mkdir(parents=True, exist_ok=True)
atomic_json_write(path, data, indent=2)
_JSON_STORE_LOCK_HOLDER = threading.local()
def _update_json_store(path: Path, update: Callable[[dict], None]) -> None:
"""Read-modify-write the JSON store at *path* under a cross-process lock.
Concurrent sandbox cleanups (delegate children, gateway sessions, the idle reaper) each
record a snapshot id; an unlocked load/mutate/save lets one cleanup drop the other's entry.
An unusable lock (no flock on the filesystem, a wedged holder) degrades to an unlocked
update: a sandbox lifecycle step (modal's restore fallback included) must not abort on it."""
from contextlib import ExitStack
from hermes_cli.auth import _file_lock
lock_path = path.with_name(f"{path.name}.lock")
with ExitStack() as stack:
try:
stack.enter_context(_file_lock(
lock_path, _JSON_STORE_LOCK_HOLDER, 10.0, f"Timed out waiting for the snapshot store lock ({path})",
))
except (OSError, TimeoutError):
logger.warning("Updating %s without its lock", path, exc_info=True)
data = read_json_or_empty(path)
before = dict(data)
update(data)
if data != before:
_save_json_store(path, data)
def _file_mtime_key(host_path: str) -> tuple[int, int, int, int] | None:
"""Return the ``file_signature`` change key, or ``None`` if unreadable.
mtime + size alone miss a same-size credential refresh landing in the same mtime tick;
inode and ctime catch the atomic replace / rewrite."""
try:
return file_signature(Path(host_path).stat())
except OSError:
return None
def env_uses_host_filesystem(env: object) -> bool:
"""Whether the file tools may touch *env*'s files on the host (``BaseEnvironment.uses_host_filesystem``).
Strict ``is True``: an environment without the declaration (``None``, a duck-typed plugin
backend, a test double) is not the host."""
return getattr(env, "uses_host_filesystem", False) is True
# ---------------------------------------------------------------------------
# BaseEnvironment
# ---------------------------------------------------------------------------
class BaseEnvironment(LongLivedProcessMixin, ABC):
"""Common interface and unified execution flow for all Hermes backends. Subclasses
implement ``_run_bash()`` and ``cleanup()``; the base provides ``execute()`` with
snapshot sourcing, CWD tracking, interrupt handling and timeout enforcement, and the
long-lived process surface (``start_process`` / ``attach_process`` / ``stop_process``,
``base_processes``)."""
# How execute() hands stdin to _run_bash: "pipe" (process stdin, default),
# "payload" (passed through as stdin_data; the backend transports it via its
# SDK -- Modal/managed Modal stream it, Daytona/Vercel stage a file) or
# "heredoc" (embedded in the command; no built-in backend, plugins only).
_stdin_mode: str = "pipe" # "pipe" | "payload" | "heredoc"
# True only when commands execute on the SAME host as the Hermes process
# (LocalEnvironment); controller-host facts then describe the execution target.
is_local: bool = False
# True when the file tools may read, write, patch and search this backend's files straight on
# the host instead of through execute() (only LocalEnvironment). A host backend that confines its
# commands (a sandbox hiding paths, mounting them read-only or giving them their own $TMPDIR) sets
# it False, so the file tools see exactly what its commands see. Read via env_uses_host_filesystem().
uses_host_filesystem: bool = False
# Snapshot creation timeout (override for slow cold-starts).
_snapshot_timeout: int = 30
# Re-dump exported env into the snapshot after each command (see __init__).
_persist_env: bool = True
# Opt in only when a timed-out probe can kill its command without tearing
# down the whole backend. SDK adapters cancel by stopping the sandbox. Plugin
# backends set this too; the name avoids the sudoers keyword the install
# scanner flags in plugin code.
_passwordless_sudo_probe_supported: bool = False
# Whether background processes spawned through this env are gone, for this env object, once
# cleanup() returns: ended with the sandbox, or no longer reachable through it (persist-mode
# Docker drops its container id, so the old object can't poll the job). Teardown then
# settles them as lost so their pollers stop. A backend whose processes outlive cleanup()
# and stay reachable through the same object (SSH reconnects) sets False so the poller keeps
# collecting the real result. Plugin contract: website/docs/developer-guide/terminal-environment-plugin.md.
cleanup_stops_processes: bool = True
# Local and Docker override this because they resolve allowlisted values
# through the active profile scope; other backends keep plain snapshots.
_profile_scoped_passthrough: bool = False
# Per-session PIPESTATUS marker set in __init__; empty means the wrapper emits none.
_pipe_marker: str = ""
# Per-session marker the wrapper prints after falling back from a dead cwd; empty = refuse.
_cwd_fallback_marker: str = ""
def get_temp_dir(self) -> str:
"""Return the backend temp directory used for session artifacts.
Most sandboxed backends use ``/tmp`` inside the target environment.
LocalEnvironment overrides this on hosts where ``/tmp`` may be missing
and ``TMPDIR`` is the portable writable location.
"""
return "/tmp" # no-tmp: ok — sandbox-side (remote container) temp dir, not the host
def __init__(self, cwd: str, timeout: int, env: dict | None = None, persist_env: bool = True):
self.cwd = cwd
self.timeout = timeout
self.env = env or {}
# False: each command sources only the login-time snapshot, so exported vars do not
# carry over (terminal.persistent_shell / TERMINAL_LOCAL_PERSISTENT). CWD still does.
self._persist_env = persist_env
self._session_id = uuid.uuid4().hex[:12]
temp_dir = self.get_temp_dir().rstrip("/") or "/"
self._snapshot_path = f"{temp_dir}/hermes-snap-{self._session_id}.sh"
self._cwd_file = f"{temp_dir}/hermes-cwd-{self._session_id}.txt"
self._cwd_marker = _cwd_marker(self._session_id)
self._pipe_marker = _pipe_marker(self._session_id)
self._cwd_fallback_marker = _cwd_fallback_marker(self._session_id)
self._snapshot_ready = False
self._snapshot_passthrough_names: set[str] = set()
# True when login bash is unusable (e.g. broken Git-for-Windows startup)
# so execute() must fall back to non-login ``bash -c``, not ``bash -l``.
self._prefer_nonlogin = False
# --- Abstract methods ---
def _run_bash(
self, cmd_string: str, *, login: bool = False, timeout: int = 120, stdin_data: str | None = None,
) -> ProcessHandle:
"""Spawn a bash process to run *cmd_string*; every backend overrides this."""
raise NotImplementedError(f"{type(self).__name__} must implement _run_bash()")
@abstractmethod
def cleanup(self):
"""Release backend resources (container, instance, connection). Raise only while the
sandbox may still be live: once it is destroyed, log later failures instead, since a
raise skips settling its background processes (see ``cleanup_stops_processes``)."""
...
# --- File extraction (backend -> host) ---
def fetch_file(self, remote_path: str, local_dest: Path, *, max_bytes: int) -> None:
"""Copy a regular file out of the backend filesystem to ``local_dest`` on the host.
One transport for every backend: base64 over the exec channel, bounded INSIDE the sandbox
(``head -c max+1`` so /dev/zero can't stream unbounded data into host memory — the same
shape ``tools.image_source`` uses to read sandbox images). The payload is fenced between
unique markers so login-shell noise in the merged stdout/stderr can't corrupt the decode.
Raises :class:`FileFetchError` on a missing/unreadable/oversized file.
"""
import base64
import binascii
marker = f"__HERMES_FETCH_{uuid.uuid4().hex[:12]}__"
quoted = shlex.quote(remote_path)
# ``[ -f ]`` follows symlinks, so a link to a denied host file is judged by the CALLER on
# ``readlink -f`` output before any bytes move.
result = self.execute(
f"[ -f {quoted} ] && echo {marker} && head -c {max_bytes + 1} < {quoted} | base64 && echo {marker}",
timeout=_FETCH_TIMEOUT_SECONDS, rewrite_compound_background=False)
output = result.get("output") or ""
first, last = output.find(marker), output.rfind(marker)
if int(result.get("returncode") or 0) != 0 or first == -1 or last <= first:
raise FileFetchError(f"could not read {remote_path!r} in the sandbox (missing, not a regular file, or unreadable)")
try:
data = base64.b64decode("".join(output[first + len(marker):last].split()), validate=True)
except (ValueError, binascii.Error) as exc:
raise FileFetchError(f"transfer of {remote_path!r} was corrupted in transit: {exc}") from exc
if len(data) > max_bytes:
raise FileFetchError(f"{remote_path!r} exceeds the {max_bytes // (1024 * 1024)} MB delivery limit")
Path(local_dest).write_bytes(data)
def fetch_realpath(self, remote_path: str) -> str | None:
"""``readlink -f`` inside the backend, or None when it cannot be resolved."""
result = self.execute(f"readlink -f {shlex.quote(remote_path)} 2>/dev/null", rewrite_compound_background=False)
if int(result.get("returncode") or 0) != 0:
return None
return next((ln.strip() for ln in reversed((result.get("output") or "").splitlines()) if ln.strip().startswith("/")), None)
# --- Session snapshot (init_session) ---
def _additional_profile_scoped_passthrough_names(self) -> Iterable[str]:
"""Return backend-specific names that must not persist in snapshots."""
return ()
def _snapshot_excluded_passthrough_names(self) -> tuple[str, ...]:
"""Profile-scoped names that must not persist in the snapshot. Monotonic for the
environment lifetime: an allowlist can be cleared after a value was captured, and
retaining the exclusion keeps that old value from leaking to a later profile."""
if not self._profile_scoped_passthrough:
return ()
try:
from agent.secret_scope import is_multiplex_active
if is_multiplex_active():
from tools.env_passthrough import get_all_passthrough
names = (*get_all_passthrough(), *self._additional_profile_scoped_passthrough_names())
self._snapshot_passthrough_names.update(
name for name in names if isinstance(name, str) and _SHELL_ENV_NAME_RE.fullmatch(name))
except Exception:
logger.debug("Could not refresh profile-scoped snapshot exclusions", exc_info=True)
return tuple(sorted(self._snapshot_passthrough_names))
def _task_snapshot_path(self, task_id: str) -> str:
"""Per-task snapshot beside the environment's own (``cleanup`` globs the prefix)."""
digest = hashlib.sha256(task_id.encode()).hexdigest()[:16]
return f"{self._snapshot_path[:-len('.sh')]}-{digest}.sh"
def _snapshot_script_kwargs(self, cwd: str, snap: str | None = None) -> dict:
"""Quoting inputs shared by the bootstrap and per-command wrapper scripts.
``_quote_cwd_for_cd`` / ``_quote_shell_path`` (not bare shlex.quote) let the Windows
subclass rewrite ``C:\\...`` to ``/c/...`` so ``cd`` resolves and MSYS doesn't choke."""
return dict(
quoted_cwd=self._quote_cwd_for_cd(cwd),
quoted_snap=self._quote_shell_path(snap or self._snapshot_path),
snap_tmp_template=self._quote_shell_path((snap or self._snapshot_path) + _SNAP_TMP_SUFFIX),
cwd_marker=self._cwd_marker)
def init_session(self):
"""Capture the login shell environment into the snapshot file (once, after construction).
On success ``_snapshot_ready`` is set so commands source the snapshot instead of running
under ``bash -l``. On failure, fall back to ``bash -l`` per command — unless a non-login
probe shows login bash itself is dead, in which case prefer ``bash -c``."""
bootstrap = _snapshot_bootstrap_script(
excluded_names=self._snapshot_excluded_passthrough_names(), **self._snapshot_script_kwargs(self.cwd))
try:
proc = self._run_bash(bootstrap, login=True, timeout=self._snapshot_timeout)
result = self._wait_for_process(proc, timeout=self._snapshot_timeout)
if int(result.get("returncode") or 0) != 0:
raise RuntimeError(f"snapshot bootstrap failed with exit code {result.get('returncode')}")
self._snapshot_ready = True
self._update_cwd(result)
logger.info("Session snapshot created (session=%s, cwd=%s)", self._session_id, self.cwd)
except Exception as exc:
self._snapshot_ready = False
self._prefer_nonlogin, detail = self._probe_nonlogin_fallback(str(exc))
if self._prefer_nonlogin:
logger.warning(
"init_session failed (session=%s): %s — "
"login bash unusable; falling back to non-login bash -c",
self._session_id, exc)
else:
logger.warning(
"init_session failed (session=%s): %s — falling back to bash -l per command",
self._session_id, detail)
def _probe_nonlogin_fallback(self, detail: str) -> tuple[bool, str]:
"""Run ``true`` under non-login bash; return ``(prefer_nonlogin, detail)``."""
probe_timeout = min(15, self._snapshot_timeout)
try:
probe = self._run_bash("true", login=False, timeout=probe_timeout)
probe_result = self._wait_for_process(probe, timeout=probe_timeout)
prefer_nonlogin = int(probe_result.get("returncode") or 0) == 0
if not prefer_nonlogin:
detail = (probe_result.get("stdout") or detail).strip() or detail
return prefer_nonlogin, detail
except Exception as probe_exc:
return False, f"{detail}; non-login probe: {probe_exc}"
# --- Command wrapping ---
@staticmethod
def _quote_cwd_for_cd(cwd: str) -> str:
"""Quote a ``cd`` target while preserving ``~`` expansion (``~/...``
goes through ``$HOME`` so suffixes with spaces stay one word)."""
if cwd == "~":
return cwd
if cwd == "~/":
return "$HOME"
if cwd.startswith("~/"):
return f"$HOME/{shlex.quote(cwd[2:])}"
return shlex.quote(cwd)
def _quote_shell_path(self, path: str) -> str:
"""Quote *path* for a bash script. LocalEnvironment overrides this to
rewrite native/mixed Windows paths to ``/c/...``; remote backends are POSIX."""
return shlex.quote(path)
def _wrap_command(self, command: str, cwd: str, *, cwd_fallback: bool = False) -> str:
"""Full bash script: source snapshot, cd, run, re-dump env, emit CWD markers.
``cwd_fallback`` lets the ``cd`` walk up from a missing *cwd* (see ``execute``)."""
chain = [self._task_snapshot_path(t) for t in _shell_state_chain.get()]
seeds = [self._quote_shell_path(p) for p in (*chain[1:], self._snapshot_path)] if chain else []
return _wrap_command_script(
command,
passthrough_names=self._snapshot_excluded_passthrough_names(),
snapshot_ready=self._snapshot_ready,
persist_env=self._persist_env,
pipe_marker=self._pipe_marker,
seed_snaps=seeds,
cwd_fallback_marker=self._cwd_fallback_marker if cwd_fallback else "",
**self._snapshot_script_kwargs(cwd, chain[0] if chain else None))
@staticmethod
def _embed_stdin_heredoc(command: str, stdin_data: str) -> str:
"""Redirect stdin_data to the complete command with a shell heredoc (SDK backends).
A heredoc body always ends in a newline stdin_data may lack, and write_file verifies a
byte-exact hash, so a process substitution re-emits the body minus that last character.
Redirections apply left to right, so the substitution inherits the heredoc as its stdin;
the heredoc stays outside ``<( )`` because bash 3.2 mis-parses bodies inside it. ``|| :``
keeps an inherited ``set -e`` from killing the reader on read's EOF status."""
delimiter = f"HERMES_STDIN_{uuid.uuid4().hex[:12]}"
return (f"{{\n{command}\n}} << '{delimiter}' < <(IFS= read -r -d '' s || :; printf '%s' \"${{s%?}}\")\n"
f"{stdin_data}\n{delimiter}")
def _staged_stdin_path(self) -> str:
"""Unique sandbox path for staging a payload-mode stdin file."""
temp_dir = self.get_temp_dir().rstrip("/") or "/"
return f"{temp_dir}/.hermes-stdin-{uuid.uuid4().hex}"
@staticmethod
def _redirect_stdin_from_file(command: str, path: str) -> str:
"""Prefix ``command`` so the shell reads stdin from the staged ``path`` and
unlinks it before ``command`` runs (the shell then owns the payload)."""
quoted = shlex.quote(path)
return f"exec 0< {quoted} || exit $?\nrm -f -- {quoted} || exit $?\n{command}"
# --- Process lifecycle ---
def _wait_for_process(
self, proc: ProcessHandle, timeout: int = 120, *,
bounded_capture: bool = False, watch_interrupt_tid: int | None = None,
output=None, yield_handler: Callable[[ProcessHandle, str], dict] | None = None) -> dict:
"""Poll-based wait with interrupt checking and stdout draining (shared, not overridden).
``yield_handler(proc, output_so_far)``: when the tool thread is asked to yield
(``tools.interrupt.request_yield`` — a user message arrived mid-command), the drain
thread is stopped, the still-running process is handed to the handler and its dict
is returned as the result; the process is NOT killed.
``bounded_capture=True`` (foreground terminal-tool path only) retains at most
``tool_output.max_bytes`` in a head/tail window so a verbose subprocess cannot OOM the
process; the default keeps full fidelity for internal consumers. Fires the activity
callback every 10s so the gateway's inactivity timeout doesn't kill long commands.
``watch_interrupt_tid`` is the tool-worker thread that submitted this wait: ``execute()``
may move the wait onto a ``run_bounded_sync`` worker while ``/stop`` still interrupts the
original tid, so both bits are honored. ``KeyboardInterrupt``/``SystemExit`` mid-poll
kills the process first — the local backend spawns into its own process group, so an
unkilled child would be orphaned.
The default (False) preserves full-fidelity capture for internal consumers — file-operation ``cat``
reads feeding the patch engine, code-execution RPC reads, log reads — where truncation would corrupt
data. See #64435.
"""
if output is None:
output = _new_output_collector(proc, bounded_capture)
drain_stop = threading.Event() if yield_handler is not None else None
drain_thread = _start_drain_thread(proc, output, drain_stop)
_now = time.monotonic()
deadline = _now + timeout
_activity_state = {"last_touch": _now, "start": _now}
trace = _WaitTrace(proc, timeout, enabled=_DEBUG_INTERRUPT, logger=logger)
trace.enter()
def _kill_and_join():
self._kill_process(proc)
drain_thread.join(timeout=2)
try:
# Adaptive poll: start at 5ms so fast commands return in ~6ms, back
# off exponentially toward 200ms so long builds don't pay poll CPU.
_poll_sleep = 0.005
while proc.poll() is None:
trace.iterations += 1
if is_interrupted() or is_thread_interrupted(watch_interrupt_tid):
trace.interrupted()
_kill_and_join()
return {**self._finalize_wait_result(output, output.render(suffix="\n[Command interrupted]"), 130),
"hermes_interrupted": True}
if yield_handler is not None and consume_yield(watch_interrupt_tid):
drain_stop.set()
drain_thread.join(timeout=1)
try:
handed = yield_handler(proc, output.render())
except Exception:
logger.warning("yield-to-background handoff failed; continuing to wait", exc_info=True)
handed = None
if handed is not None:
output.close_spill()
return handed
drain_stop.clear()
drain_thread = _start_drain_thread(proc, output, drain_stop)
if time.monotonic() > deadline:
trace.timed_out()
_kill_and_join()
rendered = output.render(suffix=f"\n[Command timed out after {timeout}s]")
if output.total_chars == 0:
rendered = rendered.lstrip()
# The flag tells Hermes' own deadline apart from a command's own ``exit 124``.
return {**self._finalize_wait_result(output, rendered, 124), "hermes_timed_out": True}
touch_activity_if_due(_activity_state, "terminal command running")
trace.heartbeat()
time.sleep(_poll_sleep)
if _poll_sleep < 0.2:
_poll_sleep = min(_poll_sleep * 1.5, 0.2)
except (KeyboardInterrupt, SystemExit):
trace.exception_exit()
try:
_kill_and_join()
except Exception:
pass # cleanup is best-effort
raise
# The drain thread exits promptly after bash does (~300ms idle check);
# a long join here would itself indicate a bug in the drain loop.
drain_thread.join(timeout=2)
try:
proc.stdout.close()
except Exception:
pass
trace.natural_exit(proc.returncode)
# Join the stdin writer before reading its error list: a child that exits without
# reading stdin can otherwise race ahead of a recorded encode failure. The timeout
# is a pure safety net (write raises BrokenPipeError once the pipe closes).
stdin_thread = getattr(proc, "_hermes_stdin_thread", None)
if stdin_thread is not None:
stdin_thread.join(timeout=5)
rendered = output.render()
result = self._finalize_wait_result(output, rendered, proc.returncode)
if stdin_errors := getattr(proc, "_hermes_stdin_errors", None):
result["stdin_error"] = err = str(stdin_errors[0])
result["output"] = rendered + f"\n[stdin write failed: {err}]"
return result
_finalize_wait_result = staticmethod(_finalize_wait_result)
def _kill_process(self, proc: ProcessHandle):
"""Terminate a process. Subclasses may override for process-group kill."""
try:
proc.kill()
except (ProcessLookupError, PermissionError, OSError):
pass
def _force_kill_process(self, proc: ProcessHandle):
"""Kill without waiting, for a host that hard-exits next. Subclasses kill the whole tree."""
self._kill_process(proc)
# --- CWD extraction ---
def _update_cwd(self, result: dict, *, move_session: bool = True):
"""Parse the cwd marker into *result*; only a command that may move the session
(``move_session``) then publishes it to ``self.cwd``. Parsing never writes ``self.cwd``:
the environment is shared, so a housekeeping command run from ``/`` (the background
poller) must never expose ``/`` to a concurrent file op, even briefly."""
self._parse_cwd_marker(result)
if move_session and result.get("cwd_observed"):
self.cwd = result["cwd"]
def _parse_cwd_marker(self, result: dict):
"""Strip the ``__HERMES_CWD_{session}__`` (and PIPESTATUS) marker lines from
``result["output"]``. ``result["cwd_observed"]``/``["cwd"]`` are set only when THIS
command emitted a marker: a killed/timed-out command emits none. Concurrent callers
must read ``result["cwd"]`` rather than ``self.cwd``."""
pipe_split = _split_cwd_marker(result.get("output", ""), self._pipe_marker) if self._pipe_marker else None
if pipe_split is not None:
statuses, cleaned = pipe_split
result["output"] = cleaned
try:
result["pipeline_status"] = [int(s) for s in (statuses or "").split()]
except ValueError:
pass
split = _split_cwd_marker(result.get("output", ""), self._cwd_marker)
if split is None:
return
cwd_path, cleaned = split
if cwd_path:
result["cwd_observed"] = True
result["cwd"] = cwd_path
result["output"] = cleaned
# --- Hooks ---
def _before_execute(self) -> None:
"""Hook before each command. Remote backends (SSH, Modal, Daytona)
trigger their FileSyncManager here; bind-mount backends and Local don't."""
def _mark_recreated(self) -> None:
"""Flag that the live container/sandbox was replaced while serving the
current command. ``execute`` folds the flag into the result as
``environment_recreated`` so the tool layer can warn the model that
background processes died and non-persisted files may be gone —
without this the recovery is silent and the model keeps assuming the
old workspace state (lobehub/lobehub#19329 class)."""
self._recreated_notice_pending = True
def _usable_cwd(self, cwd: str) -> tuple[str, dict | None]:
"""Directory the command may actually start in, plus ``{"requested", "used"}`` when it
differs. Local swaps a deleted/unenterable one host-side; a sandbox's filesystem is only
visible to its own shell, so its wrapper walks up instead (``_cd_with_fallback``) and
``execute`` reports that. Returned, never stored on ``self``: concurrent commands share
one environment."""
return cwd, None
# --- Unified execute() ---
def execute(
self,
command: str,
cwd: str = "",
*,
timeout: int | None = None,
stdin_data: str | None = None,
rewrite_compound_background: bool = True,
bounded_capture: bool = False,
yield_handler: Callable[[ProcessHandle, str], dict] | None = None,
allow_cwd_fallback: bool = False) -> dict:
"""Execute a command, return {"output": str, "returncode": int}. ``bounded_capture=True``
caps retention at ``tool_output.max_bytes`` WHILE draining; only the foreground terminal
tool may set it — internal full-fidelity consumers (file-op ``cat`` reads feeding the
patch engine, RPC reads, log reads) MUST leave it False or data is corrupted. The wait is
bounded by ``agent.deadline.run_bounded_sync`` so a wedged poll loop cannot hang past
``timeout`` and silently disable every asyncio timer in the process.
``bounded_capture=True`` caps stdout/stderr retention at ``tool_output.max_bytes`` WHILE the stream
is drained (head/tail window) instead of holding the full output in memory (#64435).
See #94285.
``allow_cwd_fallback=True`` runs in the nearest usable ancestor when the cwd is gone and
reports it as ``cwd_fallback``. Only the terminal tool's own commands (session cwd, no
``workdir``) may set it, since it tells the model; every other caller fails at ``cd``
instead (exit 126), because a file op's relative path would silently resolve against the
ancestor and read or write a different file.
"""
self._before_execute()
exec_command, sudo_stdin = self._prepare_command(command)
# Guard against the `A && B &` subshell-wait trap by default; callers
# that already produce shell-safe wrappers (spawn_via_env) pass False.
if rewrite_compound_background:
from tools.terminal_tool_sudo import _rewrite_compound_background
exec_command = _rewrite_compound_background(exec_command)
effective_timeout = timeout or self.timeout
exec_command = fill_timeout_token(exec_command, effective_timeout)
# An explicit ``cwd`` (the session's recorded cd state) can be just as dead as
# ``self.cwd``; resolving only the latter left every call exiting 126 at ``cd``.
effective_cwd, cwd_fallback = (
self._usable_cwd(cwd or self.cwd) if allow_cwd_fallback else (cwd or self.cwd, None))
moves_session = allow_cwd_fallback or not cwd or cwd == self.cwd
# Merge sudo stdin with caller stdin.
effective_stdin = sudo_stdin + (stdin_data or "") if sudo_stdin is not None else stdin_data
if effective_stdin and self._stdin_mode == "heredoc":
exec_command = self._embed_stdin_heredoc(exec_command, effective_stdin)
effective_stdin = None
wrapped = self._wrap_command(exec_command, effective_cwd, cwd_fallback=allow_cwd_fallback)
# Login shell if the snapshot failed (so the user's profile still
# loads), unless login itself is broken — then non-login is the only path.
login = not self._snapshot_ready and not self._prefer_nonlogin
parent_tid = threading.current_thread().ident
# The activity callback is thread-local and the wait runs on the
# deadline worker, so copy it across or long commands look idle.
parent_activity_cb = get_activity_callback()
proc_holder: list = []
output_holder: list = []
def _spawn_and_wait() -> dict:
if parent_activity_cb is not None:
set_activity_callback(parent_activity_cb)
if not _enter_foreground_spawn():
return {"output": "[host is exiting: command not started]", "returncode": 130}
spawned = None
try:
spawned = self._run_bash(wrapped, login=login, timeout=effective_timeout, stdin_data=effective_stdin)
finally:
fenced = _leave_foreground_spawn(self, spawned)
proc_holder.append(spawned)
if fenced: # the hard-exit kill may have stopped waiting for us before we registered
self._force_kill_process(spawned)
# The fallback line leads the stream; pulled out before capture so truncation can't cut it.
output = _new_output_collector(
spawned, bounded_capture, control_marker=self._cwd_fallback_marker if allow_cwd_fallback else "")
output_holder.append(output)
try:
return self._wait_for_process(
spawned, timeout=effective_timeout, bounded_capture=bounded_capture,
watch_interrupt_tid=parent_tid, output=output,
**({"yield_handler": yield_handler} if yield_handler is not None else {}))
finally:
with _live_foreground_cond:
_live_foreground.pop(id(spawned), None)
def _on_timeout() -> None:
if proc_holder:
self._kill_spawned_tree(proc_holder[0])
# Hard wall-clock backstop: ``_wait_for_process`` polls to ``effective_timeout`` on the
# tool thread; if that is the event-loop thread, or the wait never returns (Windows
# pipe/poll hang), every asyncio timer is silently disabled. ``run_bounded_sync`` drives
# expiry from a daemon worker + ``Event.wait`` so a blocked loop cannot disable it; the
# grace lets the inner loop return the partial-output 124 path.
# See #94285.
from agent.deadline import run_bounded_sync
try:
bound_s = float(effective_timeout)
except (TypeError, ValueError):
bound_s = 120.0 # a non-numeric timeout must not disable the backstop
bound_s += _EXECUTE_WAIT_BOUND_GRACE_S
try:
bounded = run_bounded_sync(
_spawn_and_wait, bound_s, label=f"terminal.wait:{type(self).__name__}", on_timeout=_on_timeout)
except (KeyboardInterrupt, SystemExit):
_on_timeout()
raise
if bounded.timed_out:
suffix = f"\n[Command timed out after {effective_timeout}s]"
if output_holder:
collector = output_holder[0]
result = self._finalize_wait_result(collector, collector.render(suffix=suffix).lstrip("\n"), 124)
else:
result = {"output": suffix.lstrip(), "returncode": 124}
result["hermes_timed_out"] = True
else:
result = bounded.value
# Housekeeping from "/" (the background poller) or a one-off workdir must not move the session.
self._update_cwd(result, move_session=moves_session)
used = output_holder[0].control_value if output_holder else None
if allow_cwd_fallback and not used and (split := _split_cwd_marker(result.get("output", ""), self._cwd_fallback_marker)):
used, result["output"] = split # login-shell noise printed ahead of the wrapper's line
if used:
cwd_fallback = cwd_fallback or {"requested": effective_cwd, "used": used}
if getattr(self, "_recreated_notice_pending", False):
self._recreated_notice_pending = False
result["environment_recreated"] = True
if cwd_fallback:
result["cwd_fallback"] = cwd_fallback
return result
def _kill_spawned_tree(self, spawned) -> None:
"""Best-effort kill of a wedged spawned process and its tree (backstop path)."""
try:
self._kill_process(spawned)
except Exception:
logger.debug("terminal wait-bound kill_process failed", exc_info=True)
pid = getattr(spawned, "pid", None)
if not pid:
return
try:
from agent.deadline import kill_process_tree
kill_process_tree(int(pid))
except Exception:
logger.debug("terminal wait-bound kill_process_tree failed", exc_info=True)
# --- Shared helpers ---
def __del__(self):
try:
self.cleanup()
except Exception:
pass
def _prepare_command(self, command: str) -> tuple[str, str | None]:
"""Rewrite sudo for an askpass helper or a piped password (untouched under NOPASSWD) and
apply the macOS ``open`` raise-ladder (no-op off Darwin) so opened files come to the front."""
from tools.environments.sudo_password_note import note_sudo_password_sent
from tools.terminal_tool_macos_open import _transform_macos_open_command
from tools.terminal_tool_sudo import _transform_sudo_command
prepared = _transform_sudo_command(
_transform_macos_open_command(command),
sudo_nopasswd_check=self._sudo_nopasswd_works,
askpass_supported=self._sudo_askpass_supported,
)
note_sudo_password_sent(self, prepared[1]) # once per non-local backend class
return prepared
_sudo_askpass_supported = sudo_askpass_supported_once
_SUDO_PROBE_TIMEOUT_S = 3
def _sudo_nopasswd_works(self) -> bool:
"""``sudo -n true`` inside THIS backend (host sudo state must not leak into a sandbox).
Fails closed: any error or a timed-out probe means "assume a password is needed"."""
if not self._passwordless_sudo_probe_supported:
return False
try:
proc = self._run_bash("sudo -n true", timeout=self._SUDO_PROBE_TIMEOUT_S)
return self._wait_for_process(proc, timeout=self._SUDO_PROBE_TIMEOUT_S).get("returncode") == 0
except Exception:
return False