mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-18 20:04:07 +03:00
065f5abbb3
Background polls run from '/' on the shared environment: _update_cwd wrote self.cwd from their marker and execute() restored it afterwards, so a concurrent file op could resolve a relative path against '/' (and the restore could clobber a concurrent cd). Marker parsing now only fills the result; self.cwd is assigned only when the command may move the session. The cwd-fallback line leads the stream, so a small tool_output.max_bytes or a timeout could cut or drop it. The output collector now strips it before the head/tail window applies.
904 lines
46 KiB
Python
904 lines
46 KiB
Python
"""Base class for all Hermes execution environment backends.
|
|
|
|
Unified spawn-per-call model: every command spawns a fresh ``bash -c`` process.
|
|
A session snapshot (env vars, functions, aliases) is captured once at init and
|
|
re-sourced before each command. CWD persists via in-band stdout markers (remote)
|
|
or a temp file (local). Cohesive pieces live in sibling modules (``base_output``,
|
|
``base_session_env``, ``base_wait``, ``path_utils``).
|
|
"""
|
|
|
|
import contextvars
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from abc import ABC, abstractmethod
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Callable, Iterable
|
|
|
|
from hermes_constants import get_hermes_home
|
|
from tools.interrupt import consume_yield, is_interrupted, is_thread_interrupted
|
|
from tools.environments.base_output import (
|
|
ProcessHandle, _finalize_wait_result, _new_output_collector, _start_drain_thread,
|
|
)
|
|
from tools.environments.base_session_env import (
|
|
_SHELL_ENV_NAME_RE, _SNAP_TMP_SUFFIX, _cwd_fallback_marker, _cwd_marker, _pipe_marker, _snapshot_bootstrap_script, _split_cwd_marker,
|
|
_wrap_command_script,
|
|
)
|
|
from tools.environments.base_processes import LongLivedProcessMixin
|
|
from tools.environments.base_wait import _WaitTrace
|
|
from tools.terminal_tool_sudo_askpass import fill_timeout_token, sudo_askpass_supported_once
|
|
from utils import atomic_json_write, env_var_enabled, file_signature, read_json_or_empty
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Opt-in debug tracing for the interrupt/activity/poll machinery
|
|
# (HERMES_DEBUG_INTERRUPT=1). Off by default to avoid flooding gateway logs.
|
|
_DEBUG_INTERRUPT = env_var_enabled("HERMES_DEBUG_INTERRUPT")
|
|
|
|
# Extra seconds the ``run_bounded_sync`` backstop waits past the inner ``_wait_for_process``
|
|
# deadline: the inner loop returns partial output + 124; the outer bound only fires when that
|
|
# loop never returns. Keep small so a healthy timeout still comes from the inner path.
|
|
# The inner poll loop is what returns partial output + returncode 124; this outer bound only exists for when
|
|
# that loop itself never returns (family A of #94285: a blocked wait that silently disables asyncio timers).
|
|
_EXECUTE_WAIT_BOUND_GRACE_S = 2.0
|
|
|
|
if _DEBUG_INTERRUPT:
|
|
# quiet_mode forces the `tools` logger to ERROR on CLI startup, which would
|
|
# swallow every trace; force this logger back to INFO in the opt-in case.
|
|
logger.setLevel(logging.INFO)
|
|
|
|
# Thread-local activity callback: the agent sets it before a tool call so
|
|
# long-running _wait_for_process loops can report liveness to the gateway.
|
|
_activity_callback_local = threading.local()
|
|
|
|
# Foreground commands in flight in THIS process, across every environment. Each runs in its
|
|
# own session/process group, so a host that exits mid-command (TUI client gone, SIGTERM) would
|
|
# orphan the whole tree; the process-exit funnel ``cleanup_all_environments`` kills them.
|
|
_live_foreground: "dict[int, tuple[BaseEnvironment, ProcessHandle]]" = {}
|
|
# Reentrant, and the hard-exit path only ever takes it with a timeout: a signal handler can run
|
|
# on a thread that already holds it.
|
|
_live_foreground_cond = threading.Condition(threading.RLock())
|
|
_exit_fenced = False # one-way, set by the hard-exit kill: no foreground command spawns after it
|
|
_spawns_in_flight = 0 # past the fence check, child maybe alive, not yet in _live_foreground
|
|
_HARD_KILL_BUDGET_S = 0.5
|
|
|
|
# Task ids whose shell state a command reads and writes, own id first, then its delegation
|
|
# ancestors; empty means the environment's own snapshot. One environment serves a parent and
|
|
# its delegate_task children (shared sandbox, #16177), so without this a child's ``export``
|
|
# became the parent's shell state (#84536).
|
|
_shell_state_chain: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar(
|
|
"terminal_shell_state_chain", default=())
|
|
|
|
|
|
@contextmanager
|
|
def shell_state_scope(chain: tuple[str, ...]):
|
|
"""Run commands against the per-task snapshot of ``chain[0]`` (seeded from its ancestors)."""
|
|
token = _shell_state_chain.set(tuple(chain))
|
|
try:
|
|
yield
|
|
finally:
|
|
_shell_state_chain.reset(token)
|
|
|
|
|
|
def _enter_foreground_spawn() -> bool:
|
|
global _spawns_in_flight
|
|
with _live_foreground_cond:
|
|
if _exit_fenced:
|
|
return False
|
|
_spawns_in_flight += 1
|
|
return True
|
|
|
|
|
|
def _leave_foreground_spawn(env: "BaseEnvironment", spawned) -> bool:
|
|
"""Publish ``spawned`` (None: the spawn failed); True when the exit fence went up meanwhile."""
|
|
global _spawns_in_flight
|
|
with _live_foreground_cond:
|
|
_spawns_in_flight -= 1
|
|
if spawned is not None:
|
|
_live_foreground[id(spawned)] = (env, spawned)
|
|
_live_foreground_cond.notify_all()
|
|
return _exit_fenced
|
|
|
|
|
|
def _quiet_kill(kill: Callable, proc) -> None:
|
|
try:
|
|
kill(proc)
|
|
except Exception:
|
|
logger.debug("exit-time kill of a foreground command failed", exc_info=True)
|
|
|
|
|
|
def kill_live_foreground_processes(*, now: bool = False) -> int:
|
|
"""Kill every in-flight foreground command's process tree; returns how many were signalled.
|
|
|
|
``now=True`` is for a caller about to ``os._exit``: the graceful kill TERMs, waits and only then
|
|
KILLs, so a SIGTERM-ignoring command outlives a hard exit that lands inside that window. It also
|
|
raises the exit fence and waits for spawns already past it to register, so no command started
|
|
around the snapshot survives, and it never blocks past ``_HARD_KILL_BUDGET_S``: SDK cancels
|
|
(Modal, Daytona, Vercel) run on daemon threads under that one deadline."""
|
|
global _exit_fenced
|
|
if not now:
|
|
with _live_foreground_cond:
|
|
live = list(_live_foreground.values())
|
|
for env, proc in live:
|
|
_quiet_kill(env._kill_process, proc)
|
|
return len(live)
|
|
deadline = time.monotonic() + _HARD_KILL_BUDGET_S
|
|
_exit_fenced = True
|
|
if _live_foreground_cond.acquire(timeout=_HARD_KILL_BUDGET_S):
|
|
try:
|
|
_live_foreground_cond.wait_for(lambda: _spawns_in_flight == 0, max(0.0, deadline - time.monotonic()))
|
|
live = list(_live_foreground.values())
|
|
finally:
|
|
_live_foreground_cond.release()
|
|
else: # the holder is stuck under our signal: a lock-free copy beats hanging the exit
|
|
live = list(_live_foreground.values())
|
|
remote = []
|
|
for env, proc in live:
|
|
if isinstance(proc, subprocess.Popen): # killpg/kill: never blocks
|
|
_quiet_kill(env._force_kill_process, proc)
|
|
else:
|
|
remote.append(threading.Thread(target=_quiet_kill, args=(env._force_kill_process, proc), daemon=True))
|
|
remote[-1].start()
|
|
for t in remote:
|
|
t.join(max(0.0, deadline - time.monotonic()))
|
|
return len(live)
|
|
|
|
|
|
class FileFetchError(RuntimeError):
|
|
"""A file could not be extracted from the backend filesystem."""
|
|
|
|
|
|
# Pulling a large artifact over a slow exec channel can outlast the per-command terminal timeout.
|
|
_FETCH_TIMEOUT_SECONDS = 300
|
|
|
|
|
|
class EnvironmentConnectionError(RuntimeError):
|
|
"""Infrastructure/connection-class failure of a terminal backend (SSH host down, Docker
|
|
daemon not running, remote sync on a dead link) — never a command that merely exited
|
|
nonzero. Subclassing RuntimeError keeps every ``except RuntimeError`` catcher working.
|
|
``terminal_tool`` turns this into a structured ``status: "degraded"`` result; the failed
|
|
backend is never cached, so a later call retries from scratch."""
|
|
|
|
def __init__(self, reason: str, *, retry_hint: str = ""):
|
|
super().__init__(reason)
|
|
self.reason = reason
|
|
self.retry_hint = retry_hint or (
|
|
"This is an infrastructure failure, not a command failure. "
|
|
"Verify the backend is reachable (network, service running, "
|
|
"credentials), then retry the same command — recovery is "
|
|
"automatic once the backend is back.")
|
|
|
|
|
|
def set_activity_callback(cb: Callable[[str], None] | None) -> None:
|
|
"""Register a callback that _wait_for_process fires periodically."""
|
|
_activity_callback_local.callback = cb
|
|
|
|
|
|
def get_activity_callback() -> Callable[[str], None] | None:
|
|
"""Thread-local activity callback; capture it before handing work to another thread.
|
|
|
|
Public accessor for callers outside this module that need to capture the calling thread's callback
|
|
before handing work to another thread (the callback is thread-local, so a freshly spawned thread cannot
|
|
read it back) — e.g. the manual cron-run heartbeat (#76502).
|
|
"""
|
|
return getattr(_activity_callback_local, "callback", None)
|
|
|
|
|
|
def touch_activity_if_due(state: dict, label: str) -> None:
|
|
"""Fire the activity callback at most once every ``state['interval']`` (default 10 s).
|
|
*state* holds ``last_touch``/``start`` monotonic timestamps. Swallows all exceptions."""
|
|
now = time.monotonic()
|
|
if now - state["last_touch"] < state.get("interval", 10.0):
|
|
return
|
|
state["last_touch"] = now
|
|
try:
|
|
cb = get_activity_callback()
|
|
if cb:
|
|
cb(f"{label} ({int(now - state['start'])}s elapsed)")
|
|
except Exception:
|
|
logger.debug("activity callback failed during a long-running command", exc_info=True)
|
|
|
|
|
|
def get_sandbox_dir() -> Path:
|
|
"""Host-side root for all sandbox storage (Docker workspaces, plugin backends'
|
|
host-side state). ``TERMINAL_SANDBOX_DIR`` overrides ``{HERMES_HOME}/sandboxes``."""
|
|
custom = os.getenv("TERMINAL_SANDBOX_DIR")
|
|
p = Path(custom) if custom else get_hermes_home() / "sandboxes"
|
|
p.mkdir(parents=True, exist_ok=True)
|
|
return p
|
|
|
|
|
|
def _save_json_store(path: Path, data: dict) -> None:
|
|
"""Write *data* as pretty-printed JSON to *path* atomically (readers never see a torn file)."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
atomic_json_write(path, data, indent=2)
|
|
|
|
|
|
_JSON_STORE_LOCK_HOLDER = threading.local()
|
|
|
|
|
|
def _update_json_store(path: Path, update: Callable[[dict], None]) -> None:
|
|
"""Read-modify-write the JSON store at *path* under a cross-process lock.
|
|
|
|
Concurrent sandbox cleanups (delegate children, gateway sessions, the idle reaper) each
|
|
record a snapshot id; an unlocked load/mutate/save lets one cleanup drop the other's entry.
|
|
An unusable lock (no flock on the filesystem, a wedged holder) degrades to an unlocked
|
|
update: a sandbox lifecycle step (modal's restore fallback included) must not abort on it."""
|
|
from contextlib import ExitStack
|
|
|
|
from hermes_cli.auth import _file_lock
|
|
|
|
lock_path = path.with_name(f"{path.name}.lock")
|
|
with ExitStack() as stack:
|
|
try:
|
|
stack.enter_context(_file_lock(
|
|
lock_path, _JSON_STORE_LOCK_HOLDER, 10.0, f"Timed out waiting for the snapshot store lock ({path})",
|
|
))
|
|
except (OSError, TimeoutError):
|
|
logger.warning("Updating %s without its lock", path, exc_info=True)
|
|
data = read_json_or_empty(path)
|
|
before = dict(data)
|
|
update(data)
|
|
if data != before:
|
|
_save_json_store(path, data)
|
|
|
|
|
|
def _file_mtime_key(host_path: str) -> tuple[int, int, int, int] | None:
|
|
"""Return the ``file_signature`` change key, or ``None`` if unreadable.
|
|
|
|
mtime + size alone miss a same-size credential refresh landing in the same mtime tick;
|
|
inode and ctime catch the atomic replace / rewrite."""
|
|
try:
|
|
return file_signature(Path(host_path).stat())
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def env_uses_host_filesystem(env: object) -> bool:
|
|
"""Whether the file tools may touch *env*'s files on the host (``BaseEnvironment.uses_host_filesystem``).
|
|
Strict ``is True``: an environment without the declaration (``None``, a duck-typed plugin
|
|
backend, a test double) is not the host."""
|
|
return getattr(env, "uses_host_filesystem", False) is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# BaseEnvironment
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class BaseEnvironment(LongLivedProcessMixin, ABC):
|
|
"""Common interface and unified execution flow for all Hermes backends. Subclasses
|
|
implement ``_run_bash()`` and ``cleanup()``; the base provides ``execute()`` with
|
|
snapshot sourcing, CWD tracking, interrupt handling and timeout enforcement, and the
|
|
long-lived process surface (``start_process`` / ``attach_process`` / ``stop_process``,
|
|
``base_processes``)."""
|
|
|
|
# How execute() hands stdin to _run_bash: "pipe" (process stdin, default),
|
|
# "payload" (passed through as stdin_data; the backend transports it via its
|
|
# SDK -- Modal/managed Modal stream it, Daytona/Vercel stage a file) or
|
|
# "heredoc" (embedded in the command; no built-in backend, plugins only).
|
|
_stdin_mode: str = "pipe" # "pipe" | "payload" | "heredoc"
|
|
|
|
# True only when commands execute on the SAME host as the Hermes process
|
|
# (LocalEnvironment); controller-host facts then describe the execution target.
|
|
is_local: bool = False
|
|
|
|
# True when the file tools may read, write, patch and search this backend's files straight on
|
|
# the host instead of through execute() (only LocalEnvironment). A host backend that confines its
|
|
# commands (a sandbox hiding paths, mounting them read-only or giving them their own $TMPDIR) sets
|
|
# it False, so the file tools see exactly what its commands see. Read via env_uses_host_filesystem().
|
|
uses_host_filesystem: bool = False
|
|
|
|
# Snapshot creation timeout (override for slow cold-starts).
|
|
_snapshot_timeout: int = 30
|
|
|
|
# Re-dump exported env into the snapshot after each command (see __init__).
|
|
_persist_env: bool = True
|
|
|
|
# Opt in only when a timed-out probe can kill its command without tearing
|
|
# down the whole backend. SDK adapters cancel by stopping the sandbox. Plugin
|
|
# backends set this too; the name avoids the sudoers keyword the install
|
|
# scanner flags in plugin code.
|
|
_passwordless_sudo_probe_supported: bool = False
|
|
|
|
# Whether background processes spawned through this env are gone, for this env object, once
|
|
# cleanup() returns: ended with the sandbox, or no longer reachable through it (persist-mode
|
|
# Docker drops its container id, so the old object can't poll the job). Teardown then
|
|
# settles them as lost so their pollers stop. A backend whose processes outlive cleanup()
|
|
# and stay reachable through the same object (SSH reconnects) sets False so the poller keeps
|
|
# collecting the real result. Plugin contract: website/docs/developer-guide/terminal-environment-plugin.md.
|
|
cleanup_stops_processes: bool = True
|
|
|
|
# Local and Docker override this because they resolve allowlisted values
|
|
# through the active profile scope; other backends keep plain snapshots.
|
|
_profile_scoped_passthrough: bool = False
|
|
|
|
# Per-session PIPESTATUS marker set in __init__; empty means the wrapper emits none.
|
|
_pipe_marker: str = ""
|
|
# Per-session marker the wrapper prints after falling back from a dead cwd; empty = refuse.
|
|
_cwd_fallback_marker: str = ""
|
|
|
|
def get_temp_dir(self) -> str:
|
|
"""Return the backend temp directory used for session artifacts.
|
|
|
|
Most sandboxed backends use ``/tmp`` inside the target environment.
|
|
LocalEnvironment overrides this on hosts where ``/tmp`` may be missing
|
|
and ``TMPDIR`` is the portable writable location.
|
|
"""
|
|
return "/tmp" # no-tmp: ok — sandbox-side (remote container) temp dir, not the host
|
|
|
|
def __init__(self, cwd: str, timeout: int, env: dict | None = None, persist_env: bool = True):
|
|
self.cwd = cwd
|
|
self.timeout = timeout
|
|
self.env = env or {}
|
|
# False: each command sources only the login-time snapshot, so exported vars do not
|
|
# carry over (terminal.persistent_shell / TERMINAL_LOCAL_PERSISTENT). CWD still does.
|
|
self._persist_env = persist_env
|
|
|
|
self._session_id = uuid.uuid4().hex[:12]
|
|
temp_dir = self.get_temp_dir().rstrip("/") or "/"
|
|
self._snapshot_path = f"{temp_dir}/hermes-snap-{self._session_id}.sh"
|
|
self._cwd_file = f"{temp_dir}/hermes-cwd-{self._session_id}.txt"
|
|
self._cwd_marker = _cwd_marker(self._session_id)
|
|
self._pipe_marker = _pipe_marker(self._session_id)
|
|
self._cwd_fallback_marker = _cwd_fallback_marker(self._session_id)
|
|
self._snapshot_ready = False
|
|
self._snapshot_passthrough_names: set[str] = set()
|
|
# True when login bash is unusable (e.g. broken Git-for-Windows startup)
|
|
# so execute() must fall back to non-login ``bash -c``, not ``bash -l``.
|
|
self._prefer_nonlogin = False
|
|
|
|
# --- Abstract methods ---
|
|
def _run_bash(
|
|
self, cmd_string: str, *, login: bool = False, timeout: int = 120, stdin_data: str | None = None,
|
|
) -> ProcessHandle:
|
|
"""Spawn a bash process to run *cmd_string*; every backend overrides this."""
|
|
raise NotImplementedError(f"{type(self).__name__} must implement _run_bash()")
|
|
|
|
@abstractmethod
|
|
def cleanup(self):
|
|
"""Release backend resources (container, instance, connection). Raise only while the
|
|
sandbox may still be live: once it is destroyed, log later failures instead, since a
|
|
raise skips settling its background processes (see ``cleanup_stops_processes``)."""
|
|
...
|
|
|
|
# --- File extraction (backend -> host) ---
|
|
def fetch_file(self, remote_path: str, local_dest: Path, *, max_bytes: int) -> None:
|
|
"""Copy a regular file out of the backend filesystem to ``local_dest`` on the host.
|
|
|
|
One transport for every backend: base64 over the exec channel, bounded INSIDE the sandbox
|
|
(``head -c max+1`` so /dev/zero can't stream unbounded data into host memory — the same
|
|
shape ``tools.image_source`` uses to read sandbox images). The payload is fenced between
|
|
unique markers so login-shell noise in the merged stdout/stderr can't corrupt the decode.
|
|
Raises :class:`FileFetchError` on a missing/unreadable/oversized file.
|
|
"""
|
|
import base64
|
|
import binascii
|
|
marker = f"__HERMES_FETCH_{uuid.uuid4().hex[:12]}__"
|
|
quoted = shlex.quote(remote_path)
|
|
# ``[ -f ]`` follows symlinks, so a link to a denied host file is judged by the CALLER on
|
|
# ``readlink -f`` output before any bytes move.
|
|
result = self.execute(
|
|
f"[ -f {quoted} ] && echo {marker} && head -c {max_bytes + 1} < {quoted} | base64 && echo {marker}",
|
|
timeout=_FETCH_TIMEOUT_SECONDS, rewrite_compound_background=False)
|
|
output = result.get("output") or ""
|
|
first, last = output.find(marker), output.rfind(marker)
|
|
if int(result.get("returncode") or 0) != 0 or first == -1 or last <= first:
|
|
raise FileFetchError(f"could not read {remote_path!r} in the sandbox (missing, not a regular file, or unreadable)")
|
|
try:
|
|
data = base64.b64decode("".join(output[first + len(marker):last].split()), validate=True)
|
|
except (ValueError, binascii.Error) as exc:
|
|
raise FileFetchError(f"transfer of {remote_path!r} was corrupted in transit: {exc}") from exc
|
|
if len(data) > max_bytes:
|
|
raise FileFetchError(f"{remote_path!r} exceeds the {max_bytes // (1024 * 1024)} MB delivery limit")
|
|
Path(local_dest).write_bytes(data)
|
|
|
|
def fetch_realpath(self, remote_path: str) -> str | None:
|
|
"""``readlink -f`` inside the backend, or None when it cannot be resolved."""
|
|
result = self.execute(f"readlink -f {shlex.quote(remote_path)} 2>/dev/null", rewrite_compound_background=False)
|
|
if int(result.get("returncode") or 0) != 0:
|
|
return None
|
|
return next((ln.strip() for ln in reversed((result.get("output") or "").splitlines()) if ln.strip().startswith("/")), None)
|
|
|
|
# --- Session snapshot (init_session) ---
|
|
def _additional_profile_scoped_passthrough_names(self) -> Iterable[str]:
|
|
"""Return backend-specific names that must not persist in snapshots."""
|
|
return ()
|
|
|
|
def _snapshot_excluded_passthrough_names(self) -> tuple[str, ...]:
|
|
"""Profile-scoped names that must not persist in the snapshot. Monotonic for the
|
|
environment lifetime: an allowlist can be cleared after a value was captured, and
|
|
retaining the exclusion keeps that old value from leaking to a later profile."""
|
|
if not self._profile_scoped_passthrough:
|
|
return ()
|
|
try:
|
|
from agent.secret_scope import is_multiplex_active
|
|
if is_multiplex_active():
|
|
from tools.env_passthrough import get_all_passthrough
|
|
names = (*get_all_passthrough(), *self._additional_profile_scoped_passthrough_names())
|
|
self._snapshot_passthrough_names.update(
|
|
name for name in names if isinstance(name, str) and _SHELL_ENV_NAME_RE.fullmatch(name))
|
|
except Exception:
|
|
logger.debug("Could not refresh profile-scoped snapshot exclusions", exc_info=True)
|
|
return tuple(sorted(self._snapshot_passthrough_names))
|
|
|
|
def _task_snapshot_path(self, task_id: str) -> str:
|
|
"""Per-task snapshot beside the environment's own (``cleanup`` globs the prefix)."""
|
|
digest = hashlib.sha256(task_id.encode()).hexdigest()[:16]
|
|
return f"{self._snapshot_path[:-len('.sh')]}-{digest}.sh"
|
|
|
|
def _snapshot_script_kwargs(self, cwd: str, snap: str | None = None) -> dict:
|
|
"""Quoting inputs shared by the bootstrap and per-command wrapper scripts.
|
|
``_quote_cwd_for_cd`` / ``_quote_shell_path`` (not bare shlex.quote) let the Windows
|
|
subclass rewrite ``C:\\...`` to ``/c/...`` so ``cd`` resolves and MSYS doesn't choke."""
|
|
return dict(
|
|
quoted_cwd=self._quote_cwd_for_cd(cwd),
|
|
quoted_snap=self._quote_shell_path(snap or self._snapshot_path),
|
|
snap_tmp_template=self._quote_shell_path((snap or self._snapshot_path) + _SNAP_TMP_SUFFIX),
|
|
cwd_marker=self._cwd_marker)
|
|
|
|
def init_session(self):
|
|
"""Capture the login shell environment into the snapshot file (once, after construction).
|
|
On success ``_snapshot_ready`` is set so commands source the snapshot instead of running
|
|
under ``bash -l``. On failure, fall back to ``bash -l`` per command — unless a non-login
|
|
probe shows login bash itself is dead, in which case prefer ``bash -c``."""
|
|
bootstrap = _snapshot_bootstrap_script(
|
|
excluded_names=self._snapshot_excluded_passthrough_names(), **self._snapshot_script_kwargs(self.cwd))
|
|
try:
|
|
proc = self._run_bash(bootstrap, login=True, timeout=self._snapshot_timeout)
|
|
result = self._wait_for_process(proc, timeout=self._snapshot_timeout)
|
|
if int(result.get("returncode") or 0) != 0:
|
|
raise RuntimeError(f"snapshot bootstrap failed with exit code {result.get('returncode')}")
|
|
self._snapshot_ready = True
|
|
self._update_cwd(result)
|
|
logger.info("Session snapshot created (session=%s, cwd=%s)", self._session_id, self.cwd)
|
|
except Exception as exc:
|
|
self._snapshot_ready = False
|
|
self._prefer_nonlogin, detail = self._probe_nonlogin_fallback(str(exc))
|
|
if self._prefer_nonlogin:
|
|
logger.warning(
|
|
"init_session failed (session=%s): %s — "
|
|
"login bash unusable; falling back to non-login bash -c",
|
|
self._session_id, exc)
|
|
else:
|
|
logger.warning(
|
|
"init_session failed (session=%s): %s — falling back to bash -l per command",
|
|
self._session_id, detail)
|
|
|
|
def _probe_nonlogin_fallback(self, detail: str) -> tuple[bool, str]:
|
|
"""Run ``true`` under non-login bash; return ``(prefer_nonlogin, detail)``."""
|
|
probe_timeout = min(15, self._snapshot_timeout)
|
|
try:
|
|
probe = self._run_bash("true", login=False, timeout=probe_timeout)
|
|
probe_result = self._wait_for_process(probe, timeout=probe_timeout)
|
|
prefer_nonlogin = int(probe_result.get("returncode") or 0) == 0
|
|
if not prefer_nonlogin:
|
|
detail = (probe_result.get("stdout") or detail).strip() or detail
|
|
return prefer_nonlogin, detail
|
|
except Exception as probe_exc:
|
|
return False, f"{detail}; non-login probe: {probe_exc}"
|
|
|
|
# --- Command wrapping ---
|
|
@staticmethod
|
|
def _quote_cwd_for_cd(cwd: str) -> str:
|
|
"""Quote a ``cd`` target while preserving ``~`` expansion (``~/...``
|
|
goes through ``$HOME`` so suffixes with spaces stay one word)."""
|
|
if cwd == "~":
|
|
return cwd
|
|
if cwd == "~/":
|
|
return "$HOME"
|
|
if cwd.startswith("~/"):
|
|
return f"$HOME/{shlex.quote(cwd[2:])}"
|
|
return shlex.quote(cwd)
|
|
|
|
def _quote_shell_path(self, path: str) -> str:
|
|
"""Quote *path* for a bash script. LocalEnvironment overrides this to
|
|
rewrite native/mixed Windows paths to ``/c/...``; remote backends are POSIX."""
|
|
return shlex.quote(path)
|
|
|
|
def _wrap_command(self, command: str, cwd: str, *, cwd_fallback: bool = False) -> str:
|
|
"""Full bash script: source snapshot, cd, run, re-dump env, emit CWD markers.
|
|
``cwd_fallback`` lets the ``cd`` walk up from a missing *cwd* (see ``execute``)."""
|
|
chain = [self._task_snapshot_path(t) for t in _shell_state_chain.get()]
|
|
seeds = [self._quote_shell_path(p) for p in (*chain[1:], self._snapshot_path)] if chain else []
|
|
return _wrap_command_script(
|
|
command,
|
|
passthrough_names=self._snapshot_excluded_passthrough_names(),
|
|
snapshot_ready=self._snapshot_ready,
|
|
persist_env=self._persist_env,
|
|
pipe_marker=self._pipe_marker,
|
|
seed_snaps=seeds,
|
|
cwd_fallback_marker=self._cwd_fallback_marker if cwd_fallback else "",
|
|
**self._snapshot_script_kwargs(cwd, chain[0] if chain else None))
|
|
|
|
@staticmethod
|
|
def _embed_stdin_heredoc(command: str, stdin_data: str) -> str:
|
|
"""Redirect stdin_data to the complete command with a shell heredoc (SDK backends).
|
|
A heredoc body always ends in a newline stdin_data may lack, and write_file verifies a
|
|
byte-exact hash, so a process substitution re-emits the body minus that last character.
|
|
Redirections apply left to right, so the substitution inherits the heredoc as its stdin;
|
|
the heredoc stays outside ``<( )`` because bash 3.2 mis-parses bodies inside it. ``|| :``
|
|
keeps an inherited ``set -e`` from killing the reader on read's EOF status."""
|
|
delimiter = f"HERMES_STDIN_{uuid.uuid4().hex[:12]}"
|
|
return (f"{{\n{command}\n}} << '{delimiter}' < <(IFS= read -r -d '' s || :; printf '%s' \"${{s%?}}\")\n"
|
|
f"{stdin_data}\n{delimiter}")
|
|
|
|
def _staged_stdin_path(self) -> str:
|
|
"""Unique sandbox path for staging a payload-mode stdin file."""
|
|
temp_dir = self.get_temp_dir().rstrip("/") or "/"
|
|
return f"{temp_dir}/.hermes-stdin-{uuid.uuid4().hex}"
|
|
|
|
@staticmethod
|
|
def _redirect_stdin_from_file(command: str, path: str) -> str:
|
|
"""Prefix ``command`` so the shell reads stdin from the staged ``path`` and
|
|
unlinks it before ``command`` runs (the shell then owns the payload)."""
|
|
quoted = shlex.quote(path)
|
|
return f"exec 0< {quoted} || exit $?\nrm -f -- {quoted} || exit $?\n{command}"
|
|
|
|
# --- Process lifecycle ---
|
|
def _wait_for_process(
|
|
self, proc: ProcessHandle, timeout: int = 120, *,
|
|
bounded_capture: bool = False, watch_interrupt_tid: int | None = None,
|
|
output=None, yield_handler: Callable[[ProcessHandle, str], dict] | None = None) -> dict:
|
|
"""Poll-based wait with interrupt checking and stdout draining (shared, not overridden).
|
|
``yield_handler(proc, output_so_far)``: when the tool thread is asked to yield
|
|
(``tools.interrupt.request_yield`` — a user message arrived mid-command), the drain
|
|
thread is stopped, the still-running process is handed to the handler and its dict
|
|
is returned as the result; the process is NOT killed.
|
|
``bounded_capture=True`` (foreground terminal-tool path only) retains at most
|
|
``tool_output.max_bytes`` in a head/tail window so a verbose subprocess cannot OOM the
|
|
process; the default keeps full fidelity for internal consumers. Fires the activity
|
|
callback every 10s so the gateway's inactivity timeout doesn't kill long commands.
|
|
``watch_interrupt_tid`` is the tool-worker thread that submitted this wait: ``execute()``
|
|
may move the wait onto a ``run_bounded_sync`` worker while ``/stop`` still interrupts the
|
|
original tid, so both bits are honored. ``KeyboardInterrupt``/``SystemExit`` mid-poll
|
|
kills the process first — the local backend spawns into its own process group, so an
|
|
unkilled child would be orphaned.
|
|
|
|
The default (False) preserves full-fidelity capture for internal consumers — file-operation ``cat``
|
|
reads feeding the patch engine, code-execution RPC reads, log reads — where truncation would corrupt
|
|
data. See #64435.
|
|
"""
|
|
if output is None:
|
|
output = _new_output_collector(proc, bounded_capture)
|
|
drain_stop = threading.Event() if yield_handler is not None else None
|
|
drain_thread = _start_drain_thread(proc, output, drain_stop)
|
|
_now = time.monotonic()
|
|
deadline = _now + timeout
|
|
_activity_state = {"last_touch": _now, "start": _now}
|
|
trace = _WaitTrace(proc, timeout, enabled=_DEBUG_INTERRUPT, logger=logger)
|
|
trace.enter()
|
|
|
|
def _kill_and_join():
|
|
self._kill_process(proc)
|
|
drain_thread.join(timeout=2)
|
|
|
|
try:
|
|
# Adaptive poll: start at 5ms so fast commands return in ~6ms, back
|
|
# off exponentially toward 200ms so long builds don't pay poll CPU.
|
|
_poll_sleep = 0.005
|
|
while proc.poll() is None:
|
|
trace.iterations += 1
|
|
if is_interrupted() or is_thread_interrupted(watch_interrupt_tid):
|
|
trace.interrupted()
|
|
_kill_and_join()
|
|
return {**self._finalize_wait_result(output, output.render(suffix="\n[Command interrupted]"), 130),
|
|
"hermes_interrupted": True}
|
|
if yield_handler is not None and consume_yield(watch_interrupt_tid):
|
|
drain_stop.set()
|
|
drain_thread.join(timeout=1)
|
|
try:
|
|
handed = yield_handler(proc, output.render())
|
|
except Exception:
|
|
logger.warning("yield-to-background handoff failed; continuing to wait", exc_info=True)
|
|
handed = None
|
|
if handed is not None:
|
|
output.close_spill()
|
|
return handed
|
|
drain_stop.clear()
|
|
drain_thread = _start_drain_thread(proc, output, drain_stop)
|
|
if time.monotonic() > deadline:
|
|
trace.timed_out()
|
|
_kill_and_join()
|
|
rendered = output.render(suffix=f"\n[Command timed out after {timeout}s]")
|
|
if output.total_chars == 0:
|
|
rendered = rendered.lstrip()
|
|
# The flag tells Hermes' own deadline apart from a command's own ``exit 124``.
|
|
return {**self._finalize_wait_result(output, rendered, 124), "hermes_timed_out": True}
|
|
touch_activity_if_due(_activity_state, "terminal command running")
|
|
trace.heartbeat()
|
|
time.sleep(_poll_sleep)
|
|
if _poll_sleep < 0.2:
|
|
_poll_sleep = min(_poll_sleep * 1.5, 0.2)
|
|
except (KeyboardInterrupt, SystemExit):
|
|
trace.exception_exit()
|
|
try:
|
|
_kill_and_join()
|
|
except Exception:
|
|
pass # cleanup is best-effort
|
|
raise
|
|
|
|
# The drain thread exits promptly after bash does (~300ms idle check);
|
|
# a long join here would itself indicate a bug in the drain loop.
|
|
drain_thread.join(timeout=2)
|
|
try:
|
|
proc.stdout.close()
|
|
except Exception:
|
|
pass
|
|
trace.natural_exit(proc.returncode)
|
|
|
|
# Join the stdin writer before reading its error list: a child that exits without
|
|
# reading stdin can otherwise race ahead of a recorded encode failure. The timeout
|
|
# is a pure safety net (write raises BrokenPipeError once the pipe closes).
|
|
stdin_thread = getattr(proc, "_hermes_stdin_thread", None)
|
|
if stdin_thread is not None:
|
|
stdin_thread.join(timeout=5)
|
|
rendered = output.render()
|
|
result = self._finalize_wait_result(output, rendered, proc.returncode)
|
|
if stdin_errors := getattr(proc, "_hermes_stdin_errors", None):
|
|
result["stdin_error"] = err = str(stdin_errors[0])
|
|
result["output"] = rendered + f"\n[stdin write failed: {err}]"
|
|
return result
|
|
|
|
_finalize_wait_result = staticmethod(_finalize_wait_result)
|
|
|
|
def _kill_process(self, proc: ProcessHandle):
|
|
"""Terminate a process. Subclasses may override for process-group kill."""
|
|
try:
|
|
proc.kill()
|
|
except (ProcessLookupError, PermissionError, OSError):
|
|
pass
|
|
|
|
def _force_kill_process(self, proc: ProcessHandle):
|
|
"""Kill without waiting, for a host that hard-exits next. Subclasses kill the whole tree."""
|
|
self._kill_process(proc)
|
|
|
|
# --- CWD extraction ---
|
|
def _update_cwd(self, result: dict, *, move_session: bool = True):
|
|
"""Parse the cwd marker into *result*; only a command that may move the session
|
|
(``move_session``) then publishes it to ``self.cwd``. Parsing never writes ``self.cwd``:
|
|
the environment is shared, so a housekeeping command run from ``/`` (the background
|
|
poller) must never expose ``/`` to a concurrent file op, even briefly."""
|
|
self._parse_cwd_marker(result)
|
|
if move_session and result.get("cwd_observed"):
|
|
self.cwd = result["cwd"]
|
|
|
|
def _parse_cwd_marker(self, result: dict):
|
|
"""Strip the ``__HERMES_CWD_{session}__`` (and PIPESTATUS) marker lines from
|
|
``result["output"]``. ``result["cwd_observed"]``/``["cwd"]`` are set only when THIS
|
|
command emitted a marker: a killed/timed-out command emits none. Concurrent callers
|
|
must read ``result["cwd"]`` rather than ``self.cwd``."""
|
|
pipe_split = _split_cwd_marker(result.get("output", ""), self._pipe_marker) if self._pipe_marker else None
|
|
if pipe_split is not None:
|
|
statuses, cleaned = pipe_split
|
|
result["output"] = cleaned
|
|
try:
|
|
result["pipeline_status"] = [int(s) for s in (statuses or "").split()]
|
|
except ValueError:
|
|
pass
|
|
split = _split_cwd_marker(result.get("output", ""), self._cwd_marker)
|
|
if split is None:
|
|
return
|
|
cwd_path, cleaned = split
|
|
if cwd_path:
|
|
result["cwd_observed"] = True
|
|
result["cwd"] = cwd_path
|
|
result["output"] = cleaned
|
|
|
|
# --- Hooks ---
|
|
def _before_execute(self) -> None:
|
|
"""Hook before each command. Remote backends (SSH, Modal, Daytona)
|
|
trigger their FileSyncManager here; bind-mount backends and Local don't."""
|
|
|
|
def _mark_recreated(self) -> None:
|
|
"""Flag that the live container/sandbox was replaced while serving the
|
|
current command. ``execute`` folds the flag into the result as
|
|
``environment_recreated`` so the tool layer can warn the model that
|
|
background processes died and non-persisted files may be gone —
|
|
without this the recovery is silent and the model keeps assuming the
|
|
old workspace state (lobehub/lobehub#19329 class)."""
|
|
self._recreated_notice_pending = True
|
|
|
|
def _usable_cwd(self, cwd: str) -> tuple[str, dict | None]:
|
|
"""Directory the command may actually start in, plus ``{"requested", "used"}`` when it
|
|
differs. Local swaps a deleted/unenterable one host-side; a sandbox's filesystem is only
|
|
visible to its own shell, so its wrapper walks up instead (``_cd_with_fallback``) and
|
|
``execute`` reports that. Returned, never stored on ``self``: concurrent commands share
|
|
one environment."""
|
|
return cwd, None
|
|
|
|
# --- Unified execute() ---
|
|
def execute(
|
|
self,
|
|
command: str,
|
|
cwd: str = "",
|
|
*,
|
|
timeout: int | None = None,
|
|
stdin_data: str | None = None,
|
|
rewrite_compound_background: bool = True,
|
|
bounded_capture: bool = False,
|
|
yield_handler: Callable[[ProcessHandle, str], dict] | None = None,
|
|
allow_cwd_fallback: bool = False) -> dict:
|
|
"""Execute a command, return {"output": str, "returncode": int}. ``bounded_capture=True``
|
|
caps retention at ``tool_output.max_bytes`` WHILE draining; only the foreground terminal
|
|
tool may set it — internal full-fidelity consumers (file-op ``cat`` reads feeding the
|
|
patch engine, RPC reads, log reads) MUST leave it False or data is corrupted. The wait is
|
|
bounded by ``agent.deadline.run_bounded_sync`` so a wedged poll loop cannot hang past
|
|
``timeout`` and silently disable every asyncio timer in the process.
|
|
|
|
``bounded_capture=True`` caps stdout/stderr retention at ``tool_output.max_bytes`` WHILE the stream
|
|
is drained (head/tail window) instead of holding the full output in memory (#64435).
|
|
See #94285.
|
|
|
|
``allow_cwd_fallback=True`` runs in the nearest usable ancestor when the cwd is gone and
|
|
reports it as ``cwd_fallback``. Only the terminal tool's own commands (session cwd, no
|
|
``workdir``) may set it, since it tells the model; every other caller fails at ``cd``
|
|
instead (exit 126), because a file op's relative path would silently resolve against the
|
|
ancestor and read or write a different file.
|
|
"""
|
|
self._before_execute()
|
|
|
|
exec_command, sudo_stdin = self._prepare_command(command)
|
|
# Guard against the `A && B &` subshell-wait trap by default; callers
|
|
# that already produce shell-safe wrappers (spawn_via_env) pass False.
|
|
if rewrite_compound_background:
|
|
from tools.terminal_tool_sudo import _rewrite_compound_background
|
|
exec_command = _rewrite_compound_background(exec_command)
|
|
effective_timeout = timeout or self.timeout
|
|
exec_command = fill_timeout_token(exec_command, effective_timeout)
|
|
# An explicit ``cwd`` (the session's recorded cd state) can be just as dead as
|
|
# ``self.cwd``; resolving only the latter left every call exiting 126 at ``cd``.
|
|
effective_cwd, cwd_fallback = (
|
|
self._usable_cwd(cwd or self.cwd) if allow_cwd_fallback else (cwd or self.cwd, None))
|
|
moves_session = allow_cwd_fallback or not cwd or cwd == self.cwd
|
|
|
|
# Merge sudo stdin with caller stdin.
|
|
effective_stdin = sudo_stdin + (stdin_data or "") if sudo_stdin is not None else stdin_data
|
|
if effective_stdin and self._stdin_mode == "heredoc":
|
|
exec_command = self._embed_stdin_heredoc(exec_command, effective_stdin)
|
|
effective_stdin = None
|
|
|
|
wrapped = self._wrap_command(exec_command, effective_cwd, cwd_fallback=allow_cwd_fallback)
|
|
|
|
# Login shell if the snapshot failed (so the user's profile still
|
|
# loads), unless login itself is broken — then non-login is the only path.
|
|
login = not self._snapshot_ready and not self._prefer_nonlogin
|
|
|
|
parent_tid = threading.current_thread().ident
|
|
# The activity callback is thread-local and the wait runs on the
|
|
# deadline worker, so copy it across or long commands look idle.
|
|
parent_activity_cb = get_activity_callback()
|
|
proc_holder: list = []
|
|
output_holder: list = []
|
|
|
|
def _spawn_and_wait() -> dict:
|
|
if parent_activity_cb is not None:
|
|
set_activity_callback(parent_activity_cb)
|
|
if not _enter_foreground_spawn():
|
|
return {"output": "[host is exiting: command not started]", "returncode": 130}
|
|
spawned = None
|
|
try:
|
|
spawned = self._run_bash(wrapped, login=login, timeout=effective_timeout, stdin_data=effective_stdin)
|
|
finally:
|
|
fenced = _leave_foreground_spawn(self, spawned)
|
|
proc_holder.append(spawned)
|
|
if fenced: # the hard-exit kill may have stopped waiting for us before we registered
|
|
self._force_kill_process(spawned)
|
|
# The fallback line leads the stream; pulled out before capture so truncation can't cut it.
|
|
output = _new_output_collector(
|
|
spawned, bounded_capture, control_marker=self._cwd_fallback_marker if allow_cwd_fallback else "")
|
|
output_holder.append(output)
|
|
try:
|
|
return self._wait_for_process(
|
|
spawned, timeout=effective_timeout, bounded_capture=bounded_capture,
|
|
watch_interrupt_tid=parent_tid, output=output,
|
|
**({"yield_handler": yield_handler} if yield_handler is not None else {}))
|
|
finally:
|
|
with _live_foreground_cond:
|
|
_live_foreground.pop(id(spawned), None)
|
|
|
|
def _on_timeout() -> None:
|
|
if proc_holder:
|
|
self._kill_spawned_tree(proc_holder[0])
|
|
|
|
# Hard wall-clock backstop: ``_wait_for_process`` polls to ``effective_timeout`` on the
|
|
# tool thread; if that is the event-loop thread, or the wait never returns (Windows
|
|
# pipe/poll hang), every asyncio timer is silently disabled. ``run_bounded_sync`` drives
|
|
# expiry from a daemon worker + ``Event.wait`` so a blocked loop cannot disable it; the
|
|
# grace lets the inner loop return the partial-output 124 path.
|
|
# See #94285.
|
|
from agent.deadline import run_bounded_sync
|
|
|
|
try:
|
|
bound_s = float(effective_timeout)
|
|
except (TypeError, ValueError):
|
|
bound_s = 120.0 # a non-numeric timeout must not disable the backstop
|
|
bound_s += _EXECUTE_WAIT_BOUND_GRACE_S
|
|
|
|
try:
|
|
bounded = run_bounded_sync(
|
|
_spawn_and_wait, bound_s, label=f"terminal.wait:{type(self).__name__}", on_timeout=_on_timeout)
|
|
except (KeyboardInterrupt, SystemExit):
|
|
_on_timeout()
|
|
raise
|
|
|
|
if bounded.timed_out:
|
|
suffix = f"\n[Command timed out after {effective_timeout}s]"
|
|
if output_holder:
|
|
collector = output_holder[0]
|
|
result = self._finalize_wait_result(collector, collector.render(suffix=suffix).lstrip("\n"), 124)
|
|
else:
|
|
result = {"output": suffix.lstrip(), "returncode": 124}
|
|
result["hermes_timed_out"] = True
|
|
else:
|
|
result = bounded.value
|
|
# Housekeeping from "/" (the background poller) or a one-off workdir must not move the session.
|
|
self._update_cwd(result, move_session=moves_session)
|
|
used = output_holder[0].control_value if output_holder else None
|
|
if allow_cwd_fallback and not used and (split := _split_cwd_marker(result.get("output", ""), self._cwd_fallback_marker)):
|
|
used, result["output"] = split # login-shell noise printed ahead of the wrapper's line
|
|
if used:
|
|
cwd_fallback = cwd_fallback or {"requested": effective_cwd, "used": used}
|
|
if getattr(self, "_recreated_notice_pending", False):
|
|
self._recreated_notice_pending = False
|
|
result["environment_recreated"] = True
|
|
if cwd_fallback:
|
|
result["cwd_fallback"] = cwd_fallback
|
|
return result
|
|
|
|
def _kill_spawned_tree(self, spawned) -> None:
|
|
"""Best-effort kill of a wedged spawned process and its tree (backstop path)."""
|
|
try:
|
|
self._kill_process(spawned)
|
|
except Exception:
|
|
logger.debug("terminal wait-bound kill_process failed", exc_info=True)
|
|
pid = getattr(spawned, "pid", None)
|
|
if not pid:
|
|
return
|
|
try:
|
|
from agent.deadline import kill_process_tree
|
|
kill_process_tree(int(pid))
|
|
except Exception:
|
|
logger.debug("terminal wait-bound kill_process_tree failed", exc_info=True)
|
|
|
|
# --- Shared helpers ---
|
|
def __del__(self):
|
|
try:
|
|
self.cleanup()
|
|
except Exception:
|
|
pass
|
|
|
|
def _prepare_command(self, command: str) -> tuple[str, str | None]:
|
|
"""Rewrite sudo for an askpass helper or a piped password (untouched under NOPASSWD) and
|
|
apply the macOS ``open`` raise-ladder (no-op off Darwin) so opened files come to the front."""
|
|
from tools.environments.sudo_password_note import note_sudo_password_sent
|
|
from tools.terminal_tool_macos_open import _transform_macos_open_command
|
|
from tools.terminal_tool_sudo import _transform_sudo_command
|
|
prepared = _transform_sudo_command(
|
|
_transform_macos_open_command(command),
|
|
sudo_nopasswd_check=self._sudo_nopasswd_works,
|
|
askpass_supported=self._sudo_askpass_supported,
|
|
)
|
|
note_sudo_password_sent(self, prepared[1]) # once per non-local backend class
|
|
return prepared
|
|
|
|
_sudo_askpass_supported = sudo_askpass_supported_once
|
|
_SUDO_PROBE_TIMEOUT_S = 3
|
|
|
|
def _sudo_nopasswd_works(self) -> bool:
|
|
"""``sudo -n true`` inside THIS backend (host sudo state must not leak into a sandbox).
|
|
Fails closed: any error or a timed-out probe means "assume a password is needed"."""
|
|
if not self._passwordless_sudo_probe_supported:
|
|
return False
|
|
try:
|
|
proc = self._run_bash("sudo -n true", timeout=self._SUDO_PROBE_TIMEOUT_S)
|
|
return self._wait_for_process(proc, timeout=self._SUDO_PROBE_TIMEOUT_S).get("returncode") == 0
|
|
except Exception:
|
|
return False
|