Files
Maciej Lisiewski 8d998ee865 fix(redact): unregister a routed profile's masks when every source is disabled
apply_all returns an empty report for an all-disabled secrets: block; the routed path returned before publishing, so revoked values stayed masked until restart.

Refs #77165

Co-authored-by: BearHuddleston <196404565+BearHuddleston@users.noreply.github.com>
2026-11-15 01:38:42 +01:00

1322 lines
70 KiB
Python

"""Helpers for loading Hermes .env files consistently across entrypoints."""
from __future__ import annotations
import codecs
import io
import itertools
import logging
import os
import sys
import threading
from pathlib import Path
# Kept at module level on purpose: importing this module must fail when the dotenv install is
# wiped (#57828) so early recovery provably runs before third-party imports (test_early_recovery).
# The parser internals are imported lazily below because gateway tests stub ``sys.modules["dotenv"]``.
import dotenv
from hermes_constants import hermes_home_key, is_launch_only_env_key
from hermes_cli.left_core_migration import profile_env_names
from utils import atomic_replace, mkstemp_beside
logger = logging.getLogger(__name__)
# The ONLY env vars sanitized on load: credentials must be pure ASCII (they become HTTP header values);
# arbitrary user env vars are never silently altered.
_CREDENTIAL_SUFFIXES = ("_API_KEY", "_TOKEN", "_SECRET", "_KEY")
# Once-per-process guards: load_hermes_dotenv() runs repeatedly (user + project env, gateway hot-reload,
# lazy imports mid-turn, tests) so warnings/logs fire once per key/path/home.
_WARNED_KEYS: set[str] = set() # credential names already given the non-ASCII warning
_WARNED_UTF32_PATHS: set[str] = set() # .env paths already given the UTF-32 refuse-to-mangle warning
_SCOPED_SKIP_LOGGED: set[str] = set() # routed profile homes whose multiplex dotenv skip was logged
_WARNED_UNREADABLE_PATHS: set[str] = set() # unreadable profile .env paths already given the warning
# env-var name → source label ("bitwarden", …) for externally injected credentials; setup / `hermes
# model` tell users WHERE a key came from when .env lacks it.
_SECRET_SOURCES: dict[str, str] = {}
# Every env-var name an external source SUPPLIED for some home, whether it was applied or lost to a
# pre-existing process value (``skipped_existing``). ``_SECRET_SOURCES`` is provenance metadata and only
# names applied values; the scrub that keeps a launch profile's source-supplied names out of a routed
# child must see the skipped ones too, or a name already in the process env leaks with the launch value.
_SOURCE_SUPPLIED_NAMES: set[str] = set()
# Every KEY name a dotenv file loaded into ``os.environ`` during this process's lifetime. A key removed
# or renamed in the launch ``.env`` after boot stays in ``os.environ`` (dotenv never unsets), but a
# re-parse of the current file no longer names it — so the launch-residue strip for a routed child must
# work from what was LOADED, not from what the file says now. Additive for the process lifetime.
_LOADED_DOTENV_KEYS: set[str] = set()
# KEY names loaded from the administrator-managed ``.env`` (``_apply_managed_env``). Kept OUT of the launch
# residue: those values are policy that beats the user's own ``.env`` for every profile, so a routed child
# must keep them — and keep them LAST, over the routed profile's scope (review on f5f88d5058).
_MANAGED_DOTENV_KEYS: set[str] = set()
# Immutable per-home snapshots: os.environ is shared across profiles and a later home's apply may overwrite it.
_SECRET_SOURCE_VALUES_BY_HOME: dict[str, dict[str, str]] = {}
# Per home: the subset of the snapshot a dotenv reload may re-assert — see ``AppliedVar.authoritative`` (#74265).
_SECRET_SOURCE_RESTORE_BY_HOME: dict[str, dict[str, str]] = {}
# Per home: what the process-global path WROTE into ``os.environ`` for an external source — name →
# (source name, value written, value the name held before the first write or None). The frozen launch
# env keeps its own record of the writes it holds (``launch_profile_policy._source_writes``). The per-home
# snapshot above is republished on every pass, but ``os.environ`` is not: without this record a value a
# since-removed or disabled source injected stayed in the process env, and single-profile
# ``get_secret()`` kept serving it through its ``os.environ`` fallback until restart. Deliberately NOT
# cleared by a per-home ``reset_secret_source_cache``: it describes ``os.environ``, which a reset does not
# touch, and the next pass needs it to revoke.
_SECRET_SOURCE_WRITES_BY_HOME: dict[str, dict[str, tuple[str, str, str | None]]] = {}
# Per home: YAML-bridge claims a source released when it supplied the name (applied OR skipped as equal)
# — name → (source name, bridge marker digest). Revoking that source hands the value back to the bridge
# while it is still the bridged value, or a later YAML tightening would read it as the operator's.
_RELEASED_BRIDGE_CLAIMS_BY_HOME: dict[str, dict[str, tuple[str, str]]] = {}
# HERMES_HOME paths already pulled external secrets for: load_hermes_dotenv() runs at import time from
# several hot modules, so without this the Bitwarden status line prints 3-5x per startup and the config
# re-parse + ASCII sweep re-run each time (Bitwarden's own cache only saves the network call).
_APPLIED_HOMES: set[str] = set()
_SECRET_SOURCE_CACHE_LOCK = threading.RLock()
# What THIS process has published from dotenv files, per variable: (value before our first publish, or
# None if absent; last value we published; load pass that published it). Reloads run per gateway turn and
# per cron fire, and a line like ``PATH=/x:${PATH}`` interpolated against an environ that already holds the
# previous reload's output grows by ``/x:`` every time until child spawns fail with E2BIG (#109902). A new
# pass resolves against the baseline instead — but only where the environ still holds exactly what we
# published, so a value the shell, config bridge, or an external secret source changed since is not frozen.
# One process-wide record (not per home/project scope): the environ is process-wide, so alternating
# callers (gateway with a project .env, cron without; home A then B) must peel each other's output too.
_DOTENV_PUBLISHED: dict[str, tuple[str | None, str, int]] = {}
_DOTENV_PASSES = itertools.count()
_DOTENV_LOCK = threading.RLock()
# Per-process credentials a parent mints and injects into the child's environment (the Desktop shell /
# a link-style launcher spawns `hermes dashboard` with a fresh HERMES_DASHBOARD_SESSION_TOKEN and keeps
# the same token for its own /api probes). They are never .env configuration, so a persisted value in
# ~/.hermes/.env must not replace an injected one — the parent would then 401 against its own child
# (#115955). A value an earlier dotenv pass published is still reloaded normally.
_SPAWN_CREDENTIAL_KEYS: frozenset[str] = frozenset({"HERMES_DASHBOARD_SESSION_TOKEN"})
# Behavioral routing keys a parent Hermes process injects into child env that silently redirect a profile
# onto the wrong provider path; these — and ONLY these — are scrubbed at startup when absent from the
# profile's .env. Credentials are excluded: shell exports are a documented way to supply them, and
# read-time secret-scope checks (agent/secret_scope.py) own cross-profile credential isolation.
# Providers that left core keep theirs listed on their ``LEFT_CORE`` row (``profile_env``), read statically:
# this runs before provider discovery, and with the plugin absent nothing else names them.
_PROFILE_MANAGED_ENV_KEYS: frozenset[str] = frozenset({
"HERMES_ACP_AUTH_METHOD", "HERMES_ACP_AUTO_APPROVE",
}) | profile_env_names()
def _env_keys_defined_in_dotenv(path: Path) -> set[str]:
"""KEY names assigned in a dotenv file (including empty ``KEY=``), via the same tokenizer that installs
profile scopes — a key the installer sees is a key the dashboard scrub sees (BOM'd first line included)."""
from agent.secret_scope import load_env_file
return set(load_env_file(path))
def _clear_known_keys_missing_from_dotenv(path: Path) -> None:
"""After ``.env`` loaded with override, delete inherited ``_PROFILE_MANAGED_ENV_KEYS`` it does not
define. Deliberately NARROW: only keys that change *which provider path* is used.
Does **not** run when the ``.env`` file does not exist (bare-profile case, which follows ``#66930`` /
``#67027`` semantics).
"""
if not path.exists():
return
defined = _env_keys_defined_in_dotenv(path)
for key in _PROFILE_MANAGED_ENV_KEYS:
if key not in defined and key in os.environ:
del os.environ[key]
def get_secret_source(env_var: str) -> str | None:
"""Source label that supplied ``env_var`` (``"bitwarden"`` …), None for .env/shell keys. Metadata only —
never authorization to persist the raw value."""
return _SECRET_SOURCES.get(env_var)
def _record_supplied_names(report) -> set[str]:
"""Every name *report*'s sources supplied — applied, or skipped because a value already existed —
recorded into ``_SOURCE_SUPPLIED_NAMES`` so the routed-child scrub knows the source owns it."""
supplied = set(report.provenance)
for src in report.sources:
supplied.update(src.skipped_existing)
_SOURCE_SUPPLIED_NAMES.update(supplied)
return supplied
def secret_source_names() -> tuple[str, ...]:
"""Every env-var name some profile's external secret source APPLIED (names only — the map is
process-wide, so a value must be resolved through the active profile's secret scope). Consumers that
forward source values into a child (MCP stdio env) want exactly these; see ``source_supplied_names``
for the wider set the routed-child scrub needs."""
return tuple(_SECRET_SOURCES)
def source_supplied_names() -> tuple[str, ...]:
"""Every env-var name an external source supplied for any home — applied, or lost to a pre-existing
process value (``skipped_existing``). The launch value in ``os.environ`` is still not a routed
profile's to inherit, so the strip must see the skipped names too."""
return tuple(sorted(set(_SECRET_SOURCES) | _SOURCE_SUPPLIED_NAMES))
def launch_dotenv_keys() -> frozenset[str]:
"""KEY names any NON-managed dotenv file loaded into this process's ``os.environ`` so far (see
``_LOADED_DOTENV_KEYS``); the launch profile's residue set for routed children."""
return frozenset(_LOADED_DOTENV_KEYS)
def dotenv_published_values(name: str) -> frozenset[str]:
"""The values dotenv loads in this process last wrote to ``name``, its name matched case-insensitively:
on Windows ``https_proxy`` in a .env and an inherited ``HTTPS_PROXY`` are one environment variable."""
folded = name.upper()
with _DOTENV_LOCK:
return frozenset(value for key, (_base, value, _pass) in _DOTENV_PUBLISHED.items() if key.upper() == folded)
def has_nonblank_dotenv_value(name: str) -> bool:
"""Whether dotenv supplied an override, not merely an empty assignment later filled by YAML.
A known key without a publication receipt is conservatively treated as explicit provenance.
On Windows the name matches case-insensitively: a ``telegram_reactions`` line in ``.env`` and
``TELEGRAM_REACTIONS`` are one environment variable there, and two distinct ones elsewhere.
"""
from hermes_platform.host.facts import os_family
fold = str.upper if os_family() == "win32" else str
with _DOTENV_LOCK:
keys = [key for key in (*_LOADED_DOTENV_KEYS, *_MANAGED_DOTENV_KEYS) if fold(key) == fold(name)]
if not keys:
return False
published = [_DOTENV_PUBLISHED[key] for key in keys if key in _DOTENV_PUBLISHED]
return not published or any(record[1].strip() for record in published)
def managed_dotenv_keys() -> frozenset[str]:
"""KEY names the administrator-managed ``.env`` loaded (see ``_MANAGED_DOTENV_KEYS``). Policy for
every profile: never stripped from a routed child, and re-applied over the routed scope."""
return frozenset(_MANAGED_DOTENV_KEYS)
def env_provenance() -> dict[str, list[str]]:
"""Names only: which keys in ``os.environ`` this process's sources, dotenvs and managed ``.env`` put
there. A fresh interpreter handed this environ inherits the values but not where they came from, so
one that builds routed-child envs (the update's abort recovery) adopts this with ``adopt_env_provenance``."""
return {"source_supplied": list(source_supplied_names()), "dotenv_loaded": sorted(_LOADED_DOTENV_KEYS),
"managed": sorted(_MANAGED_DOTENV_KEYS)}
def adopt_env_provenance(provenance: dict[str, list[str]]) -> None:
"""Record the parent's ``env_provenance`` as this process's own (additive, like a load)."""
_SOURCE_SUPPLIED_NAMES.update(provenance.get("source_supplied", ()))
_LOADED_DOTENV_KEYS.update(provenance.get("dotenv_loaded", ()))
_MANAGED_DOTENV_KEYS.update(provenance.get("managed", ()))
def get_secret_source_values(hermes_home: str | os.PathLike) -> dict[str, str]:
"""Return the external-secret value snapshot for ``hermes_home``."""
return dict(_SECRET_SOURCE_VALUES_BY_HOME.get(hermes_home_key(hermes_home), {}))
def hydrate_profile_secret_sources(hermes_home: str | os.PathLike) -> dict[str, str]:
"""Resolve one profile's configured sources without mutating ``os.environ``: multiplex gateways route
turns to profiles that never ran the process-global dotenv path, so resolve against a private mapping
seeded from that ``.env`` and record the per-home snapshot for ``build_profile_secret_scope()``.
Fail-open / once-per-home like ``_apply_external_secret_sources``; never returns plaintext .env entries.
The one fail-closed case is D32's: under the remote config backend, a source that supplies a
protected name raises :class:`ConfigBackendUnavailable` and records nothing for the home."""
with _SECRET_SOURCE_CACHE_LOCK:
return _hydrate_profile_secret_sources(Path(hermes_home))
def _hydrate_profile_secret_sources(home: Path) -> dict[str, str]:
"""Locked implementation for :func:`hydrate_profile_secret_sources`."""
from agent.safe_worker_policy import worker_config_snapshot
if worker_config_snapshot() is not None:
return {}
home_key = hermes_home_key(home)
if home_key in _APPLIED_HOMES:
return get_secret_source_values(home)
# A retry must not keep serving a partial result after the source is removed, disabled, or can no
# longer be evaluated. Publish only the snapshot established by this attempt.
_SECRET_SOURCE_VALUES_BY_HOME.pop(home_key, None)
_SECRET_SOURCE_RESTORE_BY_HOME.pop(home_key, None)
try:
cfg = _load_secrets_config(home)
except Exception:
return {}
if not cfg: # removed: its earlier values are no longer applied, so stop masking them
from agent.redact_secret_sources import publish_secret_source_values
publish_secret_source_values(home, {})
return {}
try:
from agent.secret_scope import _is_global_env, load_env_file
from agent.secret_sources.registry import apply_all
local_env = {name: value for name, value in os.environ.items() if _is_global_env(name)}
local_env.update(load_env_file(home / ".env"))
# Mirror load_hermes_dotenv()'s .op.env bootstrap (1Password token lives in gitignored .op.env)
# or cold profiles fail 1Password hydration. .env wins.
# Without seeding it here a cold profile configured for the supported .op.env flow fails 1Password
# hydration (sweeper review on #74549). .env values win — never override an existing key.
op_env = home / ".op.env"
if op_env.exists():
for _name, _value in load_env_file(op_env).items():
local_env.setdefault(_name, _value)
local_env["HERMES_HOME"] = str(home)
env_before = dict(local_env)
report = apply_all(cfg, home, environ=local_env)
except Exception:
return {}
if not report.sources: # every source disabled: nothing is applied any more, so stop masking it
from agent.redact_secret_sources import publish_secret_source_values
publish_secret_source_values(home, {})
return {}
# D32, as on the process-global path: a source must never supply the config backend's own
# deployment or plane credential, for a routed profile either (this snapshot feeds that
# profile's secret scope, which the Portal/plane resolvers read). Checked before anything of
# this attempt is recorded: the remote backend refuses the profile (raises, nothing cached, so
# every retry refuses again until the mapping is removed); the file backend drops the selector.
from hermes_cli.config_backend import get_config_backend
_refuse_protected_env_from_sources(report, get_config_backend(), env_before, env=local_env, routed_home=home)
# Routed profiles have no runtime reset path. Keep a failed source retryable so correcting its
# profile-local bootstrap credentials takes effect on the next turn; successful sources from a
# mixed report are still snapshotted below and can be used while the failed source recovers.
if all(src.result.ok for src in report.sources):
_APPLIED_HOMES.add(home_key)
# Same ownership bookkeeping as the process-global path: a name this profile's source supplied — applied,
# or skipped because the private mapping already had it — is a source-owned name the routed-child scrub
# must know about, or a sibling still inherits the launch value for it (review on f5f88d5058).
_record_supplied_names(report)
values: dict[str, str] = {}
for name, applied in report.provenance.items():
value = local_env.get(name)
if value is None:
continue
_SECRET_SOURCES[name] = applied.source
values[name] = value
_SECRET_SOURCE_VALUES_BY_HOME[home_key] = values
from agent.redact_secret_sources import publish_secret_source_values
publish_secret_source_values(home, values)
return dict(values)
def reset_secret_source_cache(hermes_home: str | os.PathLike | None = None) -> None:
"""Forget applied homes so the next load re-pulls (tests, long-running processes after config edits).
``hermes_home`` limits the reset to ONE home: a multiplex gateway keeps every profile's snapshot in
this process, and a per-fire cron re-pull or a plugin-discovery refresh for one home must not wipe
a sibling's hydrated snapshot — the sibling's next scope build would run empty until it re-hydrated
(#102041)."""
if hermes_home is None:
_APPLIED_HOMES.clear()
_SECRET_SOURCES.clear()
_SOURCE_SUPPLIED_NAMES.clear()
_SECRET_SOURCE_VALUES_BY_HOME.clear()
_SECRET_SOURCE_RESTORE_BY_HOME.clear()
_SECRET_SOURCE_WRITES_BY_HOME.clear()
_RELEASED_BRIDGE_CLAIMS_BY_HOME.clear()
return
home_key = hermes_home_key(hermes_home)
_APPLIED_HOMES.discard(home_key)
_SECRET_SOURCE_VALUES_BY_HOME.pop(home_key, None)
_SECRET_SOURCE_RESTORE_BY_HOME.pop(home_key, None)
def format_secret_source_suffix(env_var: str) -> str:
"""``" (from Bitwarden)"``-style suffix; ``""`` for .env/shell keys (only external sources are named)."""
source = get_secret_source(env_var)
if not source:
return ""
if source == "bitwarden":
return " (from Bitwarden)"
# Registry label (e.g. "1Password"); raw name for unknown sources (uninstalled plugin, tests).
try:
from agent.secret_sources.registry import get_source
registered = get_source(source)
if registered is not None and registered.label:
return f" (from {registered.label})"
except Exception:
pass
return f" (from {source})"
def _format_offending_chars(value: str, limit: int = 3) -> str:
"""Compact ``U+XXXX ('c'), ...`` summary of non-ASCII codepoints."""
seen: list[str] = []
for ch in value:
if ord(ch) > 127:
label = f"U+{ord(ch):04X}"
if ch.isprintable():
label += f" ({ch!r})"
if label not in seen:
seen.append(label)
if len(seen) >= limit:
break
return ", ".join(seen)
def _sanitize_loaded_credentials() -> None:
"""Strip non-ASCII from credential env vars (``_CREDENTIAL_SUFFIXES``) so the codebase never sees them.
Emits a one-line warning to stderr when characters are stripped. Silent stripping would mask copy-paste
corruption (Unicode lookalike glyphs from PDFs / rich-text editors, ZWSP from web pages) as opaque
provider-side "invalid API key" errors (see #6843).
"""
for key, value in list(os.environ.items()):
if not any(key.endswith(suffix) for suffix in _CREDENTIAL_SUFFIXES):
continue
if value.isascii():
continue
cleaned = value.encode("ascii", errors="ignore").decode("ascii")
os.environ[key] = cleaned
if key in _WARNED_KEYS:
continue
_WARNED_KEYS.add(key)
stripped = len(value) - len(cleaned)
detail = _format_offending_chars(value) or "non-printable"
print(f" Warning: {key} contained {stripped} non-ASCII character"
f"{'s' if stripped != 1 else ''} ({detail}) — stripped so the "
f"key can be sent as an HTTP header.", file=sys.stderr)
print(
" This usually means the key was copy-pasted from a PDF, "
"rich-text editor, or web page that substituted lookalike\n"
" Unicode glyphs for ASCII letters. If authentication fails "
"(e.g. \"API key not valid\"), re-copy the key from the\n"
" provider's dashboard and run `hermes setup` (or edit the "
".env file in a plain-text editor).",
file=sys.stderr,
)
def _parse_dotenv_file(path: Path) -> list[tuple[str, str | None]]:
"""``(name, raw value)`` assignments of one dotenv file, uninterpolated, in file order."""
raw = path.read_bytes()
try:
# utf-8-sig strips a leading BOM (PowerShell 5.1 / Notepad); plain utf-8 would keep U+FEFF on the
# first key name and silently drop it from os.environ under its canonical name.
text = raw.decode("utf-8-sig")
except UnicodeDecodeError:
raw = raw.removeprefix(codecs.BOM_UTF8)
text = raw.decode("latin-1")
# Imported here, not at module level: gateway tests stub ``sys.modules["dotenv"]`` with a bare module
# exposing only ``load_dotenv``, and ``gateway.run`` imports this module at import time.
from dotenv.main import DotEnv
return list(DotEnv(dotenv_path=None, stream=io.StringIO(text), interpolate=False).parse())
def _dotenv_lookup_env(load_pass: int | None) -> dict[str, str]:
"""``os.environ`` with every value an earlier pass (any pass, for ``None``) published peeled back to
its baseline — what ``${VAR}`` resolves against (see ``_DOTENV_PUBLISHED``). Caller holds the lock."""
lookup_env = dict(os.environ)
for name, (baseline, published, published_pass) in _DOTENV_PUBLISHED.items():
if published_pass != load_pass and lookup_env.get(name) == published:
if baseline is None:
del lookup_env[name] # absent, so ``${VAR:-default}`` takes the default again
else:
lookup_env[name] = baseline
return lookup_env
def _resolve_dotenv_assignments(
assignments: list[tuple[str, str | None]], lookup_env: dict[str, str], *, override: bool,
) -> dict[str, str | None]:
"""Expand ``${VAR}`` / ``${VAR:-default}`` like ``dotenv.main.resolve_variables``, minus the live
``os.environ``: a line sees *lookup_env* and the file's earlier lines (earlier lines win under
*override*, the environment otherwise)."""
from dotenv.variables import parse_variables
resolved: dict[str, str | None] = {}
for name, value in assignments:
if value is not None:
lookup = {**lookup_env, **resolved} if override else {**resolved, **lookup_env}
value = "".join(atom.resolve(lookup) for atom in parse_variables(value))
resolved[name] = value
return resolved
def resolve_home_dotenv(home: Path, *, lookup_env: dict[str, str] | None = None) -> dict[str, str]:
"""What ``load_hermes_dotenv`` would publish from *home*'s dotenv layers, read-only: the home's
``.env`` then the administrator-managed ``.env``, both with override and ``${VAR}`` expanded by the
same rules against the process's own environment (this process's dotenv output peeled back), each
layer seeing the one before. External secret sources and the project ``.env`` are not included.
*lookup_env* replaces that environment: another profile's file must not resolve against this one's."""
from hermes_cli import managed_scope
managed_dir = managed_scope.get_managed_dir()
layers = [home / ".env", *([managed_dir / ".env"] if managed_dir is not None else [])]
if lookup_env is not None:
lookup_env = dict(lookup_env)
else:
with _DOTENV_LOCK:
lookup_env = _dotenv_lookup_env(None)
out: dict[str, str] = {}
for path in layers:
try:
assignments = _parse_dotenv_file(path)
except OSError:
continue
for name, value in _resolve_dotenv_assignments(assignments, lookup_env, override=True).items():
if value is not None:
lookup_env[name] = out[name] = value
return out
def _release_bridge_claim(name: str) -> None:
"""An operator source now owns *name*: drop the YAML bridge's marker for it, or a value equal to an
earlier bridged one would still read as the bridge's and be scrubbed from child envs."""
from gateway.bridged_env import BRIDGED_ENV_MARKER_PREFIX
os.environ.pop(BRIDGED_ENV_MARKER_PREFIX + name, None)
def _load_dotenv_with_fallback(
path: Path, *, override: bool, load_pass: int | None = None, managed: bool = False,
) -> None:
"""Load one dotenv file into ``os.environ`` like ``dotenv.load_dotenv`` — same parser, same
``${VAR}`` / ``${VAR:-default}`` / precedence rules — except that ``${VAR}`` resolves against the value
VAR had before this process's earlier passes published it (see ``_DOTENV_PUBLISHED``).
``load_pass`` groups the layered files of one ``load_hermes_dotenv`` call: within a pass a later layer
(project, managed) still sees the earlier layer's output, as it always did; only OTHER passes' output
is peeled. A bare call (``hermes send``'s direct reload) is its own pass."""
assignments = _parse_dotenv_file(path)
with _DOTENV_LOCK:
if load_pass is None:
load_pass = next(_DOTENV_PASSES)
resolved = _resolve_dotenv_assignments(assignments, _dotenv_lookup_env(load_pass), override=override)
for name, value in resolved.items():
if value is None or (not override and name in os.environ):
continue
if is_launch_only_env_key(name):
_warn_launch_only_key(f"dotenv {path}", name)
continue # deployment bootstrap knob (#135200, #144685)
current = os.environ.get(name)
record = _DOTENV_PUBLISHED.get(name)
ours = record is not None and current == record[1]
if name in _SPAWN_CREDENTIAL_KEYS and current and not ours:
continue # parent-minted per-process credential: .env must not split it from the parent
# Ours and untouched since → keep the original baseline; anything else is a newer outside value.
baseline = record[0] if ours else current
os.environ[name] = value
_DOTENV_PUBLISHED[name] = (baseline, value, load_pass)
if value.strip():
_release_bridge_claim(name)
# Every key this file defines, for the launch-residue strip: dotenv never unsets, so a key later
# removed from the launch .env stays in os.environ and a re-parse of the file no longer names it.
# Managed keys are recorded separately: they are administrator policy, not launch-profile residue.
# Launch-only keys are recorded nowhere: they were never published, so a leftover file entry must
# not strip the launch value from a routed child either (#135200).
(_MANAGED_DOTENV_KEYS if managed else _LOADED_DOTENV_KEYS).update(
name for name, _value in assignments if not is_launch_only_env_key(name))
_sanitize_loaded_credentials() # httpx encodes headers as ASCII
_LAUNCH_ONLY_WARNED: set[tuple[str, str, str]] = set()
def _warn_launch_only_key(source: str, name: str) -> None:
"""Log a skipped launch-only key once per (profile home, source, name): the gateway reloads
``.env`` every turn, and a multiplex process serves several homes that each deserve the warning."""
marker = (hermes_home_key(), source, name)
if marker not in _LAUNCH_ONLY_WARNED:
_LAUNCH_ONLY_WARNED.add(marker)
logger.warning("%s: ignoring launch-only key %s — only the launch environment may set it",
source, name)
def bridge_config_scalars(cfg: dict, target) -> None:
"""Copy top-level ``config.yaml`` scalars into ``target`` (env or scope mapping) where unset.
Launch-only keys are refused like a ``.env`` entry: config.yaml is user-writable, so a
``HERMES_MANAGED_DIR:`` line there must not repoint the managed scope (#135200, #144685)."""
for name, value in cfg.items():
if not isinstance(value, (str, int, float, bool)) or name in target:
continue
if is_launch_only_env_key(name):
_warn_launch_only_key("config.yaml", name)
continue
target[name] = str(value)
def _dotenv_assignments(path: Path) -> list:
"""``(name, raw value)`` pairs of one dotenv file as the loader will see it once
``_sanitize_env_file_if_needed`` has run: :func:`_parse_dotenv_file` plus the sanitizer's UTF-16
rewrite and NUL strip, for the pre-boot bootstrap reads that run before the sanitizer."""
raw = path.read_bytes()
if raw.startswith(codecs.BOM_UTF16_LE) or raw.startswith(codecs.BOM_UTF16_BE):
# Notepad "Unicode": ``_sanitize_env_file_if_needed`` rewrites it as UTF-8 before the loader
# reads it, so parse it as that rewrite will -- bootstrap / stage2 run before the sanitizer
# and must select the same backend the runtime then boots.
try:
with io.TextIOWrapper(io.BytesIO(raw), encoding="utf-16", newline=None) as f:
text = f.read()
except UnicodeDecodeError:
text = raw.decode("latin-1")
else:
try:
# utf-8-sig strips a leading BOM (PowerShell 5.1 / Notepad); plain utf-8 would keep U+FEFF on the
# first key name and silently drop it from os.environ under its canonical name.
text = raw.decode("utf-8-sig")
except UnicodeDecodeError:
raw = raw.removeprefix(codecs.BOM_UTF8)
text = raw.decode("latin-1")
text = text.replace("\x00", "") # the sanitizer strips NULs too (BOM-less UTF-16 / NUL-padded ASCII)
# Imported here, not at module level: gateway tests stub ``sys.modules["dotenv"]`` with a bare module
# exposing only ``load_dotenv``, and ``gateway.run`` imports this module at import time.
from dotenv.main import DotEnv
return list(DotEnv(dotenv_path=None, stream=io.StringIO(text), interpolate=False).parse())
def _bootstrap_env(home: Path | None = None, project_env: Path | None = None) -> dict:
"""The process env as ``load_hermes_dotenv`` leaves it for the names its dotenv layers define:
the home's ``.env`` (override), ``.op.env`` (fills gaps, only without a process
``OP_SERVICE_ACCOUNT_TOKEN``), the caller's *project_env* (fills gaps when the home has a
``.env``, else overrides) and the managed ``.env`` (override, last); ``${VAR}`` resolved, the
loader's parser. Read-only: nothing is published and no file is rewritten (sanitizing imports
hermes_cli.config, whose import-time config read must not run before the backend is decided)."""
home = Path(home) if home is not None else _process_hermes_home()
env = dict(os.environ)
user_env = home / ".env"
layers: list[tuple[Path, bool]] = [(user_env, True)]
if not os.environ.get("OP_SERVICE_ACCOUNT_TOKEN"):
layers.append((home / ".op.env", False))
if project_env is not None:
layers.append((Path(project_env), not user_env.exists()))
try:
from hermes_cli import managed_scope
managed_dir = managed_scope.get_managed_dir()
except Exception: # health: allow BLE001 -- managed scope never blocks startup, as in _apply_managed_env
managed_dir = None
if managed_dir is not None:
layers.append((managed_dir / ".env", True))
for path, override in layers:
try:
assignments = _dotenv_assignments(path)
except OSError:
continue
resolved = _resolve_dotenv_assignments(assignments, env, override=override)
for name, value in resolved.items():
if value is not None and (override or name not in env):
env[name] = value
return env
def selected_config_backend_name(home: Path | None = None) -> str:
"""The config backend ``load_hermes_dotenv`` selects for *home*, decided without loading or
booting anything. For a caller that must decide before Hermes starts (Docker stage2 skips the
config-file migration in remote mode, D12)."""
from hermes_cli.config_backend import BACKEND_ENV
return (_bootstrap_env(home).get(BACKEND_ENV) or "").strip().lower() or "file"
def apply_config_bootstrap_env(backend_names, home: Path | None = None, project_env: Path | None = None) -> None:
"""Publish ``backend_names(kind)`` (the selected backend's deployment and credential, D26/D32) from the
launch home's ``.env`` and the managed ``.env``, with the values ``load_hermes_dotenv`` gives
them. Called before the first config read of a process, which can come before
``load_hermes_dotenv`` (``hermes_cli.config`` reads config at import time): the backend is
selected and authenticated from the same deployment either way. A file deployment gets only its
selector (the file backend needs nothing else before boot); values the process env holds and
no file defines are left alone; ``load_hermes_dotenv`` later publishes the same values."""
from agent.safe_worker_policy import worker_config_snapshot
from hermes_cli.config_backend import BACKEND_ENV
# A frozen-policy worker reads no profile dotenv (``load_hermes_dotenv`` returns early there):
# its bootstrap supplied the process env explicitly, so nothing is published from the files.
if worker_config_snapshot() is not None:
return
env = _bootstrap_env(home, project_env)
# The selector itself always, so a file deployment is not read as another kind meanwhile.
names = {BACKEND_ENV} | set(backend_names((env.get(BACKEND_ENV) or "").strip().lower() or "file"))
for name in names:
value = env.get(name)
if value is not None and os.environ.get(name) != value:
os.environ[name] = value
def _sanitize_env_file_if_needed(path: Path) -> None:
"""Pre-sanitize a .env file before python-dotenv reads it. Sniffs a leading BOM *before* any text
decode: UTF-16 (Notepad "Unicode") is rewritten as clean UTF-8; UTF-32 is refused (left untouched) so
we never fall through to the errors=replace corruption path."""
if not path.exists():
return
try:
from hermes_cli.config import _sanitize_env_lines
except ImportError:
return # early bootstrap — config module not available yet
try:
raw = path.read_bytes()
except Exception:
return
# ORDER MATTERS: BOM_UTF32_LE (FF FE 00 00) startswith BOM_UTF16_LE (FF FE); UTF-16 first would mangle it.
force_utf8_rewrite = False
if raw.startswith(codecs.BOM_UTF32_LE) or raw.startswith(codecs.BOM_UTF32_BE):
# Lazy import keeps the module import block identical to #65124's codecs/io additions so the two PRs
# auto-merge either order.
path_key = str(path.resolve())
if path_key not in _WARNED_UTF32_PATHS:
_WARNED_UTF32_PATHS.add(path_key)
logger.warning("Skipping .env sanitize for %s: UTF-32 BOM detected; "
"leaving file untouched to avoid corruption", path)
return
if raw.startswith(codecs.BOM_UTF16_LE) or raw.startswith(codecs.BOM_UTF16_BE):
# "utf-16" uses the BOM for endianness and strips it; newline=None matches open()'s universal
# newlines (not splitlines()'s extra boundaries like U+2028) so sanitize sees the same lines.
try:
with io.TextIOWrapper(io.BytesIO(raw), encoding="utf-16", newline=None) as f:
original = f.readlines()
except UnicodeDecodeError:
return
force_utf8_rewrite = True # always rewrite UTF-16 as UTF-8 so the dotenv load sees a canonical file
else:
# utf-8-sig strips a UTF-8 BOM; errors=replace so embedded NULs can be stripped below.
try:
with open(path, encoding="utf-8-sig", errors="replace") as f:
original = f.readlines()
except Exception:
return
# errors=replace turns undecodable leading bytes into U+FFFD; persisting would glue them onto
# the first key name permanently — leave the file untouched instead.
if original and original[0].startswith("\ufffd"):
return
try:
# Strip NULs (os.environ raises ValueError on them); also repairs BOM-less UTF-16 (NUL-padded ASCII).
stripped = [line.replace("\x00", "") for line in original]
sanitized = _sanitize_env_lines(stripped)
if sanitized != original or force_utf8_rewrite:
fd, tmp = mkstemp_beside(path, suffix=".tmp", prefix=".env_")
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
f.writelines(sanitized)
f.flush()
os.fsync(f.fileno())
atomic_replace(tmp, path)
except BaseException:
try:
os.unlink(tmp)
except OSError:
pass
raise
except Exception:
pass # best-effort — don't block gateway startup
def dotenv_unreadable(path: Path) -> bool:
"""True when *path* exists but this process may not read it."""
try:
with open(path, "rb"):
return False
except PermissionError:
return True
except OSError: # missing, or a failure that is not a refusal
return False
def _load_user_dotenv(path: Path, *, load_pass: int) -> bool:
"""Load the profile ``.env``; False when it exists but cannot be read and managed scope pins
``terminal.backend``. A sandbox launcher withholds the file, passes the keys it allows in as
environment and pins the backend. Without that pin the ``PermissionError`` propagates: the file may
be the only place the backend is set, and skipping it would run terminal commands on the host."""
try:
_load_dotenv_with_fallback(path, override=True, load_pass=load_pass)
except PermissionError:
from hermes_cli import managed_scope
if managed_scope.pinned_terminal_backend() is None:
raise
if str(path) not in _WARNED_UNREADABLE_PATHS:
_WARNED_UNREADABLE_PATHS.add(str(path))
logger.warning("Cannot read %s; managed scope pins terminal.backend, so it loads as absent", path)
return False
return True
def load_hermes_dotenv(
*,
hermes_home: str | os.PathLike | None = None,
project_env: str | os.PathLike | None = None,
load_external_secrets: bool = True,
) -> list[Path]:
"""Load Hermes env files: ``~/.hermes/.env`` overrides stale shell exports; project ``.env`` is a dev
fallback that only fills gaps when the user env exists (and overrides shell vars when it does not)."""
from agent.safe_worker_policy import worker_config_snapshot
# The private bootstrap supplies credentials explicitly; do not rehydrate
# profile behavior, external secret plugins, or managed env in this worker.
if worker_config_snapshot() is not None:
return []
# Process home on purpose (never the per-turn override): a startup .env load must not follow a routed
# profile — see the multiplex guard below.
from hermes_constants import get_process_hermes_home
home_path = Path(hermes_home) if hermes_home else get_process_hermes_home()
# Multiplex gateway: while a routed profile-home override is active, copying that profile's .env
# into os.environ would expose its credentials to sibling turns and every spawned child. The launch
# home's own .env is process configuration and still loads: the launch profile's scoped bodies bind
# an override naming the launch home too, and skipping it hid launch-only credentials such as a
# fallback_providers key from the process env (#125530). Both the load's target AND the active
# home must be the launch home: a launch-targeted load inside a FOREIGN turn re-bridges terminal.*
# from the config the override resolves to, i.e. the routed profile's cwd into the shared env.
# External sources still refresh against the profile mapping.
# (``is_multiplex_active()`` is also true, context-locally, for a routed cron fire in the desktop
# backend — see ``cron.scheduler_provider.routed_profile_fire``.)
from agent.secret_scope import is_multiplex_active
from hermes_constants import get_hermes_home, get_hermes_home_override
launch_home = _process_hermes_home().resolve()
if (is_multiplex_active() and get_hermes_home_override() is not None
and (home_path.resolve() != launch_home or get_hermes_home().resolve() != launch_home)):
home_key = hermes_home_key(home_path)
if home_key not in _SCOPED_SKIP_LOGGED:
_SCOPED_SKIP_LOGGED.add(home_key)
logger.debug("multiplex: skipping process-global dotenv load for routed "
"profile home %s (credentials resolve via the profile scope)", home_path)
if load_external_secrets:
from hermes_cli import _early_recovery
if not _early_recovery._should_skip_external_secret_sources():
hydrate_profile_secret_sources(home_path)
return []
loaded: list[Path] = []
user_env = home_path / ".env"
project_env_path = Path(project_env) if project_env else None
load_pass = next(_DOTENV_PASSES) # one pass: later layers below see the earlier layers' output
# Before anything below imports hermes_cli.config (the .env sanitizer does), whose import-time
# config read selects and boots the backend: publish the deployment and plane credential from
# THIS load's layers, the project .env included, with the precedence the loads below give them.
from hermes_cli.config_backend import bootstrap_deployment
bootstrap_deployment(home_path, project_env_path)
if user_env.exists(): # normalize formatting / strip NULs before parsing
_sanitize_env_file_if_needed(user_env)
if project_env_path and project_env_path.exists():
_sanitize_env_file_if_needed(project_env_path)
if user_env.exists() and _load_user_dotenv(user_env, load_pass=load_pass):
loaded.append(user_env)
_clear_known_keys_missing_from_dotenv(user_env) # mirrors reload_env(): inherited keys must not leak
# .op.env AFTER .env so .env wins, but the bootstrap OP_SERVICE_ACCOUNT_TOKEN reaches
# apply_onepassword_secrets() even in cron with no shell state; gitignored so the token never enters
# the committed .env. override=False lets a systemd `EnvironmentFile=-…/.op.env` token win.
op_env = home_path / ".op.env"
if op_env.exists() and not os.environ.get("OP_SERVICE_ACCOUNT_TOKEN"):
_load_dotenv_with_fallback(op_env, override=False, load_pass=load_pass)
if project_env_path and project_env_path.exists():
_load_dotenv_with_fallback(project_env_path, override=not user_env.exists(), load_pass=load_pass)
loaded.append(project_env_path)
# The override=True loads above wrote the raw .env line (``__BITWARDEN_MANAGED__`` placeholder, stale
# token) back over a value an external source resolved on an earlier call, and the source pass below is a
# once-per-home no-op — so the clobber stuck for the life of the process (#74265). Re-assert only what the
# source is authoritative for; managed scope, applied last with override=True, still wins on purpose.
if _SECRET_SOURCE_RESTORE_BY_HOME:
for name, value in _SECRET_SOURCE_RESTORE_BY_HOME.get(hermes_home_key(home_path), {}).items():
if os.environ.get(name) != value:
os.environ[name] = value
# The config backend is selected here — after the .env files, before the first config.yaml read
# (``_apply_external_secret_sources`` reads ``secrets:``) — so an unavailable backend stops the
# process before any reader could fall back to defaults (config-config design §4.4, D11/D32). A
# remote backend fetches this home's config here, failing closed; the file backend does nothing.
from hermes_cli.config_backend import get_config_backend
# The administrator-managed .env is part of the deployment the backend boots from (it may be
# where the selector, plane URL or instance id live). The override=True loads above may have
# replaced those names with the home's values, so re-publish the deployment names (only those)
# with the bootstrap precedence -- managed last -- before the boot. The full managed overlay
# still applies once, last below, where it keeps its precedence over secret-source results.
bootstrap_deployment(home_path, project_env_path)
get_config_backend().boot(home_path)
# External sources are skipped for the updater (dotenv + managed env still load): ``update`` must not
# import optional secret-manager libs (Bitwarden → cryptography → _rust.pyd) into the process replacing
# that env on Windows, and a fresh retry after a deferred dependency install would otherwise make the
# self-lock preflight exit 2 again.
from hermes_cli import _early_recovery
# External secret sources are skipped in two updater situations: 1. ``load_external_secrets=False`` —
# the caller is an ``update`` invocation that must not import optional secret-manager libraries
# (Bitwarden → cryptography → ``_rust.pyd``) into the process that replaces that same environment on
# Windows (#73381, #86735). 2. A fresh ``hermes update`` retry just completed a deferred dependency
# install before importing this module. Do not remap native secret-source dependencies in that same
# updater process or the self-lock preflight will recreate the marker and exit 2 again. Dotenv and
# managed env still load in both cases; only external source resolution is unnecessary for the updater.
if load_external_secrets and not _early_recovery._should_skip_external_secret_sources():
_apply_external_secret_sources(home_path)
_apply_managed_env(load_pass=load_pass)
# config.yaml owns terminal.*, but the override=True loads above let a stale TERMINAL_ENV=docker in
# ~/.hermes/.env win on every reload and flip the backend mid-session in long-lived processes.
# Re-apply the explicit terminal keys LAST, after the managed overlay, so the merged config lands.
# config.yaml is the documented source of truth for terminal.* settings, but the dotenv loads above run
# with override=True — so a stale TERMINAL_ENV=docker left in ~/.hermes/.env (e.g. written by an older
# `hermes setup` before the user switched terminal.backend in config.yaml) silently wins again on every
# reload. Startup launchers bridge config→env once, but long-lived processes (gateway per-turn reload,
# cron standalone runs) call load_hermes_dotenv() repeatedly and used to flip the effective backend back
# to the stale .env value mid-session (#29186, #67323).
_reapply_terminal_config_bridge(home_path)
return loaded
def _reapply_terminal_config_bridge(home_path: Path) -> None:
"""Re-assert config.yaml's explicit ``terminal.*`` keys over reloaded .env via the single shared bridge
``apply_terminal_config_to_env`` (also used by terminal_tool and the TUI/dashboard launchers) so the
semantics can't drift between sites."""
try:
if Path(home_path).resolve() != _process_hermes_home().resolve():
return
from hermes_cli.config import apply_terminal_config_to_env
apply_terminal_config_to_env(env=None)
except Exception:
pass
def _apply_managed_env(*, load_pass: int | None = None) -> None:
"""Apply the managed-scope .env last, with override, so it beats user/shell. Does NOT stop the agent
from later mutating os.environ (v1 relies on filesystem permissions). Fail-open: never blocks startup."""
try:
from hermes_cli import managed_scope
managed_dir = managed_scope.get_managed_dir()
except Exception:
return
if managed_dir is None:
return
managed_env = managed_dir / ".env"
if not managed_env.exists():
return
_sanitize_env_file_if_needed(managed_env)
_load_dotenv_with_fallback(managed_env, override=True, load_pass=load_pass, managed=True)
def _revoke_secret_source_writes(home_path: Path, *, keep) -> None:
"""Take back what a source wrote into ``os.environ`` for *home_path* once nothing backs it.
``keep(name, source)`` says which recorded writes are still owned. Every other write is revoked,
but only while ``os.environ`` still holds exactly the value the source wrote: a value another owner
replaced since (the profile's ``.env``, the administrator-managed ``.env``, a shell export, a later
source) is theirs and stays. A revoked name gets back the value it held before the source's first
write, or is removed when it had none."""
home_key = hermes_home_key(home_path)
writes = _SECRET_SOURCE_WRITES_BY_HOME.get(home_key)
claims = _RELEASED_BRIDGE_CLAIMS_BY_HOME.get(home_key)
# The frozen launch env's copies of this home's writes, revoked by their own record: the
# os.environ record may be gone or retargeted since (another writer, a failed fetch).
policy = _launch_policy()
if policy is not None:
for name, source in policy._launch_env_source_writes(home_key).items():
if not keep(name, source):
policy._revoke_launch_env_write(home_key, name)
if not writes and not claims:
return
try:
from agent.secret_scope import load_env_file
owned_by_dotenv = set(load_env_file(Path(home_path) / ".env")) | _MANAGED_DOTENV_KEYS
except Exception:
owned_by_dotenv = set(_MANAGED_DOTENV_KEYS)
_revoke_value_writes(home_key, writes or {}, keep, owned_by_dotenv)
_restore_bridge_claims(home_key, claims or {}, keep, owned_by_dotenv)
def _restore_bridge_claims(home_key: str, claims: dict, keep, owned_by_dotenv: set) -> None:
"""Give a revoked source's name back to the YAML bridge when it again holds the bridged value."""
from gateway.bridged_env import BRIDGED_ENV_MARKER_PREFIX, bridged_env_digest
kept = {name: claim for name, claim in claims.items() if keep(name, claim[0])}
for name, (_source, marker) in claims.items():
if name not in kept and name not in owned_by_dotenv and bridged_env_digest(os.environ.get(name)) == marker:
os.environ[BRIDGED_ENV_MARKER_PREFIX + name] = marker
if kept:
_RELEASED_BRIDGE_CLAIMS_BY_HOME[home_key] = kept
else:
_RELEASED_BRIDGE_CLAIMS_BY_HOME.pop(home_key, None)
def _revoke_value_writes(home_key: str, writes: dict, keep, owned_by_dotenv: set) -> None:
if not writes:
return
kept: dict[str, tuple[str, str, str | None]] = {}
for name, (source, value, prior) in writes.items():
if keep(name, source):
kept[name] = (source, value, prior)
continue
if name not in owned_by_dotenv and os.environ.get(name) == value:
if prior is None:
os.environ.pop(name, None)
else:
os.environ[name] = prior
still_attributed = any(
other.get(name, ("",))[0] == source
for key, other in _SECRET_SOURCE_WRITES_BY_HOME.items() if key != home_key)
if _SECRET_SOURCES.get(name) == source and not still_attributed:
_SECRET_SOURCES.pop(name, None)
if kept:
_SECRET_SOURCE_WRITES_BY_HOME[home_key] = kept
else:
_SECRET_SOURCE_WRITES_BY_HOME.pop(home_key, None)
def _refuse_protected_env_from_sources(report, backend=None, environ_before=None, env=None, *, routed_home=None) -> None:
"""D32: the config backend's own credential (which plane, which agent, which token) must never
come from a secret source — a source that could change it could point config at another plane.
Exits (the backend is unusable), even when the pre-existing value won.
*backend* is the backend selected BEFORE the sources ran: judging by the backend selected after
them would let a source that writes ``HERMES_CONFIG_BACKEND=file`` switch remote mode off and
disarm this very check. The selector itself is never a source's to change, under any backend:
a source's write to it is reverted in *env* (the mapping the sources wrote into; default
``os.environ``) to its *environ_before* value before anything else.
Startup runs the sources against a private staging copy of the environment and calls this
BEFORE publishing it (:func:`_publish_staged_env`), so a refused value is never visible to a
concurrent request — the poller reads the plane URL and credential live from ``os.environ``."""
if env is None:
env = os.environ
from hermes_cli.config_backend import BACKEND_ENV, ConfigBackendUnavailable, get_config_backend
if backend is None:
backend = get_config_backend()
supplied = set(report.provenance)
for src in report.sources:
supplied.update(getattr(src, "skipped_existing", ()) or ())
protected = backend.protected_env_names() | {BACKEND_ENV}
clash = sorted(supplied & protected)
if not clash:
return
if environ_before is not None: # undo the source's writes to protected names before deciding
for name in clash:
if name in environ_before:
env[name] = environ_before[name]
else:
env.pop(name, None)
if not backend.protected_env_names():
# The file backend has no plane credential: only the selector clashed. Reverted, and dropped
# from the report so no provenance/snapshot re-applies it later; keep going.
for name in clash:
report.provenance.pop(name, None)
for src in report.sources:
for bucket in (getattr(src, "applied", None), getattr(src, "skipped_existing", None)):
if isinstance(bucket, list) and name in bucket:
bucket.remove(name)
print(f" Secret sources: ignored {BACKEND_ENV} from a secrets: source (the config backend is "
"chosen by .env or the process environment only).", file=sys.stderr)
return
outcome = (f"Hermes does not serve profile home {routed_home} with it." if routed_home is not None
else "Hermes does not start with it.")
raise ConfigBackendUnavailable(
f"A secrets: source supplies {', '.join(clash)}, which the {backend.name!r} config "
"backend uses to reach its config plane. That credential must come from auth.json or .env, never "
f"from a secret source; remove the mapping. {outcome}")
def _publish_staged_env(staged: dict, environ_before: dict) -> None:
"""Copy into ``os.environ`` exactly the names the sources set in *staged* (value differs from
*environ_before*). Sources only ever set names, so nothing is removed; a name nobody's source
touched is left alone even if another thread changed it meanwhile."""
for name, value in staged.items():
if environ_before.get(name) != value:
os.environ[name] = value
_release_bridge_claim(name)
def _apply_external_secret_sources(home_path: Path) -> None:
"""Pull secrets from every enabled external source into env — AFTER dotenv (sources need .env bootstrap
tokens), BEFORE Hermes reads credentials; failures never block startup. Precedence/conflicts/provenance
live in ``registry.apply_all``; this wrapper owns the once-per-home guard, the post-apply ASCII sweep,
the ``_SECRET_SOURCES`` map and status lines."""
home_key = hermes_home_key(home_path)
if home_key in _APPLIED_HOMES:
return
# Neither early return marks the home applied: a malformed config.yaml would otherwise permanently
# disable secret loading for this process, and an unmarked home picks up a config change on the next
# load (the signature-cached read is cheap).
try:
cfg = _load_secrets_config(home_path)
except Exception:
# See #40597.
return
# No source configured / enabled any more: whatever one wrote earlier is no longer backed (#126982
# review). A config READ failure above keeps them — fail-open, same as the fetch-failure path below.
if not cfg:
_revoke_secret_source_writes(home_path, keep=lambda _name, _source: False)
_publish_live_source_values(home_key, home_path, {})
return
# Defer the registry import until a source is enabled — bitwarden eagerly loads cryptography._rust.pyd,
# which makes the Windows updater self-lock before its preflight. Detect by *shape* (dict with enabled
# flag), not names, so plugin/test sources pass and a plain dict entry never forces the crypto load.
any_enabled = any(isinstance(v, dict) and v.get("enabled") is True for v in cfg.values())
if not any_enabled:
_revoke_secret_source_writes(home_path, keep=lambda _name, _source: False)
_publish_live_source_values(home_key, home_path, {})
return
try:
from agent.secret_sources.registry import apply_all, enabled_source_names
except ImportError:
return
# Revoke a removed/disabled source's writes BEFORE the pass: left in place, its stale value would make
# a still-enabled source that supplies the same name skip it as pre-existing (``skipped_existing``).
try:
active = enabled_source_names(cfg, home_path)
except Exception:
active = None
if active is not None:
_revoke_secret_source_writes(home_path, keep=lambda _name, source: source in active)
environ_before = dict(os.environ)
# Read the capture state AFTER the copy (and before the publish: see _record_secret_source_writes):
# an activation landing first is then seen as captured, so an adopted prior always predates capture.
policy = _launch_policy()
launch_env_captured = policy is not None and policy._launch_env_captured()
# Latched before the sources run: a source must not be able to re-select the backend (D32).
from hermes_cli.config_backend import get_config_backend
backend_before = get_config_backend()
# The sources write into a private staging copy, published only after the D32 check: written
# straight into os.environ, a forbidden plane URL or credential would be live for any request
# (the config poller, another thread's write) until the check reverted it.
staged = dict(environ_before)
try:
report = apply_all(cfg, home_path, environ=staged)
except Exception:
return
if not report.sources: # no source enabled: keep retrying cheaply so flipping one on takes effect
_publish_live_source_values(home_key, home_path, {})
return
_refuse_protected_env_from_sources(report, backend_before, environ_before, env=staged)
_publish_staged_env(staged, environ_before)
# A real fetch attempt happened (success OR error): mark the home so the 3-5 import-time calls per
# startup don't re-fetch / re-print (error retries are opt-in via reset_secret_source_cache()).
# Marking AFTER the attempt keeps the earlier failure paths retryable.
_APPLIED_HOMES.add(home_key)
if report.applied_any:
_sanitize_loaded_credentials() # vault values carry the same copy-paste corruption risk as .env
for name, applied in report.provenance.items():
_SECRET_SOURCES[name] = applied.source
# Snapshot EVERY name a source supplied, not just the newly applied ones. A name the source supplied
# but the pre-existing process value won (``skipped_existing``) is still this home's effective value
# for it — and on the unscoped path os.environ IS this home's environment. Skipping those names
# latched an EMPTY snapshot whenever the key was already in the env: after the first cron/plugin
# re-pull (the previous apply's own write-back shadows every key) or from boot under systemd
# ``EnvironmentFile=``. Under multiplex the scope is the only credential source, so an empty
# snapshot failed every default-profile turn for the process lifetime (#102041).
values: dict[str, str] = {}
supplied = _record_supplied_names(report)
_record_released_bridge_claims(home_key, report, supplied, environ_before)
for name in supplied:
if name in os.environ:
values[name] = os.environ[name]
_release_bridge_claim(name) # a value a source supplies is the operator's, even one it skipped
_record_secret_source_writes(home_key, report, supplied, environ_before, staged, launch_env_captured)
if values:
_SECRET_SOURCE_VALUES_BY_HOME[home_key] = values
_publish_live_source_values(home_key, home_path, values)
_SECRET_SOURCE_RESTORE_BY_HOME[home_key] = {
n: values[n] for n, a in report.provenance.items() if a.authoritative and n in values}
for src in report.sources:
if src.applied:
print(f" {src.label}: applied {len(src.applied)} "
f"secret{'s' if len(src.applied) != 1 else ''}", file=sys.stderr)
if src.result.error:
print(f" {src.label}: {src.result.error}", file=sys.stderr)
hint = _remediation_hint(src.name, src.result.error_kind, cfg, scope=home_key)
if hint:
print(f" {src.label}: → {hint}", file=sys.stderr)
for warn in src.result.warnings:
print(f" {src.label}: {warn}", file=sys.stderr)
for conflict in report.conflicts:
print(f" Secret sources: {conflict}", file=sys.stderr)
def _record_released_bridge_claims(home_key: str, report, supplied: set[str], environ_before: dict) -> None:
"""Remember each bridge-owned value (marker matched before this pass) a source now supplies."""
from gateway.bridged_env import BRIDGED_ENV_MARKER_PREFIX, bridged_env_digest
owner = {name: src.name for src in report.sources for name in src.skipped_existing}
owner.update({name: applied.source for name, applied in report.provenance.items()})
claims = _RELEASED_BRIDGE_CLAIMS_BY_HOME.setdefault(home_key, {})
for name in supplied:
marker = environ_before.get(BRIDGED_ENV_MARKER_PREFIX + name)
if marker and name in owner and bridged_env_digest(environ_before.get(name)) == marker:
claims[name] = (owner[name], marker)
if not claims:
_RELEASED_BRIDGE_CLAIMS_BY_HOME.pop(home_key, None)
def _publish_live_source_values(home_key: str, home_path: Path, values: dict[str, str]) -> None:
"""Hand the redactor every source value live in ``os.environ`` for this home. A source whose re-pull
FAILED keeps its earlier write (not a revocation), so its value must stay masked: publish every
recorded write still holding its value, not only this pass's supplied names. Revoked writes were
already dropped from the record, so they unregister here — an empty set included."""
from agent.redact_secret_sources import publish_secret_source_values
live = {name: rec[1] for name, rec in _SECRET_SOURCE_WRITES_BY_HOME.get(home_key, {}).items()
if os.environ.get(name) == rec[1]}
live.update(values)
publish_secret_source_values(home_path, live)
def _published_value(name: str, staged: dict) -> str | None:
"""The value this pass published for *name*, in the form ``_sanitize_loaded_credentials`` leaves it."""
value = staged.get(name)
if value is None or value.isascii() or not name.endswith(_CREDENTIAL_SUFFIXES):
return value
return value.encode("ascii", errors="ignore").decode("ascii")
def _launch_policy():
"""``tui_gateway.launch_profile_policy`` once something imported it, else None: until then no
launch env is frozen, and a boot path must not import the gateway package just to learn that."""
return sys.modules.get("tui_gateway.launch_profile_policy")
def _recorded_environ_writes() -> list[tuple[str, str, str, str, str | None]]:
"""For ``launch_profile_policy`` at capture: every recorded write as
``(home key, name, source, value written, os.environ prior)``."""
return [(home, name, *rec) for home, writes in list(_SECRET_SOURCE_WRITES_BY_HOME.items())
for name, rec in list(writes.items())]
def _record_secret_source_writes(home_key: str, report, supplied: set[str], environ_before: dict,
staged: dict, launch_env_captured: bool = True) -> None:
"""Refresh ``_SECRET_SOURCE_WRITES_BY_HOME`` after one process-global pass, revoking a write its
still-enabled source has stopped supplying (rotated out, mapping removed). A source whose fetch
FAILED this pass keeps its earlier writes: a transient vault outage is not a revocation.
The frozen launch env keeps its own record of the writes it holds and follows only the value
the source published (*staged*): another thread may write the same name into os.environ during
the fetch, and that value must never enter the snapshot. *launch_env_captured* is whether the
launch env was frozen before this pass published: when activation landed mid-pass, the snapshot
may hold this pass's write without any record of it yet."""
failed = {src.name for src in report.sources if not src.result.ok}
previous = _SECRET_SOURCE_WRITES_BY_HOME.get(home_key, {})
_revoke_secret_source_writes(
Path(home_key),
keep=lambda name, source: name in report.provenance or name in supplied or source in failed)
writes = {name: rec for name, rec in _SECRET_SOURCE_WRITES_BY_HOME.get(home_key, {}).items()
if name not in report.provenance and os.environ.get(name) == rec[1]}
held: list[str] = [] # names whose published value os.environ still holds
for name, applied in report.provenance.items():
if name not in os.environ:
continue
before = environ_before.get(name)
earlier = previous.get(name)
# Re-applied over our own earlier write: the pre-source value is the one recorded then.
prior = earlier[2] if earlier is not None and earlier[1] == before else before
writes[name] = (applied.source, _published_value(name, staged), prior)
if os.environ[name] == writes[name][1]:
held.append(name)
if writes:
_SECRET_SOURCE_WRITES_BY_HOME[home_key] = writes
else:
_SECRET_SOURCE_WRITES_BY_HOME.pop(home_key, None)
# Tracked only once the record is published: an activation landing before this point adopts the
# writes (adopt=), one landing after it finds them in the record at capture.
policy = _launch_policy()
for name in held if policy is not None else ():
policy._track_launch_env_write(home_key, name, *writes[name], adopt=not launch_env_captured)
def _remediation_hint(source_name: str, error_kind, secrets_cfg: dict, *, scope: str | None = None) -> str:
"""The failed source's one-line fix-it hint; a plugin remediation() could raise and startup must not."""
try:
from agent.secret_sources.registry import get_source
source = get_source(source_name, scope=scope)
if source is None:
return ""
src_cfg = secrets_cfg.get(source_name)
src_cfg = src_cfg if isinstance(src_cfg, dict) else {}
return str(source.remediation(error_kind, src_cfg) or "").strip()
except Exception:
return ""
def _load_secrets_config(home_path: Path) -> dict:
"""Read just the ``secrets:`` section of config.yaml, isolated so a malformed config can't break dotenv."""
from hermes_cli.config_backend import config_exists, read_config_doc_readonly
config_path = home_path / "config.yaml"
if not config_exists(config_path):
return {}
# Prefer the shared raw-config cache: this is the first config.yaml read of a normal startup, so
# populating it lets main.py's early bridge and hermes_logging reuse one parse instead of 3-4.
if home_path == _process_hermes_home():
try:
from hermes_cli.config import read_raw_config
data = read_raw_config() or {}
return data.get("secrets") or {}
except Exception:
pass
# Routed profiles re-enter their scope on every poll/turn; only re-parse after the file changed.
try:
data = read_config_doc_readonly(config_path) or {}
except Exception:
return {}
return data.get("secrets") or {}
def _process_hermes_home() -> Path:
"""The HERMES_HOME the running process was launched under.
Must be the *true* process home, ignoring any context-local
``set_hermes_home_override`` a per-request task has installed. Both
callers depend on that:
* ``_reapply_terminal_config_bridge`` guards "only re-bridge config into
the shared ``os.environ`` when THIS load is for the process's own
profile". If this followed the task override, a per-turn handler scoped
to a *secondary* profile (the multiplex dashboard serving every profile
from one process) would satisfy the guard and bridge that profile's
``terminal.backend`` into the shared env — e.g. a ``local`` sibling
profile clobbering the launch profile's ``TERMINAL_ENV=ssh`` while
leaving its ``TERMINAL_SSH_*`` untouched, so the session silently runs
commands locally (cross-profile terminal-backend leak).
* ``_load_secrets_config`` uses it to decide whether the shared
(mtime,size)-keyed config cache is safe to reuse; under an override it
must fall through to an isolated parse of the scoped profile.
``hermes_constants.get_routing_process_hermes_home()`` is the override-immune
resolver built for exactly this; delegate to it. It is also immune to a host
that mirrors the served profile into the live ``HERMES_HOME`` env var
(``pin_process_hermes_home``): without that, the mirrored profile satisfied
the guard above and bridged ITS ``terminal.*`` into the shared env.
"""
try:
from hermes_constants import get_routing_process_hermes_home
return get_routing_process_hermes_home()
except Exception:
return Path.home() / ".hermes"