"""``hermes plugins validate`` — admission checks for a plugin directory. This is the command the plugin-catalog admission CI (and the ``.github/actions/plugin-validate`` composite action) runs against a candidate plugin. It performs static manifest checks plus a subprocess-isolated capability probe: the plugin is imported and its ``register(ctx)`` called against a minimal recording stub context in a scratch child process (with a throwaway ``HERMES_HOME``), so a crashing or malicious plugin cannot take down the CLI, and the *actually registered* tools/hooks/middleware are compared against the manifest's declared ``provides_*`` lists. """ from __future__ import annotations import json import os import re import subprocess import sys import tempfile from dataclasses import dataclass, field from functools import cached_property from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from hermes_cli.plugin_validate_core_override import check_core_override from hermes_cli.plugin_validate_desktop import check_desktop_surface from hermes_cli.plugin_validate_private_imports import check_private_imports from hermes_cli.plugin_validate_locales import check_language_packs from hermes_cli.plugins_manifest import _CONFIG_SCHEMA_TYPES from hermes_cli._startup_fast import project_root_str _UPPER_SNAKE_RE = re.compile(r"^[A-Z][A-Z0-9_]*$") # Admission accepts exactly the ``config_schema`` types the loader type-checks at load time (and the # Desktop settings renderer keys its field table on) — a private copy drifted and rejected ``secret``. _CONFIG_TYPES = frozenset(_CONFIG_SCHEMA_TYPES) _PROBE_TIMEOUT = 30 _PROBE_SENTINEL = "HERMES_VALIDATE_JSON:" @dataclass class ValidationReport: """Result of validating one plugin directory.""" checks: list[tuple[str, bool, str]] = field(default_factory=list) warnings: list[str] = field(default_factory=list) isolation: Optional[dict[str, Any]] = None # plugin-host readiness; informational, never fails @property def failures(self) -> list[str]: return [detail or name for name, ok, detail in self.checks if not ok] @property def ok(self) -> bool: return all(ok for _name, ok, _detail in self.checks) @property def exit_code(self) -> int: return 0 if self.ok else 1 def add(self, name: str, ok: bool, detail: str = "") -> None: self.checks.append((name, ok, detail)) def warn(self, message: str) -> None: self.warnings.append(message) def to_dict(self) -> dict[str, Any]: return { "ok": self.ok, "checks": [ {"name": name, "ok": ok, "detail": detail} for name, ok, detail in self.checks ], "warnings": list(self.warnings), "isolation": self.isolation, } # ─── Static checks ─────────────────────────────────────────────────────────── def _requires_hermes_spec_valid(spec: str) -> bool: """Strictly validate a ``requires_hermes`` spec. Unlike :func:`hermes_cli.plugins_manifest.version_satisfies` (permissive at load time), validation REJECTS clauses whose version segment doesn't parse — a typo'd spec should fail admission, not silently gate nothing. """ from hermes_cli.plugins_manifest import _VERSION_COMPARATOR_RE, _version_tuple for clause in spec.split(","): clause = clause.strip() if not clause: continue m = _VERSION_COMPARATOR_RE.match(clause) target = m.group(2) if m else clause if _version_tuple(target) is None: return False return True def _check_manifest_fields(report: ValidationReport, manifest: dict) -> None: missing = [ f for f in ("name", "version", "description") if not manifest.get(f) ] if missing: report.add( "manifest fields", False, f"plugin.yaml missing required field(s): {', '.join(missing)}", ) else: report.add("manifest fields", True, "name, version, description present") def _check_requires_hermes(report: ValidationReport, manifest: dict) -> None: spec = str(manifest.get("requires_hermes") or "").strip() if not spec: report.add("requires_hermes", True, "not declared") return if _requires_hermes_spec_valid(spec): report.add("requires_hermes", True, f"spec {spec!r} parses") else: report.add( "requires_hermes", False, f"requires_hermes spec {spec!r} does not parse " "(expected e.g. \">=0.19\" or \">=0.19, <1.0\")", ) def _check_config_spec(report: ValidationReport, manifest: dict) -> None: """Validate the manifest ``config_schema`` mapping (#64165 format).""" raw = manifest.get("config_schema") if raw in (None, [], {}): report.add("config schema", True, "not declared") return problems: list[str] = [] if not isinstance(raw, dict): problems.append("config_schema: must be a mapping of key -> spec") else: for skey, spec in raw.items(): if not isinstance(spec, dict): problems.append( f"config_schema.{skey}: must be a mapping (e.g. {{type: str}})" ) continue typ = spec.get("type") if typ is not None and str(typ).lower() not in _CONFIG_TYPES: problems.append( f"config_schema.{skey}: type must be one of " f"{'/'.join(sorted(_CONFIG_TYPES))}" ) required = spec.get("required") if required is not None and not isinstance(required, bool): problems.append( f"config_schema.{skey}: required must be a boolean" ) if problems: report.add("config schema", False, "; ".join(problems)) else: report.add("config schema", True, "shape valid") def _check_requires_env(report: ValidationReport, manifest: dict) -> None: raw = manifest.get("requires_env") or [] problems: list[str] = [] if not isinstance(raw, list): problems.append("requires_env: must be a list") raw = [] for i, entry in enumerate(raw): if isinstance(entry, str): name = entry elif isinstance(entry, dict): name = str(entry.get("name") or "") else: problems.append(f"requires_env[{i}]: must be a string or mapping") continue if not _UPPER_SNAKE_RE.match(name): problems.append( f"requires_env[{i}]: {name!r} is not UPPER_SNAKE_CASE" ) if problems: report.add("requires_env", False, "; ".join(problems)) else: report.add("requires_env", True, "all entries UPPER_SNAKE") def _check_oauth(report: ValidationReport, manifest: dict, plugin_dir: Path) -> None: """``oauth:`` clients (hermes_cli.plugin_oauth): shape, https and declared variables fail admission; a host outside the IdP's domains and plugin code that names a token URL are reviewer warnings.""" from hermes_cli.plugin_oauth import oauth_admission_problems, oauth_admission_warnings if manifest.get("oauth") in (None, []): report.add("oauth", True, "not declared") return problems = oauth_admission_problems(manifest, plugin_dir) report.add("oauth", not problems, "; ".join(problems) or "clients valid") for warning in oauth_admission_warnings(manifest, plugin_dir): report.warn(warning) # ─── Capability probe (subprocess-isolated) ────────────────────────────────── # Self-contained harness run in a scratch child process. Imports the plugin # module using the same file-location mechanics PluginManager uses, calls # register() against a recording stub ctx, and prints a sentinel-prefixed # JSON line of what was actually registered. Deliberately imports NOTHING # from hermes so a hostile plugin only sees a bare interpreter — the exceptions are the # stdlib-only compat finder (old module names keep resolving) and `providers` for # `kind: model-provider`, whose contract IS calling providers.register_provider at import. _PROBE_SCRIPT = r""" import importlib.util import json import sys from hermes_agent.compat import install install() plugin_dir = sys.argv[1] sentinel = sys.argv[2] options = json.loads(sys.argv[3]) # Public method names of the real PluginContext, computed by the parent so the # stub's attribute surface cannot drift from the class plugins run against. context_methods = set(options["context_methods"]) # Service properties (``ctx.llm``, ``ctx.state``, ...): plugins capture them in register() and # use them later, so the probe hands back an inert service whose methods are no-ops. context_properties = set(options.get("context_properties", ())) provider_kind = options["kind"] == "model-provider" recorded = {"tools": [], "hooks": [], "middleware": [], "commands": [], "providers": [], "trusted_inbound": []} class RecordingContext: plugin_config = {} profile_name = "default" plugin_id = "hermes_validate_probe_plugin" def register_tool(self, name, *args, **kwargs): recorded["tools"].append(str(name)) def register_hook(self, hook_name, callback, *, hold_stream=False): recorded["hooks"].append(str(hook_name)) def register_middleware(self, kind, callback): recorded["middleware"].append(str(kind)) def register_command(self, name, *args, **kwargs): recorded["commands"].append(str(name)) def register_cli_command(self, name, *args, **kwargs): recorded["commands"].append(str(name)) def register_platform(self, name, *args, **kwargs): if kwargs.get("trusted_inbound"): recorded["trusted_inbound"].append(str(name)) def get_config(self, key, default=None): # Mirrors PluginContext.get_config with no config on disk: the DEFAULT, never None — # plugins do `int(ctx.get_config("timeout", 180))` in register(). return default def __getattr__(self, name): # Any other REAL registration surface (platforms, providers, skills, # context engines, ...) is accepted as a no-op — the probe only audits # the declared-capability categories. Names the real PluginContext does # not have raise AttributeError exactly like it would; handing back a # callable made `getattr(ctx, "profile_path", None)` truthy and crashed # register() in the probe alone. if name in context_methods: def _noop(*args, **kwargs): return None return _noop if name == "state": return _ProbeState() if name in context_properties: return _InertService() raise AttributeError(name) class _ProbeState: # Same shape as PluginState (real Path values, get() returns the default) over an empty store # in the probe's scratch HERMES_HOME, which the parent deletes after the run. def __init__(self): import os from pathlib import Path self.data_dir = Path(os.environ["HERMES_HOME"]) / "plugin-data" / "probe" self.data_dir.mkdir(parents=True, exist_ok=True) self.path = self.data_dir / "state.json" self.quota_bytes = 1024 * 1024 self._data = {} def get(self, key, default=None): return self._data.get(key, default) def set(self, key, value): self._data[key] = value class _InertService: def __getattr__(self, name): if name.startswith("__"): raise AttributeError(name) def _noop(*args, **kwargs): return None return _noop def emit(payload): print(sentinel + json.dumps(payload)) if provider_kind: # `kind: model-provider` plugins register at import via # providers.register_provider(ProviderProfile) — the PluginManager never # calls a register(ctx) on them (plugins_discovery skips the kind), so the # probe records that call instead of demanding an entry point that would # be dead code. try: import providers as _providers except Exception as exc: emit({"error": "model-provider probe could not import providers: %s" % exc}) sys.exit(0) _real_register_provider = _providers.register_provider def _recording_register_provider(profile): recorded["providers"].append(str(getattr(profile, "name", profile))) return _real_register_provider(profile) _providers.register_provider = _recording_register_provider try: spec = importlib.util.spec_from_file_location( "hermes_validate_probe_plugin", plugin_dir + "/__init__.py", submodule_search_locations=[plugin_dir], ) module = importlib.util.module_from_spec(spec) module.__path__ = [plugin_dir] sys.modules[spec.name] = module spec.loader.exec_module(module) except Exception as exc: emit({"error": "import failed: %s" % exc}) sys.exit(0) if provider_kind: if not recorded["providers"]: emit({"error": "model-provider plugin registered no ProviderProfile at import"}) else: emit(recorded) sys.exit(0) register = getattr(module, "register", None) if register is None: emit({"error": "no register() function"}) sys.exit(0) try: register(RecordingContext()) except Exception as exc: emit({"error": "register() raised: %s" % exc}) sys.exit(0) emit(recorded) """ def _probe_options(manifest: dict) -> dict: from hermes_cli.plugins import PluginContext public = [n for n in dir(PluginContext) if not n.startswith("_")] return { "kind": str(manifest.get("kind") or ""), "context_methods": sorted(n for n in public if callable(getattr(PluginContext, n))), "context_properties": sorted( n for n in public if isinstance(getattr(PluginContext, n), (property, cached_property)) ), } def _run_capability_probe( plugin_dir: Path, manifest: dict, probe: Optional[tuple[list[str], dict[str, str]]] = None, ) -> tuple[Optional[dict], str]: """Run the recording probe in a scratch subprocess. *probe* is ``(python argv prefix, env)`` of the dependency environment to import the plugin from (``pm.environments.venv_command``); None probes this interpreter. Returns ``(recorded, error)`` — exactly one is meaningful: *recorded* is the ``{tools, hooks, middleware, commands, providers}`` dict on success, and *error* is a human-readable failure description otherwise. """ with tempfile.TemporaryDirectory(prefix="hermes-validate-") as scratch: env = dict(probe[1] if probe else os.environ) env["HERMES_HOME"] = scratch # The probe imports this tree's ``providers``; name the tree instead of relying on the cwd. repo_root = project_root_str() env["PYTHONPATH"] = os.pathsep.join(p for p in (repo_root, env.get("PYTHONPATH", "")) if p) python, *options = probe[0] if probe else [sys.executable] try: result = subprocess.run( [ # -P: ``-c`` would put the user's cwd ahead of Hermes and the plugin's imports. python, "-P", *options, "-c", _PROBE_SCRIPT, str(plugin_dir), _PROBE_SENTINEL, json.dumps(_probe_options(manifest)), ], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=_PROBE_TIMEOUT, env=env, check=False, ) except subprocess.TimeoutExpired: return None, f"capability probe timed out after {_PROBE_TIMEOUT}s" payload: Optional[dict] = None for line in (result.stdout or "").splitlines(): if line.startswith(_PROBE_SENTINEL): try: payload = json.loads(line[len(_PROBE_SENTINEL):]) except json.JSONDecodeError: payload = None if payload is None: err = (result.stderr or "").strip() return None, ( "capability probe produced no result " f"(exit {result.returncode})" + (f": {err}" if err else "") ) if "error" in payload: return None, str(payload["error"]) return payload, "" def _declared_list(manifest: dict, key: str) -> list[str]: raw = manifest.get(key) or [] if not isinstance(raw, list): return [] return [str(item) for item in raw if isinstance(item, str)] def _check_capabilities( report: ValidationReport, manifest: dict, plugin_dir: Path, probe: Optional[tuple[Path, dict[str, str]]] = None, ) -> Optional[dict]: """Probe actual registrations and diff against declared capabilities. Returns the recorded dict (for the built-in collision check) or None when the probe failed / was skipped. """ if not (plugin_dir / "__init__.py").is_file(): report.warn( "no __init__.py — capability probe skipped (manifest-only plugin)" ) report.add("capability probe", True, "skipped (no __init__.py)") return None recorded, error = _run_capability_probe(plugin_dir, manifest, probe) if recorded is None: report.add("capability probe", False, error) return None if recorded.get("providers"): report.add( "capability probe", True, "import registered provider(s): " + ", ".join(recorded["providers"]), ) else: report.add("capability probe", True, "register() ran in isolation") for kind, manifest_key in ( ("tools", "provides_tools"), ("hooks", "provides_hooks"), ("middleware", "provides_middleware"), ): declared = set(_declared_list(manifest, manifest_key)) actual = set(recorded.get(kind) or []) undeclared = sorted(actual - declared) unregistered = sorted(declared - actual) if undeclared: report.add( f"declared {kind}", False, f"undeclared {kind} registered (not in {manifest_key}): " f"{', '.join(undeclared)}", ) else: report.add(f"declared {kind}", True, "matches registrations") if unregistered: report.warn( f"{manifest_key} declares {', '.join(unregistered)} " f"but register() did not register them" ) return recorded def _builtin_tool_names() -> list[str]: """Return the built-in tool registry names (discovery-timing safe). ``tools.registry`` starts empty — built-in tool modules self-register on import, so we must run ``discover_builtin_tools()`` first (idempotent; see the AGENTS.md discover_plugins timing pitfall). """ try: from tools.registry import discover_builtin_tools, registry discover_builtin_tools() return list(registry.get_all_tool_names()) except Exception: return [] def _check_builtin_collisions( report: ValidationReport, manifest: dict, recorded: Optional[dict] ) -> None: candidate_tools = set(_declared_list(manifest, "provides_tools")) if recorded: candidate_tools.update(recorded.get("tools") or []) if not candidate_tools: report.add("built-in tool collisions", True, "no tools to check") return builtin = set(_builtin_tool_names()) collisions = sorted(candidate_tools & builtin) if collisions: report.add( "built-in tool collisions", False, "tool name(s) collide with built-in tools: " f"{', '.join(collisions)}", ) else: report.add("built-in tool collisions", True, "no collisions") # ─── Entry point ───────────────────────────────────────────────────────────── def validate_plugin_dir( plugin_dir: Path, probe: Optional[tuple[list[str], dict[str, str]]] = None, ) -> ValidationReport: """Run every admission check against *plugin_dir* and return the report. *probe* is ``(python argv prefix, env)`` for the capability probe (see ``_run_capability_probe``).""" report = ValidationReport() plugin_dir = Path(plugin_dir) if not plugin_dir.is_dir(): report.add( "plugin directory", False, f"{plugin_dir} is not a directory" ) return report manifest_file = plugin_dir / "plugin.yaml" if not manifest_file.is_file(): manifest_file = plugin_dir / "plugin.yml" if not manifest_file.is_file(): # Portable Agent Plugins v1 (#81196): a plugin.json-only package is # admissible — validated through the portable manifest reader. When a # package carries both manifests the native plugin.yaml always wins # (this branch is only reached when no native manifest exists). portable_file = plugin_dir / "plugin.json" if portable_file.is_file(): return _validate_portable_plugin(report, plugin_dir) report.add( "manifest", False, "no plugin.yaml (or portable plugin.json) in the plugin directory", ) return report import hermes_yaml as yaml try: manifest = yaml.safe_load( manifest_file.read_text(encoding="utf-8-sig") ) except Exception as exc: report.add("manifest", False, f"plugin.yaml failed to parse: {exc}") return report if not isinstance(manifest, dict): report.add("manifest", False, "plugin.yaml must be a mapping") return report report.add("manifest", True, "plugin.yaml parses") _check_manifest_fields(report, manifest) _check_requires_hermes(report, manifest) _check_config_spec(report, manifest) _check_requires_env(report, manifest) _check_oauth(report, manifest, plugin_dir) _check_loadable(report, plugin_dir, manifest) _check_python_dependencies(report, plugin_dir) recorded = _check_capabilities(report, manifest, plugin_dir, probe) _check_builtin_collisions(report, manifest, recorded) _check_trusted_inbound(report, recorded) _check_security_scan(report, plugin_dir) check_core_override(report, plugin_dir) check_private_imports(report, plugin_dir) check_desktop_surface(report, plugin_dir) check_language_packs(report, manifest, plugin_dir) from hermes_cli.plugin_isolation_audit import audit_plugin_dir report.isolation = audit_plugin_dir(plugin_dir, manifest).to_dict() return report def _check_trusted_inbound(report: ValidationReport, recorded: Optional[dict]) -> None: """Surface ``trusted_inbound`` platforms (their events skip user allowlists and pairing); one naming a core platform fails, as the loader refuses it.""" from gateway.platform_registry import core_ships_platform for name in (recorded or {}).get("trusted_inbound") or []: if core_ships_platform(name): report.add("trusted inbound", False, f"platform '{name}' ships with core; trusted_inbound is refused for it") else: report.add("trusted inbound", True, f"platform '{name}' is trusted_inbound: its events skip user allowlists and pairing") _LOADABLE_ENTRYPOINTS = ("__init__.py", "desktop/plugin.js", "plugin.json") def _check_loadable(report: ValidationReport, plugin_dir: Path, manifest: Optional[dict] = None) -> None: """A plugin.yaml with nothing beside it that Hermes can load (no ``register()`` module, no desktop bundle, no portable manifest, no declared language pack) installs "successfully" and does nothing — a pip-layout repo whose code lives under ``src/`` behind an entry point is the usual shape.""" present = [rel for rel in _LOADABLE_ENTRYPOINTS if (plugin_dir / rel).is_file()] if (manifest or {}).get("provides_locales") and (plugin_dir / "locales").is_dir(): present.append("locales/ (language pack)") report.add( "loadable", bool(present), f"entry: {', '.join(present)}" if present else "nothing to load: no __init__.py, desktop/plugin.js, plugin.json or provides_locales + locales/ beside " "plugin.yaml (pip-layout packages need a directory-plugin wrapper with a pyproject.toml declaring the deps)", ) def _check_python_dependencies(report: ValidationReport, plugin_dir: Path) -> None: """Declared deps (pyproject ``[project].dependencies`` or manifest ``python_dependencies``) must be well-formed PEP 508 specs the installer will accept; a plugin opting out with ``python_runtime: external`` declares none.""" from pm.plugin_declarations import read_python_declaration, unsupported_requirements try: decl = read_python_declaration(plugin_dir) if decl.external: report.add("python dependencies", True, "external runtime (plugin manages its own)") return urls = unsupported_requirements(decl.requirements) installable = decl.install_requirements except (ValueError, OSError) as exc: report.add("python dependencies", False, f"declaration invalid: {exc}") return if urls: report.warn("python dependencies: direct URL requirement(s) are not managed by PM; " f"use a plugin-owned external runtime: {', '.join(urls)}") source = "pyproject" if decl.pyproject is not None else "manifest" detail = f"{len(installable)} requirements from {source}" if decl.requirements else "none declared" report.add("python dependencies", True, detail) def _check_security_scan(report: ValidationReport, plugin_dir: Path) -> None: """Run the install-time scanner at admission, so a pin a reviewer approves is one the installer will accept: ``dangerous`` fails the entry; ``caution`` findings surface as warnings for the reviewer (the installer trusts them once the pin is merged).""" from tools.plugin_guard import scan_plugin result = scan_plugin(plugin_dir) flagged = [f for f in result.findings if f.severity in ("critical", "high")] summary = ", ".join(sorted({f"{f.pattern_id} ({Path(f.file).name}:{f.line})" for f in flagged})) or "no findings" if result.verdict == "dangerous": report.add("security scan", False, f"dangerous: {summary}") return report.add("security scan", True, result.verdict) if result.verdict == "caution": report.warn(f"security scan caution: {summary}") def _validate_portable_plugin(report: ValidationReport, plugin_dir: Path) -> ValidationReport: """Admission checks for a portable Agent Plugins v1 (plugin.json) package. Portable packages have no register() entry point, so the capability probe does not apply; validation is the manifest reader's own diagnostics (schema shape, name, supported subset). """ try: from hermes_cli.agent_plugins import load_agent_plugin from hermes_platform.resolver.availability import availability with tempfile.TemporaryDirectory() as data_root: package = load_agent_plugin(plugin_dir, Path(data_root)) manifest = package.manifest diagnostics = package.diagnostics except Exception as exc: report.add("portable manifest", False, f"plugin.json failed validation: {exc}") return report for diag in diagnostics: scope = getattr(diag, "scope", "") message = getattr(diag, "message", str(diag)) report.warnings.append(f"{scope}: {message}" if scope else message) report.add("portable manifest", True, "plugin.json parses (Agent Plugins v1)") name = str(manifest.get("name") or "").strip() report.add( "manifest fields", bool(name), "name present" if name else "plugin.json missing required 'name'", ) from hermes_cli.agent_plugins import manifest_config_schema _check_config_spec(report, {"config_schema": manifest_config_schema(manifest)}) for server_name, config in package.mcp_servers.items(): if config.get("trust") == "untrusted": report.add(f"server trust: {server_name}", True, "untrusted (Hermes asks before every write-capable tool call)") if config.get("tool_costs"): limit = config.get("spend_limit") or 0 report.add(f"server costs: {server_name}", True, f"{len(config['tool_costs'])} costed tool(s), " + (f"spend limit {limit} {config.get('cost_unit') or 'credits'} per chat" if limit else "no spend limit")) for server_name, server_decl in package.server_declarations.items(): result = availability(server_decl.declaration) detail = result.state if result.version: detail += f", version {result.version}" if result.path: detail += f", path {result.path}" report.add(f"server availability: {server_name}", True, detail) _check_security_scan(report, plugin_dir) check_desktop_surface(report, plugin_dir) from hermes_cli.plugin_isolation_audit import audit_plugin_dir report.isolation = audit_plugin_dir(plugin_dir, manifest).to_dict() return report