Files
hermes-agent/tests/tools/test_plugin_guard_docstring_prose.py
Teknium d1b46d718f fix(plugins): a docstring critical stops at caution; header and re-encoded lines stay code
Review on the previous head found three ways prose-capped docstring text
could still run:

- a sibling module reads the docstring back
  (`os.system(payload.__doc__)`): caution where it must not be safe;
- a multi-line `def` header puts a default (`x=os.system(...)`) on the
  docstring's line: scanned safe;
- a `# coding: utf-7` cookie makes CPython run text the UTF-8 parse
  calls a docstring: scanned caution.

A docstring is runtime data any file can read back under names no
static scan enumerates, so the cap no longer tries to prove a docstring
inert. A critical inside one now stops at high (verdict caution: the
user confirms the install), never safe and never the un-overridable
dangerous that prose alone caused. Every line any non-docstring AST
node occupies is code, and a file declaring a non-UTF-8 source encoding
gets no docstring cap at all.

The previous head's per-file read-back check is dropped: the floor
covers it everywhere, and the check misfired on `argparse(description=
__doc__)` help text in 8 listed plugins.

All 2,251 listed catalog pins vs main: 0 worse, 25 of 31 dangerous
become installable (24 caution, 1 safe).
2026-11-14 21:31:35 -08:00

149 lines
6.0 KiB
Python

"""Docstring prose in a plugin's Python files is documentation, not runtime behaviour (#155823).
``plugin_guard`` already caps findings that sit in a whole-line ``#`` comment or a CHANGELOG entry
(``_is_defensive_documentation`` → ``_comment_severity``) so prose cannot make a plugin
un-overridably ``dangerous``. A Python docstring is prose too, but it is a *string literal*, so it
never matched a comment prefix: a plugin documenting its own defenses with realistic examples
(``load("../etc/passwd") -> None``) was blocked from the catalog with no override.
These tests pin the cap for genuinely non-executable docstring text, and — just as important — pin
that executable code and non-docstring string constants keep their full severity.
"""
from pathlib import Path
import pytest
from tools.plugin_guard import scan_plugin, should_allow_plugin_install
PASSPHRASE = "/etc/passwd"
def _mk_plugin(tmp_path: Path, files: dict[str, str]) -> Path:
plugin = tmp_path / "test-plugin"
plugin.mkdir()
for rel, content in files.items():
p = plugin / rel
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(content, encoding="utf-8")
return plugin
def _findings(plugin: Path, rel: str = "core/store.py"):
result = scan_plugin(plugin, source="owner/repo")
return result, [f for f in result.findings if f.file == rel]
# A defensive module: every entry point validates the id, and the docstrings say so with a
# realistic traversal example -- the shape that blocked hermes-genui (catalog pin 43de4cdb).
DEFENSIVE_MODULE = '''"""Vault store: every id is validated before it reaches the filesystem."""
import re
ID_PATTERN = re.compile(r"^[a-z0-9-]{1,64}$")
def size(block_id):
"""Return the stored byte size, or None.
size("../etc/passwd") -> None (never a path outside the store)
"""
if not ID_PATTERN.fullmatch(block_id):
return None
return 0
'''
class TestDocstringProseIsDemoted:
def test_function_docstring_example_does_not_block_install(self, tmp_path):
plugin = _mk_plugin(tmp_path, {"core/store.py": DEFENSIVE_MODULE})
result, findings = _findings(plugin)
assert findings, "the example is still reported; it is demoted, not hidden"
assert all(f.severity != "critical" for f in findings)
assert result.verdict != "dangerous"
allowed, _reason = should_allow_plugin_install(result)
assert allowed is not False
def test_module_and_class_docstrings_are_covered(self, tmp_path):
plugin = _mk_plugin(tmp_path, {"core/ids.py": f'''"""Ids are validated; load("{PASSPHRASE}") -> None."""
class Store:
"""Never opens "{PASSPHRASE}" (see load())."""
def load(self, block_id):
"""Reject anything that is not an id: load("{PASSPHRASE}") -> None."""
return None
'''})
result, findings = _findings(plugin, "core/ids.py")
assert len(findings) == 3
# A docstring is runtime data (``__doc__``), so a critical in one stops at high: the user
# confirms the install instead of it being refused, and it is never silently ``safe``.
assert all(f.severity == "high" for f in findings), [f.severity for f in findings]
assert result.verdict == "caution"
class TestExecutableContentKeepsFullSeverity:
def test_runtime_code_line_stays_critical(self, tmp_path):
plugin = _mk_plugin(tmp_path, {"core/store.py": f'''"""Vault store."""
import pathlib
def read(block_id):
return pathlib.Path("{PASSPHRASE}").read_text()
'''})
result, findings = _findings(plugin)
assert [f.severity for f in findings] == ["critical"]
assert result.verdict == "dangerous"
def test_triple_quoted_constant_is_not_a_docstring(self, tmp_path):
# A payload held in a string constant is *not* documentation: `ast` only reports a
# docstring for the first statement of a module/class/function, so this keeps severity.
plugin = _mk_plugin(tmp_path, {"core/store.py": f'''PAYLOAD = """
cat {PASSPHRASE}
"""
def run():
return PAYLOAD
'''})
result, findings = _findings(plugin)
assert [f.severity for f in findings] == ["critical"]
assert result.verdict == "dangerous"
def test_string_after_a_statement_is_not_a_docstring(self, tmp_path):
plugin = _mk_plugin(tmp_path, {"core/store.py": f'''def run():
print("reading")
"""open("{PASSPHRASE}")"""
return None
'''})
_result, findings = _findings(plugin)
assert [f.severity for f in findings] == ["critical"]
@pytest.mark.parametrize("source", [
f'"""x"""; __import__("os").system("cat {PASSPHRASE}")\n',
f'import os\ndef f(\n x=os.system("cat {PASSPHRASE}")): """docs"""\n',
f'# coding: utf-7\n"""\n+ACIAIgAi-; import os; os.system("cat {PASSPHRASE}"); +ACIAIgAi-\n"""\n',
])
def test_code_on_a_docstring_line_stays_critical(self, tmp_path, source):
"""Code sharing the docstring's line (a header default too), or a source cookie that makes CPython
run other text than the UTF-8 parse, executes: no docstring cap."""
plugin = _mk_plugin(tmp_path, {"core/store.py": source})
result, findings = _findings(plugin)
assert "critical" in [f.severity for f in findings]
assert result.verdict == "dangerous"
@pytest.mark.parametrize("files", [
{"core/store.py": f'import os\ndef f():\n """cat {PASSPHRASE}"""\nos.system(f.__doc__)\n'},
{"core/store.py": f'def payload():\n """cat {PASSPHRASE}"""\n',
"core/run.py": "import os\nfrom .store import payload\nos.system(payload.__doc__)\n"},
])
def test_docstring_read_back_is_never_safe(self, tmp_path, files):
"""A docstring is runtime data any file can read back by a name no scan enumerates, so a critical
one in it caps at high: the install needs the user's confirmation, it is never silently safe."""
plugin = _mk_plugin(tmp_path, files)
result, findings = _findings(plugin)
assert "high" in [f.severity for f in findings]
assert result.verdict == "caution"