diff --git a/CHANGELOG.md b/CHANGELOG.md index 818ca43..739e0db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ This project follows semantic-versioning guidance once recurring releases are ta ### Script CLI changes +- Added `--format json|markdown` to `skills/agent-security/scripts/score_prompt_injection_exposure.py` for review-friendly exposure score summaries while keeping JSON as the default. - Replaced ad-hoc ZIP packaging with reproducible `scripts/package_skills.py`, deterministic `dist/MANIFEST.json` release metadata, and a non-mutating `--check` drift gate while preserving `./package-skills.sh`. - Added `--format json|markdown` to `skills/agent-security/scripts/flag_prompt_injection_signals.py` for review-friendly prompt-injection signal summaries while keeping JSON as the default. - Added `--output-dir` to `skills/agent-security/scripts/summarize_prompt_injection_corpus.py` for paired JSON/Markdown prompt-corpus review packets with no manifest or fixture mutation. diff --git a/README.md b/README.md index 1f40742..2e99854 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,14 @@ python3 skills/agent-security/scripts/score_prompt_injection_exposure.py \ < examples/high-risk-agent-config.json ``` +Emit the same exposure score as Markdown for PR comments or review notes: + +```bash +python3 skills/agent-security/scripts/score_prompt_injection_exposure.py \ + --format markdown \ + < examples/high-risk-agent-config.json +``` + Flag prompt-injection language in copied webpage/email/document text: ```bash diff --git a/skills/agent-security/SKILL.md b/skills/agent-security/SKILL.md index 3ede2b3..97bd664 100644 --- a/skills/agent-security/SKILL.md +++ b/skills/agent-security/SKILL.md @@ -356,6 +356,7 @@ openclaw status --deep --json | python3 skills/agent-security/scripts/config_ris openclaw status --deep --json | python3 skills/agent-security/scripts/config_risk_summary.py --generate-baseline > agent-security-baseline.json openclaw status --deep --json | python3 skills/agent-security/scripts/config_risk_summary.py --baseline agent-security-baseline.json --fail-on-expired-baseline openclaw status --deep --json | python3 skills/agent-security/scripts/score_prompt_injection_exposure.py +openclaw status --deep --json | python3 skills/agent-security/scripts/score_prompt_injection_exposure.py --format markdown # Expected input: untrusted or suspicious text on stdin python3 skills/agent-security/scripts/flag_prompt_injection_signals.py < suspicious-content.txt diff --git a/skills/agent-security/scripts/score_prompt_injection_exposure.py b/skills/agent-security/scripts/score_prompt_injection_exposure.py index 80772dc..b10ff1e 100644 --- a/skills/agent-security/scripts/score_prompt_injection_exposure.py +++ b/skills/agent-security/scripts/score_prompt_injection_exposure.py @@ -112,9 +112,41 @@ def normalize_config_shape(config: dict[str, Any]) -> dict[str, Any]: return normalized +def markdown_cell(value: Any) -> str: + text = str(value).replace("\n", " ").replace("|", r"\|") + return text + + +def render_markdown(result: dict[str, Any]) -> str: + lines = [ + "# Prompt Injection Exposure Score", + "", + f"**Score:** {result['score']}", + f"**Severity:** {markdown_cell(result['severity'])}", + f"**Schema version:** `{SCHEMA_VERSION}`", + "", + ] + factors = result.get("factors", []) + if not factors: + lines.append("No exposure factors were detected. Scoring is additive and heuristic; absence of factors does not guarantee a safe deployment.") + else: + lines.extend(["## Risk factors", "", "| Factor | Points |", "| --- | --- |"]) + for factor in factors: + lines.append(f"| `{markdown_cell(factor.get('factor', 'unknown'))}` | {factor.get('points', 0)} |") + lines.extend(["", f"**Total points:** {result['score']}"]) + lines.extend( + [ + "", + "Exposure scoring is heuristic only and favors recall. Combine it with enforced allowlists, approval gates, sandboxing, and least-privilege tool access.", + ] + ) + return "\n".join(lines) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--compact", action="store_true", help="emit compact JSON") + parser.add_argument("--format", choices=("json", "markdown"), default="json", help="output format") args = parser.parse_args() cfg, factors = load_json() @@ -215,7 +247,10 @@ def add(points: int, name: str, **extra: Any) -> None: severity = "medium" result = {"schema_version": SCHEMA_VERSION, "score": score, "severity": severity, "factors": factors} - print(json.dumps(result, separators=(",", ":") if args.compact else None, indent=None if args.compact else 2, sort_keys=True)) + if args.format == "markdown": + sys.stdout.write(render_markdown(result)) + else: + print(json.dumps(result, separators=(",", ":") if args.compact else None, indent=None if args.compact else 2, sort_keys=True)) return 1 if severity == "error" else 0 diff --git a/tests/test_score_prompt_injection_exposure.py b/tests/test_score_prompt_injection_exposure.py index c00ac45..43240f5 100644 --- a/tests/test_score_prompt_injection_exposure.py +++ b/tests/test_score_prompt_injection_exposure.py @@ -7,9 +7,9 @@ SCRIPT = ROOT / "skills" / "agent-security" / "scripts" / "score_prompt_injection_exposure.py" -def run_script(payload): +def run_script(payload, *args): proc = subprocess.run( - [sys.executable, str(SCRIPT)], + [sys.executable, str(SCRIPT), *args], input=json.dumps(payload) if not isinstance(payload, str) else payload, text=True, capture_output=True, @@ -38,3 +38,68 @@ def test_high_exposure_scores_high_or_critical(): data = json.loads(proc.stdout) assert data["severity"] in {"high", "critical"} assert any(f["factor"] == "shared_channel_with_high_impact_tools" for f in data["factors"]) + + +def high_exposure_payload(): + return { + "channels": {"discord": {"enabled": True, "groupPolicy": "allowlist"}}, + "browser": {"enabled": True, "ssrfPolicy": {"dangerouslyAllowPrivateNetwork": True}}, + "tools": {"exec": {"security": "full"}, "elevated": {"enabled": True}, "fs": {"workspaceOnly": False}}, + "agents": {"defaults": {"model": {"fallbacks": ["ollama/qwen2.5:7b"]}}}, + "bindings": [{"agentId": "shared", "match": {"channel": "discord", "peer": {"kind": "channel"}}}], + "memory": {"enabled": True}, + } + + +def test_markdown_format_emits_summary_header_and_score(): + proc = run_script(high_exposure_payload(), "--format", "markdown") + assert proc.returncode in (0, 1) + markdown = proc.stdout + assert "# Prompt Injection Exposure Score" in markdown + assert "**Score:**" in markdown + assert "**Severity:**" in markdown + assert "**Schema version:** `1.0`" in markdown + + +def test_markdown_format_lists_risk_factors_table(): + proc = run_script(high_exposure_payload(), "--format", "markdown") + assert proc.returncode in (0, 1) + markdown = proc.stdout + assert "## Risk factors" in markdown + assert "| Factor | Points |" in markdown + assert "| --- | --- |" in markdown + assert "shared_channel_with_high_impact_tools" in markdown + assert "browser_private_network_allowed" in markdown + + +def test_markdown_format_includes_total_score_and_note(): + proc = run_script(high_exposure_payload(), "--format", "markdown") + assert proc.returncode in (0, 1) + markdown = proc.stdout + assert "**Total points:**" in markdown + assert "exposure scoring" in markdown.lower() + + +def test_markdown_format_for_empty_input_documents_error(): + proc = run_script("", "--format", "markdown") + assert proc.returncode == 1 + markdown = proc.stdout + assert "# Prompt Injection Exposure Score" in markdown + assert "**Severity:** error" in markdown + + +def test_markdown_format_for_clean_config_shows_no_factors(): + proc = run_script({}, "--format", "markdown") + assert proc.returncode == 0 + markdown = proc.stdout + assert "**Score:** 0" in markdown + assert "No exposure factors were detected" in markdown + assert "## Risk factors" not in markdown + + +def test_json_remains_default_format(): + proc = run_script(high_exposure_payload()) + data = json.loads(proc.stdout) + assert data["schema_version"] == "1.0" + assert "score" in data + assert "factors" in data