diff --git a/CHANGELOG.md b/CHANGELOG.md index 818ca43..8abd6c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ This project follows semantic-versioning guidance once recurring releases are ta ### Script CLI changes +- Added `--envelope-secret-file` to `skills/agent-security/scripts/config_risk_summary.py` for optional HMAC-SHA256 report authenticity envelopes on JSON output, plus the dependency-light `skills/agent-security/scripts/verify_report_envelope.py` verifier with `--secret-file`/`--secret-env` sources and machine-readable failure codes. - Replaced ad-hoc ZIP packaging with reproducible `scripts/package_skills.py`, deterministic `dist/MANIFEST.json` release metadata, and a non-mutating `--check` drift gate while preserving `./package-skills.sh`. - Added `--format json|markdown` to `skills/agent-security/scripts/flag_prompt_injection_signals.py` for review-friendly prompt-injection signal summaries while keeping JSON as the default. - Added `--output-dir` to `skills/agent-security/scripts/summarize_prompt_injection_corpus.py` for paired JSON/Markdown prompt-corpus review packets with no manifest or fixture mutation. diff --git a/README.md b/README.md index 1f40742..9cf2db4 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,21 @@ python3 skills/agent-security/scripts/config_risk_summary.py \ > agent-security.sarif ``` +Sign the JSON report with an optional HMAC-SHA256 authenticity envelope so downstream consumers can detect report mutation: + +```bash +python3 skills/agent-security/scripts/config_risk_summary.py \ + --envelope-secret-file agent-security-report.key \ + < examples/high-risk-agent-config.json \ + > report.json + +python3 skills/agent-security/scripts/verify_report_envelope.py \ + --secret-file agent-security-report.key \ + < report.json +``` + +See [`docs/report-envelopes.md`](docs/report-envelopes.md) for envelope fields, verifier exit codes, secret handling, and rotation guidance. + JSON, Markdown, and SARIF findings include `evidence_paths` such as `browser.ssrfPolicy.dangerouslyAllowPrivateNetwork` or `bindings[0].match.peer.kind`. JSON and SARIF also include best-effort @@ -165,6 +180,7 @@ Key files: - `skills/agent-security/scripts/config_risk_summary.py` — schema-tolerant config risk summary - `skills/agent-security/scripts/score_prompt_injection_exposure.py` — exposure scoring for agent configs - `skills/agent-security/scripts/flag_prompt_injection_signals.py` — prompt-injection text detector +- `skills/agent-security/scripts/verify_report_envelope.py` — HMAC-SHA256 authenticity envelope verifier for exported JSON reports - `docs/prompt-injection-detector-quality.md` — detector-quality notes, known false positives/negatives, and fixture guidance - `skills/agent-security/scripts/summarize_prompt_injection_corpus.py` — JSON/Markdown inventory summaries for the prompt-injection fixture manifest - `docs/config-shapes.md` — canonical config fields, supported aliases, and real-world fixture guidance diff --git a/docs/ci-integration.md b/docs/ci-integration.md index 0791e69..54ced92 100644 --- a/docs/ci-integration.md +++ b/docs/ci-integration.md @@ -61,6 +61,28 @@ python3 skills/agent-security/scripts/config_risk_summary.py \ For release branches or security-sensitive config changes, run `--strict` locally so high or critical findings fail before CI does. +## Signed report artifacts + +When a scan report is stored or forwarded for later review, sign it at scan time +and verify it at review time with an HMAC-SHA256 authenticity envelope. Store the +signing secret in your CI secret store and inject it as a file or environment +variable; never commit it. + +```yaml + - name: Scan and sign report + run: | + python3 skills/agent-security/scripts/config_risk_summary.py \ + --envelope-secret-file "$REPORT_SECRET_FILE" \ + < examples/high-risk-agent-config.json > agent-security-report.json + python3 skills/agent-security/scripts/verify_report_envelope.py \ + --secret-file "$REPORT_SECRET_FILE" \ + < agent-security-report.json +``` + +The verifier fails the job (exit 1) on tampered or unsigned reports. See +[`report-envelopes.md`](report-envelopes.md) for envelope fields, exit codes, +secret generation, and rotation guidance. + ## Minimal permissions | Integration | Minimum permissions | Notes | diff --git a/docs/report-envelopes.md b/docs/report-envelopes.md new file mode 100644 index 0000000..5eb69be --- /dev/null +++ b/docs/report-envelopes.md @@ -0,0 +1,93 @@ +# Report Authenticity Envelopes + +Phase 20 adds an optional, non-executable integrity envelope for exported JSON +reports so downstream consumers can detect accidental or malicious report +mutation between scan time and review time. + +The envelope is **an integrity control, not encryption**. It hides nothing about +the report; it only lets a reviewer with the same secret confirm the report bytes +were produced by someone holding that secret and were not modified afterwards. + +## Sign a JSON report + +Pass `--envelope-secret-file` pointing at a UTF-8 text file containing the +signing secret. At most one trailing newline is stripped, so secrets written by +common CI secret-file tooling verify correctly. + +```bash +python3 skills/agent-security/scripts/config_risk_summary.py \ + --envelope-secret-file /path/to/agent-security-report.key \ + < examples/high-risk-agent-config.json \ + > report.json +``` + +The JSON report gains an additive `report_envelope` object: + +```json +{ + "algorithm": "hmac-sha256", + "payload_sha256": "<64-char SHA-256 of the canonical report payload>", + "signature": "<64-char HMAC-SHA256 of the payload digest>", + "covered_fields": ["baseline_lifecycle", "counts", "findings", "..."] +} +``` + +`covered_fields` lists every top-level report field covered by the signature. +The envelope itself is never self-signed. Without the flag, output is unchanged +and fully backwards compatible. + +Envelope signing requires `--format json` (the default). Passing it with +`--format markdown` or `--format sarif` is a usage error (exit code 2) instead +of silently emitting unsigned output. + +## Verify a signed report + +Use the dependency-light verifier with exactly one secret source — a file or an +environment variable: + +```bash +python3 skills/agent-security/scripts/verify_report_envelope.py \ + --secret-file /path/to/agent-security-report.key \ + < report.json +``` + +```bash +python3 skills/agent-security/scripts/verify_report_envelope.py \ + --secret-env AGENT_SECURITY_ENVELOPE_SECRET \ + < report.json +``` + +Exit codes: + +| Exit | Meaning | +| --- | --- | +| `0` | Report envelope verified; payload and signature are intact. | +| `1` | Verification failed (tampering, wrong secret, missing/malformed envelope) or the secret source was invalid. | +| `2` | Usage error: zero or multiple secret sources supplied. | + +On failure the verifier prints a JSON verdict with machine-readable error codes: +`missing_envelope`, `unsupported_algorithm`, `malformed_envelope`, +`payload_digest_mismatch`, `signature_mismatch`, `invalid_secret`, or +`invalid_report`. + +- `payload_digest_mismatch` means the report content changed after signing. +- `signature_mismatch` with an intact digest means the secret does not match. + +## Secret handling + +- Generate secrets with a CSPRNG, e.g. `openssl rand -hex 32`. +- Never commit secrets to the repo; store them in your CI secret store and + inject them as files or environment variables at scan time. +- Treat the secret as a shared capability: anyone holding it can produce valid + reports. Scope CI permissions so untrusted PRs cannot both read the secret and + publish artifacts verified with it. +- To rotate a secret, generate a new one, re-sign and re-verify one report with + the new secret, then retire the old secret. Reports signed with the old secret + will fail verification with `signature_mismatch` until re-signed; keep the old + secret (offline) if you need to verify historical artifacts. + +## CI usage + +See [`ci-integration.md`](ci-integration.md) for how envelope signing composes +with strict scans, SARIF upload, and scheduled audits in GitHub Actions, and +see [`../README.md`](../README.md) for quick-start examples. diff --git a/docs/roadmap.md b/docs/roadmap.md index 988c68f..61796db 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -365,9 +365,29 @@ Before starting new roadmap work, check open PRs and avoid duplicating any branc ## Phase 20: Scanner report authenticity envelopes -**Status:** Planned +**Status:** Shipped (JSON report slice) **Goal:** Define an optional, non-executable integrity envelope for exported JSON/SARIF reports so downstream consumers can detect accidental or malicious report mutation. +### Shipped scope + +1. Added `--envelope-secret-file` to `skills/agent-security/scripts/config_risk_summary.py` for optional HMAC-SHA256 authenticity envelopes over the JSON report payload; output is unchanged without the flag. +2. Envelopes are additive (`report_envelope` with `algorithm`, `payload_sha256`, `signature`, and sorted `covered_fields`) and never self-signed; signing requires `--format json` and is a usage error otherwise. +3. Invalid, unreadable, or empty/blank secret files fail closed as structured `invalid_envelope_secret` error findings instead of unsigned or crash output. +4. Added the dependency-light `skills/agent-security/scripts/verify_report_envelope.py` verifier with mutually exclusive `--secret-file`/`--secret-env` sources, constant-time signature comparison, stable exit codes (0/1/2), and machine-readable failure codes for missing, malformed, tampered, or wrongly signed reports. +5. Documented envelope semantics, verifier usage, secret generation/storage, rotation guidance, and CI composition in [`docs/report-envelopes.md`](report-envelopes.md), with README, skill, changelog, and CI-integration cross-links. +6. Added `tests/test_phase20_report_envelopes.py` covering compatibility, determinism, secret-file newline tolerance, fail-closed secret errors, format gating, verifier acceptance/rejection matrix, and documentation sync. + +### Acceptance criteria + +- JSON output is byte-identical to previous behavior when no envelope secret is supplied. +- Envelope signatures are deterministic for identical inputs and secrets. +- Verification detects payload tampering, forged signatures, wrong secrets, missing envelopes, and unsupported algorithms with distinct error codes. + +### Remaining follow-ups + +- SARIF and Markdown envelope coverage (if needed by downstream consumers). +- Key-identifier support (`kid`) for concurrent multi-secret verification during rotations. + ## Implementation order 1. Finish or merge PRs that already cover roadmap work before starting duplicate branches. diff --git a/skills/agent-security/SKILL.md b/skills/agent-security/SKILL.md index 3ede2b3..698de98 100644 --- a/skills/agent-security/SKILL.md +++ b/skills/agent-security/SKILL.md @@ -344,6 +344,7 @@ Use these helper resources when useful: - `scripts/score_prompt_injection_exposure.py` - `scripts/flag_prompt_injection_signals.py` - `scripts/summarize_prompt_injection_corpus.py` +- `scripts/verify_report_envelope.py` - `../healthcheck/scripts/summarize_openclaw_posture.py` - `../healthcheck/scripts/parse_openclaw_audit.py` @@ -355,6 +356,8 @@ openclaw status --deep --json | python3 skills/agent-security/scripts/config_ris openclaw status --deep --json | python3 skills/agent-security/scripts/config_risk_summary.py --policy examples/policies/agent-security-policy.json openclaw status --deep --json | python3 skills/agent-security/scripts/config_risk_summary.py --generate-baseline > agent-security-baseline.json openclaw status --deep --json | python3 skills/agent-security/scripts/config_risk_summary.py --baseline agent-security-baseline.json --fail-on-expired-baseline +openclaw status --deep --json | python3 skills/agent-security/scripts/config_risk_summary.py --envelope-secret-file agent-security-report.key > report.json +python3 skills/agent-security/scripts/verify_report_envelope.py --secret-file agent-security-report.key < report.json openclaw status --deep --json | python3 skills/agent-security/scripts/score_prompt_injection_exposure.py # Expected input: untrusted or suspicious text on stdin diff --git a/skills/agent-security/scripts/config_risk_summary.py b/skills/agent-security/scripts/config_risk_summary.py index 7190dd6..98395e4 100644 --- a/skills/agent-security/scripts/config_risk_summary.py +++ b/skills/agent-security/scripts/config_risk_summary.py @@ -5,6 +5,8 @@ so wrong-type fields become findings instead of Python tracebacks. """ import argparse +import hashlib +import hmac import json import re import sys @@ -915,6 +917,63 @@ def count_by_severity(findings: list[dict[str, Any]]) -> dict[str, int]: return counts +def load_envelope_secret(path: str | None) -> tuple[bytes | None, list[dict[str, Any]]]: + """Load the raw secret for report signing. + + Returns (secret, errors). The secret is read as UTF-8 text with at most one + trailing newline stripped, mirroring common secret-file conventions. An + empty/blank secret is rejected so callers fail closed instead of signing + reports with a trivially guessable key. + """ + if not path: + return None, [] + try: + with open(path, encoding="utf-8") as fh: + raw = fh.read() + except OSError as exc: + return None, [envelope_secret_error(f"could not read envelope secret file: {exc}", path)] + except UnicodeDecodeError as exc: + return None, [envelope_secret_error(f"envelope secret file is not valid UTF-8 text: {exc}", path)] + secret = raw[:-1] if raw.endswith("\n") else raw + if not secret.strip(): + return None, [envelope_secret_error("envelope secret file is empty or blank", path)] + return secret.encode("utf-8"), [] + + +def envelope_secret_error(message: str, path: str) -> dict[str, Any]: + return { + "severity": "error", + "risk": "invalid_envelope_secret", + "field": "report_envelope", + "message": message, + "envelope_secret_path": path, + } + + +def canonical_payload_bytes(payload: Any) -> bytes: + """Serialize the signed payload deterministically (sorted keys, tight separators).""" + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def build_report_envelope(summary: dict[str, Any], secret: bytes) -> dict[str, Any]: + """Build the Phase 20 HMAC-SHA256 authenticity envelope over the JSON summary. + + The envelope signs the full summary minus the ``report_envelope`` key itself so + downstream consumers can detect accidental or malicious report mutation. The + envelope is an integrity control, not encryption: it hides nothing. + """ + payload = {key: value for key, value in summary.items() if key != "report_envelope"} + payload_bytes = canonical_payload_bytes(payload) + digest = hashlib.sha256(payload_bytes).hexdigest() + signature = hmac.new(secret, digest.encode("ascii"), hashlib.sha256).hexdigest() + return { + "algorithm": "hmac-sha256", + "payload_sha256": digest, + "signature": signature, + "covered_fields": sorted(payload.keys()), + } + + def render_sarif(summary: dict[str, Any]) -> dict[str, Any]: schema_adapter = summary.get("schema", {}).get("adapter", "canonical") findings_by_rule_id = {sarif_rule_id(finding): finding for finding in summary["findings"]} @@ -996,15 +1055,20 @@ def main() -> int: parser.add_argument("--generate-baseline", action="store_true", help="emit a baseline for current findings with TODO lifecycle metadata") parser.add_argument("--fail-on-stale-baseline", action="store_true", help="exit nonzero when baseline entries no longer match findings") parser.add_argument("--fail-on-expired-baseline", action="store_true", help="exit nonzero when baseline entries are expired") + parser.add_argument( + "--envelope-secret-file", + help="sign the JSON report with an HMAC-SHA256 authenticity envelope using the secret in this UTF-8 text file", + ) args = parser.parse_args() policy, policy_errors = load_policy(args.policy) baseline_suppressions, baseline_errors = load_baseline(args.baseline) + envelope_secret, envelope_errors = load_envelope_secret(args.envelope_secret_file) cfg, initial_findings, raw_input = load_json() schema_adapter = schema_adapter_name(cfg) if cfg is not None else "invalid" if cfg is not None and not policy_errors: cfg = normalize_config_shape(cfg) - findings: list[dict[str, Any]] = list(policy_errors) + list(baseline_errors) + list(initial_findings) + findings: list[dict[str, Any]] = list(policy_errors) + list(baseline_errors) + list(envelope_errors) + list(initial_findings) def add(severity: str, risk: str, **extra: Any) -> None: item: dict[str, Any] = {"severity": severity, "risk": risk} @@ -1164,11 +1228,16 @@ def add(severity: str, risk: str, **extra: Any) -> None: "counts": count_by_severity(suppressed_findings), }, } + if args.envelope_secret_file and args.format != "json": + print("--envelope-secret-file requires --format json; envelopes sign the JSON report payload", file=sys.stderr) + return 2 if args.format == "markdown": print(render_markdown(summary)) elif args.format == "sarif": print(json.dumps(render_sarif(summary), separators=(",", ":") if args.compact else None, indent=None if args.compact else 2, sort_keys=True)) else: + if envelope_secret is not None: + summary["report_envelope"] = build_report_envelope(summary, envelope_secret) print(json.dumps(summary, separators=(",", ":") if args.compact else None, indent=None if args.compact else 2, sort_keys=True)) fail_on = args.fail_on or ("high" if args.strict else None) diff --git a/skills/agent-security/scripts/verify_report_envelope.py b/skills/agent-security/scripts/verify_report_envelope.py new file mode 100644 index 0000000..04c86de --- /dev/null +++ b/skills/agent-security/scripts/verify_report_envelope.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Verify the HMAC-SHA256 authenticity envelope on an agent-security JSON report. + +Reads a JSON report on stdin and checks its ``report_envelope`` signature +against a secret supplied via ``--secret-file `` or +``--secret-env `` (exactly one source). Exits 0 when the report is +intact, 1 on any verification failure, and 2 on usage errors. + +This is an integrity check, not encryption: envelopes detect mutation, they +hide nothing. Dependency-light by design (standard library only). +""" +import argparse +import hashlib +import hmac +import json +import os +import sys +from typing import Any + +SUPPORTED_ALGORITHMS = {"hmac-sha256"} + + +def error(code: str, message: str) -> dict[str, str]: + return {"code": code, "message": message} + + +def read_secret_from_file(path: str) -> tuple[bytes | None, dict[str, str] | None]: + try: + with open(path, encoding="utf-8") as fh: + raw = fh.read() + except OSError as exc: + return None, error("invalid_secret", f"could not read secret file: {exc}") + except UnicodeDecodeError as exc: + return None, error("invalid_secret", f"secret file is not valid UTF-8 text: {exc}") + secret = raw[:-1] if raw.endswith("\n") else raw + if not secret.strip(): + return None, error("invalid_secret", "secret file is empty or blank") + return secret.encode("utf-8"), None + + +def read_secret_from_env(var: str) -> tuple[bytes | None, dict[str, str] | None]: + raw = os.environ.get(var) + if raw is None: + return None, error("invalid_secret", f"environment variable {var} is not set") + secret = raw[:-1] if raw.endswith("\n") else raw + if not secret.strip(): + return None, error("invalid_secret", f"environment variable {var} is empty or blank") + return secret.encode("utf-8"), None + + +def canonical_payload_bytes(payload: Any) -> bytes: + """Serialize the signed payload deterministically (sorted keys, tight separators).""" + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def verify_report(report: Any, secret: bytes) -> dict[str, Any]: + """Verify one report object; returns a verdict dict with ``ok`` and ``errors``.""" + errors: list[dict[str, str]] = [] + if not isinstance(report, dict): + return {"ok": False, "errors": [error("invalid_report", "report top-level value must be a JSON object")]} + + envelope = report.get("report_envelope") + if not isinstance(envelope, dict): + return {"ok": False, "errors": [error("missing_envelope", "report has no report_envelope object")]} + + algorithm = envelope.get("algorithm") + if algorithm not in SUPPORTED_ALGORITHMS: + errors.append(error("unsupported_algorithm", f"envelope algorithm must be one of {sorted(SUPPORTED_ALGORITHMS)}")) + + expected_digest = envelope.get("payload_sha256") + expected_signature = envelope.get("signature") + if not isinstance(expected_digest, str) or len(expected_digest) != 64: + errors.append(error("malformed_envelope", "payload_sha256 must be a 64-character hex string")) + expected_digest = "" + if not isinstance(expected_signature, str) or len(expected_signature) != 64: + errors.append(error("malformed_envelope", "signature must be a 64-character hex string")) + expected_signature = "" + if errors: + return {"ok": False, "errors": errors} + + payload = {key: value for key, value in report.items() if key != "report_envelope"} + digest = hashlib.sha256(canonical_payload_bytes(payload)).hexdigest() + if digest != expected_digest: + errors.append(error("payload_digest_mismatch", f"payload digest is {digest}, envelope claims {expected_digest}")) + return {"ok": False, "errors": errors} + + signature = hmac.new(secret, digest.encode("ascii"), hashlib.sha256).hexdigest() + if not hmac.compare_digest(signature, expected_signature): + errors.append(error("signature_mismatch", "HMAC signature does not match the supplied secret")) + return {"ok": not errors, "errors": errors} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sources = parser.add_mutually_exclusive_group(required=True) + sources.add_argument("--secret-file", help="path to a UTF-8 text file containing the signing secret") + sources.add_argument("--secret-env", help="name of an environment variable containing the signing secret") + args = parser.parse_args() + + if args.secret_file: + secret, secret_error = read_secret_from_file(args.secret_file) + else: + secret, secret_error = read_secret_from_env(args.secret_env) + if secret is None: + verdict = {"ok": False, "errors": [secret_error]} + print(json.dumps(verdict, sort_keys=True)) + return 1 + + raw = sys.stdin.read() + try: + report = json.loads(raw) + except json.JSONDecodeError as exc: + verdict = {"ok": False, "errors": [error("invalid_report", f"stdin is not valid JSON: {exc}")]} + print(json.dumps(verdict, sort_keys=True)) + return 1 + + verdict = verify_report(report, secret) + verdict.setdefault("algorithm", "hmac-sha256") + print(json.dumps(verdict, sort_keys=True)) + return 0 if verdict["ok"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_phase20_report_envelopes.py b/tests/test_phase20_report_envelopes.py new file mode 100644 index 0000000..a67fbd0 --- /dev/null +++ b/tests/test_phase20_report_envelopes.py @@ -0,0 +1,226 @@ +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "skills" / "agent-security" / "scripts" / "config_risk_summary.py" +VERIFIER = ROOT / "skills" / "agent-security" / "scripts" / "verify_report_envelope.py" +DOC = ROOT / "docs" / "report-envelopes.md" + +SECRET = "test-envelope-secret-0123456789abcdef" + +HIGH_RISK_PAYLOAD = { + "channels": {"discord": {"enabled": True}}, + "browser": {"enabled": True, "ssrfPolicy": {"dangerouslyAllowPrivateNetwork": True}}, + "tools": {"exec": {"security": "full"}}, + "bindings": [{"agentId": "shared", "match": {"channel": "discord", "peer": {"kind": "channel"}}}], +} + + +def run_script(script: Path, payload, *args: str) -> subprocess.CompletedProcess[str]: + proc = subprocess.run( + [sys.executable, str(script), *args], + input=json.dumps(payload) if not isinstance(payload, str) else payload, + text=True, + capture_output=True, + check=False, + ) + return proc + + +def write_secret(tmp_path: Path, value: str, name: str = "agent-security.key") -> Path: + secret_path = tmp_path / name + secret_path.write_text(value, encoding="utf-8") + return secret_path + + +def signed_report(tmp_path: Path, payload=HIGH_RISK_PAYLOAD, secret: str = SECRET) -> dict: + secret_path = write_secret(tmp_path, secret) + proc = run_script(SCRIPT, payload, "--envelope-secret-file", str(secret_path)) + assert proc.returncode == 0, proc.stderr + return json.loads(proc.stdout) + + +def verify(report: dict, *args: str) -> subprocess.CompletedProcess[str]: + return run_script(VERIFIER, json.dumps(report), *args) + + +def test_envelope_absent_by_default_keeps_output_compatible(): + proc = run_script(SCRIPT, HIGH_RISK_PAYLOAD) + data = json.loads(proc.stdout) + assert "report_envelope" not in data + assert data["schema_version"] == "1.0" + assert any(f["risk"] == "shared_channel_with_private_network_browser" for f in data["findings"]) + + +def test_envelope_secret_file_adds_hmac_envelope(tmp_path): + secret_path = write_secret(tmp_path, SECRET) + proc = run_script(SCRIPT, HIGH_RISK_PAYLOAD, "--envelope-secret-file", str(secret_path)) + assert proc.returncode == 0, proc.stderr + data = json.loads(proc.stdout) + envelope = data["report_envelope"] + assert envelope["algorithm"] == "hmac-sha256" + assert len(envelope["payload_sha256"]) == 64 + assert len(envelope["signature"]) == 64 + assert "findings" in envelope["covered_fields"] + assert "report_envelope" not in envelope["covered_fields"] + + +def test_envelope_signature_is_deterministic_and_stable_across_runs(tmp_path): + secret_path = write_secret(tmp_path, SECRET) + first = json.loads(run_script(SCRIPT, HIGH_RISK_PAYLOAD, "--envelope-secret-file", str(secret_path)).stdout) + second = json.loads(run_script(SCRIPT, HIGH_RISK_PAYLOAD, "--envelope-secret-file", str(secret_path)).stdout) + assert first["report_envelope"] == second["report_envelope"] + + +def test_envelope_ignores_one_trailing_newline_in_secret_file(tmp_path): + padded = signed_report(tmp_path, secret=SECRET + "\n") + plain = signed_report(tmp_path, secret=SECRET) + assert padded["report_envelope"]["signature"] == plain["report_envelope"]["signature"] + + +def test_envelope_rejects_empty_secret_with_structured_finding(tmp_path): + secret_path = write_secret(tmp_path, "\n") + proc = run_script(SCRIPT, HIGH_RISK_PAYLOAD, "--envelope-secret-file", str(secret_path)) + data = json.loads(proc.stdout) + assert "report_envelope" not in data + assert any(f["risk"] == "invalid_envelope_secret" and f["severity"] == "error" for f in data["findings"]) + assert data["ok"] is False + + strict = run_script( + SCRIPT, HIGH_RISK_PAYLOAD, "--envelope-secret-file", str(secret_path), "--strict" + ) + assert strict.returncode == 1 + + +def test_envelope_rejects_unreadable_secret_with_structured_finding(tmp_path): + missing = tmp_path / "missing.key" + proc = run_script(SCRIPT, HIGH_RISK_PAYLOAD, "--envelope-secret-file", str(missing)) + data = json.loads(proc.stdout) + assert "report_envelope" not in data + assert any(f["risk"] == "invalid_envelope_secret" for f in data["findings"]) + + +def test_envelope_flag_with_non_json_format_is_a_usage_error(tmp_path): + secret_path = write_secret(tmp_path, SECRET) + proc = run_script(SCRIPT, HIGH_RISK_PAYLOAD, "--format", "markdown", "--envelope-secret-file", str(secret_path)) + assert proc.returncode == 2 + assert "--format json" in proc.stderr + + +def test_verifier_accepts_intact_report_with_secret_file(tmp_path): + secret_path = write_secret(tmp_path, SECRET) + report = signed_report(tmp_path) + proc = verify(report, "--secret-file", str(secret_path)) + assert proc.returncode == 0, proc.stdout + proc.stderr + verdict = json.loads(proc.stdout) + assert verdict["ok"] is True + assert verdict["algorithm"] == "hmac-sha256" + + +def test_verifier_accepts_intact_report_with_secret_env(tmp_path, monkeypatch): + monkeypatch.setenv("AGENT_SECURITY_ENVELOPE_SECRET", SECRET) + report = signed_report(tmp_path) + proc = verify(report, "--secret-env", "AGENT_SECURITY_ENVELOPE_SECRET") + assert proc.returncode == 0, proc.stdout + proc.stderr + assert json.loads(proc.stdout)["ok"] is True + + +def test_verifier_detects_tampered_findings(tmp_path): + secret_path = write_secret(tmp_path, SECRET) + report = signed_report(tmp_path) + report["findings"][0]["risk"] = "tampered_low_severity" + proc = verify(report, "--secret-file", str(secret_path)) + assert proc.returncode == 1 + verdict = json.loads(proc.stdout) + assert verdict["ok"] is False + assert any(err["code"] == "payload_digest_mismatch" for err in verdict["errors"]) + + +def test_verifier_detects_forged_signature(tmp_path): + secret_path = write_secret(tmp_path, SECRET) + report = signed_report(tmp_path) + report["report_envelope"]["signature"] = "0" * 64 + proc = verify(report, "--secret-file", str(secret_path)) + assert proc.returncode == 1 + verdict = json.loads(proc.stdout) + assert any(err["code"] == "signature_mismatch" for err in verdict["errors"]) + + +def test_verifier_rejects_missing_envelope(tmp_path): + secret_path = write_secret(tmp_path, SECRET) + report = signed_report(tmp_path) + report.pop("report_envelope") + proc = verify(report, "--secret-file", str(secret_path)) + assert proc.returncode == 1 + verdict = json.loads(proc.stdout) + assert any(err["code"] == "missing_envelope" for err in verdict["errors"]) + + +def test_verifier_rejects_unsupported_algorithm(tmp_path): + secret_path = write_secret(tmp_path, SECRET) + report = signed_report(tmp_path) + report["report_envelope"]["algorithm"] = "md5" + proc = verify(report, "--secret-file", str(secret_path)) + assert proc.returncode == 1 + verdict = json.loads(proc.stdout) + assert any(err["code"] == "unsupported_algorithm" for err in verdict["errors"]) + + +def test_verifier_rejects_wrong_secret(tmp_path): + secret_path = write_secret(tmp_path, "a-completely-different-secret", name="wrong.key") + report = signed_report(tmp_path) + proc = verify(report, "--secret-file", str(secret_path)) + assert proc.returncode == 1 + verdict = json.loads(proc.stdout) + assert any(err["code"] == "signature_mismatch" for err in verdict["errors"]) + + +def test_verifier_rejects_empty_or_missing_secret(tmp_path, monkeypatch): + report = signed_report(tmp_path) + empty_secret = write_secret(tmp_path, "\n", name="empty.key") + proc = verify(report, "--secret-file", str(empty_secret)) + assert proc.returncode == 1 + assert any(err["code"] == "invalid_secret" for err in json.loads(proc.stdout)["errors"]) + + monkeypatch.delenv("AGENT_SECURITY_ENVELOPE_SECRET", raising=False) + proc = verify(report, "--secret-env", "AGENT_SECURITY_ENVELOPE_SECRET") + assert proc.returncode == 1 + assert any(err["code"] == "invalid_secret" for err in json.loads(proc.stdout)["errors"]) + + +def test_verifier_requires_exactly_one_secret_source(tmp_path): + secret_path = write_secret(tmp_path, SECRET) + proc = verify(signed_report(tmp_path), "--secret-file", str(secret_path), "--secret-env", "ANY_VAR") + assert proc.returncode == 2 + + proc = verify(signed_report(tmp_path)) + assert proc.returncode == 2 + + +def test_phase20_docs_readme_changelog_skill_and_roadmap_are_in_sync(): + doc = DOC.read_text(encoding="utf-8") + assert "hmac-sha256" in doc + assert "--envelope-secret-file" in doc + assert "verify_report_envelope.py" in doc + assert "not encryption" in doc + assert "rotate" in doc + + readme = (ROOT / "README.md").read_text(encoding="utf-8") + assert "docs/report-envelopes.md" in readme + assert "--envelope-secret-file" in readme + + changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + assert "report authenticity envelopes" in changelog + + roadmap = (ROOT / "docs" / "roadmap.md").read_text(encoding="utf-8") + phase20 = roadmap.split("## Phase 20:", 1)[1].split("## Implementation order", 1)[0] + assert "**Status:** Shipped (JSON report slice)" in phase20 + assert "tests/test_phase20_report_envelopes.py" in phase20 + + skill = (ROOT / "skills" / "agent-security" / "SKILL.md").read_text(encoding="utf-8") + assert "verify_report_envelope.py" in skill + + ci_doc = (ROOT / "docs" / "ci-integration.md").read_text(encoding="utf-8") + assert "report-envelopes.md" in ci_doc