From 0c7c9225a8f6dcf09c2afd5bfbbe06115264d5a7 Mon Sep 17 00:00:00 2001 From: Matt Partida Date: Mon, 7 Sep 2026 09:24:03 -0700 Subject: [PATCH] feat: add tagged release automation with fail-closed release gate (Phase 19) - scripts/verify_release_gate.py: verifies tag shape, tag/commit binding (annotated-tag peeling), changelog version section, exact dist inventory, manifest structure, archive digests, secret-shaped content, and the Phase 18 deterministic rebuild before any tagged release publishes. - .github/workflows/release.yml: tag-push-only workflow (v*.*.*), job-scoped contents/attestations/id-token permissions, persist-credentials: false, full quality gate, deterministic rebuild, release gate, actions/attest-build-provenance@v4 attestations, gh release --verify-tag publication with skill archives and MANIFEST.json attached. - docs/release-automation.md: workflow steps, gate checks, maintainer flow, retry path, and pull-request permission non-goals. - tests/test_phase19_release_gate.py: 13 regression tests across gate checks, annotated-tag binding, drift/tamper/secret failure modes, and workflow least-privilege shape. - roadmap Phase 19 -> Shipped; changelog + installation guide links. --- .github/workflows/release.yml | 68 ++++++ CHANGELOG.md | 2 + docs/installation-and-release.md | 9 + docs/release-automation.md | 94 ++++++++ docs/roadmap.md | 15 +- scripts/verify_release_gate.py | 327 +++++++++++++++++++++++++ tests/test_phase19_release_gate.py | 368 +++++++++++++++++++++++++++++ 7 files changed, 882 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release.yml create mode 100644 docs/release-automation.md create mode 100644 scripts/verify_release_gate.py create mode 100644 tests/test_phase19_release_gate.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..aff4440 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,68 @@ +name: Release + +on: + push: + tags: + - "v*.*.*" + +permissions: {} + +jobs: + release: + name: Verify and publish tagged release + runs-on: ubuntu-latest + permissions: + contents: write + attestations: write + id-token: write + steps: + - name: Check out the tagged commit + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Install test tools + run: python -m pip install --upgrade pip pytest ruff + + - name: Lint + run: ruff check . + + - name: Compile scripts + run: python -m compileall -q skills tests scripts + + - name: Run tests + run: pytest -q + + - name: Build skill archives + run: ./package-skills.sh + + - name: Verify release gate + run: python3 scripts/verify_release_gate.py "${GITHUB_REF_NAME}" --commit "${GITHUB_SHA}" + + - name: Attest build provenance for skill archives + uses: actions/attest-build-provenance@v4 + with: + subject-path: | + dist/agent-security.skill + dist/healthcheck.skill + + - name: Publish GitHub release + run: | + set -euo pipefail + NOTES_FILE="$(mktemp)" + printf 'See CHANGELOG.md for the %s release notes.\n' "${GITHUB_REF_NAME}" > "${NOTES_FILE}" + gh release create "${GITHUB_REF_NAME}" \ + --repo "${GITHUB_REPOSITORY}" \ + --verify-tag \ + --title "${GITHUB_REF_NAME}" \ + --notes-file "${NOTES_FILE}" + gh release upload "${GITHUB_REF_NAME}" \ + --repo "${GITHUB_REPOSITORY}" \ + dist/agent-security.skill dist/healthcheck.skill dist/MANIFEST.json + env: + GH_TOKEN: ${{ github.token }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 818ca43..07ad953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ This project follows semantic-versioning guidance once recurring releases are ta ### Script CLI changes +- Added `scripts/verify_release_gate.py` as a fail-closed pre-release gate for tagged releases: stable tag shape, tag/commit binding (including annotated-tag peeling), changelog version-section coverage, exact dist inventory, manifest validation, archive digest verification, secret-shaped archive scanning, and the Phase 18 deterministic rebuild check. Exit `0` on pass, `1` on gate failure, `2` on usage errors; no repository, dist, or remote mutation. +- Added the tag-gated Release workflow (`.github/workflows/release.yml`) that runs the quality gate, rebuilds archives, runs the release gate, attests build provenance for both `.skill` archives, and publishes the GitHub release with `--verify-tag`; job-scoped permissions only, and pull requests never receive release permissions. Documented in `docs/release-automation.md` with regression coverage in `tests/test_phase19_release_gate.py`. - Replaced ad-hoc ZIP packaging with reproducible `scripts/package_skills.py`, deterministic `dist/MANIFEST.json` release metadata, and a non-mutating `--check` drift gate while preserving `./package-skills.sh`. - Added `--format json|markdown` to `skills/agent-security/scripts/flag_prompt_injection_signals.py` for review-friendly prompt-injection signal summaries while keeping JSON as the default. - Added `--output-dir` to `skills/agent-security/scripts/summarize_prompt_injection_corpus.py` for paired JSON/Markdown prompt-corpus review packets with no manifest or fixture mutation. diff --git a/docs/installation-and-release.md b/docs/installation-and-release.md index c768eb0..70419e3 100644 --- a/docs/installation-and-release.md +++ b/docs/installation-and-release.md @@ -123,6 +123,15 @@ window, and process interruption or power loss can stop between replacements. Concurrent modification is unsupported. After any interruption, run `--check`; do not release unless the exact inventory and every byte match. +## Tagged release automation + +Tag pushes matching `v*.*.*` are verified and published by the Release workflow +described in [`docs/release-automation.md`](release-automation.md). The workflow +runs the full quality gate, rebuilds the archives deterministically, verifies them +with `scripts/verify_release_gate.py`, attests build provenance, and attaches +`agent-security.skill`, `healthcheck.skill`, and `MANIFEST.json` to the GitHub +release. See that guide for the maintainer release flow and retry path. + ## Release checklist Before tagging or publishing a release, complete this checklist from a clean checkout: diff --git a/docs/release-automation.md b/docs/release-automation.md new file mode 100644 index 0000000..6498c14 --- /dev/null +++ b/docs/release-automation.md @@ -0,0 +1,94 @@ +# Release Automation and Attestations + +Phase 19 adds a review-gated tagged-release workflow. Pushing a stable `vX.Y.Z` tag +runs the full quality gate, rebuilds the skill archives deterministically, verifies +the release with a fail-closed gate script, attests build provenance for the +archives, and publishes the GitHub release. The workflow never runs on pull +requests and does not grant pull requests (or any other context) release +permissions. + +## What the release workflow does + +[`../.github/workflows/release.yml`](../.github/workflows/release.yml) triggers +only on tag pushes matching `v*.*.*`: + +1. Checks out the tagged commit with `persist-credentials: false`. +2. Runs the same quality gate as CI: ruff, `compileall`, pytest. +3. Rebuilds the archives with `./package-skills.sh`. +4. Runs `scripts/verify_release_gate.py` (see below). Any failure blocks the release. +5. Attests build provenance for `dist/agent-security.skill` and + `dist/healthcheck.skill` with `actions/attest-build-provenance@v4`, which + requires and receives only `attestations: write` and `id-token: write` for this job. +6. Publishes the GitHub release with `gh release create --verify-tag` and uploads + `dist/agent-security.skill`, `dist/healthcheck.skill`, and `dist/MANIFEST.json`. + +The job-level permissions are exactly `contents: write` (to create the release), +`attestations: write`, and `id-token: write`. The workflow-level default is +`permissions: {}`, so every permission is explicit and job-scoped. Pull requests +never receive release permissions from this workflow. + +## The release gate + +`scripts/verify_release_gate.py [--dist-dir dist] [--commit ]` fails +closed on all of the following: + +- **Tag shape:** the tag must be a stable `vMAJOR.MINOR.PATCH` tag. Pre-release + suffixes are rejected for now. +- **Tag/commit binding:** the tag must resolve to the exact commit the workflow is + building. Both sides are peeled to commit SHAs, so an annotated tag object SHA + binds correctly to its commit. This prevents publishing a release built from a + different commit than the one tagged. +- **Changelog coverage:** `CHANGELOG.md` must contain a `## X.Y.Z` section for the + tag, with released sections ordered above older releases. This stops tagging a + release whose notes still live under `Unreleased`. +- **Artifact inventory:** `dist/` must contain exactly `agent-security.skill`, + `healthcheck.skill`, and `MANIFEST.json` — nothing missing, nothing extra. +- **Manifest validity:** `MANIFEST.json` must parse, carry a schema version, and + list exactly the two published skills. +- **Digest verification:** every archive's recomputed SHA-256 must match the + manifest, so tampered or stale archives fail before publication. +- **Secret scan:** archive bytes are scanned for secret-shaped content (GitHub + tokens, AWS access key IDs, Slack tokens, private key blocks, Google API keys). + Matches are reported by label only and never echoed. +- **Deterministic rebuild:** the Phase 18 packager `--check` gate must confirm the + built artifacts byte-match a clean rebuild of the tagged sources. + +Exit codes: `0` when every gate passes, `1` when any gate fails, `2` on usage +errors. The gate never modifies the repository, the dist tree, or remote state. + +Run it locally before tagging: + +```bash +./package-skills.sh +python3 scripts/verify_release_gate.py v0.2.0 +``` + +## Release flow for maintainers + +1. Move the intended `Unreleased` changelog entries into a new `## X.Y.Z` section + (see [versioning guidance](installation-and-release.md#versioning-guidance)). +2. Commit the changelog update and wait for CI to pass on that commit. +3. Tag the commit — annotated tags are recommended: + + ```bash + git tag -a vX.Y.Z -m "Release vX.Y.Z" + git push origin vX.Y.Z + ``` + +4. The Release workflow runs the gate, attests the archives, and publishes the + GitHub release with the three dist artifacts attached. +5. Verify the release page shows the expected artifacts, and check the attestation + via the release workflow's summary or `gh attestation verify` with the + artifact digest from `dist/MANIFEST.json`. + +If any gate step fails, no release is created. Fix the underlying issue, delete the +local tag, re-tag, and push again. Deleting and re-pushing a tag is the documented +retry path; the workflow itself never force-pushes or rewrites history. + +## Relationship to the release checklist + +The workflow automates the mechanical steps of the +[release checklist](installation-and-release.md#release-checklist) (clean rebuild, +quality gate, archive inspection, secret scan, manifest digest verification). The +human review steps — rule doc review, fixture review, and changelog wording — +still happen before tagging. diff --git a/docs/roadmap.md b/docs/roadmap.md index 988c68f..9b222f1 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -360,9 +360,22 @@ Before starting new roadmap work, check open PRs and avoid duplicating any branc ## Phase 19: Tagged release automation and attestations -**Status:** Planned +**Status:** Shipped **Goal:** Add a review-gated tagged-release workflow that publishes verified archives, manifest digests, and provenance without granting pull requests release permissions. +### Shipped scope + +1. Added [`scripts/verify_release_gate.py`](../scripts/verify_release_gate.py), a fail-closed pre-release gate that verifies stable tag shape, tag/commit binding with annotated-tag peeling, a matching `CHANGELOG.md` version section, exact dist inventory, manifest structure, archive SHA-256 digests against the manifest, a secret-shaped content scan over packaged archives, and the Phase 18 deterministic rebuild check. +2. Added [`docs/release-automation.md`](release-automation.md) covering the workflow's steps, gate checks, maintainer release flow, retry path, and explicit non-goals for pull-request permissions. +3. Added `.github/workflows/release.yml`, triggered only by `v*.*.*` tag pushes, with job-scoped `contents: write`, `attestations: write`, and `id-token: write` permissions, `persist-credentials: false` checkout, the full CI quality gate, deterministic rebuild, the release gate, `actions/attest-build-provenance@v4` attestations for both skill archives, and `gh release create --verify-tag` publication with `agent-security.skill`, `healthcheck.skill`, and `MANIFEST.json` attached. +4. Added `tests/test_phase19_release_gate.py` covering tag-shape validation, changelog section ordering, dist inventory/manifest/digest failure modes, annotated-tag commit binding, secret-scan reporting without echoing matches, packager-check drift detection, end-to-end gate pass/fail runs, and workflow least-privilege shape. + +### Acceptance criteria + +- Pushing a `vX.Y.Z` tag runs the quality gate, rebuild, release gate, and attestation steps; any failure blocks publication. +- Pull requests never gain release permissions from this workflow. +- The gate fails closed on tag/commit mismatch, missing changelog sections, artifact drift, digest mismatch, and secret-shaped archive content. + ## Phase 20: Scanner report authenticity envelopes **Status:** Planned diff --git a/scripts/verify_release_gate.py b/scripts/verify_release_gate.py new file mode 100644 index 0000000..eefb119 --- /dev/null +++ b/scripts/verify_release_gate.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +"""Verify pre-release gates for tagged agent-security releases. + +This gate runs before publishing a tagged release. It fails closed on: + +- missing or malformed release tag arguments, +- tags that do not point at the commit being released, +- unrecoverable packaging drift (via the Phase 18 ``--check`` gate), +- a ``CHANGELOG.md`` without a matching section for the tag, +- dist artifacts that disagree with the deterministic rebuild, +- secret-shaped content inside packaged release artifacts. + +The gate never modifies the repository, the dist tree, or remote state. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PACKAGER = ROOT / "scripts" / "package_skills.py" +CHANGELOG = ROOT / "CHANGELOG.md" +DEFAULT_DIST = ROOT / "dist" + +MANIFEST_NAME = "MANIFEST.json" +TAG_PATTERN = re.compile(r"^v\d+\.\d+\.\d+$") +SEMVER_PATTERN = re.compile(r"^\d+\.\d+\.\d+$") + +# High-signal, low-false-positive token shapes. Patterns are matched +# case-sensitively against artifact text and never printed with matches. +SECRET_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("github_pat_token", re.compile(r"github_pat_[A-Za-z0-9_]{36,}")), + ("github_classic_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b")), + ("aws_access_key_id", re.compile(r"\bAKIA[0-9A-Z]{16}\b")), + ("slack_token", re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b")), + ("private_key_block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")), + ("google_api_key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")), +) + +EXIT_OK = 0 +EXIT_GATE_FAILURE = 1 +EXIT_USAGE = 2 + + +def gate_failure(message: str) -> None: + print(f"release gate: {message}", file=sys.stderr) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("tag", help="release tag being verified, e.g. v0.1.0") + parser.add_argument( + "--dist-dir", + type=Path, + default=DEFAULT_DIST, + help="artifact directory to verify (default: dist)", + ) + parser.add_argument( + "--commit", + default=None, + help="commit SHA the release workflow is building (defaults to HEAD)", + ) + parser.add_argument( + "--skip-packager-check", + action="store_true", + help="skip the Phase 18 packager --check invocation (tests and offline audits)", + ) + return parser.parse_args(argv) + + +def verify_tag_shape(tag: str) -> str | None: + """Return an error string when the tag is not a stable ``vX.Y.Z`` tag.""" + if tag != tag.strip(): + return f"tag {tag!r} has surrounding whitespace" + if not TAG_PATTERN.match(tag): + return ( + f"tag {tag!r} is not a stable semantic tag of the form vMAJOR.MINOR.PATCH " + "(pre-release suffixes are not supported by this gate yet)" + ) + return None + + +def verify_tag_points_at_commit(tag: str, commit: str) -> str | None: + """Return an error string when the tag does not resolve to ``commit``. + + Both sides are peeled to commit SHAs first: for annotated tags the + workflow-provided SHA may be the tag object rather than the commit. + """ + resolved = subprocess.run( + ["git", "rev-parse", f"{tag}^{{commit}}"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + if resolved.returncode != 0: + return f"could not resolve tag {tag!r} to a commit: {resolved.stderr.strip()}" + tagged_commit = resolved.stdout.strip() + peeled = subprocess.run( + ["git", "rev-parse", f"{commit}^{{commit}}"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + if peeled.returncode != 0: + return f"could not resolve provided commit {commit!r}: {peeled.stderr.strip()}" + provided_commit = peeled.stdout.strip() + if tagged_commit != provided_commit: + return ( + f"tag {tag!r} points at {tagged_commit} but the release workflow is building {commit}; " + "re-tag at the release commit or re-run the workflow from the tagged commit" + ) + return None + + +def verify_changelog_section(tag: str) -> str | None: + """Return an error string when CHANGELOG.md lacks a released section for ``tag``.""" + try: + text = CHANGELOG.read_text(encoding="utf-8") + except OSError as exc: + return f"could not read CHANGELOG.md: {exc}" + version = tag.lstrip("v") + if not SEMVER_PATTERN.match(version): + return f"tag {tag!r} does not carry a parseable version" + heading = f"## {version}" + for line in text.splitlines(): + if line.startswith("#"): + if line.strip() == heading: + return None + if line.strip() in {"## Unreleased", "## Unreleased changes"}: + continue + # Any other heading encountered above the target section means + # the version section is missing or ordered below later entries. + if line.startswith("## "): + return ( + f"CHANGELOG.md has no '## {version}' section; add released notes for {tag} " + "above older release headings" + ) + return f"CHANGELOG.md has no '## {version}' section for tag {tag}" + + +def verify_dist_inventory(dist_dir: Path) -> list[str]: + """Return a list of inventory problems in the dist directory.""" + problems: list[str] = [] + if not dist_dir.is_dir(): + return [f"artifact directory {dist_dir} does not exist; run ./package-skills.sh first"] + expected_archives = {"agent-security.skill", "healthcheck.skill", MANIFEST_NAME} + # Directory-descriptor safety is enforced by the packager; this inventory + # pass only checks names so the gate can report actionable problems. + observed = {entry.name for entry in dist_dir.iterdir()} + missing = sorted(expected_archives - observed) + extra = sorted(observed - expected_archives) + for name in missing: + problems.append(f"missing expected artifact {name!r} in {dist_dir}") + for name in extra: + problems.append(f"unexpected extra entry {name!r} in {dist_dir}") + return problems + + +def load_manifest(dist_dir: Path) -> tuple[dict | None, str | None]: + """Load and structurally validate dist/MANIFEST.json.""" + path = dist_dir / MANIFEST_NAME + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except OSError as exc: + return None, f"could not read {path}: {exc}" + except json.JSONDecodeError as exc: + return None, f"{path} is not valid JSON: {exc}" + + problems: list[str] = [] + if not isinstance(manifest.get("schema_version"), str): + problems.append("manifest 'schema_version' is missing or not a string") + archives = manifest.get("archives") + if not isinstance(archives, list) or not archives: + return None, "manifest 'archives' is missing or empty" + names = {entry.get("name") for entry in archives if isinstance(entry, dict)} + if names != {"agent-security", "healthcheck"}: + problems.append(f"manifest archive names {sorted(map(str, names))} != ['agent-security', 'healthcheck']") + if problems: + return None, "; ".join(problems) + return manifest, None + + +def verify_artifact_digests(dist_dir: Path, manifest: dict) -> list[str]: + """Recompute SHA-256 digests for each archive and compare with the manifest.""" + problems: list[str] = [] + for entry in manifest["archives"]: + name = entry.get("name") + expected = entry.get("sha256") + archive = dist_dir / f"{name}.skill" + if not isinstance(expected, str) or not re.fullmatch(r"[0-9a-f]{64}", expected): + problems.append(f"manifest entry for {name!r} lacks a valid sha256 digest") + continue + try: + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + except OSError as exc: + problems.append(f"could not read {archive}: {exc}") + continue + if digest != expected: + problems.append( + f"archive {archive.name} digest {digest} does not match manifest sha256 {expected}" + ) + return problems + + +def scan_artifacts_for_secrets(dist_dir: Path) -> list[str]: + """Scan packaged artifact bytes for secret-shaped content; never echo matches.""" + problems: list[str] = [] + for archive in sorted(dist_dir.glob("*.skill")): + try: + data = archive.read_bytes() + except OSError: + problems.append(f"could not read {archive} for secret scanning") + continue + try: + text = data.decode("utf-8", errors="ignore") + except OSError: + problems.append(f"could not decode {archive} for secret scanning") + continue + for label, pattern in SECRET_PATTERNS: + if pattern.search(text): + problems.append( + f"{archive.name} contains secret-shaped content matching {label}; " + "inspect the archive before publishing" + ) + return problems + + +def run_packager_check(dist_dir: Path) -> str | None: + """Run the Phase 18 deterministic rebuild comparison and return an error string on drift.""" + proc = subprocess.run( + [sys.executable, str(PACKAGER), "--check", "--output-dir", str(dist_dir)], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout).strip().splitlines() + summary = detail[-1] if detail else f"exit code {proc.returncode}" + return f"packager --check failed: {summary}" + return None + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + failures: list[str] = [] + + tag_error = verify_tag_shape(args.tag) + if tag_error: + gate_failure(tag_error) + failures.append(tag_error) + valid_tag = tag_error is None + + commit = args.commit or _head_commit() + if commit is None: + head_error = "could not determine HEAD commit; pass --commit explicitly" + gate_failure(head_error) + failures.append(head_error) + else: + tag_commit_error = verify_tag_points_at_commit(args.tag, commit) + if tag_commit_error: + gate_failure(tag_commit_error) + failures.append(tag_commit_error) + + if valid_tag: + changelog_error = verify_changelog_section(args.tag) + if changelog_error: + gate_failure(changelog_error) + failures.append(changelog_error) + + inventory_problems = verify_dist_inventory(args.dist_dir) + for problem in inventory_problems: + gate_failure(problem) + failures.extend(inventory_problems) + + manifest, manifest_error = load_manifest(args.dist_dir) + if manifest_error: + gate_failure(manifest_error) + failures.append(manifest_error) + + if manifest is not None: + digest_problems = verify_artifact_digests(args.dist_dir, manifest) + for problem in digest_problems: + gate_failure(problem) + failures.extend(digest_problems) + + secret_problems = scan_artifacts_for_secrets(dist_dir=args.dist_dir) + for problem in secret_problems: + gate_failure(problem) + failures.extend(secret_problems) + + if not args.skip_packager_check: + packager_error = run_packager_check(args.dist_dir) + if packager_error: + gate_failure(packager_error) + failures.append(packager_error) + + if failures: + print(f"release gate: {len(failures)} blocking issue(s); release blocked", file=sys.stderr) + return EXIT_GATE_FAILURE + print(f"release gate passed for {args.tag}") + return EXIT_OK + + +def _head_commit() -> str | None: + proc = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_phase19_release_gate.py b/tests/test_phase19_release_gate.py new file mode 100644 index 0000000..b631219 --- /dev/null +++ b/tests/test_phase19_release_gate.py @@ -0,0 +1,368 @@ +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +GATE = ROOT / "scripts" / "verify_release_gate.py" +PACKAGER = ROOT / "scripts" / "package_skills.py" +RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "release.yml" +RELEASE_DOC = ROOT / "docs" / "release-automation.md" +INSTALL_DOC = ROOT / "docs" / "installation-and-release.md" +ROADMAP = ROOT / "docs" / "roadmap.md" +CHANGELOG = ROOT / "CHANGELOG.md" + +EXPECTED_ARTIFACTS = {"agent-security.skill", "healthcheck.skill", "MANIFEST.json"} + + +def load_gate(): + spec = importlib.util.spec_from_file_location("phase19_gate", GATE) + assert spec is not None and spec.loader is not None + gate = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = gate + spec.loader.exec_module(gate) + return gate + + +def run_packager(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(PACKAGER), *args], + cwd=ROOT, + check=True, + text=True, + capture_output=True, + ) + + +def init_release_repo(path: Path) -> str: + """Create a tiny git repo with one commit and return its HEAD SHA.""" + def git(*args: str) -> None: + subprocess.run(["git", *args], cwd=path, check=True, capture_output=True, text=True) + + path.mkdir(parents=True, exist_ok=True) + git("init", "-q") + git("config", "user.email", "release-gate@example.com") + git("config", "user.name", "Release Gate Test") + (path / "placeholder.txt").write_text("release gate test\n", encoding="utf-8") + git("add", "placeholder.txt") + git("commit", "-q", "-m", "test commit") + return subprocess.run(["git", "rev-parse", "HEAD"], cwd=path, check=True, capture_output=True, text=True).stdout.strip() + + +def build_dist(tmp_path: Path) -> Path: + dist = tmp_path / "dist" + run_packager("--output-dir", str(dist)) + return dist + + +# --- Tag shape --------------------------------------------------------------- + + +def test_tag_shape_accepts_stable_semver_and_rejects_other_tags(): + gate = load_gate() + assert gate.verify_tag_shape("v0.1.0") is None + assert gate.verify_tag_shape("v10.20.30") is None + for bad in ("v0.1", "0.1.0", "v0.1.0-rc1", " v0.1.0", "v0.1.0 ", "release-1", "vX.Y.Z"): + assert gate.verify_tag_shape(bad) is not None, bad + + +# --- Changelog section ------------------------------------------------------- + + +def test_changelog_check_detects_version_section(tmp_path): + gate = load_gate() + original = gate.CHANGELOG + try: + changelog = tmp_path / "CHANGELOG.md" + gate.CHANGELOG = changelog + + changelog.write_text("# Changelog\n\n## Unreleased\n\n- something\n", encoding="utf-8") + assert gate.verify_changelog_section("v0.2.0") is not None + + changelog.write_text("# Changelog\n\n## Unreleased\n\n- something\n\n## 0.2.0\n\n- shipped\n", encoding="utf-8") + assert gate.verify_changelog_section("v0.2.0") is None + + changelog.write_text("# Changelog\n\n## Unreleased\n\n- something\n", encoding="utf-8") + assert gate.verify_changelog_section("v0.2.0") is not None + + changelog.write_text("# Changelog\n\n## 0.2.0\n\n- shipped\n", encoding="utf-8") + assert gate.verify_changelog_section("v0.2.0") is None + finally: + gate.CHANGELOG = original + + +def test_changelog_check_requires_newest_version_section_topmost(tmp_path): + gate = load_gate() + original = gate.CHANGELOG + try: + changelog = tmp_path / "CHANGELOG.md" + gate.CHANGELOG = changelog + changelog.write_text( + "# Changelog\n\n## Unreleased\n\n- something\n\n## 0.2.0\n\n- newer\n\n## 0.1.0\n\n- older\n", + encoding="utf-8", + ) + assert gate.verify_changelog_section("v0.2.0") is None + assert gate.verify_changelog_section("v0.1.0") is not None + finally: + gate.CHANGELOG = original + + +# --- Dist inventory, manifest, digests ---------------------------------------- + + +def test_inventory_reports_missing_and_extra_artifacts(tmp_path): + gate = load_gate() + assert gate.verify_dist_inventory(tmp_path / "does-not-exist") + + dist = tmp_path / "dist" + dist.mkdir() + problems = gate.verify_dist_inventory(dist) + assert len(problems) == len(EXPECTED_ARTIFACTS) + + for name in sorted(EXPECTED_ARTIFACTS): + (dist / name).write_bytes(b"placeholder") + assert gate.verify_dist_inventory(dist) == [] + + (dist / "extra.skill").write_bytes(b"placeholder") + problems = gate.verify_dist_inventory(dist) + assert len(problems) == 1 + assert "extra.skill" in problems[0] + + +def test_manifest_validation_rejects_malformed_manifests(tmp_path): + gate = load_gate() + dist = tmp_path / "dist" + dist.mkdir() + for name in EXPECTED_ARTIFACTS: + (dist / name).write_bytes(b"placeholder") + + (dist / "MANIFEST.json").write_text("{not json", encoding="utf-8") + _, error = gate.load_manifest(dist) + assert error is not None and "JSON" in error + + (dist / "MANIFEST.json").write_text(json.dumps({"schema_version": "1.0", "archives": []}), encoding="utf-8") + _, error = gate.load_manifest(dist) + assert error is not None and "archives" in error + + (dist / "MANIFEST.json").write_text( + json.dumps({"schema_version": "1.0", "archives": [{"name": "someone-else"}]}), + encoding="utf-8", + ) + _, error = gate.load_manifest(dist) + assert error is not None and "someone-else" in error + + +def test_digest_verification_detects_tampered_archives(tmp_path): + dist = build_dist(tmp_path) + gate = load_gate() + manifest, error = gate.load_manifest(dist) + assert error is None and manifest is not None + assert gate.verify_artifact_digests(dist, manifest) == [] + + tampered = dist / "agent-security.skill" + tampered.write_bytes(tampered.read_bytes() + b"tampered") + problems = gate.verify_artifact_digests(dist, manifest) + assert len(problems) == 1 + assert "agent-security.skill" in problems[0] + + +def test_secret_scan_flags_token_shaped_artifact_content(tmp_path): + gate = load_gate() + dist = tmp_path / "dist" + dist.mkdir() + for name in ("agent-security.skill", "healthcheck.skill", "MANIFEST.json"): + (dist / name).write_bytes(b"clean") + assert gate.scan_artifacts_for_secrets(dist) == [] + + (dist / "agent-security.skill").write_text( + "leaked token ghp_0123456789abcdefghijklmnopqrstuvwxyzAB\n", encoding="utf-8" + ) + problems = gate.scan_artifacts_for_secrets(dist) + assert len(problems) == 1 + assert "secret-shaped" in problems[0] + # Match details are never echoed into gate output. + assert "ghp_0123456789" not in problems[0] + + +# --- Tag/commit binding ------------------------------------------------------- + + +def test_tag_commit_binding_peels_annotated_tags(tmp_path, monkeypatch): + gate = load_gate() + head = init_release_repo(tmp_path) + + def git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=tmp_path, capture_output=True, text=True, check=True + ).stdout.strip() + + monkeypatch.setattr(gate, "ROOT", tmp_path) + + # Lightweight tag at HEAD binds directly. + git("tag", "v1.2.3") + assert gate.verify_tag_points_at_commit("v1.2.3", head) is None + + # Annotated tag object SHA also binds after peeling. + git("tag", "-a", "v1.2.4", "-m", "annotated") + tag_object = git("rev-parse", "v1.2.4") + assert tag_object != head, "expected the annotated tag object SHA to differ from the commit" + assert gate.verify_tag_points_at_commit("v1.2.4", tag_object) is None + assert gate.verify_tag_points_at_commit("v1.2.4", head) is None + + # A tag pointing elsewhere must fail closed. + (tmp_path / "other.txt").write_text("second commit\n", encoding="utf-8") + git("add", "other.txt") + git("commit", "-q", "-m", "second") + new_head = git("rev-parse", "HEAD") + assert gate.verify_tag_points_at_commit("v1.2.3", new_head) is not None + + # Unknown tags and commits fail closed. + assert gate.verify_tag_points_at_commit("v9.9.9", head) is not None + assert gate.verify_tag_points_at_commit("v1.2.3", "not-a-commit") is not None + + +# --- End-to-end gate ---------------------------------------------------------- + + +def test_gate_passes_for_clean_tagged_release(tmp_path, monkeypatch): + gate = load_gate() + repo = tmp_path / "repo" + init_release_repo(repo) + + def git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=repo, capture_output=True, text=True, check=True + ).stdout.strip() + + git("tag", "-a", "v0.2.0", "-m", "release v0.2.0") + dist = build_dist(tmp_path) + + changelog = tmp_path / "CHANGELOG.md" + changelog.write_text("# Changelog\n\n## 0.2.0\n\n- shipped\n", encoding="utf-8") + + monkeypatch.setattr(gate, "ROOT", repo) + monkeypatch.setattr(gate, "CHANGELOG", changelog) + rc = gate.main(["v0.2.0", "--dist-dir", str(dist), "--commit", git("rev-parse", "v0.2.0")]) + assert rc == 0 + + +def test_gate_blocks_when_changelog_or_artifacts_are_wrong(tmp_path, monkeypatch, capsys): + gate = load_gate() + repo = tmp_path / "repo" + init_release_repo(repo) + + def git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=repo, capture_output=True, text=True, check=True + ).stdout.strip() + + git("tag", "v0.2.0") + dist = build_dist(tmp_path) + changelog = tmp_path / "CHANGELOG.md" + changelog.write_text("# Changelog\n\n## Unreleased\n\n- not released yet\n", encoding="utf-8") + + monkeypatch.setattr(gate, "ROOT", repo) + monkeypatch.setattr(gate, "CHANGELOG", changelog) + + # Missing changelog section blocks the release. + rc = gate.main(["v0.2.0", "--dist-dir", str(dist), "--skip-packager-check"]) + assert rc == 1 + assert "CHANGELOG.md has no '## 0.2.0' section" in capsys.readouterr().err + + # A tampered archive blocks the release via digest mismatch. + changelog.write_text("# Changelog\n\n## 0.2.0\n\n- shipped\n", encoding="utf-8") + tampered = dist / "healthcheck.skill" + tampered.write_bytes(b"tampered") + rc = gate.main(["v0.2.0", "--dist-dir", str(dist), "--skip-packager-check"]) + assert rc == 1 + assert "does not match manifest sha256" in capsys.readouterr().err + + # Invalid tag shape is a hard block regardless of other inputs. + rc = gate.main(["0.2.0", "--dist-dir", str(dist), "--skip-packager-check"]) + assert rc == 1 + assert "not a stable semantic tag" in capsys.readouterr().err + + +def test_gate_runs_packager_check_by_default_and_detects_drift(tmp_path, monkeypatch, capsys): + gate = load_gate() + repo = tmp_path / "repo" + init_release_repo(repo) + + def git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=repo, capture_output=True, text=True, check=True + ).stdout.strip() + + git("tag", "v0.2.0") + dist = build_dist(tmp_path) + changelog = tmp_path / "CHANGELOG.md" + changelog.write_text("# Changelog\n\n## 0.2.0\n\n- shipped\n", encoding="utf-8") + monkeypatch.setattr(gate, "ROOT", repo) + monkeypatch.setattr(gate, "CHANGELOG", changelog) + + # An extra artifact is both an inventory problem and packager drift. + (dist / "stale-extra.skill").write_bytes(b"stale") + rc = gate.main(["v0.2.0", "--dist-dir", str(dist)]) + assert rc == 1 + err = capsys.readouterr().err + assert "stale-extra.skill" in err + + +# --- Workflow and documentation shape ----------------------------------------- + + +def _read(path: Path) -> str: + assert path.exists(), f"missing expected Phase 19 artifact: {path.relative_to(ROOT)}" + text = path.read_text(encoding="utf-8") + assert text.endswith("\n"), f"{path.relative_to(ROOT)} should end with a newline" + return text + + +def test_release_workflow_is_tag_gated_and_least_privilege(): + text = _read(RELEASE_WORKFLOW) + assert "tags:" in text + assert '"v*.*.*"' in text + assert "on:" in text + assert "pull_request" not in text + assert "pull-requests:" not in text + assert "issues:" not in text + assert "permissions:" in text + assert "contents: write" in text + assert "attestations: write" in text + assert "id-token: write" in text + assert "persist-credentials: false" in text + assert "actions/checkout@v7" in text + assert "actions/setup-python@v7" in text + assert "actions/attest-build-provenance@v4" in text + assert "verify_release_gate.py" in text + assert "GITHUB_REF_NAME" in text + assert "--verify-tag" in text + assert "dist/agent-security.skill" in text + assert "dist/healthcheck.skill" in text + assert "dist/MANIFEST.json" in text + + +def test_release_docs_and_roadmap_coverage(): + doc = _read(RELEASE_DOC) + for phrase in ( + "verify_release_gate.py", + "annotated tag", + "gh release", + "attest", + "MANIFEST.json", + "does not grant", + "fail", + ): + assert phrase in doc, phrase + + install = _read(INSTALL_DOC) + assert "release-automation.md" in install + + roadmap = _read(ROADMAP) + assert "## Phase 19: Tagged release automation and attestations" in roadmap + assert "**Status:** Shipped" in roadmap + + changelog = _read(CHANGELOG) + assert "verify_release_gate.py" in changelog + assert "release.yml" in changelog