diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index a97c0738cb..efb4c15044 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -31,6 +31,10 @@ on: description: "Optional full merged commit supplying corrected browser fixtures" required: false type: string + validated_run_id: + description: "Publication only: reuse the successful build of an existing tagged release run" + required: false + type: string permissions: contents: read @@ -42,7 +46,7 @@ concurrency: jobs: build: - if: github.repository == 'loopx-project/loopx' + if: github.repository == 'loopx-project/loopx' && inputs.validated_run_id == '' runs-on: ubuntu-latest # Run 39196404607 completed all 71 packaged scenarios and two companion # flows at the 20m job window, leaving required release checks unrun. @@ -416,23 +420,50 @@ jobs: retention-days: 30 upload-release: - if: github.event_name != 'pull_request' + if: >- + !cancelled() && github.repository == 'loopx-project/loopx' && + github.event_name != 'pull_request' && + (needs.build.result == 'success' || + (inputs.validated_run_id != '' && needs.build.result == 'skipped')) needs: build runs-on: ubuntu-latest timeout-minutes: 10 permissions: contents: write + actions: read + attestations: read steps: - name: Check out release source uses: actions/checkout@v7 with: - ref: ${{ needs.build.outputs.release-tag }} + ref: ${{ needs.build.outputs.release-tag || inputs.tag }} + + - name: Check out reviewed publication tooling + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + path: publication-tooling + + - name: Verify the original tagged build before recovery + if: inputs.validated_run_id != '' + env: + GH_TOKEN: ${{ github.token }} + VALIDATED_RUN_ID: ${{ inputs.validated_run_id }} + BROWSER_FIXTURE_COMMIT: ${{ inputs.browser_fixture_commit }} + run: | + set -euo pipefail + test -z "${BROWSER_FIXTURE_COMMIT}" + python publication-tooling/scripts/release_artifacts.py verify-publication-run \ + --repo "$GITHUB_REPOSITORY" --run-id "$VALIDATED_RUN_ID" \ + --commit "$(git rev-parse HEAD)" - name: Download validated release bundle uses: actions/download-artifact@v7 with: - name: loopx-${{ needs.build.outputs.release-tag }} + name: loopx-${{ needs.build.outputs.release-tag || inputs.tag }} path: dist + run-id: ${{ inputs.validated_run_id || github.run_id }} + github-token: ${{ github.token }} - name: Verify checksums before upload run: >- @@ -443,28 +474,25 @@ jobs: - name: Upload immutable GitHub Release assets env: GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ needs.build.outputs.release-tag }} + VALIDATED_RUN_ID: ${{ inputs.validated_run_id }} run: | set -euo pipefail - gh release view "${RELEASE_TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null - gh release upload "${RELEASE_TAG}" \ - dist/packages/* \ - dist/SHA256SUMS \ - dist/RELEASE-SOURCES.json \ - dist/runtime/* \ - --repo "${GITHUB_REPOSITORY}" + recovery=() + if [[ -n "$VALIDATED_RUN_ID" ]]; then recovery=(--recover); fi + python publication-tooling/scripts/release_artifacts.py upload-immutable \ + --dist-dir dist --repo "$GITHUB_REPOSITORY" "${recovery[@]}" publish-pypi: if: >- - github.event_name != 'pull_request' && + !cancelled() && github.event_name != 'pull_request' && + needs.upload-release.result == 'success' && vars.PYPI_PUBLISH_ENABLED == 'true' - needs: - - build - - upload-release + needs: upload-release runs-on: ubuntu-latest timeout-minutes: 10 permissions: id-token: write + actions: read environment: name: pypi url: https://pypi.org/project/loopx/ @@ -472,8 +500,10 @@ jobs: - name: Download validated release bundle uses: actions/download-artifact@v7 with: - name: loopx-${{ needs.build.outputs.release-tag }} + name: loopx-${{ github.event.release.tag_name || inputs.tag }} path: dist + run-id: ${{ inputs.validated_run_id || github.run_id }} + github-token: ${{ github.token }} - name: Publish distributions with PyPI Trusted Publishing uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 @@ -482,7 +512,7 @@ jobs: - name: Verify the published package from PyPI env: - RELEASE_TAG: ${{ needs.build.outputs.release-tag }} + RELEASE_TAG: ${{ github.event.release.tag_name || inputs.tag }} run: | set -euo pipefail version="${RELEASE_TAG#v}" diff --git a/docs/product/release-readiness.md b/docs/product/release-readiness.md index 9e3ebeaadd..069b0cb534 100644 --- a/docs/product/release-readiness.md +++ b/docs/product/release-readiness.md @@ -239,6 +239,20 @@ workflow commit, separately from the immutable product tag passed as `tag`. Retain the tooling ref for provenance verification; do not move the product tag, relax the environment policy, or replace Trusted Publishing with a token. +When a tagged build passed but publication stopped after a partial upload, +dispatch the reviewed release workflow with the original product `tag` and its +`validated_run_id`. This path reuses the successful release-event build without +rebuilding packages or repeating qualification. It checks the repository, +workflow, tag, source and successful build job, then verifies every recovered +file's attestation against that tagged source. Existing Release assets must +match downloaded bytes exactly before any missing file is uploaded; a mismatch +or case-insensitive name collision stops publication without replacement. +Trusted Publishing and the existing environment policy still apply. + +Reserve `qualification.json` for the workflow's attested runtime qualification. +Use `RELEASE-QUALIFICATION.json` for a separately maintained release overview; +GitHub Release asset names must also be unique when compared ignoring case. + For a browser-fixture error discovered after an immutable tag was published, maintainers may dispatch the release workflow from reviewed tooling with `tag` set to that existing release and `browser_fixture_commit` set to a full, diff --git a/examples/release-artifacts-smoke.py b/examples/release-artifacts-smoke.py index e587cd085b..374dd17ac3 100644 --- a/examples/release-artifacts-smoke.py +++ b/examples/release-artifacts-smoke.py @@ -284,7 +284,7 @@ def main() -> int: "python scripts/release_artifacts.py write-checksums", "python scripts/release_artifacts.py verify-checksums", "uses: actions/attest@v4", - "gh release upload", + "scripts/release_artifacts.py upload-immutable", "vars.PYPI_PUBLISH_ENABLED == 'true'", "environment:\n name: pypi", "pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33", diff --git a/scripts/release_artifacts.py b/scripts/release_artifacts.py index 130ae7d245..c6b6712e69 100644 --- a/scripts/release_artifacts.py +++ b/scripts/release_artifacts.py @@ -9,8 +9,12 @@ import gzip import hashlib import io +import json +import re +import subprocess import sys import tarfile +import tempfile import tomllib from dataclasses import dataclass from pathlib import Path @@ -182,6 +186,87 @@ def verify_manifest(dist_dir: Path, checksum_file: Path, identity: ReleaseIdenti print(f"verified {checksum_file} for {len(files)} distributions") +def gh(*args: str) -> bytes: + return subprocess.check_output(["gh", *args]) + + +def verify_publication_run(repo: str, run_id: str, tag: str, commit: str) -> None: + """Only reuse a successful tagged build from this repository's release workflow.""" + if not re.fullmatch(r"[1-9][0-9]*", run_id): + raise ReleaseArtifactError("validated run id must be a positive integer") + run = json.loads(gh("api", f"repos/{repo}/actions/runs/{run_id}")) + if ( + run["repository"]["full_name"] != repo + or run["event"] != "release" + or run["path"] != ".github/workflows/release-artifacts.yml" + or run["head_sha"] != commit + or run["head_branch"] != tag + ): + raise ReleaseArtifactError("recovery run does not match the tagged product build") + jobs = json.loads(gh("api", f"repos/{repo}/actions/runs/{run_id}/jobs?per_page=100")) + builds = [job for job in jobs["jobs"] if job["name"] == "build"] + if len(builds) != 1 or builds[0]["conclusion"] != "success": + raise ReleaseArtifactError("recovery requires the original successful build job") + print(f"verified successful build {run_id} for {tag}@{commit}") + + +def publication_files(dist_dir: Path, identity: ReleaseIdentity) -> tuple[Path, ...]: + files = (*distribution_files(dist_dir / "packages", identity), + dist_dir / "SHA256SUMS", dist_dir / "RELEASE-SOURCES.json", + *sorted((dist_dir / "runtime").iterdir())) + if any(path.is_symlink() or not path.is_file() for path in files): + raise ReleaseArtifactError("publication inputs must be regular files") + if len({path.name.casefold() for path in files}) != len(files): + raise ReleaseArtifactError("publication names collide ignoring case") + verify_manifest(dist_dir / "packages", dist_dir / "SHA256SUMS", identity) + return tuple(files) + + +def upload_immutable(dist_dir: Path, identity: ReleaseIdentity, repo: str, + *, recover: bool = False, dry_run: bool = False) -> None: + files = publication_files(dist_dir, identity) + if recover: + commit = gh("api", f"repos/{repo}/commits/{identity.tag}", "--jq", ".sha").decode().strip() + sources = json.loads((dist_dir / "RELEASE-SOURCES.json").read_text()) + if sources["product_tag"] != identity.tag or sources["product_commit"] != commit: + raise ReleaseArtifactError("bundle product identity differs from the release tag") + for path in files: + gh("attestation", "verify", str(path), "--repo", repo, + "--signer-workflow", f"{repo}/.github/workflows/release-artifacts.yml", + "--source-digest", commit, "--source-ref", f"refs/tags/{identity.tag}", + "--deny-self-hosted-runners") + release = json.loads(gh("api", f"repos/{repo}/releases/tags/{identity.tag}")) + assets = json.loads(gh("api", "--paginate", "--slurp", + f"repos/{repo}/releases/{release['id']}/assets?per_page=100")) + remote = {} + for asset in (asset for page in assets for asset in page): + key = asset["name"].casefold() + if key in remote: + raise ReleaseArtifactError("remote release names collide ignoring case") + remote[key] = asset + missing = [] + # Check every existing asset before the first upload; never delete or clobber. + for path in files: + asset = remote.get(path.name.casefold()) + if asset is None: + missing.append(path) + continue + if asset["name"] != path.name: + raise ReleaseArtifactError(f"release name collision: {asset['name']} / {path.name}") + with tempfile.TemporaryFile() as downloaded: + subprocess.run(["gh", "api", f"repos/{repo}/releases/assets/{asset['id']}", + "-H", "Accept: application/octet-stream"], + stdout=downloaded, check=True) + downloaded.seek(0) + actual = hashlib.file_digest(downloaded, "sha256").hexdigest() + if asset["size"] != path.stat().st_size or actual != sha256(path): + raise ReleaseArtifactError(f"immutable release asset differs: {path.name}") + if missing and not dry_run: + gh("release", "upload", identity.tag, *(str(path) for path in missing), "--repo", repo) + print(f"verified {len(files) - len(missing)} existing assets; " + f"{'would upload' if dry_run else 'uploaded'} {len(missing)} missing assets") + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--project-root", type=Path, default=Path.cwd()) @@ -202,6 +287,16 @@ def build_parser() -> argparse.ArgumentParser: verify = subparsers.add_parser("verify-checksums", help="verify a canonical SHA256SUMS file") verify.add_argument("--dist-dir", type=Path, required=True) verify.add_argument("--checksum-file", type=Path, required=True) + + recovery = subparsers.add_parser("verify-publication-run", help="verify a tagged build for publication recovery") + recovery.add_argument("--repo", required=True) + recovery.add_argument("--run-id", required=True) + recovery.add_argument("--commit", required=True) + upload = subparsers.add_parser("upload-immutable", help="resume publication without replacing existing bytes") + upload.add_argument("--repo", required=True) + upload.add_argument("--dist-dir", type=Path, required=True) + upload.add_argument("--recover", action="store_true") + upload.add_argument("--dry-run", action="store_true") return parser @@ -224,6 +319,11 @@ def main(argv: list[str] | None = None) -> int: write_manifest(args.dist_dir, args.output, identity) elif args.command == "verify-checksums": verify_manifest(args.dist_dir, args.checksum_file, identity) + elif args.command == "verify-publication-run": + verify_publication_run(args.repo, args.run_id, identity.tag, args.commit) + elif args.command == "upload-immutable": + upload_immutable(args.dist_dir, identity, args.repo, + recover=args.recover, dry_run=args.dry_run) else: parser.error(f"unsupported command: {args.command}") except ReleaseArtifactError as exc: diff --git a/tests/test_release_artifact_publication.py b/tests/test_release_artifact_publication.py new file mode 100644 index 0000000000..621f0c1c04 --- /dev/null +++ b/tests/test_release_artifact_publication.py @@ -0,0 +1,138 @@ +"""Publication may resume, but must never replace already published bytes.""" + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +SPEC = importlib.util.spec_from_file_location( + "publication_artifacts", Path(__file__).parents[1] / "scripts/release_artifacts.py" +) +assert SPEC and SPEC.loader +owner = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = owner +SPEC.loader.exec_module(owner) + + +@pytest.fixture +def bundle(tmp_path): + identity = owner.ReleaseIdentity("loopx", "1.2.3") + (tmp_path / "packages").mkdir() + (tmp_path / "runtime").mkdir() + for name in ["loopx-1.2.3-py3-none-any.whl", "loopx-1.2.3.tar.gz"]: + (tmp_path / "packages" / name).write_bytes(name.encode()) + (tmp_path / "runtime/qualification.json").write_bytes(b'{"ok":true}') + (tmp_path / "RELEASE-SOURCES.json").write_text(json.dumps({ + "product_tag": identity.tag, "product_commit": "a" * 40, + })) + owner.write_manifest(tmp_path / "packages", tmp_path / "SHA256SUMS", identity) + return tmp_path, identity + + +def transport(files, *, mismatch=False, wrong_case=False, attestation_failure=False): + """Fake only the external boundary; inspect actual downloaded bytes.""" + calls = [] + assets = [{"id": i, "name": path.name, "size": path.stat().st_size} + for i, path in enumerate(files)] + if wrong_case: + assets[-1]["name"] = assets[-1]["name"].upper() + + def gh(*args): + calls.append(args) + if args[0] == "attestation" and attestation_failure: + raise subprocess.CalledProcessError(1, "attestation") + if args[0:2] == ("release", "upload") or args[0] == "attestation": + return b"" + if "/commits/" in args[1]: + return ("a" * 40).encode() + return json.dumps([assets] if "--paginate" in args else {"id": 42}).encode() + + def download(argv, *, stdout, check): + index = int(argv[2].rsplit("/", 1)[1]) + stdout.write(b"changed" if mismatch else files[index].read_bytes()) + return subprocess.CompletedProcess(argv, 0) + + return calls, gh, download + + +@pytest.mark.parametrize("dry_run", [True, False]) +def test_resume_only_uploads_missing_files(bundle, dry_run): + root, identity = bundle + files = owner.publication_files(root, identity) + existing = list(files[:-1]) + calls, gh, download = transport(existing) + with patch.object(owner, "gh", gh), patch.object(owner.subprocess, "run", download): + owner.upload_immutable(root, identity, "owner/repo", dry_run=dry_run) + uploads = [c for c in calls if c[:2] == ("release", "upload")] + assert len(uploads) == (0 if dry_run else 1) + if uploads: + assert uploads[0] == ("release", "upload", identity.tag, str(files[-1]), "--repo", "owner/repo") + + +@pytest.mark.parametrize("wrong_case,mismatch", [(True, False), (False, True)]) +def test_collision_or_changed_bytes_prevents_every_upload(bundle, wrong_case, mismatch): + root, identity = bundle + files = list(owner.publication_files(root, identity)) + calls, gh, download = transport(files[1:], mismatch=mismatch, wrong_case=wrong_case) + with patch.object(owner, "gh", gh), patch.object(owner.subprocess, "run", download): + with pytest.raises(owner.ReleaseArtifactError): + owner.upload_immutable(root, identity, "owner/repo") + assert not any(c[:2] == ("release", "upload") for c in calls) + + +def test_recovery_attestation_failure_cannot_publish(bundle): + root, identity = bundle + calls, gh, download = transport([], attestation_failure=True) + with patch.object(owner, "gh", gh), patch.object(owner.subprocess, "run", download): + with pytest.raises(subprocess.CalledProcessError): + owner.upload_immutable(root, identity, "owner/repo", recover=True) + assert not any(c[:2] == ("release", "upload") for c in calls) + + +def test_recovery_bundle_cannot_substitute_another_product(bundle): + root, identity = bundle + (root / "RELEASE-SOURCES.json").write_text(json.dumps({ + "product_tag": identity.tag, "product_commit": "b" * 40, + })) + calls, gh, download = transport([]) + with patch.object(owner, "gh", gh), patch.object(owner.subprocess, "run", download): + with pytest.raises(owner.ReleaseArtifactError, match="identity"): + owner.upload_immutable(root, identity, "owner/repo", recover=True) + assert not any(c[:2] == ("release", "upload") for c in calls) + + +@pytest.mark.parametrize("field,value", [ + ("head_sha", "b" * 40), ("head_branch", "main"), ("event", "pull_request"), + ("path", ".github/workflows/unrelated.yml"), ("repository", {"full_name": "other/repo"}), + ("build_conclusion", "failure"), +]) +def test_recovery_requires_matching_successful_tagged_build(field, value): + run = {"repository": {"full_name": "owner/repo"}, "event": "release", + "path": ".github/workflows/release-artifacts.yml", "head_sha": "a" * 40, + "head_branch": "v1.2.3"} + if field != "build_conclusion": + run[field] = value + jobs = {"jobs": [{"name": "build", "conclusion": value if field == "build_conclusion" else "success"}]} + with patch.object(owner, "gh", side_effect=[json.dumps(run).encode(), json.dumps(jobs).encode()]): + with pytest.raises(owner.ReleaseArtifactError): + owner.verify_publication_run("owner/repo", "123", "v1.2.3", "a" * 40) + + +def test_matching_successful_tagged_build_is_recoverable(): + run = {"repository": {"full_name": "owner/repo"}, "event": "release", + "path": ".github/workflows/release-artifacts.yml", "head_sha": "a" * 40, + "head_branch": "v1.2.3"} + jobs = {"jobs": [{"name": "build", "conclusion": "success"}]} + with patch.object(owner, "gh", side_effect=[json.dumps(run).encode(), json.dumps(jobs).encode()]): + owner.verify_publication_run("owner/repo", "123", "v1.2.3", "a" * 40) + + +def test_symlink_cannot_add_publication_input(bundle, tmp_path): + root, identity = bundle + (root / "runtime/link").symlink_to(root / "SHA256SUMS") + with pytest.raises(owner.ReleaseArtifactError, match="regular files"): + owner.publication_files(root, identity)