diff --git a/.github/workflows/migration-fixture-bootstrap.yml b/.github/workflows/migration-fixture-bootstrap.yml index 8e4e0394f..b74abbc13 100644 --- a/.github/workflows/migration-fixture-bootstrap.yml +++ b/.github/workflows/migration-fixture-bootstrap.yml @@ -21,8 +21,8 @@ name: Migration Fixture Bootstrap # nothing but jq and the GitHub API, and hands the bytes to the verifying job # as a same-run artifact. Delete the draft once the run has succeeded. # -# This is not capture automation: it captures nothing, runs on no schedule and -# opens no PR. A human dispatches it and records the run id in the manifest. +# Manual capture upload and recovery; scheduled renewal uses +# migration-fixture-renewal.yml and proposes updated pointers in a PR. # # USAGE, small archive: # gh workflow run migration-fixture-bootstrap.yml \ @@ -44,11 +44,8 @@ name: Migration Fixture Bootstrap # Either way, copy the values from the run's step summary into the entry's # `artifact` block in tests/migration-fixtures/manifest.json. # -# RETENTION HAZARD: artifacts expire, and 90 days is the ceiling for a public -# repository. Before the manifest's `expires_at`, re-dispatch with the same -# bytes and record the new `workflow_run_id`/`expires_at`. Once the artifact -# has expired, the migration matrix fails (download-fixtures.sh --strict) -# until that is done. +# For manual renewal, re-dispatch with the same bytes and record the returned +# pointers. Expired archives require a saved copy; the matrix fails without it. # # REVOCATION: `gh run delete ` removes the artifact together with the # run's inputs payload. diff --git a/.github/workflows/migration-fixture-renewal.yml b/.github/workflows/migration-fixture-renewal.yml new file mode 100644 index 000000000..d389a05e3 --- /dev/null +++ b/.github/workflows/migration-fixture-renewal.yml @@ -0,0 +1,210 @@ +name: Migration Fixture Renewal + +on: + schedule: + - cron: "23 5 * * 1" + workflow_dispatch: + inputs: + base_branch: + description: Branch whose manifest should be renewed + default: v1.0-dev + type: string + dry_run: + description: Verify downloads without uploading archives or opening a PR + default: true + type: boolean + force: + description: Include archives more than 30 days from expiry + default: false + type: boolean + pull_request: + paths: + - .github/workflows/migration-fixture-renewal.yml + - scripts/migration-fixtures/** + +concurrency: + group: fixture-renewal-${{ inputs.base_branch || github.event.pull_request.number || 'v1.0-dev' }} + cancel-in-progress: false + +permissions: {} + +jobs: + tests: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Test renewal contracts + run: python3 -m unittest discover -s scripts/migration-fixtures -p test_renewal.py -v + - name: Test fixture scripts + env: + TMPDIR: ${{ runner.temp }} + run: bash scripts/migration-fixtures/test-scripts.sh + + plan: + needs: tests + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + count: ${{ steps.plan.outputs.count }} + base: ${{ steps.plan.outputs.base }} + steps: + - uses: actions/checkout@v4 + with: + path: automation + persist-credentials: false + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.base_branch || 'v1.0-dev' }} + path: source + persist-credentials: false + - name: Select archives due within 30 days + id: plan + env: + GH_TOKEN: ${{ github.token }} + BASE_BRANCH: ${{ inputs.base_branch || 'v1.0-dev' }} + FORCE: ${{ inputs.force }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + git check-ref-format --branch "$BASE_BRANCH" + args=() + if [ "$FORCE" = true ]; then args+=(--force); fi + matrix=$(python3 automation/scripts/migration-fixtures/renewal.py plan \ + --manifest source/tests/migration-fixtures/manifest.json "${args[@]}") + pending=$(gh pr list --repo "$GITHUB_REPOSITORY" --base "$BASE_BRANCH" \ + --state open --limit 100 --json headRefName,url \ + --jq '.[] | select(.headRefName | startswith("ci/migration-fixture-renewal-")) | .url') + if [ -n "$pending" ] && [ "$DRY_RUN" != true ]; then + echo "Review the existing renewal PR before publishing another: $pending" >> "$GITHUB_STEP_SUMMARY" + matrix='{"include":[]}' + fi + count=$(jq '.include | length' <<< "$matrix") + { + echo "matrix=$matrix" + echo "count=$count" + echo "base=$BASE_BRANCH" + } >> "$GITHUB_OUTPUT" + echo "Selected $count archive(s) for renewal on $BASE_BRANCH." >> "$GITHUB_STEP_SUMMARY" + + renew: + needs: plan + if: needs.plan.outputs.count != '0' + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Download and verify original archive + env: + GH_TOKEN: ${{ github.token }} + FIXTURE: ${{ toJSON(matrix.fixture) }} + run: | + set -euo pipefail + jq '{schema_version: 1, fixtures: [.]}' <<< "$FIXTURE" > "$RUNNER_TEMP/selected.json" + bash scripts/migration-fixtures/download-fixtures.sh \ + --manifest "$RUNNER_TEMP/selected.json" --dest "$RUNNER_TEMP/archives" --strict + python3 scripts/migration-fixtures/renewal.py verify \ + --manifest "$RUNNER_TEMP/selected.json" --archives "$RUNNER_TEMP/archives" + - name: Upload identical archive + if: github.event_name != 'workflow_dispatch' || !inputs.dry_run + id: upload + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.fixture.artifact.artifact_name }} + path: ${{ runner.temp }}/archives/${{ matrix.fixture.artifact.archive_filename }} + if-no-files-found: error + retention-days: 90 + compression-level: 0 + overwrite: true + - name: Record actual upload expiry + if: github.event_name != 'workflow_dispatch' || !inputs.dry_run + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT_ID: ${{ steps.upload.outputs.artifact-id }} + FIXTURE: ${{ toJSON(matrix.fixture) }} + run: | + set -euo pipefail + gh api "repos/$GITHUB_REPOSITORY/actions/artifacts/$ARTIFACT_ID" > "$RUNNER_TEMP/upload.json" + mkdir -p "$RUNNER_TEMP/receipts" + jq --slurpfile uploaded "$RUNNER_TEMP/upload.json" \ + '{fixture_id: .id, source_artifact: .artifact, uploaded: $uploaded[0]}' \ + <<< "$FIXTURE" > "$RUNNER_TEMP/receipts/receipt.json" + python3 scripts/migration-fixtures/renewal.py apply \ + --manifest "$RUNNER_TEMP/selected.json" --receipts "$RUNNER_TEMP/receipts" + mv "$RUNNER_TEMP/receipts/receipt.json" "$RUNNER_TEMP/receipts/$(jq -r .id <<< "$FIXTURE").json" + - name: Share verified renewal receipt + if: github.event_name != 'workflow_dispatch' || !inputs.dry_run + uses: actions/upload-artifact@v4 + with: + name: fixture-renewal-record-${{ matrix.fixture.id }} + path: ${{ runner.temp }}/receipts/*.json + retention-days: 7 + if-no-files-found: error + overwrite: true + + propose: + needs: [plan, renew] + if: github.event_name != 'workflow_dispatch' || !inputs.dry_run + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + actions: read + steps: + - uses: actions/checkout@v4 + with: + path: automation + persist-credentials: false + - uses: actions/checkout@v4 + with: + ref: ${{ needs.plan.outputs.base }} + path: source + - uses: actions/download-artifact@v4 + with: + pattern: fixture-renewal-record-* + path: ${{ runner.temp }}/receipts + merge-multiple: true + - name: Prepare manifest-only pull request + working-directory: source + env: + GH_TOKEN: ${{ github.token }} + BASE_BRANCH: ${{ needs.plan.outputs.base }} + EXPECTED_COUNT: ${{ needs.plan.outputs.count }} + run: | + set -euo pipefail + test "$(find "$RUNNER_TEMP/receipts" -name '*.json' | wc -l)" -eq "$EXPECTED_COUNT" + python3 ../automation/scripts/migration-fixtures/renewal.py apply \ + --manifest tests/migration-fixtures/manifest.json --receipts "$RUNNER_TEMP/receipts" + branch="ci/migration-fixture-renewal-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + git switch -c "$branch" + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add tests/migration-fixtures/manifest.json + git commit -m 'ci: renew migration fixture artifact retention' + git push origin "HEAD:refs/heads/$branch" + cat > "$RUNNER_TEMP/pr-body.md" <<'EOF' + Historical migration archives are approaching expiry. This refresh preserves their exact bytes and extends their availability for migration tests. + + Only artifact run pointers and retention metadata change. Archive sizes and SHA-256 digests were verified before upload; actual expiry was read back from GitHub. No wallet capture or migration was performed. + + Review and merge this PR before the old artifacts expire. Run the migration matrix against this branch before merging. If GitHub requests approval for Actions on this bot-created PR, approve those checks. + + 🤖 Co-authored by [Claudius the Magnificent](https://github.com/lklimek/claudius) AI Agent + EOF + gh pr create --repo "$GITHUB_REPOSITORY" --base "$BASE_BRANCH" --head "$branch" \ + --draft --title 'ci: renew migration fixture artifacts' --body-file "$RUNNER_TEMP/pr-body.md" \ + | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.md b/CHANGELOG.md index 80a510315..da13fceae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- A scheduled workflow renews expiring migration fixture archives without + changing their contents and proposes updated manifest pointers in a PR. + - Migration tests also replay public user/DPNS and Evonode identities serialized by v0.9.3, checking their metadata and every public key after repeated startup. diff --git a/scripts/migration-fixtures/renewal.py b/scripts/migration-fixtures/renewal.py new file mode 100644 index 000000000..981c3016b --- /dev/null +++ b/scripts/migration-fixtures/renewal.py @@ -0,0 +1,155 @@ +"""Plan archive renewal and update only verified artifact pointers.""" + +import argparse +import copy +import hashlib +import json +import re +from datetime import datetime, timedelta, timezone +from pathlib import Path + + +def timestamp(value: str) -> datetime: + """Parse an expiry with an explicit timezone.""" + if not isinstance(value, str): + raise ValueError("missing artifact expiry") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("artifact expiry needs a timezone") + return parsed + + +def entries(manifest: dict) -> list[dict]: + """Validate immutable archive identities before any network access.""" + fixtures = manifest.get("fixtures") + if not isinstance(fixtures, list) or not fixtures: + raise ValueError("manifest must contain fixtures") + seen = {key: set() for key in ("id", "artifact_name", "archive_filename")} + for fixture in fixtures: + artifact = fixture["artifact"] + for key in seen: + value = fixture["id"] if key == "id" else artifact.get(key, "") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", value): + raise ValueError(f"invalid {key}") + if value in seen[key]: + raise ValueError(f"duplicate {key}: {value}") + seen[key].add(value) + if not artifact["archive_filename"].endswith((".tar.zst", ".tar.gz")): + raise ValueError("unsupported archive extension") + if not re.fullmatch(r"[0-9a-fA-F]{64}", artifact.get("sha256", "")): + raise ValueError("invalid archive checksum") + for key in ("bytes", "workflow_run_id"): + if type(artifact.get(key)) is not int or artifact[key] <= 0: + raise ValueError(f"invalid {key}") + timestamp(artifact.get("expires_at")) + return fixtures + + +def plan(manifest: dict, now: datetime, days: int, force: bool = False) -> dict: + """Select unexpired archives at or within the renewal threshold.""" + if not 1 <= days <= 90: + raise ValueError("renewal threshold must be between 1 and 90 days") + selected = [] + for fixture in entries(manifest): + expiry = timestamp(fixture["artifact"]["expires_at"]) + if expiry <= now: + raise ValueError( + f"{fixture['id']}: source artifact has expired; restore a saved archive" + ) + if force or expiry <= now + timedelta(days=days): + selected.append({"fixture": fixture}) + return {"include": selected} + + +def verify_archive(fixture: dict, path: Path) -> None: + """Verify exact archive size and digest without extracting its contents.""" + artifact = fixture["artifact"] + if path.name != artifact["archive_filename"] or path.is_symlink(): + raise ValueError("unexpected archive path") + if path.stat().st_size != artifact["bytes"]: + raise ValueError("archive size mismatch") + digest = hashlib.sha256() + with path.open("rb") as archive: + for block in iter(lambda: archive.read(1024 * 1024), b""): + digest.update(block) + if digest.hexdigest() != artifact["sha256"].lower(): + raise ValueError("archive checksum mismatch") + + +def renew_manifest(manifest: dict, receipts: list[dict], now: datetime) -> dict: + """Apply upload receipts only if the source pointers have not changed.""" + result = copy.deepcopy(manifest) + by_id = {fixture["id"]: fixture for fixture in entries(result)} + updated = set() + if not receipts: + raise ValueError("no renewal receipts") + for receipt in receipts: + fixture_id = receipt["fixture_id"] + if fixture_id not in by_id or fixture_id in updated: + raise ValueError("unknown or duplicate renewal receipt") + artifact = by_id[fixture_id]["artifact"] + if artifact != receipt["source_artifact"]: + raise ValueError(f"{fixture_id}: source manifest changed; rerun renewal") + uploaded = receipt["uploaded"] + run_id = uploaded["workflow_run"]["id"] + expiry = timestamp(uploaded["expires_at"]) + if ( + uploaded["name"] != artifact["artifact_name"] + or uploaded["expired"] is not False + or type(run_id) is not int + or run_id <= 0 + or run_id == artifact["workflow_run_id"] + or expiry <= max(now, timestamp(artifact["expires_at"])) + ): + raise ValueError( + f"{fixture_id}: upload does not extend this archive's retention" + ) + artifact.update( + workflow_name="migration-fixture-renewal.yml", + workflow_run_id=run_id, + retention_days=90, + expires_at=uploaded["expires_at"], + ) + updated.add(fixture_id) + return result + + +def main() -> None: + """Expose planning, downloaded archive verification and manifest updates.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("plan", "verify", "apply")) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--days", type=int, default=30) + parser.add_argument("--force", action="store_true") + parser.add_argument("--archives", type=Path) + parser.add_argument("--receipts", type=Path) + args = parser.parse_args() + manifest = json.loads(args.manifest.read_text()) + now = datetime.now(timezone.utc) + if args.command == "plan": + print( + json.dumps( + plan(manifest, now, args.days, args.force), separators=(",", ":") + ) + ) + elif args.command == "verify": + if args.archives is None: + parser.error("verify requires --archives") + for fixture in entries(manifest): + verify_archive( + fixture, args.archives / fixture["artifact"]["archive_filename"] + ) + print(f"Verified exact archive bytes: {fixture['id']}") + else: + if args.receipts is None: + parser.error("apply requires --receipts") + receipts = [ + json.loads(path.read_text()) + for path in sorted(args.receipts.glob("*.json")) + ] + result = renew_manifest(manifest, receipts, now) + args.manifest.write_text(json.dumps(result, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/migration-fixtures/test_renewal.py b/scripts/migration-fixtures/test_renewal.py new file mode 100644 index 000000000..0988f49a6 --- /dev/null +++ b/scripts/migration-fixtures/test_renewal.py @@ -0,0 +1,136 @@ +"""Offline contract tests for fixture renewal planning and manifest updates.""" + +import copy +import hashlib +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path + +import renewal + + +class RenewalTests(unittest.TestCase): + def setUp(self): + self.now = datetime(2026, 11, 15, tzinfo=timezone.utc) + self.entry = { + "id": "historical-wallet", + "contents": {"wallets": [{"password_env": "FIXTURE_PASSWORD"}]}, + "artifact": { + "workflow_name": "migration-fixture-bootstrap.yml", + "workflow_run_id": 10, + "artifact_name": "migration-fixture-historical-wallet", + "archive_filename": "historical.tar.zst", + "sha256": hashlib.sha256(b"archive").hexdigest(), + "bytes": 7, + "expires_at": "2026-12-10T00:00:00Z", + "retention_days": 90, + }, + } + self.manifest = {"schema_version": 1, "fixtures": [self.entry]} + self.receipt = { + "fixture_id": self.entry["id"], + "source_artifact": copy.deepcopy(self.entry["artifact"]), + "uploaded": { + "name": self.entry["artifact"]["artifact_name"], + "expired": False, + "expires_at": "2027-02-13T00:00:00Z", + "workflow_run": {"id": 20}, + }, + } + + def test_due_boundary_and_force(self): + self.assertEqual( + renewal.plan(self.manifest, self.now, 25)["include"], + [{"fixture": self.entry}], + ) + self.assertEqual(renewal.plan(self.manifest, self.now, 24)["include"], []) + self.assertEqual( + len(renewal.plan(self.manifest, self.now, 1, force=True)["include"]), 1 + ) + + def test_expired_source_is_an_error_even_with_force(self): + self.entry["artifact"]["expires_at"] = "2026-11-14T00:00:00Z" + with self.assertRaisesRegex(ValueError, "expired"): + renewal.plan(self.manifest, self.now, 30, force=True) + + def test_invalid_manifest_rejected_before_network(self): + for field, value in [ + ("sha256", ""), + ("bytes", 0), + ("workflow_run_id", None), + ("archive_filename", "../escape.tar.zst"), + ("expires_at", "invalid"), + ]: + with self.subTest(field=field): + manifest = copy.deepcopy(self.manifest) + manifest["fixtures"][0]["artifact"][field] = value + with self.assertRaises(ValueError): + renewal.plan(manifest, self.now, 30) + + def test_duplicate_artifact_names_rejected(self): + duplicate = copy.deepcopy(self.entry) + duplicate["id"] = "another-wallet" + self.manifest["fixtures"].append(duplicate) + with self.assertRaisesRegex(ValueError, "duplicate"): + renewal.plan(self.manifest, self.now, 30) + + def test_empty_manifest_is_an_error(self): + with self.assertRaises(ValueError): + renewal.plan({"fixtures": []}, self.now, 30) + + def test_archive_bytes_and_digest_are_verified(self): + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "historical.tar.zst" + archive.write_bytes(b"archive") + renewal.verify_archive(self.entry, archive) + archive.write_bytes(b"changed") + with self.assertRaisesRegex(ValueError, "checksum"): + renewal.verify_archive(self.entry, archive) + archive.write_bytes(b"short") + with self.assertRaisesRegex(ValueError, "size"): + renewal.verify_archive(self.entry, archive) + + def test_only_artifact_pointers_change(self): + original = copy.deepcopy(self.manifest) + result = renewal.renew_manifest(self.manifest, [self.receipt], self.now) + expected = copy.deepcopy(original) + expected["fixtures"][0]["artifact"].update( + workflow_name="migration-fixture-renewal.yml", + workflow_run_id=20, + expires_at="2027-02-13T00:00:00Z", + retention_days=90, + ) + self.assertEqual(result, expected) + self.assertEqual(self.manifest, original) + + def test_stale_manifest_is_not_overwritten(self): + self.entry["artifact"]["workflow_run_id"] = 11 + with self.assertRaisesRegex(ValueError, "changed"): + renewal.renew_manifest(self.manifest, [self.receipt], self.now) + + def test_bad_receipts_rejected(self): + for field, value in [ + ("name", "other"), + ("expired", True), + ("expires_at", "2026-12-01T00:00:00Z"), + ("workflow_run", {"id": 10}), + ]: + with self.subTest(field=field): + receipt = copy.deepcopy(self.receipt) + receipt["uploaded"][field] = value + with self.assertRaises(ValueError): + renewal.renew_manifest(self.manifest, [receipt], self.now) + + def test_duplicate_or_unknown_receipts_rejected(self): + with self.assertRaises(ValueError): + renewal.renew_manifest( + self.manifest, [self.receipt, self.receipt], self.now + ) + self.receipt["fixture_id"] = "unknown" + with self.assertRaises(ValueError): + renewal.renew_manifest(self.manifest, [self.receipt], self.now) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/migration-fixtures/README.md b/tests/migration-fixtures/README.md index ca9dd22a5..313cc97e4 100644 --- a/tests/migration-fixtures/README.md +++ b/tests/migration-fixtures/README.md @@ -107,12 +107,52 @@ Consequences that any workflow consuming this directory must handle: base64-encoded in the `archive_b64` input. A larger one travels through a temporary **draft** release: attach the archive to a draft, dispatch with `release_tag` instead, and delete the draft once the run has succeeded (the - workflow header has the commands). Re-dispatching the same bytes before - `expires_at` is the refresh path. + workflow header has the commands). Re-dispatching the same bytes is also + the manual recovery path when scheduled renewal cannot download an archive. - **Artifact name convention:** `migration-fixture--` — e.g. `migration-fixture-v0.9.3-wallet-identity-dpns`. The template also lives in the manifest as `artifact_name_template` so tooling does not re-derive it. +## Automatic archive renewal + +`Migration Fixture Renewal` (`.github/workflows/migration-fixture-renewal.yml`) +checks the `v1.0-dev` manifest each Monday. It selects archives expiring within +30 days, downloads the exact recorded artifacts, checks SHA-256 and byte size, +and uploads the unchanged archives with a requested 90-day retention. It never +extracts archives, captures wallets, or needs the fixture wallet password. +Actual upload expiry is read back from GitHub. The workflow opens a draft PR +changing only artifact run pointers and retention metadata in the manifest. +It refuses to overwrite source pointers that changed during the run. + +Review and merge the renewal PR before the original expiry. An existing open +renewal PR suppresses duplicate uploads/PRs; close it to request a replacement. +Missing, expired or mismatched archives fail visibly and require restoring the +same bytes through the bootstrap workflow. Uploaded replacements alone do not +renew the manifest: the PR must be merged. Run the migration matrix on that PR +before merging; GitHub may require approval for checks on bot-created PRs. + +Scheduled runs require the workflow file on the repository's **default branch**, +even though it checks out `v1.0-dev`. Before that, dispatch it explicitly. +The repository Actions setting **Allow GitHub Actions to create and approve +pull requests** must permit PR creation. The workflow grants write permissions +only to the final PR job and never merges its own PR. + +Safe trial against a branch containing the fixture manifest: + +```bash +gh workflow run migration-fixture-renewal.yml --repo dashpay/dash-evo-tool \ + --ref ci/migration-fixture-renewal \ + -f base_branch=feat/migration-test-matrix -F dry_run=true -F force=true +``` + +`--ref` selects the workflow implementation; use a merged branch after the +feature branch is removed. `base_branch` selects the manifest and target of the +generated PR. `dry_run=true` verifies downloads without publishing anything +(the manual default); `force=true` includes archives outside the 30-day window. +To publish a renewal PR, dispatch with `dry_run=false`. Scheduled runs publish +only when needed. Offline tests: `python3 -m unittest discover -s +scripts/migration-fixtures -p test_renewal.py -v`. + ## What goes into an archive Pack the isolated data directory the capture ran against, with these