Retry transient release package installs and verify them independently

.github/workflows/ci.yml

@@ -51,6 +51,24 @@ jobs:
       - name: Run release-smoke tests
         run: python3 -m unittest discover -s tests/ReleaseSmokeTests -p 'test_*.py'
 
+  release-installation-retries:
+    name: Release installation retries (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.x"
+      - name: Verify installation failure handling
+        run: python -m unittest discover -s tests/ReleaseSmokeTests -p test_install_release_package.py
+
   worker-process-identity:
     name: Worker process identity (${{ matrix.os }})
     runs-on: ${{ matrix.os }}

.github/workflows/release.yml

@@ -12,8 +12,9 @@ on:
           - credential-preflight
           - prepare-release
           - verify-published
+          - verify-installation
       tag:
-        description: Existing empty draft release tag (required for prepare-release)
+        description: Draft tag for prepare-release, or published tag for verification
         required: false
         type: string
       publication:
@@ -28,13 +29,13 @@ on:
     types: [published]
 
 concurrency:
-  group: release-${{ inputs.tag || github.event.release.tag_name || 'credential-preflight' }}
+  group: ${{ inputs.operation == 'verify-installation' && 'installation' || 'release' }}-${{ inputs.tag || github.event.release.tag_name || 'credential-preflight' }}
   cancel-in-progress: false
 
 jobs:
   preflight-release-app:
     name: Verify release App
-    if: github.event_name == 'workflow_dispatch'
+    if: github.event_name == 'workflow_dispatch' && inputs.operation != 'verify-installation'
     runs-on: ubuntu-latest
     timeout-minutes: 5
     environment: release
@@ -802,10 +803,59 @@ jobs:
             )
           done
 
+  installation-target:
+    name: Resolve existing release for installation checks
+    if: github.event_name == 'workflow_dispatch' && inputs.operation == 'verify-installation'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    outputs:
+      source-sha: ${{ steps.target.outputs.source-sha }}
+      version: ${{ steps.target.outputs.version }}
+    steps:
+      - name: Resolve and verify published release
+        id: target
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          TAG: ${{ inputs.tag }}
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import json
+          import os
+          import re
+          import subprocess
+
+          tag = os.environ["TAG"]
+          numeric = r"(?:0|[1-9][0-9]*)"
+          identifier = rf"(?:{numeric}|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)"
+          pattern = rf"v{numeric}\.{numeric}\.{numeric}(?:-{identifier}(?:\.{identifier})*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?"
+          if not re.fullmatch(pattern, tag):
+              raise SystemExit("Installation verification requires an existing v<SemVer> tag.")
+          release = json.loads(subprocess.check_output(
+              ["gh", "release", "view", tag, "--json", "isDraft,isImmutable,tagName"], text=True))
+          if release["isDraft"] or not release["isImmutable"] or release["tagName"] != tag:
+              raise SystemExit("Installation verification requires a published immutable release.")
+          subprocess.run(["gh", "release", "verify", tag], check=True)
+          sha = subprocess.check_output(
+              ["gh", "api", f"repos/{os.environ['GH_REPO']}/commits/{tag}", "--jq", ".sha"],
+              text=True).strip()
+          if not re.fullmatch(r"[0-9a-f]{40}", sha):
+              raise SystemExit("Could not resolve the release source commit.")
+          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
+              output.write(f"source-sha={sha}\nversion={tag[1:]}\n")
+          print(f"Verify installed {tag} from {sha}; installation code comes from the dispatched workflow ref.")
+          PY
+
   smoke-homebrew-installation:
     name: Smoke Homebrew on ${{ matrix.os }}
-    if: ${{ !cancelled() && needs.update-package-managers.result == 'success' }}
-    needs: [published-release, update-package-managers]
+    if: >-
+      !cancelled() &&
+      (needs.update-package-managers.result == 'success' || needs.installation-target.result == 'success')
+    needs: [published-release, update-package-managers, installation-target]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 15
     permissions:
@@ -819,13 +869,16 @@ jobs:
       - name: Check out smoke test
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
-          ref: ${{ needs.published-release.outputs.source-sha }}
+          ref: ${{ inputs.operation == 'verify-installation' && github.sha || needs.published-release.outputs.source-sha }}
 
       - name: Set up Python
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.x"
 
+      - name: Verify installation failure handling
+        run: python -m unittest discover -s tests/ReleaseSmokeTests -p test_install_release_package.py
+
       - name: Prepare Homebrew
         shell: bash
         run: |
@@ -839,20 +892,37 @@ jobs:
 
           brew --version
 
+      - name: Tap Highbyte formulas
+        shell: bash
+        run: brew tap highbyte/tap
+
       - name: Install Wrighty from Homebrew
         shell: bash
+        env:
+          SOURCE_SHA: ${{ needs.installation-target.outputs.source-sha || needs.published-release.outputs.source-sha }}
+          VERSION: ${{ needs.installation-target.outputs.version || needs.published-release.outputs.version }}
         run: |
           set -euo pipefail
+          python3 scripts/install-release-package.py \
+            --manager homebrew \
+            --version "$VERSION" \
+            --source-sha "$SOURCE_SHA" \
+            --log-directory "$RUNNER_TEMP/package-install-logs"
 
-          brew tap highbyte/tap
-          brew install highbyte/tap/wrighty
-          command -v wrighty
+      - name: Retain installation diagnostics
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: homebrew-install-${{ matrix.os }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/package-install-logs
+          if-no-files-found: ignore
+          retention-days: 7
 
       - name: Smoke installed Wrighty
         shell: bash
         env:
-          SOURCE_SHA: ${{ needs.published-release.outputs.source-sha }}
-          VERSION: ${{ needs.published-release.outputs.version }}
+          SOURCE_SHA: ${{ needs.installation-target.outputs.source-sha || needs.published-release.outputs.source-sha }}
+          VERSION: ${{ needs.installation-target.outputs.version || needs.published-release.outputs.version }}
         run: |
           python3 scripts/smoke-release-cli.py \
             --cli "$(command -v wrighty)" \
@@ -874,8 +944,10 @@ jobs:
 
   smoke-scoop-installation:
     name: Smoke Scoop on Windows
-    if: ${{ !cancelled() && needs.update-package-managers.result == 'success' }}
-    needs: [published-release, update-package-managers]
+    if: >-
+      !cancelled() &&
+      (needs.update-package-managers.result == 'success' || needs.installation-target.result == 'success')
+    needs: [published-release, update-package-managers, installation-target]
     runs-on: windows-latest
     timeout-minutes: 15
     permissions:
@@ -885,13 +957,16 @@ jobs:
       - name: Check out smoke test
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
-          ref: ${{ needs.published-release.outputs.source-sha }}
+          ref: ${{ inputs.operation == 'verify-installation' && github.sha || needs.published-release.outputs.source-sha }}
 
       - name: Set up Python
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.x"
 
+      - name: Verify installation failure handling
+        run: python -m unittest discover -s tests/ReleaseSmokeTests -p test_install_release_package.py
+
       - name: Install Scoop
         id: scoop
         shell: pwsh
@@ -932,15 +1007,22 @@ jobs:
           SCOOP: ${{ steps.scoop.outputs.root }}
           SCOOP_CMD: ${{ steps.scoop.outputs.scoop }}
           SCOOP_ROOT: ${{ steps.scoop.outputs.root }}
+          SOURCE_SHA: ${{ needs.installation-target.outputs.source-sha || needs.published-release.outputs.source-sha }}
+          VERSION: ${{ needs.installation-target.outputs.version || needs.published-release.outputs.version }}
         run: |
           & $env:SCOOP_CMD bucket add highbyte https://github.com/highbyte/scoop-bucket
           if ($LASTEXITCODE -ne 0) {
             throw "Could not add the Highbyte Scoop bucket."
           }
 
-          & $env:SCOOP_CMD install highbyte/wrighty
+          python scripts/install-release-package.py `
+            --manager scoop `
+            --scoop-root $env:SCOOP_ROOT `
+            --version $env:VERSION `
+            --source-sha $env:SOURCE_SHA `
+            --log-directory (Join-Path $env:RUNNER_TEMP 'package-install-logs')
           if ($LASTEXITCODE -ne 0) {
-            throw "Could not install Wrighty from Scoop."
+            throw "Could not install and verify Wrighty from Scoop. See installation diagnostics."
           }
 
           $cli = Join-Path $env:SCOOP_ROOT 'shims\wrighty.exe'
@@ -949,12 +1031,21 @@ jobs:
           }
           "cli=$cli" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
 
+      - name: Retain installation diagnostics
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: scoop-install-windows-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/package-install-logs
+          if-no-files-found: ignore
+          retention-days: 7
+
       - name: Smoke installed Wrighty
         shell: pwsh
         env:
           CLI: ${{ steps.wrighty.outputs.cli }}
-          SOURCE_SHA: ${{ needs.published-release.outputs.source-sha }}
-          VERSION: ${{ needs.published-release.outputs.version }}
+          SOURCE_SHA: ${{ needs.installation-target.outputs.source-sha || needs.published-release.outputs.source-sha }}
+          VERSION: ${{ needs.installation-target.outputs.version || needs.published-release.outputs.version }}
         run: |
           python scripts/smoke-release-cli.py `
             --cli $env:CLI `

docs/development/repository-maintenance.md

@@ -232,6 +232,44 @@ Immediate publication makes the completed draft immutable, verifies the release
 updates the Homebrew tap and Scoop bucket with the App token, installs the packages on their
 supported runners, repeats the Local Markdown smoke test, and uninstalls the package.
 
+The Homebrew and Scoop Wrighty installation steps allow at most three attempts, waiting 10 and
+30 seconds after recognized transient failures such as broken pipes, connection resets, DNS
+failures, timeouts, or HTTP 502/503/504 responses. Unknown errors, authentication/permission
+denials, and integrity failures stop immediately. These retries cover the Wrighty package install;
+package-manager bootstrap, tap/bucket setup, release downloads, publication, and repository pushes
+are not retried by this helper.
+
+Each installation check requires a clean disposable runner. Before retrying, it inspects the
+package directory and executable. If installation completed despite a transient command failure,
+the exact CLI version and source commit must match before proceeding to the separate functional
+smoke test. Partial installations or version mismatches stop instead of being repaired or retried.
+The job timeout still applies, and the existing cleanup step runs on failure as well as success.
+
+The Actions job summary records attempts and recovery, and seven-day `homebrew-install-*` or
+`scoop-install-*` artifacts retain each attempt's exit code and output. Homebrew installation uses
+verbose output; the helper does not dump the environment. Automatic retries never bypass checksum,
+attestation, version, or functional checks. A terminal workflow failure still requires maintainer
+approval before a manual rerun.
+
+To verify installation workflow changes without publishing another release, dispatch the existing
+Release workflow with `operation=verify-installation`, selecting the branch that contains the
+changes and the published tag currently served by the Homebrew tap and Scoop bucket:
+
+```shell
+gh workflow run release.yml --repo highbyte/wrighty --ref YOUR_BRANCH \
+  -f operation=verify-installation -f tag=v0.19.0-alpha -f publication=draft
+```
+
+This operation uses only read access to Wrighty's repository and does not access the release App
+or modify releases, tags, the tap, or the bucket. It verifies the existing immutable release and
+resolves its source commit, then runs the shared Homebrew and Scoop installation jobs using the
+helper and tests from the dispatched workflow commit. The `publication` input is ignored for this
+operation. Installation failure-handling tests, real package installs, exact version/commit checks,
+Local Markdown smoke tests, and cleanup run on Linux, macOS, and Windows. Selecting a release
+different from the version currently served by the package managers fails the version check; it
+does not silently test a different version. This also works before the branch is merged, because
+`release.yml` already exists on the default branch.
+
 If the maintainer chooses a draft, the workflow stops after verified assets are attached. Later
 publishing that draft through GitHub triggers public-release verification, package-manager
 updates, installation smoke tests, and cleanup.