diff --git a/.github/workflows/docs_nightly.yaml b/.github/workflows/docs_nightly.yaml index 881fc57..e37b06c 100644 --- a/.github/workflows/docs_nightly.yaml +++ b/.github/workflows/docs_nightly.yaml @@ -1,10 +1,10 @@ -name: docs / nightly deploy +name: docs / deploy # Builds the docs site and publishes it to the sdk.genlayer.com Pages repo, # which serves one directory per version out of `_site/` and lists them in # `_site/versions.json` (the switcher every page links to). Nightly runs -# refresh `main`; `workflow_dispatch` can publish any other directory, which is -# how a release train (`v0.6.x`) gets its own copy. +# refresh `main`; branch pushes refresh their train, and releases dispatch +# this workflow at the exact release tag. # # GitHub only schedules workflows from the DEFAULT branch, so this file has to # reach it before the cron fires — merging it into a dev branch is not enough. @@ -14,6 +14,8 @@ name: docs / nightly deploy # deploy-pages workflow, which is what actually publishes and purges the CDN. on: + push: + branches: [v0.6, v0.6-dev] schedule: # 03:17 UTC — off the hour, where scheduling backlogs are smaller. - cron: "17 3 * * *" @@ -29,19 +31,16 @@ on: required: false default: false -# Two runs pushing to the site repo would race on the same branch; the loser -# would just fail, so serialize rather than cancel (a half-finished deploy is -# worse than a slow one). concurrency: - group: docs-deploy - cancel-in-progress: false + group: docs-deploy-${{ inputs.version || (github.event_name == 'push' && (github.ref_name == 'v0.6' && 'v0.6.x' || github.ref_name)) || 'main' }} + cancel-in-progress: true permissions: contents: read env: SITE_REPO: genlayerlabs/sdk.genlayer.com - DOCS_VERSION: ${{ inputs.version || 'main' }} + DOCS_VERSION: ${{ inputs.version || (github.event_name == 'push' && (github.ref_name == 'v0.6' && 'v0.6.x' || github.ref_name)) || 'main' }} # GitHub's stock `bash -e {0}` has no `pipefail`. defaults: @@ -54,6 +53,8 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: ${{ github.event_name == 'schedule' && 'main' || github.sha }} # The docs build spans the manager and every active executor line, each # in its own `gen-docs` nix shell, so it needs submodules and nix. @@ -78,25 +79,10 @@ jobs: path: build/site ssh-key: ${{ secrets.DOCS_DEPLOY_KEY }} - - name: Stage the built docs + - name: Publish the built docs run: | ./support/ci/run.sh tool deploy-docs \ --site build/site \ --version "$DOCS_VERSION" \ + --push \ ${{ inputs.preferred && '--preferred' || '' }} - - # An unchanged nightly must not push an empty commit: the site repo's - # deploy workflow runs on every push, so that would redeploy and purge the - # CDN for nothing. - - name: Commit and push - working-directory: build/site - run: | - git config user.email "ci@genlayerlabs.com" - git config user.name "CI worker" - git add -A _site - if git diff --cached --quiet; then - echo "docs are unchanged; nothing to deploy" - exit 0 - fi - git commit -m "Deploy documentation version $DOCS_VERSION" - git push origin HEAD diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 57b296d..fbbf6ec 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -153,3 +153,17 @@ jobs: body_path: artifacts/notes/${{ needs.build-and-test.outputs.notes_file }} draft: false prerelease: ${{ contains(needs.build-and-test.outputs.tag, '-') }} + + docs-dispatch: + needs: [build-and-test, release-publish] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + steps: + - name: Dispatch release docs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TAG: ${{ needs.build-and-test.outputs.tag }} + run: gh workflow run docs_nightly.yaml --ref "$TAG" -f version="$TAG" diff --git a/docs/contributing/howto/building/docs.md b/docs/contributing/howto/building/docs.md index d7a48f6..6289a59 100644 --- a/docs/contributing/howto/building/docs.md +++ b/docs/contributing/howto/building/docs.md @@ -41,10 +41,30 @@ The site is `sdk.genlayer.com` — GitHub Pages out of `_site/versions.json` driving the version switcher; pushing that repo deploys it -`.github/workflows/docs_nightly.yaml` republishes `main` nightly and takes a -version/preferred pair on `workflow_dispatch` for a release train. It needs the -`DOCS_DEPLOY_KEY` secret, and — since GitHub schedules only from the default -branch — the workflow file must reach that branch before the cron can fire +`.github/workflows/docs_nightly.yaml` publishes these versions: + +| Trigger | Source | Site Directory | +|---|---|---| +| Successful release publication | Exact release tag | Exact release tag | +| Push to `v0.6` | Pushed commit | `v0.6.x` | +| Push to `v0.6-dev` | Pushed commit | `v0.6-dev` | +| Nightly schedule | `main` | `main` | +| Manual dispatch | Selected ref | `version` input (default `main`) | + +The release workflow dispatches docs only after its final publishing job +succeeds, using `actions: write` on that dispatch job. A docs failure can be +retried by dispatching the docs workflow at the release tag with the same +`version`, without republishing the release + +Concurrency is per target version and cancels the previous run for that version +Different versions may publish together: `deploy-docs --push` retries a rejected +push up to 5 attempts, each time reapplying the built tree and version entry to +the latest site `main`. This preserves concurrent changes to other versions and +`versions.json`; unchanged output produces no commit + +The workflow needs `DOCS_DEPLOY_KEY` with write access to the site repo +Schedules and manual dispatch require the workflow on the repository's default +branch; push triggers require it on the pushed branch By hand, into a checkout of the site repo: @@ -55,7 +75,9 @@ By hand, into a checkout of the site repo: The version directory is replaced wholesale, so deleted pages do not linger; `_site/versions.json` keeps its curated order, a missing version is inserted -after `main`. Committing and pushing the site repo is left to you +after `main`. Committing and pushing the site repo is left to you unless you +pass `--push`, which requires a clean, disposable checkout and resets it to the +site repo's `origin/main` on each attempt ## ADRs diff --git a/support/ci/tools/deploy_docs.py b/support/ci/tools/deploy_docs.py index 2a78e5a..a8f25e0 100644 --- a/support/ci/tools/deploy_docs.py +++ b/support/ci/tools/deploy_docs.py @@ -5,8 +5,7 @@ That repo publishes `_site/` to GitHub Pages, one directory per docs version (`_site/main`, `_site/v0.3.x`, ...) plus a root `_site/versions.json` that feeds the version switcher every built page links to. This tool only rewrites those -files; committing and pushing is left to the caller, so a nightly run and a -local `--site ../sdk.genlayer.com` do exactly the same thing. +files by default; `--push` publishes from a disposable, clean site checkout. The version directory is replaced wholesale rather than merged, so pages deleted upstream do not linger. @@ -21,6 +20,8 @@ import pathlib import re import shutil +import subprocess +import time import ci_lib @@ -100,6 +101,11 @@ def add_to(self, parser: argparse.ArgumentParser) -> None: default=str(ci_lib.ROOT_DIR / 'build' / 'doc' / 'html'), help='built docs tree to publish (default: build/doc/html)', ) + parser.add_argument( + '--push', + action='store_true', + help='commit and push to origin/main, retrying races (disposable clean checkout only)', + ) parser.add_argument( '--preferred', action='store_true', @@ -128,6 +134,12 @@ def handler(self, args: argparse.Namespace) -> int: print(f'{site} does not exist; is --site an sdk.genlayer.com checkout?') return 1 + if args.push: + return self.publish(args, html, site) + self.stage(args, html, site) + return 0 + + def stage(self, args, html, site): target = site / args.version if target.exists(): shutil.rmtree(target) @@ -136,7 +148,37 @@ def handler(self, args: argparse.Namespace) -> int: print(f'published {html} -> {target}') update_versions(site / VERSIONS_JSON, args.version, args.preferred) - return 0 + + def publish(self, args, html, site): + def git(*command, **kwargs): + return subprocess.run(['git', *command], cwd=site.parent, check=True, **kwargs) + + if git('status', '--porcelain', capture_output=True, text=True).stdout: + print('--push requires a clean, disposable site checkout') + return 1 + git('config', 'user.email', 'ci@genlayerlabs.com') + git('config', 'user.name', 'CI worker') + for attempt in range(5): + git('fetch', 'origin', 'refs/heads/main') + git('reset', '--hard', 'FETCH_HEAD') + # Reapply to the latest tree so concurrent versions.json edits survive. + self.stage(args, html, site) + git('add', '-A', '_site') + if not git( + 'diff', '--cached', '--name-only', capture_output=True, text=True + ).stdout: + print('docs are unchanged; nothing to deploy') + return 0 + git('commit', '-m', f'Deploy documentation version {args.version}') + try: + git('push', 'origin', 'HEAD:refs/heads/main') + return 0 + except subprocess.CalledProcessError: + if attempt == 4: + print('docs push failed after 5 attempts') + return 1 + print(f'docs push failed; retrying publication (attempt {attempt + 2}/5)') + time.sleep(2**attempt) COMMANDS = [DeployDocs()] diff --git a/support/ci/unit_tests/test_deploy_docs.py b/support/ci/unit_tests/test_deploy_docs.py new file mode 100644 index 0000000..9b9e54b --- /dev/null +++ b/support/ci/unit_tests/test_deploy_docs.py @@ -0,0 +1,129 @@ +import argparse +import json +import subprocess + +import pytest + +from tools import deploy_docs as deploy + + +@pytest.fixture +def publication(tmp_path, monkeypatch): + run = subprocess.run + + def git(path, *args): + return run( + ['git', '-C', str(path), *args], check=True, capture_output=True, text=True + ).stdout.strip() + + remote = tmp_path / 'remote.git' + run(['git', 'init', '--bare', '--initial-branch=main', str(remote)], check=True) + site = tmp_path / 'site' + run(['git', 'clone', str(remote), str(site)], check=True) + git(site, 'config', 'user.email', 'test@example.com') + git(site, 'config', 'user.name', 'Test') + (site / '_site' / 'v0.6.x').mkdir(parents=True) + (site / '_site' / 'v0.6.x' / 'obsolete.html').write_text('old') + (site / '_site' / 'versions.json').write_text( + json.dumps([deploy.entry_for('main', True)]) + ) + git(site, 'add', '.') + git(site, 'commit', '-m', 'Initial site') + git(site, 'push', 'origin', 'main') + html = tmp_path / 'html' + html.mkdir() + (html / 'index.html').write_text('new docs') + (html / '.doctrees').mkdir() + (html / '.doctrees' / 'cache').write_text('cache') + parser = argparse.ArgumentParser() + tool = deploy.DeployDocs() + tool.add_to(parser) + args = parser.parse_args( + ['--site', str(site), '--html', str(html), '--version', 'v0.6.x', '--push'] + ) + monkeypatch.setattr(deploy.time, 'sleep', lambda _: None) + return tool, args, git, remote, site, run + + +def test_publish_replaces_tree_and_skips_unchanged(publication): + tool, args, git, remote, site, _ = publication + assert tool.handler(args) == 0 + head = git(remote, 'rev-parse', 'main') + assert (site / '_site' / args.version / 'index.html').read_text() == 'new docs' + assert not (site / '_site' / args.version / 'obsolete.html').exists() + assert not (site / '_site' / args.version / '.doctrees').exists() + assert tool.handler(args) == 0 + assert git(remote, 'rev-parse', 'main') == head + + +@pytest.mark.parametrize('preferred', [False, True]) +def test_publish_retries_concurrent_version_update( + publication, monkeypatch, tmp_path, preferred +): + tool, args, git, remote, site, run = publication + args.preferred = preferred + competitor = tmp_path / 'competitor' + run(['git', 'clone', str(remote), str(competitor)], check=True) + git(competitor, 'config', 'user.email', 'other@example.com') + git(competitor, 'config', 'user.name', 'Other') + pushes = 0 + + def race(command, **kwargs): + nonlocal pushes + if command[:2] == ['git', 'push']: + pushes += 1 + if pushes == 1: + (competitor / '_site' / 'v0.6-dev').mkdir() + (competitor / '_site' / 'v0.6-dev' / 'index.html').write_text('concurrent docs') + deploy.update_versions(competitor / '_site' / 'versions.json', 'v0.6-dev', True) + git(competitor, 'add', '.') + git(competitor, 'commit', '-m', 'Concurrent version') + git(competitor, 'push', 'origin', 'main') + return run(command, **kwargs) + + monkeypatch.setattr(deploy.subprocess, 'run', race) + assert tool.handler(args) == 0 + assert pushes == 2 + assert git(remote, 'show', 'main:_site/v0.6-dev/index.html') == 'concurrent docs' + assert git(remote, 'show', 'main:_site/v0.6.x/index.html') == 'new docs' + versions = json.loads(git(remote, 'show', 'main:_site/versions.json')) + assert {entry['version'] for entry in versions} == {'main', 'v0.6-dev', 'v0.6.x'} + assert [entry['version'] for entry in versions if entry['preferred']] == [ + 'v0.6.x' if preferred else 'v0.6-dev' + ] + + +def test_publish_stops_after_repeated_rejection(publication, monkeypatch): + tool, args, git, remote, _, run = publication + head = git(remote, 'rev-parse', 'main') + pushes = 0 + + def reject(command, **kwargs): + nonlocal pushes + if command[:2] == ['git', 'push']: + pushes += 1 + raise subprocess.CalledProcessError(1, command) + return run(command, **kwargs) + + monkeypatch.setattr(deploy.subprocess, 'run', reject) + assert tool.handler(args) == 1 + assert pushes == 5 + assert git(remote, 'rev-parse', 'main') == head + + +def test_publish_refuses_dirty_checkout(publication): + tool, args, git, remote, site, _ = publication + head = git(remote, 'rev-parse', 'main') + (site / 'keep.txt').write_text('local work') + assert tool.handler(args) == 1 + assert (site / 'keep.txt').read_text() == 'local work' + assert git(remote, 'rev-parse', 'main') == head + + +def test_stage_does_not_push(publication): + tool, args, git, remote, site, _ = publication + head = git(remote, 'rev-parse', 'main') + args.push = False + assert tool.handler(args) == 0 + assert (site / '_site' / args.version / 'index.html').exists() + assert git(remote, 'rev-parse', 'main') == head