diff --git a/.github/workflows/backup-production.yml b/.github/workflows/backup-production.yml index 81f412e8..a89f0fd9 100644 --- a/.github/workflows/backup-production.yml +++ b/.github/workflows/backup-production.yml @@ -1,8 +1,5 @@ name: Production Backup -# The nightly run is a fixed host crontab entry (see deploy/production/README.md): -# scheduled workflows start hours late and then overlap daytime traffic. This workflow -# is for manual backups. on: workflow_dispatch: @@ -10,71 +7,47 @@ concurrency: group: production-backup cancel-in-progress: false -permissions: {} +permissions: + contents: read + id-token: write jobs: backup: name: Back up production state runs-on: ubuntu-22.04 + environment: production timeout-minutes: 180 - permissions: - contents: read steps: - - name: Require repository owner for manual runs - if: ${{ github.event_name == 'workflow_dispatch' }} - shell: bash - run: | - set -euo pipefail - if [[ "${{ github.actor }}" != "iamwavecut" || "${{ github.triggering_actor }}" != "iamwavecut" ]]; then - echo "Production backup can only be started or re-run manually by iamwavecut" >&2 - exit 1 - fi - - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Configure SSH - shell: bash + - name: Require repository owner and main env: - GETA_SSH_PRIVATE_KEY: ${{ secrets.GETA_SSH_PRIVATE_KEY }} - GETA_SSH_KNOWN_HOSTS: ${{ secrets.GETA_SSH_KNOWN_HOSTS }} - GETA_SSH_PORT: ${{ vars.GETA_SSH_PORT || '22022' }} + ACTOR: ${{ github.actor }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + RELEASE_REF: ${{ github.ref }} run: | set -euo pipefail - test -n "$GETA_SSH_PRIVATE_KEY" - test -n "$GETA_SSH_KNOWN_HOSTS" - install -m 700 -d "$HOME/.ssh" - printf '%s\n' "$GETA_SSH_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts" - chmod 600 "$HOME/.ssh/known_hosts" - printf 'Host geta.moe 207.244.239.212 2605:a140:2260:6080::1\n Port %s\n' "$GETA_SSH_PORT" > "$HOME/.ssh/config" - chmod 600 "$HOME/.ssh/config" - eval "$(ssh-agent -s)" - printf '%s\n' "$GETA_SSH_PRIVATE_KEY" | ssh-add - - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" - echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - - name: Upload backup script - shell: bash + [[ "$ACTOR" == iamwavecut && "$TRIGGERING_ACTOR" == iamwavecut ]] + [[ "$RELEASE_REF" == refs/heads/main ]] + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Configure pinned SSH identity env: - GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }} - GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }} + SSH_KEY: ${{ secrets.GETA_SSH_PRIVATE_KEY }} + KNOWN_HOSTS: ${{ secrets.GETA_SSH_KNOWN_HOSTS }} + SSH_PORT: ${{ vars.GETA_SSH_PORT || '22022' }} run: | set -euo pipefail - # shellcheck disable=SC2029 - ssh "$GETA_SSH_TARGET" "install -d -m 755 '$GETA_DEPLOY_ROOT'" - scp deploy/production/backup-production.sh "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/backup-production.sh" - # shellcheck disable=SC2029 - ssh "$GETA_SSH_TARGET" "chmod 755 '$GETA_DEPLOY_ROOT/backup-production.sh'" - - - name: Create production backup - shell: bash + install -m 700 -d ~/.ssh + printf '%s\n' "$SSH_KEY" > ~/.ssh/geta_deploy + chmod 600 ~/.ssh/geta_deploy + printf '%s\n' "$KNOWN_HOSTS" > ~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + printf 'Host geta.moe\n Port %s\n IdentitiesOnly yes\n BatchMode yes\n StrictHostKeyChecking yes\n' "$SSH_PORT" > ~/.ssh/config + - name: Request fixed host backup procedure env: - GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }} - GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }} + DEPLOY_OPERATION: backup run: | set -euo pipefail - ssh \ - -o ServerAliveInterval=30 \ - -o ServerAliveCountMax=120 \ - "$GETA_SSH_TARGET" \ - "'$GETA_DEPLOY_ROOT/backup-production.sh'" + python3 scripts/geta-deploy-request.py | + ssh -T -i ~/.ssh/geta_deploy -o ServerAliveInterval=30 -o ServerAliveCountMax=360 \ + deploy@geta.moe backup diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 233808d8..9e1b7f8a 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -8,43 +8,49 @@ env: permissions: {} +concurrency: + group: openplotva-production + cancel-in-progress: false + jobs: authorize: name: Authorize deployment operator runs-on: ubuntu-latest permissions: {} steps: - - name: Require repository owner + - name: Require repository owner and main shell: bash + env: + ACTOR: ${{ github.actor }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + RELEASE_REF: ${{ github.ref }} run: | set -euo pipefail - if [[ "${{ github.actor }}" != "iamwavecut" || "${{ github.triggering_actor }}" != "iamwavecut" ]]; then - echo "Production deploy can only be started or re-run by iamwavecut" >&2 - exit 1 - fi + [[ "$ACTOR" == iamwavecut && "$TRIGGERING_ACTOR" == iamwavecut ]] + [[ "$RELEASE_REF" == refs/heads/main ]] build-and-deploy: name: Build image and deploy needs: authorize runs-on: ubuntu-22.04 + environment: production + timeout-minutes: 60 permissions: actions: read contents: read packages: write + id-token: write steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - name: Select GHCR image tag + - name: Select immutable application image shell: bash run: | set -euo pipefail - image_name="ghcr.io/${GITHUB_REPOSITORY}" - image_name="$(printf '%s' "$image_name" | tr '[:upper:]' '[:lower:]')" - echo "OPENPLOTVA_IMAGE=${image_name}:${GITHUB_SHA}" >> "$GITHUB_ENV" - echo "OPENPLOTVA_SANDBOX_IMAGE=${image_name}-sandbox:${GITHUB_SHA}" >> "$GITHUB_ENV" + echo "OPENPLOTVA_IMAGE=ghcr.io/iamwavecut/openplotva:${GITHUB_SHA}" >> "$GITHUB_ENV" - name: Install Rust run: | @@ -56,8 +62,8 @@ jobs: with: shared-key: rust-${{ runner.os }}-ubuntu-22.04-${{ env.RUST_VERSION }}-deploy - - name: Build release binaries - run: cargo build --locked --release -p openplotva-app -p openplotva-sandbox + - name: Build application release binary + run: cargo build --locked --release -p openplotva-app - name: Log in to GHCR uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 @@ -69,7 +75,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 - - name: Build and push GHCR image + - name: Build and push application image + id: image uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 env: DOCKER_BUILD_RECORD_UPLOAD: "false" @@ -80,28 +87,12 @@ jobs: labels: | org.opencontainers.image.revision=${{ github.sha }} org.opencontainers.image.source=https://github.com/${{ github.repository }} + io.geta-deploy.data-version=1 tags: ${{ env.OPENPLOTVA_IMAGE }} push: true cache-from: type=gha,scope=openplotva-runtime-prebuilt cache-to: type=gha,mode=min,scope=openplotva-runtime-prebuilt - - name: Build and push sandbox image - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 - env: - DOCKER_BUILD_RECORD_UPLOAD: "false" - with: - context: . - file: sandbox/Dockerfile - target: runtime-prebuilt - labels: | - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.source=https://github.com/${{ github.repository }} - tags: ${{ env.OPENPLOTVA_SANDBOX_IMAGE }} - push: true - # mode=max keeps the nsjail and package layers between deploys. - cache-from: type=gha,scope=openplotva-sandbox - cache-to: type=gha,mode=max,scope=openplotva-sandbox - - name: Wait for green CI on this commit shell: bash env: @@ -114,128 +105,45 @@ jobs: --jq '.workflow_runs[0] // empty | "\(.status) \(.conclusion // "none") \(.html_url)"')" elapsed=$((SECONDS - started)) if [[ -z "$run" ]]; then - echo "No push CI run for ${GITHUB_SHA} after ${elapsed}s" - (( elapsed < 180 )) || { echo "Only commits that CI ran on after a push can be deployed" >&2; exit 1; } + (( elapsed < 180 )) || { echo 'No push CI found for release' >&2; exit 1; } else read -r status conclusion url <<<"$run" - echo "CI ${status}/${conclusion} after ${elapsed}s: ${url}" + echo "CI ${status}/${conclusion}: ${url}" if [[ "$status" == completed ]]; then - [[ "$conclusion" == success ]] && exit 0 - echo "CI for ${GITHUB_SHA} concluded ${conclusion}; not deploying" >&2 + [[ "$conclusion" == success ]] && break + echo 'CI did not pass; not deploying' >&2 exit 1 fi - (( elapsed < 1800 )) || { echo "CI for ${GITHUB_SHA} did not finish within 30 minutes" >&2; exit 1; } + (( elapsed < 1800 )) || { echo 'CI timeout' >&2; exit 1; } fi sleep 20 done - - name: Configure SSH + - name: Configure pinned SSH identity shell: bash env: - GETA_SSH_PRIVATE_KEY: ${{ secrets.GETA_SSH_PRIVATE_KEY }} - GETA_SSH_KNOWN_HOSTS: ${{ secrets.GETA_SSH_KNOWN_HOSTS }} - GETA_SSH_PORT: ${{ vars.GETA_SSH_PORT || '22022' }} + SSH_KEY: ${{ secrets.GETA_SSH_PRIVATE_KEY }} + KNOWN_HOSTS: ${{ secrets.GETA_SSH_KNOWN_HOSTS }} + SSH_PORT: ${{ vars.GETA_SSH_PORT || '22022' }} run: | set -euo pipefail - test -n "$GETA_SSH_PRIVATE_KEY" - test -n "$GETA_SSH_KNOWN_HOSTS" - install -m 700 -d "$HOME/.ssh" - printf '%s\n' "$GETA_SSH_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts" - chmod 600 "$HOME/.ssh/known_hosts" - printf 'Host geta.moe 207.244.239.212 2605:a140:2260:6080::1\n Port %s\n' "$GETA_SSH_PORT" > "$HOME/.ssh/config" - chmod 600 "$HOME/.ssh/config" - eval "$(ssh-agent -s)" - printf '%s\n' "$GETA_SSH_PRIVATE_KEY" | ssh-add - - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV" - echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV" - - - name: Upload deploy assets - shell: bash + install -m 700 -d ~/.ssh + printf '%s\n' "$SSH_KEY" > ~/.ssh/geta_deploy + chmod 600 ~/.ssh/geta_deploy + printf '%s\n' "$KNOWN_HOSTS" > ~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + printf 'Host geta.moe\n Port %s\n IdentitiesOnly yes\n BatchMode yes\n StrictHostKeyChecking yes\n' "$SSH_PORT" > ~/.ssh/config + + - name: Request application-only deployment env: - GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }} - GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }} + DEPLOY_SHA: ${{ github.sha }} + DEPLOY_OPERATION: deploy + OPENPLOTVA_IMAGE_DIGEST: ${{ steps.image.outputs.digest }} run: | set -euo pipefail - # shellcheck disable=SC2029 - ssh "$GETA_SSH_TARGET" "install -d -m 755 '$GETA_DEPLOY_ROOT'" - scp deploy/production/compose.production.yml "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/compose.production.yml" - scp deploy/maintenance/compose.maintenance.yml "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/compose.maintenance.yml" - scp deploy/production/configure-gradius-generation.py "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/configure-gradius-generation.py" - scp deploy/production/deploy-production.sh "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/deploy-production.sh" - # shellcheck disable=SC2029 - ssh "$GETA_SSH_TARGET" "chmod 644 '$GETA_DEPLOY_ROOT/compose.maintenance.yml'; chmod 755 '$GETA_DEPLOY_ROOT/deploy-production.sh'" - - - name: Install generation advertising key - shell: bash - env: - GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }} - GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }} - GRADIUS_GENERATION_API_KEY: ${{ secrets.GRADIUS_GENERATION_API_KEY }} - OPENPLOTVA_PRODUCTION_ENV_B64: ${{ secrets.OPENPLOTVA_PRODUCTION_ENV_B64 }} - run: | - set -euo pipefail - if [[ -n "${GRADIUS_GENERATION_API_KEY:-}" ]]; then - # shellcheck disable=SC2029 - python3 -c 'import json, os; print(json.dumps({"key": os.environ["GRADIUS_GENERATION_API_KEY"], "production_env_b64": os.environ.get("OPENPLOTVA_PRODUCTION_ENV_B64", "")}))' | - ssh "$GETA_SSH_TARGET" "python3 '$GETA_DEPLOY_ROOT/configure-gradius-generation.py' '$GETA_DEPLOY_ROOT/.env.production'" - fi + python3 scripts/geta-deploy-request.py | + ssh -T -i ~/.ssh/geta_deploy -o ServerAliveInterval=30 -o ServerAliveCountMax=40 \ + deploy@geta.moe "deploy $DEPLOY_SHA" - - name: Apply production deployment - shell: bash - env: - GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }} - GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }} - GHCR_PULL_TOKEN: ${{ secrets.GHCR_PULL_TOKEN }} - OPENPLOTVA_PRODUCTION_ENV_B64: ${{ secrets.OPENPLOTVA_PRODUCTION_ENV_B64 }} - run: | - set -euo pipefail - test -n "$GHCR_PULL_TOKEN" - printf -v image_q '%q' "$OPENPLOTVA_IMAGE" - printf -v sandbox_image_q '%q' "$OPENPLOTVA_SANDBOX_IMAGE" - printf -v token_q '%q' "$GHCR_PULL_TOKEN" - printf -v actor_q '%q' "$GITHUB_ACTOR" - printf -v env_q '%q' "${OPENPLOTVA_PRODUCTION_ENV_B64:-}" - ssh \ - -o ServerAliveInterval=30 \ - -o ServerAliveCountMax=20 \ - "$GETA_SSH_TARGET" \ - "OPENPLOTVA_DEPLOY_IMAGE=$image_q OPENPLOTVA_SANDBOX_IMAGE=$sandbox_image_q GHCR_USERNAME=$actor_q GHCR_PULL_TOKEN=$token_q OPENPLOTVA_PRODUCTION_ENV_B64=$env_q '$GETA_DEPLOY_ROOT/deploy-production.sh'" - - - name: Remove non-current GHCR images - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - test -n "${GH_TOKEN:-}" || { - echo "GITHUB_TOKEN is required for post-deploy package cleanup" >&2 - exit 1 - } - - owner="${GITHUB_REPOSITORY_OWNER,,}" - repository="${GITHUB_REPOSITORY#*/}" - repository="${repository,,}" - current_tag="${GITHUB_SHA}" - - for package in "$repository" "${repository}-sandbox"; do - versions_file="$(mktemp)" - echo "Cleaning GHCR package ${owner}/${package}; keeping only current tag ${current_tag}" - gh api --paginate --slurp "/users/${owner}/packages/container/${package}/versions?per_page=100" > "$versions_file" - - deleted=0 - while IFS= read -r version; do - id="$(jq -r '.id' <<<"$version")" - created_at="$(jq -r '.created_at' <<<"$version")" - tags="$(jq -r '[.metadata.container.tags[]?] | join(",")' <<<"$version")" - - if jq -e --arg tag "$current_tag" '.metadata.container.tags // [] | index($tag)' <<<"$version" >/dev/null; then - continue - fi - - echo "Deleting GHCR version id=${id} created_at=${created_at} tags=${tags:-}" - gh api -X DELETE "/users/${owner}/packages/container/${package}/versions/${id}" - deleted=$((deleted + 1)) - done < <(jq -c '.[][]' "$versions_file") - - echo "Deleted ${deleted} old version(s) of ${package}" - done + - name: Verify broker runtime status + run: ssh -T -i ~/.ssh/geta_deploy deploy@geta.moe status diff --git a/Cargo.lock b/Cargo.lock index 7f9b58b9..04a41f13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4707,27 +4707,27 @@ checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" -version = "0.13.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.2.4" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 9261147f..e1f02145 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,7 +74,7 @@ tracing-opentelemetry = "0.33.0" tracing-subscriber = "0.3.23" url = "2.5.8" utoipa = "5.5.0" -zstd = "0.13.3" +zstd = "0.14.3" [workspace.lints.rust] unsafe_code = "forbid" diff --git a/deploy/geta-deploy.json b/deploy/geta-deploy.json new file mode 100644 index 00000000..3062fb0f --- /dev/null +++ b/deploy/geta-deploy.json @@ -0,0 +1,7 @@ +{ + "data_version": "1", + "files": ["deploy/production/compose.production.yml", "deploy/maintenance/compose.maintenance.yml", "sandbox/Dockerfile", "sandbox/nsjail/python.cfg", "sandbox/requirements.txt"], + "trees": ["migrations", "crates/openplotva-sandbox/src"], + "images": {"openplotva": "OPENPLOTVA_IMAGE_DIGEST"}, + "registry_auth": false +} diff --git a/deploy/production/README.md b/deploy/production/README.md index 2a691224..3552066d 100644 --- a/deploy/production/README.md +++ b/deploy/production/README.md @@ -5,35 +5,55 @@ workflow never starts, waits for, or checks a backup. It pulls the selected imag recreates the application, and succeeds after `/api/health` and `/api/ready` pass. Every deploy builds the release binary and runtime image from the exact dispatched -`main` commit and pushes it to GHCR tagged with that commit SHA. Pull request CI does -not build images. The release build cache is scoped to `main`, so a deploy with an +`main` commit and pushes it to GHCR tagged with that commit SHA. The existing +`ghcr.io/iamwavecut/openplotva` application package is public, so the host pulls it +without registry credentials. Pull request CI does not build images. The release build cache is scoped to `main`, so a deploy with an unchanged `Cargo.lock` recompiles only the workspace crates. Before touching the host, the deploy waits for the CI run of the same commit after its push to `main` and stops unless that run succeeded. -After a successful deploy, local Docker and GHCR cleanup keep only the current -main-SHA image; they do not maintain rollback image history. - -The nightly backup runs from the deploy user's crontab on the host at 01:00 UTC (04:00 -Moscow), the quietest hour of dialog traffic. The line is installed once by hand; -deploys never touch it or the backup script: - -``` -0 1 * * * flock -n /.backup.lock /backup-production.sh > /backup-cron.log 2>&1 +The runner sends the immutable application digest and a short-lived GitHub OIDC +identity to the host's deployment broker. Its SSH key has no arbitrary shell, +sudo, or Docker socket access. Compose settings, runtime secrets and bind mounts +are fixed in an administrator-owned profile. CI cannot upload replacement host +scripts or production environment files. + +The broker preserves the previous pinned application image for rollback. Do not +delete current/previous registry versions or their OCI child manifests as a +post-deploy cleanup step. The protected release contract in `deploy/geta-deploy.json` +includes runtime templates, database migrations and sandbox protocol/code inputs. +A changed contract requires administrator review before another release. + +The privileged Python sandbox is an administrator-managed dependency. Application +CI does not build, publish or activate its replacement. An administrator must +review and apply sandbox updates separately, using personal host access. Ordinary +application releases leave the existing sandbox, databases, Caddy and host +configuration unchanged. + +The nightly backup runs from the administrator's crontab at 01:00 UTC, the quietest +hour of dialog traffic. The installed command invokes the fixed broker operation: + +```cron +0 1 * * * sudo -n /usr/local/libexec/geta-deploy admin --project openplotva --operation backup >> /backup-cron.log 2>&1 ``` GitHub's scheduled workflows start hours late, so `.github/workflows/backup-production.yml` -is for manual backups only. Each manual run uploads the current `backup-production.sh`, -which is how a changed script reaches the host. The backup is stored under -`${OPENPLOTVA_BACKUP_ROOT:-/backups}` and contains: - -- `postgres.dump` — PostgreSQL custom-format logical dump; -- `dragonfly-snapshot.tar.gz` — native Dragonfly DFS snapshot set; -- `redis-ingress.rdb` — durable Valkey ingress snapshot; -- `openplotva-state.tar.gz` — runtime TLS/application state when the volume exists; -- `SHA256SUMS` — checksums verified before the backup is accepted. - -Only the newest complete backup directory (`scheduled-*` or `predeploy-*`) is kept by default. -Override it with `OPENPLOTVA_BACKUP_KEEP`. +is for manual backups only. It requests `backup` over the restricted SSH identity +with OIDC from the separately authorized backup workflow. It cannot upload or +modify the installed backup helper. The broker serializes backup and application +rollout operations. New backups are root-only under +`/var/lib/geta-deploy/backups/openplotva` and contain: + +- `postgres.dump`: PostgreSQL custom-format logical dump; +- `dragonfly-snapshot.tar.gz`: native Dragonfly DFS snapshot set; +- `redis-ingress.rdb`: durable Valkey ingress snapshot; +- `openplotva-state.tar.gz`: runtime TLS/application state; +- `deployment-control.tar.gz` and `broker-project.json`: this project's protected + runtime profile and exact image history; +- `SHA256SUMS`: checksums verified before the backup is accepted. + +The previous complete backup is retained until its replacement is verified. Only +the newest verified backup is then kept in the new backup root. Existing legacy +backups are not deleted during migration. The uploader's media volume has a separate daily retention job on the production host: @@ -76,11 +96,12 @@ enable AOF with `CONFIG SET appendonly yes` and wait for ## Image generation advertising Create a separate Gradius project of type **Generation model** for `@PlotvoBot`. -Store its key in the repository Actions secret `GRADIUS_GENERATION_API_KEY`. -The deployment workflow streams this key over SSH on stdin, installs it into -`.env.production` with mode 0600, and enables `GRADIUS_UTILITY_IMAGE_ENABLED`. -It preserves the existing dialogue key and other runtime settings. The Gradius -master switch (`GRADIUS_ENABLED`) must also be enabled. If the generation key is +An administrator stores its `GRADIUS_GENERATION_API_KEY` in the protected runtime +profile and enables `GRADIUS_UTILITY_IMAGE_ENABLED`. Application CI does not +provision or overwrite these credentials. Changes to a legacy `.env.production` +do not update an already frozen broker profile. Preserve the existing dialogue +key and other runtime settings when changing the profile. The Gradius master +switch (`GRADIUS_ENABLED`) must also be enabled. If the generation key is missing, image ads stay disabled; the dialogue key is never used as a fallback. Images use `POST /v1/native/generation_model/chat` with a redacted prompt and diff --git a/scripts/geta-deploy-request.py b/scripts/geta-deploy-request.py new file mode 100644 index 00000000..c2ce9030 --- /dev/null +++ b/scripts/geta-deploy-request.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Prepare a bounded deployment request. Credentials go to SSH stdin only.""" +import argparse +import hashlib +import http.client +import json +import os +from pathlib import Path +import re +import sys +import urllib.parse + +CONTRACT = 'deploy/geta-deploy.json' + + +def schema_hashes(directory): + root = Path(directory) + return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(root.rglob('*.sql')) if p.is_file() and not p.is_symlink()} + + +def contract_digest(root): + root = Path(root).resolve() + contract = json.loads((root / CONTRACT).read_text()) + names = {CONTRACT, *contract.get('files', [])} + for tree in contract.get('trees', []): + path = (root / tree).resolve() + if not path.is_relative_to(root) or not path.is_dir(): + raise ValueError('invalid contract tree') + names.update(str(p.relative_to(root)) for p in path.rglob('*') if p.is_file()) + digest = hashlib.sha256() + for name in sorted(names): + path = root / name + if path.is_symlink() or not path.resolve().is_relative_to(root) or not path.is_file(): + raise ValueError('invalid contract file') + digest.update(name.encode() + b'\0' + path.read_bytes() + b'\0') + return digest.hexdigest(), contract + + +def identity(): + endpoint = urllib.parse.urlsplit(os.environ['ACTIONS_ID_TOKEN_REQUEST_URL']) + if (endpoint.scheme != 'https' or not endpoint.hostname + or not endpoint.hostname.endswith('.actions.githubusercontent.com') + or endpoint.username is not None or endpoint.password is not None + or endpoint.port not in (None, 443) or endpoint.fragment): + raise ValueError('identity endpoint must be a GitHub Actions HTTPS origin') + query = [(key, value) for key, value in urllib.parse.parse_qsl(endpoint.query) + if key != 'audience'] + query.append(('audience', 'geta-deploy')) + target = endpoint.path + '?' + urllib.parse.urlencode(query) + connection = http.client.HTTPSConnection(endpoint.hostname, timeout=15) + try: + connection.request('GET', target, headers={ + 'Authorization': 'Bearer ' + os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}) + response = connection.getresponse() + if response.status != 200: + raise ValueError('GitHub Actions identity request failed') + body = response.read(32769) + if len(body) > 32768: + raise ValueError('GitHub Actions identity response is too large') + value = json.loads(body).get('value') + finally: + connection.close() + if not isinstance(value, str) or not value or len(value) > 16000: + raise ValueError('invalid identity') + return value + + +def payload(root, environment, get_identity): + operation = environment.get('DEPLOY_OPERATION', 'deploy') + if operation in ('rollback', 'backup'): + return {'token': get_identity()} + if operation != 'deploy' or not re.fullmatch(r'[0-9a-f]{40}', environment.get('DEPLOY_SHA', '')): + raise ValueError('a deploy operation and immutable source commit are required') + digest, contract = contract_digest(root) + images = {service: environment.get(variable, '') for service, variable in contract['images'].items()} + if not images or any(not re.fullmatch(r'sha256:[0-9a-f]{64}', value) for value in images.values()): + raise ValueError('immutable image digests are required') + result = {'token': get_identity(), 'images': images, + 'data_version': contract['data_version'], 'profile_digest': digest} + if contract.get('registry_auth'): + credential = environment.get('GETA_REGISTRY_TOKEN', '') + if not credential: + raise ValueError('scoped workflow registry credential required') + result['registry_token'] = credential + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--root', type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument('--digest', action='store_true') + parser.add_argument('--check-schema', type=Path) + args = parser.parse_args() + if args.check_schema: + _, contract = contract_digest(args.root) + approved_name = contract.get('approved_schema') + if not isinstance(approved_name, str) or not approved_name: + parser.error('--check-schema requires approved_schema in the deployment contract') + approved_path = args.root / approved_name + if (approved_path.is_symlink() or not approved_path.is_file() + or not approved_path.resolve().is_relative_to(args.root.resolve())): + parser.error('approved_schema must be a regular file inside the source root') + approved = json.loads(approved_path.read_text()) + if not approved or approved != schema_hashes(args.check_schema): + raise ValueError('database schema changed; coordinated administrator migration required') + print('Approved database migration set verified') + elif args.digest: + print(contract_digest(args.root)[0]) + else: + json.dump(payload(args.root, os.environ, identity), sys.stdout) + sys.stdout.write('\n') + + +if __name__ == '__main__': + try: + main() + except Exception: + print('deployment contract or authenticated request validation failed', file=sys.stderr) + raise SystemExit(1) diff --git a/tools/maintenance/tests/test_deploy.py b/tools/maintenance/tests/test_deploy.py index 9dc60ed6..35ddbff9 100644 --- a/tools/maintenance/tests/test_deploy.py +++ b/tools/maintenance/tests/test_deploy.py @@ -119,15 +119,18 @@ def test_overlay_uses_protected_snapshot_and_keeps_loopback_binding(self): overlay = OVERLAY.read_text() self.assertIn('"127.0.0.1:9092:9092"', overlay) - def test_workflow_uploads_overlay_without_credentials(self): + def test_workflow_keeps_runtime_configuration_on_the_host(self): workflow = (ROOT / ".github/workflows/deploy-production.yml").read_text() script = SCRIPT.read_text() - self.assertIn( - 'scp deploy/maintenance/compose.maintenance.yml "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/compose.maintenance.yml"', - workflow, - ) - self.assertNotIn("MAINTENANCE_TOKEN", workflow) - self.assertNotIn("runtime.env", workflow) + self.assertIn("id-token: write", workflow) + self.assertIn("environment: production", workflow) + self.assertIn("OPENPLOTVA_IMAGE_DIGEST: ${{ steps.image.outputs.digest }}", workflow) + self.assertIn('deploy@geta.moe "deploy $DEPLOY_SHA"', workflow) + for forbidden in ("scp ", "OPENPLOTVA_PRODUCTION_ENV_B64", "GHCR_PULL_TOKEN", "MAINTENANCE_TOKEN", "runtime.env", "Build and push sandbox image", "Remove non-current GHCR images"): + self.assertNotIn(forbidden, workflow) + contract = (ROOT / "deploy/geta-deploy.json").read_text() + self.assertIn("deploy/maintenance/compose.maintenance.yml", contract) + self.assertIn("crates/openplotva-sandbox/src", contract) self.assertIn( 'maintenance_env_file="${OPENPLOTVA_MAINTENANCE_ENV_FILE:-/etc/openplotva-maintenance/runtime.env}"', script, diff --git a/tools/maintenance/tests/test_release_contract.py b/tools/maintenance/tests/test_release_contract.py new file mode 100644 index 00000000..dbcd80f0 --- /dev/null +++ b/tools/maintenance/tests/test_release_contract.py @@ -0,0 +1,49 @@ +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[3] +spec = importlib.util.spec_from_file_location('release_request', ROOT / 'scripts/geta-deploy-request.py') +release = importlib.util.module_from_spec(spec) +spec.loader.exec_module(release) + + +class ReleaseContractTests(unittest.TestCase): + def test_schema_and_sandbox_changes_require_profile_review(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'deploy').mkdir() + (root / 'migrations').mkdir() + (root / 'sandbox').mkdir() + contract = {'data_version': '1', 'files': [], 'trees': ['migrations', 'sandbox'], 'images': {'openplotva': 'IMAGE'}} + (root / release.CONTRACT).write_text(json.dumps(contract)) + (root / 'migrations/001.sql').write_text('SELECT 1;') + before, _ = release.contract_digest(root) + (root / 'sandbox/protocol.rs').write_text('changed protocol') + after, _ = release.contract_digest(root) + self.assertNotEqual(before, after) + (root / 'migrations/002.sql').write_text('SELECT 2;') + final, _ = release.contract_digest(root) + self.assertNotEqual(after, final) + + def test_request_cannot_select_a_host_path_or_sandbox_image(self): + environment = {'DEPLOY_OPERATION': 'deploy', 'DEPLOY_SHA': 'a' * 40, 'OPENPLOTVA_IMAGE_DIGEST': 'sha256:' + 'b' * 64} + payload = release.payload(ROOT, environment, lambda: 'synthetic-identity') + self.assertEqual(set(payload['images']), {'openplotva'}) + self.assertNotIn('path', payload) + self.assertNotIn('registry_token', payload) + self.assertNotIn('sandbox', payload['images']) + + def test_backup_uses_a_fixed_server_operation(self): + workflow = (ROOT / '.github/workflows/backup-production.yml').read_text() + self.assertIn('deploy@geta.moe backup', workflow) + self.assertNotIn('scp ', workflow) + self.assertNotIn('backup-production.sh', workflow) + payload = release.payload(ROOT, {'DEPLOY_OPERATION': 'backup'}, lambda: 'synthetic-identity') + self.assertEqual(payload, {'token': 'synthetic-identity'}) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/maintenance/tests/test_release_identity.py b/tools/maintenance/tests/test_release_identity.py new file mode 100644 index 00000000..8852177a --- /dev/null +++ b/tools/maintenance/tests/test_release_identity.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +HELPER = next(path for parent in Path(__file__).resolve().parents + for name in ('scripts/geta-deploy-request.py', 'workflow-tools/release-request.py') + if (path := parent / name).is_file()) +spec = importlib.util.spec_from_file_location('identity_request', HELPER) +release = importlib.util.module_from_spec(spec) +spec.loader.exec_module(release) +ENDPOINT = 'https://vstoken.actions.githubusercontent.com/oidc?api-version=2.0&audience=old' +ENVIRONMENT = {'ACTIONS_ID_TOKEN_REQUEST_URL': ENDPOINT, + 'ACTIONS_ID_TOKEN_REQUEST_TOKEN': 'synthetic-request-token'} + + +class IdentityTransportTests(unittest.TestCase): + def response(self, factory, status, body): + response = io.BytesIO(body) + response.status = status + connection = Mock() + connection.getresponse.return_value = response + factory.return_value = connection + return connection + + def test_untrusted_origins_never_receive_credentials(self): + for address in ('file:///tmp/data', 'http://vstoken.actions.githubusercontent.com/oidc', + 'https://example.invalid/oidc', 'https://actions.githubusercontent.com.example.invalid/', + 'https://user@vstoken.actions.githubusercontent.com/oidc', + 'https://vstoken.actions.githubusercontent.com:444/oidc', + ENDPOINT + '#fragment'): + with self.subTest(address=address), patch.dict(release.os.environ, { + **ENVIRONMENT, 'ACTIONS_ID_TOKEN_REQUEST_URL': address}, clear=True), \ + patch.object(release.http.client, 'HTTPSConnection') as factory: + with self.assertRaises(ValueError): + release.identity() + factory.assert_not_called() + + def test_https_request_pins_audience_and_closes_connection(self): + with patch.dict(release.os.environ, ENVIRONMENT, clear=True), \ + patch.object(release.http.client, 'HTTPSConnection') as factory: + connection = self.response(factory, 200, b'{"value":"synthetic-identity"}') + self.assertEqual(release.identity(), 'synthetic-identity') + factory.assert_called_once_with('vstoken.actions.githubusercontent.com', timeout=15) + connection.request.assert_called_once_with( + 'GET', '/oidc?api-version=2.0&audience=geta-deploy', + headers={'Authorization': 'Bearer synthetic-request-token'}) + connection.close.assert_called_once() + + def test_redirects_are_not_followed(self): + for status in (301, 302, 403, 500): + with self.subTest(status=status), patch.dict(release.os.environ, ENVIRONMENT, clear=True), \ + patch.object(release.http.client, 'HTTPSConnection') as factory: + connection = self.response(factory, status, b'synthetic-private-response') + with self.assertRaisesRegex(ValueError, 'identity request failed'): + release.identity() + factory.assert_called_once() + connection.request.assert_called_once() + connection.close.assert_called_once() + + def test_response_size_is_bounded(self): + with patch.dict(release.os.environ, ENVIRONMENT, clear=True), \ + patch.object(release.http.client, 'HTTPSConnection') as factory: + connection = self.response(factory, 200, b'x' * 32769) + with self.assertRaisesRegex(ValueError, 'response is too large'): + release.identity() + connection.close.assert_called_once() + + def test_invalid_identities_are_rejected(self): + for value in (None, 123, '', 'x' * 16001): + with self.subTest(value_type=type(value).__name__), \ + patch.dict(release.os.environ, ENVIRONMENT, clear=True), \ + patch.object(release.http.client, 'HTTPSConnection') as factory: + connection = self.response(factory, 200, json.dumps({'value': value}).encode()) + with self.assertRaisesRegex(ValueError, 'invalid identity'): + release.identity() + connection.close.assert_called_once() + + def test_schema_mode_requires_an_explicit_schema_contract(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'deploy').mkdir() + (root / release.CONTRACT).write_text('{"files":[],"trees":[]}') + result = subprocess.run([sys.executable, str(HELPER), '--root', str(root), + '--check-schema', str(root)], capture_output=True, text=True) + self.assertEqual(result.returncode, 2) + self.assertIn('--check-schema requires approved_schema', result.stderr) + + +if __name__ == '__main__': + unittest.main()