Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 28 additions & 55 deletions .github/workflows/backup-production.yml
Original file line number Diff line number Diff line change
@@ -1,80 +1,53 @@
name: Production Backup

# The nightly run is a fixed host crontab entry (see deploy/production/README.md):
# scheduled workflows start hours late and then overlap daytime traffic. This workflow
# is for manual backups.
on:
workflow_dispatch:

concurrency:
group: production-backup
cancel-in-progress: false

permissions: {}
permissions:
contents: read
id-token: write

jobs:
backup:
name: Back up production state
runs-on: ubuntu-22.04
environment: production
timeout-minutes: 180
permissions:
contents: read
steps:
- name: Require repository owner for manual runs
if: ${{ github.event_name == 'workflow_dispatch' }}
shell: bash
run: |
set -euo pipefail
if [[ "${{ github.actor }}" != "iamwavecut" || "${{ github.triggering_actor }}" != "iamwavecut" ]]; then
echo "Production backup can only be started or re-run manually by iamwavecut" >&2
exit 1
fi

- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Configure SSH
shell: bash
- name: Require repository owner and main
env:
GETA_SSH_PRIVATE_KEY: ${{ secrets.GETA_SSH_PRIVATE_KEY }}
GETA_SSH_KNOWN_HOSTS: ${{ secrets.GETA_SSH_KNOWN_HOSTS }}
GETA_SSH_PORT: ${{ vars.GETA_SSH_PORT || '22022' }}
ACTOR: ${{ github.actor }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
RELEASE_REF: ${{ github.ref }}
run: |
set -euo pipefail
test -n "$GETA_SSH_PRIVATE_KEY"
test -n "$GETA_SSH_KNOWN_HOSTS"
install -m 700 -d "$HOME/.ssh"
printf '%s\n' "$GETA_SSH_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts"
chmod 600 "$HOME/.ssh/known_hosts"
printf 'Host geta.moe 207.244.239.212 2605:a140:2260:6080::1\n Port %s\n' "$GETA_SSH_PORT" > "$HOME/.ssh/config"
chmod 600 "$HOME/.ssh/config"
eval "$(ssh-agent -s)"
printf '%s\n' "$GETA_SSH_PRIVATE_KEY" | ssh-add -
echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV"
echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV"

- name: Upload backup script
shell: bash
[[ "$ACTOR" == iamwavecut && "$TRIGGERING_ACTOR" == iamwavecut ]]
[[ "$RELEASE_REF" == refs/heads/main ]]
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Configure pinned SSH identity
env:
GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }}
GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }}
SSH_KEY: ${{ secrets.GETA_SSH_PRIVATE_KEY }}
KNOWN_HOSTS: ${{ secrets.GETA_SSH_KNOWN_HOSTS }}
SSH_PORT: ${{ vars.GETA_SSH_PORT || '22022' }}
run: |
set -euo pipefail
# shellcheck disable=SC2029
ssh "$GETA_SSH_TARGET" "install -d -m 755 '$GETA_DEPLOY_ROOT'"
scp deploy/production/backup-production.sh "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/backup-production.sh"
# shellcheck disable=SC2029
ssh "$GETA_SSH_TARGET" "chmod 755 '$GETA_DEPLOY_ROOT/backup-production.sh'"

- name: Create production backup
shell: bash
install -m 700 -d ~/.ssh
printf '%s\n' "$SSH_KEY" > ~/.ssh/geta_deploy
chmod 600 ~/.ssh/geta_deploy
printf '%s\n' "$KNOWN_HOSTS" > ~/.ssh/known_hosts
chmod 600 ~/.ssh/known_hosts
printf 'Host geta.moe\n Port %s\n IdentitiesOnly yes\n BatchMode yes\n StrictHostKeyChecking yes\n' "$SSH_PORT" > ~/.ssh/config
- name: Request fixed host backup procedure
env:
GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }}
GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }}
DEPLOY_OPERATION: backup
run: |
set -euo pipefail
ssh \
-o ServerAliveInterval=30 \
-o ServerAliveCountMax=120 \
"$GETA_SSH_TARGET" \
"'$GETA_DEPLOY_ROOT/backup-production.sh'"
python3 scripts/geta-deploy-request.py |
ssh -T -i ~/.ssh/geta_deploy -o ServerAliveInterval=30 -o ServerAliveCountMax=360 \
deploy@geta.moe backup
184 changes: 46 additions & 138 deletions .github/workflows/deploy-production.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,43 +8,49 @@ env:

permissions: {}

concurrency:
group: openplotva-production
cancel-in-progress: false

jobs:
authorize:
name: Authorize deployment operator
runs-on: ubuntu-latest
permissions: {}
steps:
- name: Require repository owner
- name: Require repository owner and main
shell: bash
env:
ACTOR: ${{ github.actor }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
RELEASE_REF: ${{ github.ref }}
run: |
set -euo pipefail
if [[ "${{ github.actor }}" != "iamwavecut" || "${{ github.triggering_actor }}" != "iamwavecut" ]]; then
echo "Production deploy can only be started or re-run by iamwavecut" >&2
exit 1
fi
[[ "$ACTOR" == iamwavecut && "$TRIGGERING_ACTOR" == iamwavecut ]]
[[ "$RELEASE_REF" == refs/heads/main ]]

build-and-deploy:
name: Build image and deploy
needs: authorize
runs-on: ubuntu-22.04
environment: production
timeout-minutes: 60
permissions:
actions: read
contents: read
packages: write
id-token: write
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false

- name: Select GHCR image tag
- name: Select immutable application image
shell: bash
run: |
set -euo pipefail
image_name="ghcr.io/${GITHUB_REPOSITORY}"
image_name="$(printf '%s' "$image_name" | tr '[:upper:]' '[:lower:]')"
echo "OPENPLOTVA_IMAGE=${image_name}:${GITHUB_SHA}" >> "$GITHUB_ENV"
echo "OPENPLOTVA_SANDBOX_IMAGE=${image_name}-sandbox:${GITHUB_SHA}" >> "$GITHUB_ENV"
echo "OPENPLOTVA_IMAGE=ghcr.io/iamwavecut/openplotva:${GITHUB_SHA}" >> "$GITHUB_ENV"

- name: Install Rust
run: |
Expand All @@ -56,8 +62,8 @@ jobs:
with:
shared-key: rust-${{ runner.os }}-ubuntu-22.04-${{ env.RUST_VERSION }}-deploy

- name: Build release binaries
run: cargo build --locked --release -p openplotva-app -p openplotva-sandbox
- name: Build application release binary
run: cargo build --locked --release -p openplotva-app

- name: Log in to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
Expand All @@ -69,7 +75,8 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4

- name: Build and push GHCR image
- name: Build and push application image
id: image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
env:
DOCKER_BUILD_RECORD_UPLOAD: "false"
Expand All @@ -80,28 +87,12 @@ jobs:
labels: |
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
io.geta-deploy.data-version=1
tags: ${{ env.OPENPLOTVA_IMAGE }}
push: true
cache-from: type=gha,scope=openplotva-runtime-prebuilt
cache-to: type=gha,mode=min,scope=openplotva-runtime-prebuilt

- name: Build and push sandbox image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
env:
DOCKER_BUILD_RECORD_UPLOAD: "false"
with:
context: .
file: sandbox/Dockerfile
target: runtime-prebuilt
labels: |
org.opencontainers.image.revision=${{ github.sha }}
org.opencontainers.image.source=https://github.com/${{ github.repository }}
tags: ${{ env.OPENPLOTVA_SANDBOX_IMAGE }}
push: true
# mode=max keeps the nsjail and package layers between deploys.
cache-from: type=gha,scope=openplotva-sandbox
cache-to: type=gha,mode=max,scope=openplotva-sandbox

- name: Wait for green CI on this commit
shell: bash
env:
Expand All @@ -114,128 +105,45 @@ jobs:
--jq '.workflow_runs[0] // empty | "\(.status) \(.conclusion // "none") \(.html_url)"')"
elapsed=$((SECONDS - started))
if [[ -z "$run" ]]; then
echo "No push CI run for ${GITHUB_SHA} after ${elapsed}s"
(( elapsed < 180 )) || { echo "Only commits that CI ran on after a push can be deployed" >&2; exit 1; }
(( elapsed < 180 )) || { echo 'No push CI found for release' >&2; exit 1; }
else
read -r status conclusion url <<<"$run"
echo "CI ${status}/${conclusion} after ${elapsed}s: ${url}"
echo "CI ${status}/${conclusion}: ${url}"
if [[ "$status" == completed ]]; then
[[ "$conclusion" == success ]] && exit 0
echo "CI for ${GITHUB_SHA} concluded ${conclusion}; not deploying" >&2
[[ "$conclusion" == success ]] && break
echo 'CI did not pass; not deploying' >&2
exit 1
fi
(( elapsed < 1800 )) || { echo "CI for ${GITHUB_SHA} did not finish within 30 minutes" >&2; exit 1; }
(( elapsed < 1800 )) || { echo 'CI timeout' >&2; exit 1; }
fi
sleep 20
done

- name: Configure SSH
- name: Configure pinned SSH identity
shell: bash
env:
GETA_SSH_PRIVATE_KEY: ${{ secrets.GETA_SSH_PRIVATE_KEY }}
GETA_SSH_KNOWN_HOSTS: ${{ secrets.GETA_SSH_KNOWN_HOSTS }}
GETA_SSH_PORT: ${{ vars.GETA_SSH_PORT || '22022' }}
SSH_KEY: ${{ secrets.GETA_SSH_PRIVATE_KEY }}
KNOWN_HOSTS: ${{ secrets.GETA_SSH_KNOWN_HOSTS }}
SSH_PORT: ${{ vars.GETA_SSH_PORT || '22022' }}
run: |
set -euo pipefail
test -n "$GETA_SSH_PRIVATE_KEY"
test -n "$GETA_SSH_KNOWN_HOSTS"
install -m 700 -d "$HOME/.ssh"
printf '%s\n' "$GETA_SSH_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts"
chmod 600 "$HOME/.ssh/known_hosts"
printf 'Host geta.moe 207.244.239.212 2605:a140:2260:6080::1\n Port %s\n' "$GETA_SSH_PORT" > "$HOME/.ssh/config"
chmod 600 "$HOME/.ssh/config"
eval "$(ssh-agent -s)"
printf '%s\n' "$GETA_SSH_PRIVATE_KEY" | ssh-add -
echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> "$GITHUB_ENV"
echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> "$GITHUB_ENV"

- name: Upload deploy assets
shell: bash
install -m 700 -d ~/.ssh
printf '%s\n' "$SSH_KEY" > ~/.ssh/geta_deploy
chmod 600 ~/.ssh/geta_deploy
printf '%s\n' "$KNOWN_HOSTS" > ~/.ssh/known_hosts
chmod 600 ~/.ssh/known_hosts
printf 'Host geta.moe\n Port %s\n IdentitiesOnly yes\n BatchMode yes\n StrictHostKeyChecking yes\n' "$SSH_PORT" > ~/.ssh/config

- name: Request application-only deployment
env:
GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }}
GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }}
DEPLOY_SHA: ${{ github.sha }}
DEPLOY_OPERATION: deploy
OPENPLOTVA_IMAGE_DIGEST: ${{ steps.image.outputs.digest }}
Comment on lines +137 to +141

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggestion: The GHCR pull token is no longer sent and the contract sets registry_auth: false,
but the image is pushed under ghcr.io/iamwavecut/openplotva using the ephemeral
GITHUB_TOKEN. If that package is private, the host broker cannot pull it and the
deployment will fail. Confirm the package is public or arrange an authenticated
pull path on the host. [general, importance: 5]

Suggested change
- name: Request application-only deployment
env:
GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }}
GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }}
DEPLOY_SHA: ${{ github.sha }}
DEPLOY_OPERATION: deploy
OPENPLOTVA_IMAGE_DIGEST: ${{ steps.image.outputs.digest }}
- name: Request application-only deployment
env:
DEPLOY_SHA: ${{ github.sha }}
DEPLOY_OPERATION: deploy
OPENPLOTVA_IMAGE_DIGEST: ${{ steps.image.outputs.digest }}
run: |
set -euo pipefail
python3 scripts/geta-deploy-request.py |
ssh -T -i ~/.ssh/geta_deploy -o ServerAliveInterval=30 -o ServerAliveCountMax=40 \
deploy@geta.moe "deploy $DEPLOY_SHA"

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed through the package API: ghcr.io/iamwavecut/openplotva is public. Anonymous host pulls are intentional; the production README now states this explicitly.

run: |
set -euo pipefail
# shellcheck disable=SC2029
ssh "$GETA_SSH_TARGET" "install -d -m 755 '$GETA_DEPLOY_ROOT'"
scp deploy/production/compose.production.yml "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/compose.production.yml"
scp deploy/maintenance/compose.maintenance.yml "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/compose.maintenance.yml"
scp deploy/production/configure-gradius-generation.py "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/configure-gradius-generation.py"
scp deploy/production/deploy-production.sh "$GETA_SSH_TARGET:$GETA_DEPLOY_ROOT/deploy-production.sh"
# shellcheck disable=SC2029
ssh "$GETA_SSH_TARGET" "chmod 644 '$GETA_DEPLOY_ROOT/compose.maintenance.yml'; chmod 755 '$GETA_DEPLOY_ROOT/deploy-production.sh'"

- name: Install generation advertising key
shell: bash
env:
GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }}
GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }}
GRADIUS_GENERATION_API_KEY: ${{ secrets.GRADIUS_GENERATION_API_KEY }}
OPENPLOTVA_PRODUCTION_ENV_B64: ${{ secrets.OPENPLOTVA_PRODUCTION_ENV_B64 }}
run: |
set -euo pipefail
if [[ -n "${GRADIUS_GENERATION_API_KEY:-}" ]]; then
# shellcheck disable=SC2029
python3 -c 'import json, os; print(json.dumps({"key": os.environ["GRADIUS_GENERATION_API_KEY"], "production_env_b64": os.environ.get("OPENPLOTVA_PRODUCTION_ENV_B64", "")}))' |
ssh "$GETA_SSH_TARGET" "python3 '$GETA_DEPLOY_ROOT/configure-gradius-generation.py' '$GETA_DEPLOY_ROOT/.env.production'"
fi
python3 scripts/geta-deploy-request.py |
ssh -T -i ~/.ssh/geta_deploy -o ServerAliveInterval=30 -o ServerAliveCountMax=40 \
deploy@geta.moe "deploy $DEPLOY_SHA"

- name: Apply production deployment
shell: bash
env:
GETA_SSH_TARGET: ${{ secrets.GETA_SSH_TARGET }}
GETA_DEPLOY_ROOT: ${{ secrets.GETA_DEPLOY_ROOT }}
GHCR_PULL_TOKEN: ${{ secrets.GHCR_PULL_TOKEN }}
OPENPLOTVA_PRODUCTION_ENV_B64: ${{ secrets.OPENPLOTVA_PRODUCTION_ENV_B64 }}
run: |
set -euo pipefail
test -n "$GHCR_PULL_TOKEN"
printf -v image_q '%q' "$OPENPLOTVA_IMAGE"
printf -v sandbox_image_q '%q' "$OPENPLOTVA_SANDBOX_IMAGE"
printf -v token_q '%q' "$GHCR_PULL_TOKEN"
printf -v actor_q '%q' "$GITHUB_ACTOR"
printf -v env_q '%q' "${OPENPLOTVA_PRODUCTION_ENV_B64:-}"
ssh \
-o ServerAliveInterval=30 \
-o ServerAliveCountMax=20 \
"$GETA_SSH_TARGET" \
"OPENPLOTVA_DEPLOY_IMAGE=$image_q OPENPLOTVA_SANDBOX_IMAGE=$sandbox_image_q GHCR_USERNAME=$actor_q GHCR_PULL_TOKEN=$token_q OPENPLOTVA_PRODUCTION_ENV_B64=$env_q '$GETA_DEPLOY_ROOT/deploy-production.sh'"

- name: Remove non-current GHCR images
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
test -n "${GH_TOKEN:-}" || {
echo "GITHUB_TOKEN is required for post-deploy package cleanup" >&2
exit 1
}

owner="${GITHUB_REPOSITORY_OWNER,,}"
repository="${GITHUB_REPOSITORY#*/}"
repository="${repository,,}"
current_tag="${GITHUB_SHA}"

for package in "$repository" "${repository}-sandbox"; do
versions_file="$(mktemp)"
echo "Cleaning GHCR package ${owner}/${package}; keeping only current tag ${current_tag}"
gh api --paginate --slurp "/users/${owner}/packages/container/${package}/versions?per_page=100" > "$versions_file"

deleted=0
while IFS= read -r version; do
id="$(jq -r '.id' <<<"$version")"
created_at="$(jq -r '.created_at' <<<"$version")"
tags="$(jq -r '[.metadata.container.tags[]?] | join(",")' <<<"$version")"

if jq -e --arg tag "$current_tag" '.metadata.container.tags // [] | index($tag)' <<<"$version" >/dev/null; then
continue
fi

echo "Deleting GHCR version id=${id} created_at=${created_at} tags=${tags:-<none>}"
gh api -X DELETE "/users/${owner}/packages/container/${package}/versions/${id}"
deleted=$((deleted + 1))
done < <(jq -c '.[][]' "$versions_file")

echo "Deleted ${deleted} old version(s) of ${package}"
done
- name: Verify broker runtime status
run: ssh -T -i ~/.ssh/geta_deploy deploy@geta.moe status
Loading
Loading