diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 768b00e..5546a37 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,7 @@ RabbitMQ as a queue is used to distribute the worker jobs to its workers. [RustFS](https://github.com/rustfs/rustfs) is used to store the artifacts (screenshots, videos, downloads) and delete them automatically after a few minutes. It is S3-compatible, so the file service talks to it with the existing S3 client. -After upgrading an existing cluster, delete leftover MinIO resources (`kubectl delete deployment,service minio --ignore-not-found`) before or after `kubectl apply -f k8/`. `k8/generate.sh` also removes stale `k8/generated-minio-*.yaml` files from earlier checkouts. +After upgrading an existing cluster, delete leftover MinIO resources (`kubectl delete deployment,service minio --ignore-not-found`) before or after `kubectl apply -f k8/`. `upgrade_k3s_containers.sh` does this automatically via `RETIRED_RESOURCES`. `k8/generate.sh` also removes stale `k8/generated-minio-*.yaml` files from earlier checkouts. ### Etcd diff --git a/README.md b/README.md index aa7fd01..4e9f9d4 100644 --- a/README.md +++ b/README.md @@ -16,12 +16,25 @@ git clone https://github.com/mxschmitt/try-playwright.git openssl req -x509 -nodes -days 730 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=try.playwright.tech/O=try.playwright.tech" kubectl create secret tls try-playwright-cf-tls-cert --key=tls.key --cert=tls.crt cd try-playwright -export RUSTFS_ACCESS_KEY="tryplaywright" -export RUSTFS_SECRET_KEY=$(openssl rand -base64 32) +# Settings for k8/generate.sh. upgrade_k3s_containers.sh reads the same file on every deploy. +mkdir -p /etc/try-playwright +(umask 077; cat > /etc/try-playwright/env < +RUSTFS_ACCESS_KEY=tryplaywright +RUSTFS_SECRET_KEY=$(openssl rand -hex 32) +LOG_AGGREGATOR_ENABLED=false +LOG_AGGREGATOR_URL= +EOF +) +set -a; . /etc/try-playwright/env; set +a bash k8/generate.sh # Remove MinIO if this cluster was created before the RustFS migration. kubectl delete deployment,service minio --ignore-not-found kubectl apply -f k8/ ``` +### Automatic updates + +`upgrade_k3s_containers.sh` (run from cron every 15 minutes) deploys `origin/main` once its CI run is green: it regenerates and applies `k8/` with `/etc/try-playwright/env`, deletes resources listed in `RETIRED_RESOURCES`, pulls the images, restarts the app deployments and checks control and object storage health. When removing a manifest from `k8/`, add its resources to `RETIRED_RESOURCES`. + For more information about the infrastructure and contributing, see [here](./CONTRIBUTING.md). diff --git a/k8/file-deployment.yaml.tpl b/k8/file-deployment.yaml.tpl index d90992f..76e9d43 100644 --- a/k8/file-deployment.yaml.tpl +++ b/k8/file-deployment.yaml.tpl @@ -34,4 +34,15 @@ spec: image: ghcr.io/mxschmitt/try-playwright/file-service:${DOCKER_TAG} imagePullPolicy: IfNotPresent name: file + ports: + - name: http + containerPort: 8080 + # The HTTP server only starts after the storage bucket check passed, so + # a pod that cannot reach RustFS never becomes ready. + readinessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: 2 + periodSeconds: 5 restartPolicy: Always diff --git a/upgrade_k3s_containers.sh b/upgrade_k3s_containers.sh index 581681b..e9f4106 100755 --- a/upgrade_k3s_containers.sh +++ b/upgrade_k3s_containers.sh @@ -1,9 +1,12 @@ #!/bin/bash -# Pull origin/main every 15 minutes. If a new commit is fully built, refresh images and restart. +# Pull origin/main every 15 minutes. If a new commit is fully built, apply the +# k8/ manifests, refresh images and restart. set -euo pipefail REPO="${TRY_PLAYWRIGHT_REPO:-/root/try-playwright}" GITHUB_REPO="mxschmitt/try-playwright" +# Secrets and settings for k8/generate.sh (RUSTFS_*, TURNSTILE_SECRET_KEY, ...). +ENV_FILE="${TRY_PLAYWRIGHT_ENV_FILE:-/etc/try-playwright/env}" STATE_DIR="${TRY_PLAYWRIGHT_STATE_DIR:-/var/lib/try-playwright}" STATE_FILE="${STATE_DIR}/last-deployed-sha" LOCK_FILE="${STATE_DIR}/autoupdate.lock" @@ -18,9 +21,17 @@ IMAGES=( ghcr.io/mxschmitt/try-playwright/worker-java:latest ghcr.io/mxschmitt/try-playwright/worker-python:latest ghcr.io/mxschmitt/try-playwright/worker-csharp:latest + ghcr.io/mxschmitt/try-playwright/log-aggregator:latest ) -APP_DEPLOYMENTS=(frontend file control squid) +APP_DEPLOYMENTS=(frontend file control squid log-aggregator) + +# Resources whose manifests were removed from k8/. `kubectl apply` never deletes +# anything, so list them here to have them cleaned up on the next deploy. +RETIRED_RESOURCES=( + deployment/minio + service/minio +) FORCE=0 DRY_RUN=0 @@ -150,6 +161,7 @@ if [[ -z "$CONTINUE_SHA" ]]; then if [[ "$DRY_RUN" -eq 1 ]]; then log "dry-run: would reset ${REPO} to ${AFTER}" + log "dry-run: would apply k8/ manifests with ${ENV_FILE} and delete ${RETIRED_RESOURCES[*]}" log "dry-run: would pull ${#IMAGES[@]} images and restart ${APP_DEPLOYMENTS[*]}" exit 0 fi @@ -175,6 +187,22 @@ for image in "${IMAGES[@]}"; do "${CRICTL[@]}" pull "$image" done +# Without this, manifest changes (new services, env vars, ingress routes) never +# reach the cluster and only image updates would be deployed. +[[ -f "$ENV_FILE" ]] || die "env file ${ENV_FILE} not found; see README.md for the required variables" +log "generating k8/ manifests with ${ENV_FILE}" +set -a +# shellcheck source=/dev/null +. "$ENV_FILE" +set +a +(cd "$REPO" && bash k8/generate.sh) || die "k8/generate.sh failed" + +log "applying k8/ manifests" +kubectl apply -f "$REPO/k8/" +log "deleting retired resources: ${RETIRED_RESOURCES[*]}" +kubectl delete "${RETIRED_RESOURCES[@]}" --ignore-not-found +kubectl rollout status deployment/rustfs --timeout=180s + # Delete workers before bouncing control so the new control process creates a # fresh pool. Do not delete workers after control is ready. log "removing worker pods before control restart" @@ -186,27 +214,38 @@ for deploy in "${APP_DEPLOYMENTS[@]}"; do restart_args+=("deployment/${deploy}") done kubectl rollout restart "${restart_args[@]}" +# rollout status waits until the new pods are ready. Avoid `kubectl wait -l` +# here: it also matches old pods that are still terminating and then fails +# with NotFound once they are gone. +log "waiting for application pods" for deploy in "${APP_DEPLOYMENTS[@]}"; do kubectl rollout status "deployment/${deploy}" --timeout=180s done -log "waiting for application pods" -kubectl wait --timeout=180s --for=condition=ready pod -l io.kompose.service=frontend -kubectl wait --timeout=180s --for=condition=ready pod -l io.kompose.service=file -kubectl wait --timeout=180s --for=condition=ready pod -l io.kompose.service=control -kubectl wait --timeout=180s --for=condition=ready pod -l io.kompose.service=squid - FRONTEND_PORT="$(kubectl get svc frontend -o jsonpath='{.spec.ports[0].nodePort}' 2>/dev/null || true)" if [[ -n "$FRONTEND_PORT" ]]; then - HEALTH_URL="http://127.0.0.1:${FRONTEND_PORT}/service/control/health" + BASE_URL="http://127.0.0.1:${FRONTEND_PORT}" else - HEALTH_URL="http://127.0.0.1:8080/service/control/health" + BASE_URL="http://127.0.0.1:8080" fi +HEALTH_URL="${BASE_URL}/service/control/health" log "checking health at ${HEALTH_URL}" curl -fsS --retry 12 --retry-all-errors --retry-delay 5 "$HEALTH_URL" >/dev/null \ || die "health check failed" +# A missing object returns 404 only when file-service can reach object storage; +# a storage outage surfaces as 5xx. +STORAGE_URL="${BASE_URL}/file-uploads/00000000-0000-0000-0000-000000000000.png" +log "checking object storage via ${STORAGE_URL}" +storage_status="" +for _ in $(seq 1 12); do + storage_status="$(curl -s -o /dev/null -w '%{http_code}' -I "$STORAGE_URL" || true)" + [[ "$storage_status" == "404" ]] && break + sleep 5 +done +[[ "$storage_status" == "404" ]] || die "object storage check failed (HTTP ${storage_status:-none})" + printf '%s\n' "$AFTER" > "$STATE_FILE" touch "$FILE_ROLL_MARKER" log "deployed ${AFTER} successfully"