diff --git a/README.md b/README.md index 711e4f4..eccbdd8 100644 --- a/README.md +++ b/README.md @@ -22,3 +22,26 @@ Runners keep repo checkouts and gem/node caches in `/tmp/github-runner/ger3-N` b `host/clean-runner-workdirs.sh` empties them every Sunday at 04:00 UTC. It is installed on ger3 as `/usr/local/bin/clean-runner-workdirs` with `/etc/cron.d/clean-runner-workdirs`, and logs to `/var/log/clean-runner-workdirs.log`. Edit it here and copy it back to ger3; the host doesn't pull it. + +## Weekly load report + +`host/ci-load-sample.sh` runs every 5 minutes on ger3 and appends the load average and each +runner's busy state to `/var/log/ci-load.tsv` (35 days kept). `host/ci-load-report.sh` emails a 7-day +summary every Monday at 13:00 UTC, from alerts@ericdahl.dev through the `mail-bridge` container +(ger3 blocks outbound SMTP). Its settings are in `/etc/ci-load-report.env` (root only: +`MAIL_BRIDGE_SMTP_PASSWORD` from Doppler `ericdahl-dev/prd`, and `REPORT_TO`). Cron: +`/etc/cron.d/ci-load-report`, log `/var/log/ci-load-report.log`. + +## Setup facts and workflow rules + +- Runners: Coolify service `h4esa4ronft1iqni8yrlu8ze` on ger3, two ephemeral org runners `ger3-1` + and `ger3-2` with `cpu_shares: 256` so the apps on ger3 win CPU contention. The registration + token is the fine-grained PAT `GH_RUNNER_PAT` in Doppler, copied to the service's `ACCESS_TOKEN`. +- A new image only reaches the runners on a Coolify restart with `?latest=true`, and a restart + kills running jobs: wait until both runners are idle. +- In workflows: no `services:` with host ports and no job-level `container:`. Start Postgres as a + step with `docker run --network "container:$HOSTNAME"` and remove it in an `if: always()` step + (ericdahl-ops #254 is the reference). Put `ger3` in cache keys, because caches saved on + GitHub-hosted runners don't restore here. Give test jobs `timeout-minutes`. +- The Sunday cleanup pauses each runner, so Docker shows it unhealthy for a few minutes. That's + expected. diff --git a/host/ci-load-report.sh b/host/ci-load-report.sh new file mode 100755 index 0000000..30140bb --- /dev/null +++ b/host/ci-load-report.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# Mondays: email a 7-day summary of ger3 load and runner use, from the samples +# ci-load-sample collects. Mail goes through the mail-bridge on the coolify network +# (ger3 blocks outbound SMTP). Needs /etc/ci-load-report.env (root, 600) with +# MAIL_BRIDGE_SMTP_PASSWORD and REPORT_TO. +# Installed as /usr/local/bin/ci-load-report, cron in /etc/cron.d/ci-load-report. +set -eu +. /etc/ci-load-report.env +LOG=/var/log/ci-load.tsv +CORES=$(nproc) +since=$(( $(date +%s) - 7*86400 )) + +summary=$(awk -v s="$since" -v cores="$CORES" ' + $1 >= s { + n++; sum += $3; if ($3 > peak) { peak = $3; peakt = $1 } + if ($3 > cores) over++ + b = $4 + $5; if (b > 0) anyb++; if (b == 2) both++ + } + END { + if (n == 0) { print "No samples in the last 7 days."; exit } + printf "Samples: %d (every 5 minutes)\n\n", n + printf "Load (5-min average) on %d cores\n", cores + printf " average: %.2f\n", sum / n + printf " peak: %.2f at %s UTC\n", peak, strftime("%a %Y-%m-%d %H:%M", peakt, 1) + printf " above %d: %.1f%% of the time\n\n", cores, 100 * over / n + printf "Runners\n" + printf " at least one busy: %.1f%% of the time (~%.1f hours)\n", 100 * anyb / n, anyb * 5 / 60 + printf " both busy: %.1f%% of the time (~%.1f hours)\n\n", 100 * both / n, both * 5 / 60 + printf "Both busy is when new jobs queue. If it is a large share, or load often\n" + printf "sits above the core count, revisit runner count or limits.\n" + }' "$LOG") + +body=$(printf 'From: ger3 CI \r\nTo: %s\r\nSubject: ger3 CI load, week ending %s\r\n\r\n%s\r\n' \ + "$REPORT_TO" "$(date -u +%Y-%m-%d)" "$summary") + +printf '%s' "$body" | docker run --rm -i --network coolify curlimages/curl:latest -sS \ + --url smtp://mail-bridge:2525 --user "glitchtip:$MAIL_BRIDGE_SMTP_PASSWORD" \ + --mail-from alerts@ericdahl.dev --mail-rcpt "$REPORT_TO" --upload-file - +echo "sent to $REPORT_TO" diff --git a/host/ci-load-sample.sh b/host/ci-load-sample.sh new file mode 100755 index 0000000..7e62cb3 --- /dev/null +++ b/host/ci-load-sample.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Every 5 minutes: record ger3's load and whether each runner is mid-job, for the +# weekly report. Columns: epoch load1 load5 runner1_busy runner2_busy. +# Installed as /usr/local/bin/ci-load-sample, cron in /etc/cron.d/ci-load-report. +SERVICE=h4esa4ronft1iqni8yrlu8ze +LOG=/var/log/ci-load.tsv +busy() { docker top "runner-$1-$SERVICE" -eo pid,args 2>/dev/null | grep -c '[R]unner.Worker'; } +read l1 l5 _ < /proc/loadavg +printf '%s\t%s\t%s\t%s\t%s\n' "$(date +%s)" "$l1" "$l5" "$(busy 1)" "$(busy 2)" >> "$LOG" +# Keep 35 days. +cutoff=$(( $(date +%s) - 35*86400 )) +awk -v c="$cutoff" '$1 >= c' "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"