From b3f7178838cb8f59ccac672d3d4857e2220e0753 Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 11:41:39 +0300 Subject: [PATCH 01/17] security: pin image versions, add Baserow healthcheck, resource limits, and container hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace :latest tags with pinned versions (n8n 2.35.7, Baserow 2.3.3, Postgres 15.15-alpine3.24, Caddy 2.11.4-alpine) so upgrades are deliberate, not silent on the next pull. - Add a healthcheck to Baserow (postgres and n8n already had one) using its /api/_health/ endpoint. - Add per-service memory limits and rotated/capped JSON logging so a runaway execution or import can't OOM or fill the disk on a small VPS. - Harden every service with no-new-privileges; drop all Linux capabilities on Caddy/n8n/Postgres and re-add only what each specifically needs. Baserow is deliberately left uncapped for now — its process supervisor's root requirements aren't confirmed safe to guess at without live testing. --- docker-compose.yml | 83 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7f4b8d4..e801ef0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: postgres: - image: postgres:15-alpine + image: postgres:15.15-alpine3.24 restart: unless-stopped environment: POSTGRES_USER: ${POSTGRES_USER} @@ -13,11 +13,34 @@ services: interval: 10s timeout: 5s retries: 5 + security_opt: + - no-new-privileges:true + # cap_drop/cap_add: official postgres entrypoint starts as root to fix + # data-directory ownership on first init, then drops via gosu. These five + # caps are the documented minimal set — verify a fresh-volume init still + # works cleanly before trusting this on a real deploy. + cap_drop: + - ALL + cap_add: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - SETUID + - SETGID + deploy: + resources: + limits: + memory: 512m + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" networks: - backend-network n8n: - image: n8nio/n8n:latest + image: n8nio/n8n:2.35.7 restart: unless-stopped depends_on: postgres: @@ -44,12 +67,27 @@ services: timeout: 10s retries: 5 start_period: 30s + security_opt: + - no-new-privileges:true + # Safe to fully drop: the official image already runs as the + # unprivileged `node` user and never binds a privileged port. + cap_drop: + - ALL + deploy: + resources: + limits: + memory: 1g + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" networks: - proxy-network - backend-network baserow: - image: baserow/baserow:latest + image: baserow/baserow:2.3.3 restart: unless-stopped depends_on: postgres: @@ -59,12 +97,33 @@ services: SECRET_KEY: ${BASEROW_SECRET_KEY} volumes: - baserow_data:/baserow/data + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost/api/_health/"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 45s + security_opt: + - no-new-privileges:true + # No cap_drop here yet: Baserow's all-in-one image runs nginx + + # gunicorn/daphne + celery via an internal process supervisor whose root + # requirements aren't confirmed. Drop capabilities in a follow-up pass + # once this base hardening is proven in CI, not blindly in this one. + deploy: + resources: + limits: + memory: 1g + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" networks: - proxy-network - backend-network caddy: - image: caddy:2-alpine + image: caddy:2.11.4-alpine restart: unless-stopped ports: - "80:80" @@ -78,6 +137,22 @@ services: - ./Caddyfile:/etc/caddy/Caddyfile:ro - caddy_data:/data - caddy_config:/config + security_opt: + - no-new-privileges:true + # NET_BIND_SERVICE is required to bind ports 80/443 as non-root. + cap_drop: + - ALL + cap_add: + - NET_BIND_SERVICE + deploy: + resources: + limits: + memory: 128m + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" networks: - proxy-network From 58aae0260654c3892c0bfaf6a15e1123bdcbd71c Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 11:41:44 +0300 Subject: [PATCH 02/17] ci: add Dependabot for pinned image tags and github-actions Tracks the version pins just added to docker-compose.yml (docker ecosystem, GA support for docker-compose.yml since Feb 2025) and third-party Action versions (github-actions ecosystem) weekly. --- .github/dependabot.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..6ef9a61 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 +updates: + # Third-party GitHub Actions — keeps every workflow action current and + # off stale/floating tags, which is exactly the class of risk the + # aquasecurity/trivy-action supply-chain compromise (March 2026) exposed. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + # Tracks the pinned image tags in docker-compose.yml directly (GA since + # Feb 2025) and opens a PR when n8n/Baserow/Postgres/Caddy cut a new + # release. + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" From ec4dd1230666e6d5240ddf2a6fdff498ad94bec7 Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 11:41:55 +0300 Subject: [PATCH 03/17] ci: add real smoke-test job and new Makefile targets validate.yml now has a second job that boots the dev stack for real (docker compose up --wait) and curls both health endpoints externally, instead of only checking that the compose files parse. Also pins actions/checkout by commit SHA rather than a floating tag. Makefile gains test, restore, backup-test, and security-scan targets so the new verification tooling has a local entry point, not just a CI-only one. --- .github/workflows/validate.yml | 40 ++++++++++++++++++++++++++++++++-- Makefile | 39 +++++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index ef167cf..5986de3 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Copy env template run: cp .env.example .env @@ -24,4 +24,40 @@ jobs: - name: Lint shell scripts run: | sudo apt-get install -y shellcheck - shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh + shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh scripts/restore.sh scripts/backup-drill.sh + + smoke-test: + runs-on: ubuntu-latest + needs: validate + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Prepare .env + # .env.example ships with N8N_ENCRYPTION_KEY and BASEROW_SECRET_KEY + # blank — Baserow's Django backend won't boot with an empty + # SECRET_KEY. Generate real values the same way scripts/setup.sh + # does for a real deploy. + run: | + cp .env.example .env + sed -i "s|^N8N_ENCRYPTION_KEY=.*|N8N_ENCRYPTION_KEY=$(openssl rand -hex 32)|" .env + sed -i "s|^BASEROW_SECRET_KEY=.*|BASEROW_SECRET_KEY=$(openssl rand -hex 32)|" .env + + - name: Boot dev stack and wait for healthchecks + run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait --wait-timeout 180 + + - name: Check n8n health endpoint (external, through the mapped port) + run: curl -sf http://localhost:5678/healthz + + - name: Check Baserow health endpoint (external, through the mapped port) + run: curl -sf http://localhost:8080/api/_health/ + + - name: Dump service status and logs on failure + if: failure() + run: | + docker compose ps + docker compose logs + + - name: Tear down + if: always() + run: docker compose down -v diff --git a/Makefile b/Makefile index 2d77044..b5358f6 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,24 @@ .DEFAULT_GOAL := help -.PHONY: help up down restart dev logs backup update setup +.PHONY: help up down restart dev logs backup update setup test restore backup-test security-scan help: @echo "Self-Hosted Automation Stack" @echo "" @echo "Usage: make " @echo "" - @echo " setup Interactive first-run setup (writes .env, starts stack)" - @echo " up Start the production stack" - @echo " down Stop and remove containers" - @echo " restart Restart all services" - @echo " dev Start in dev mode (direct ports, no Caddy)" - @echo " logs Follow logs for all services" - @echo " backup Back up all volumes to ./backups/" - @echo " update Pull latest images and redeploy" + @echo " setup Interactive first-run setup (writes .env, starts stack)" + @echo " up Start the production stack" + @echo " down Stop and remove containers" + @echo " restart Restart all services" + @echo " dev Start in dev mode (direct ports, no Caddy)" + @echo " logs Follow logs for all services" + @echo " backup Back up all volumes to ./backups/" + @echo " update Pull latest images and redeploy" + @echo " test Boot the dev stack and verify health endpoints respond" + @echo " restore Restore from a backup: make restore TIMESTAMP=20260823_140000" + @echo " backup-test Full backup -> destroy -> restore -> verify drill (needs a running stack)" + @echo " security-scan Scan the four pinned images with the Trivy CLI (must be installed locally)" setup: bash scripts/setup.sh @@ -39,3 +43,20 @@ backup: update: bash scripts/update.sh + +test: + docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait --wait-timeout 180 + curl -sf http://localhost:5678/healthz && echo "n8n: healthy" + curl -sf http://localhost:8080/api/_health/ && echo "baserow: healthy" + +restore: + bash scripts/restore.sh $(TIMESTAMP) + +backup-test: + bash scripts/backup-drill.sh + +security-scan: + @for img in n8nio/n8n:2.35.7 baserow/baserow:2.3.3 postgres:15.15-alpine3.24 caddy:2.11.4-alpine; do \ + echo "--- $$img ---"; \ + trivy image --severity CRITICAL,HIGH --ignore-unfixed "$$img"; \ + done From 92e431e4543da46ff9bd4dce7d933de04da4e454 Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 11:42:01 +0300 Subject: [PATCH 04/17] ci: add weekly Trivy vulnerability scan of the four pinned images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scans images, not the compose file — Trivy has no native docker-compose misconfiguration scanner (confirmed against its own issue tracker), so this intentionally doesn't claim IaC coverage it doesn't have. Non-blocking (visibility via the Security tab, not a merge gate) since some CVEs in upstream images won't be fixable from this repo. aquasecurity/trivy-action is pinned by commit SHA, not a tag — 75 of 76 of its version tags were force-pushed with malware in March 2026. --- .github/workflows/security.yml | 57 ++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/security.yml diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..b91ed19 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,57 @@ +name: Security Scan + +on: + push: + branches: [main] + pull_request: + schedule: + # Weekly — catches newly-disclosed CVEs in images that were already + # pinned and clean when they were merged. + - cron: "0 6 * * 1" + workflow_dispatch: + +# Trivy scans the four pinned images for known CVEs. It does NOT scan +# docker-compose.yml itself — Trivy has no native docker-compose misconfig +# scanner (confirmed against its own issue tracker), so claiming IaC +# coverage here would be inaccurate. +jobs: + trivy: + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + include: + - name: n8n + image: n8nio/n8n:2.35.7 + - name: baserow + image: baserow/baserow:2.3.3 + - name: postgres + image: postgres:15.15-alpine3.24 + - name: caddy + image: caddy:2.11.4-alpine + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Pinned to a commit SHA, not a version tag. aquasecurity/trivy-action + # had 75 of 76 tags force-pushed with malware in March 2026 — tags are + # mutable, SHAs aren't. Re-resolve this SHA from a current release + # before bumping, never trust a floating tag for a third-party action. + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ${{ matrix.image }} + format: sarif + output: trivy-results.sarif + severity: CRITICAL,HIGH + ignore-unfixed: true + continue-on-error: true # visibility, not a merge gate — some CVEs are unfixable from this repo + + - name: Upload SARIF to GitHub code scanning + uses: github/codeql-action/upload-sarif@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + with: + sarif_file: trivy-results.sarif + category: "trivy-${{ matrix.name }}" From 7958912d652814d58e5c0ff438f133aa4967635b Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 11:42:08 +0300 Subject: [PATCH 05/17] feat: add restore script and automated backup/restore verification drill scripts/restore.sh restores all three volumes from a backup.sh snapshot (mirrors its dynamic project-name detection). scripts/backup-drill.sh proves the cycle actually recovers data: writes a marker row, backs up, destroys the volumes, restores, and fails if the marker isn't back. Shared between `make backup-test` (local) and the new backup-drill.yml workflow (weekly + on-demand in CI) so the two can't drift apart. docs/backup-and-restore.md updated to document both. --- .github/workflows/backup-drill.yml | 46 +++++++++++++++ docs/backup-and-restore.md | 24 ++++++++ scripts/backup-drill.sh | 68 ++++++++++++++++++++++ scripts/restore.sh | 92 ++++++++++++++++++++++++++++++ 4 files changed, 230 insertions(+) create mode 100644 .github/workflows/backup-drill.yml create mode 100755 scripts/backup-drill.sh create mode 100755 scripts/restore.sh diff --git a/.github/workflows/backup-drill.yml b/.github/workflows/backup-drill.yml new file mode 100644 index 0000000..ce8526b --- /dev/null +++ b/.github/workflows/backup-drill.yml @@ -0,0 +1,46 @@ +name: Backup Drill + +on: + schedule: + - cron: "0 7 * * 1" # weekly + workflow_dispatch: + +# Proves the restore path actually works, not just that backup.sh runs +# without error — see scripts/backup-drill.sh for the drill logic itself +# (also runnable locally via `make backup-test`). +# +# COMPOSE_FILE layers the dev override on top of the production compose +# file for every plain `docker compose` call in this job — including the +# ones inside the scripts — without those scripts needing to know about CI +# at all. This is the standard Compose CLI mechanism for that. +env: + COMPOSE_FILE: docker-compose.yml:docker-compose.dev.yml + +jobs: + backup-restore-verify: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Prepare .env + run: | + cp .env.example .env + sed -i "s|^N8N_ENCRYPTION_KEY=.*|N8N_ENCRYPTION_KEY=$(openssl rand -hex 32)|" .env + sed -i "s|^BASEROW_SECRET_KEY=.*|BASEROW_SECRET_KEY=$(openssl rand -hex 32)|" .env + + - name: Boot the stack + run: docker compose up -d --wait --wait-timeout 180 + + - name: Run the backup/restore drill + run: bash scripts/backup-drill.sh + + - name: Dump service status and logs on failure + if: failure() + run: | + docker compose ps + docker compose logs + + - name: Tear down + if: always() + run: docker compose down -v diff --git a/docs/backup-and-restore.md b/docs/backup-and-restore.md index 1d537c4..ffd5158 100644 --- a/docs/backup-and-restore.md +++ b/docs/backup-and-restore.md @@ -67,6 +67,19 @@ For weekly backups (Sundays at 2 AM): ## Restoring from a Backup +The fastest path is `scripts/restore.sh`, which wraps the manual steps below: + +```bash +bash scripts/restore.sh 20260422_140000 +# or: make restore TIMESTAMP=20260422_140000 +``` + +It detects the Compose project name the same way `backup.sh` does, confirms all three backup files for that timestamp exist before touching anything, prompts for confirmation (`--force` skips the prompt — used by the automated drill below), stops the stack, removes and recreates each volume, restores the tarball into it, and starts the stack back up. + +### Manual restore (what the script does, step by step) + +Useful if you need to restore a single volume rather than all three, or want to understand exactly what's happening before running it against production data. + First, capture your project name into a variable — all volume operations below use it: ```bash @@ -117,6 +130,17 @@ Open your n8n and Baserow URLs and confirm workflows and databases are intact. C docker compose logs --tail=100 ``` +## Verifying the Restore Path Actually Works + +A backup you've never restored is a hope, not a backup. `scripts/backup-drill.sh` proves the full cycle works: it writes a marker row into Postgres, runs a real backup, **destroys the volumes**, restores from the tarball just taken, and fails loudly if the marker row isn't back. + +```bash +make backup-test +# needs a running stack — boot one first with `make dev` or `make up` +``` + +The same script runs on a schedule in CI ([`.github/workflows/backup-drill.yml`](../.github/workflows/backup-drill.yml), weekly + on-demand via `workflow_dispatch`), against a throwaway stack it boots itself — so the restore path gets exercised regularly even if nobody runs it by hand. Check the Actions tab for the `Backup Drill` workflow's history. + ## Transferring Backups Off-Server Keep at least one copy off the VPS to protect against server loss. diff --git a/scripts/backup-drill.sh b/scripts/backup-drill.sh new file mode 100755 index 0000000..e4f477f --- /dev/null +++ b/scripts/backup-drill.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -e + +# ------------------------------------------------------- +# backup-drill.sh — Prove the backup/restore cycle actually recovers data +# +# Usage: ./scripts/backup-drill.sh +# +# Precondition: the stack must already be up and healthy +# (`docker compose up -d --wait`) and `.env` must exist. +# +# Writes a marker row into Postgres, backs up, destroys the volumes, restores +# from the tarball just taken, and fails if the marker row isn't back. Used +# by both `make backup-test` (local) and the scheduled Backup Drill workflow +# (CI) — the drill logic lives here once so the two don't drift apart. +# +# Respects COMPOSE_FILE like any other `docker compose` invocation — export +# it (e.g. to include the dev override) before running this script. +# ------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + +cd "$PROJECT_DIR" + +if [ ! -f .env ]; then + echo "ERROR: .env not found. Run this against a configured stack." + exit 1 +fi + +set -a +# shellcheck disable=SC1091 +source .env +set +a + +MARKER="backup-drill-$(date +%s)-$$" + +echo "Writing marker row: $MARKER" +docker compose exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + -c "CREATE TABLE IF NOT EXISTS backup_drill (marker text); INSERT INTO backup_drill VALUES ('${MARKER}');" + +echo "Running backup..." +bash "$SCRIPT_DIR/backup.sh" + +echo "Destroying the stack and its volumes (this is the point of the drill)..." +docker compose down -v + +TIMESTAMP=$(find "$PROJECT_DIR/backups" -name "*_postgres_data_*.tar.gz" | sed -E 's/.*_([0-9]{8}_[0-9]{6})\.tar\.gz/\1/' | sort | tail -1) +if [ -z "$TIMESTAMP" ]; then + echo "ERROR: no backup archive found after running backup.sh" + exit 1 +fi + +echo "Restoring from backup timestamped: $TIMESTAMP" +bash "$SCRIPT_DIR/restore.sh" "$TIMESTAMP" --force + +echo "Waiting for the restored stack to become healthy..." +docker compose up -d --wait --wait-timeout 180 + +RESULT=$(docker compose exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -t -c \ + "SELECT 1 FROM backup_drill WHERE marker = '${MARKER}';" | tr -d '[:space:]') + +if [ "$RESULT" != "1" ]; then + echo "FAIL: marker row '${MARKER}' did not survive the backup/restore cycle." + exit 1 +fi + +echo "PASS: marker row confirmed present after restore. Backup/restore cycle verified." diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 0000000..5126333 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -e + +# ------------------------------------------------------- +# restore.sh — Restore all Docker volumes from a backup.sh snapshot +# +# Usage: ./scripts/restore.sh [--force] +# Matches the suffix on files in ./backups/, e.g. 20260823_140000 +# --force Skip the confirmation prompt (used by the automated backup drill) +# +# WARNING: this replaces the current contents of n8n_data, baserow_data, +# and postgres_data. There is no undo once the existing volumes are removed. +# +# Respects the standard COMPOSE_FILE env var, same as any `docker compose` +# invocation — set it if you need to restore against a non-default compose +# file combination (e.g. the dev override). +# ------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +BACKUP_DIR="$PROJECT_DIR/backups" + +TIMESTAMP="${1:-}" +FORCE_FLAG="${2:-}" + +if [ -z "$TIMESTAMP" ]; then + echo "Usage: $0 [--force]" + echo "" + echo "Available backup timestamps:" + find "$BACKUP_DIR" -name "*.tar.gz" 2>/dev/null | sed -E 's/.*_([0-9]{8}_[0-9]{6})\.tar\.gz/\1/' | sort -u + exit 1 +fi + +cd "$PROJECT_DIR" + +COMPOSE_PROJECT=$(docker compose config 2>/dev/null | grep -m1 '^name:' | awk '{print $2}') +if [ -z "$COMPOSE_PROJECT" ]; then + echo "ERROR: Could not determine compose project name." + echo "Make sure .env exists and docker compose config runs without errors." + exit 1 +fi + +VOLUMES=("n8n_data" "baserow_data" "postgres_data") + +for VOL in "${VOLUMES[@]}"; do + ARCHIVE="$BACKUP_DIR/${COMPOSE_PROJECT}_${VOL}_${TIMESTAMP}.tar.gz" + if [ ! -f "$ARCHIVE" ]; then + echo "ERROR: Backup file not found: $ARCHIVE" + exit 1 + fi +done + +if [ "$FORCE_FLAG" != "--force" ]; then + echo "This will STOP the stack and REPLACE the current contents of:" + for VOL in "${VOLUMES[@]}"; do + echo " - ${COMPOSE_PROJECT}_${VOL}" + done + echo "with the backup timestamped ${TIMESTAMP}. This cannot be undone." + read -rp "Type 'restore' to continue: " CONFIRM + if [ "$CONFIRM" != "restore" ]; then + echo "Aborted." + exit 1 + fi +fi + +echo "Stopping stack..." +docker compose down + +restore_volume() { + local VOL="$1" + local VOLUME_NAME="${COMPOSE_PROJECT}_${VOL}" + local ARCHIVE="$BACKUP_DIR/${VOLUME_NAME}_${TIMESTAMP}.tar.gz" + + echo "Restoring volume: $VOLUME_NAME <- $ARCHIVE" + docker volume rm "$VOLUME_NAME" >/dev/null 2>&1 || true + docker volume create "$VOLUME_NAME" >/dev/null + docker run --rm \ + -v "${VOLUME_NAME}:/data" \ + -v "$BACKUP_DIR:/backup:ro" \ + alpine \ + tar xzf "/backup/$(basename "$ARCHIVE")" -C /data +} + +for VOL in "${VOLUMES[@]}"; do + restore_volume "$VOL" +done + +echo "Starting stack..." +docker compose up -d + +echo "" +echo "Restore complete from backup timestamped: ${TIMESTAMP}" From d9f061b3066bd78b3c4c4c302d4587cedd1455ce Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 11:42:25 +0300 Subject: [PATCH 06/17] docs: add ADRs, Mermaid diagram, and reflect the hardening pass - Replace the ASCII architecture diagram with a Mermaid graph (renders natively on GitHub, no added tooling). - Add three ADRs: why Caddy, why one shared Postgres instance, why two networks instead of one. - New README "Security & Reliability" section summarizing the hardening, scanning, and verified-backup work in one place for a reviewer. - Update docs/updating-services.md and docs/troubleshooting.md for the version-pinning/Dependabot change and the new capability-hardening failure mode respectively. - Update project CLAUDE.md with the new make targets, workflows, the Baserow capability-hardening caveat, and the blank-secrets-in-.env.example gotcha for anything that boots the stack directly from the template. --- CLAUDE.md | 20 +++++- README.md | 84 +++++++++++++++-------- docs/adr/0001-caddy-as-reverse-proxy.md | 18 +++++ docs/adr/0002-shared-postgres-instance.md | 18 +++++ docs/adr/0003-two-network-topology.md | 21 ++++++ docs/troubleshooting.md | 22 ++++++ docs/updating-services.md | 18 ++--- 7 files changed, 158 insertions(+), 43 deletions(-) create mode 100644 docs/adr/0001-caddy-as-reverse-proxy.md create mode 100644 docs/adr/0002-shared-postgres-instance.md create mode 100644 docs/adr/0003-two-network-topology.md diff --git a/CLAUDE.md b/CLAUDE.md index 5d990bf..dedb10d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,10 @@ make logs # docker compose logs -f make backup # stops stack → snapshots volumes → restarts make update # pulls latest images, redeploys, prunes old images make restart # docker compose restart +make test # boots the dev stack and curls both health endpoints (real smoke test) +make restore # restores a backup: make restore TIMESTAMP=20260823_140000 +make backup-test # full backup → destroy volumes → restore → verify drill (needs a running stack) +make security-scan # scans the four pinned images with the local Trivy CLI ``` Raw equivalents when needed: @@ -44,7 +48,14 @@ Internet → Caddy (80/443) ──proxy-network──► n8n (5678) - **`docker-compose.yml`** — production definition; Caddy is the only public-facing service - **`docker-compose.dev.yml`** — override that ports n8n and Baserow directly and puts Caddy behind a `production` profile (so it doesn't start) - **`Caddyfile`** — reads domain/subdomain from env vars at runtime via `{$VAR}` syntax; security headers include HSTS (`max-age=31536000; includeSubDomains`), X-Frame-Options, X-Content-Type-Options, Referrer-Policy; gzip on both virtual hosts -- **`.github/workflows/validate.yml`** — CI runs on every push to main and every PR: validates both compose files and shellchecks all scripts in `scripts/` +- **`.github/workflows/validate.yml`** — CI runs on every push to main and every PR: validates both compose files, shellchecks all scripts in `scripts/`, then actually boots the dev stack (`--wait`) and curls both health endpoints externally — this is real runtime verification, not just YAML syntax checking +- **`.github/workflows/security.yml`** — weekly + on-push Trivy CVE scan of the four pinned images (not the compose file — Trivy has no native docker-compose misconfig scanner), results land in the repo's Security tab. Non-blocking (`continue-on-error`). +- **`.github/workflows/backup-drill.yml`** — weekly + on-demand: writes a marker row, backs up, destroys the volumes, restores, and fails if the marker isn't back. Proves the restore path works instead of just documenting it. Logic lives in `scripts/backup-drill.sh` (also runnable locally via `make backup-test`), shared between CI and local use so they can't drift apart. +- **`.github/dependabot.yml`** — tracks the pinned image tags in `docker-compose.yml` (`docker` ecosystem) and third-party Action versions (`github-actions` ecosystem) weekly. +- **Third-party GitHub Actions are pinned by commit SHA, not by tag**, in every workflow file — `aquasecurity/trivy-action` had 75 of 76 version tags force-pushed with malware in March 2026 (tags are mutable; SHAs aren't). Apply the same pattern to any new third-party Action added to this repo; re-resolve the current SHA at the time you add it, don't copy an old one. +- **`scripts/restore.sh`** — restores all three volumes from a `backup.sh` snapshot; respects `COMPOSE_FILE` like any other `docker compose` call, so it works against the dev override too (see `backup-drill.yml` for how CI sets that). +- **Container hardening in `docker-compose.yml`** — every service has `no-new-privileges`; Caddy/n8n/Postgres have `cap_drop: [ALL]` plus only the specific caps each needs. **Baserow is deliberately not capability-dropped** — its all-in-one process supervisor's root requirements weren't confirmed safe to guess at from outside a live container. Don't attempt to add `cap_drop` to Baserow without verifying against a real running container first (see `docs/troubleshooting.md` § Container Won't Start After a Capability Change for the diagnostic approach). +- **`docs/adr/`** — architecture decision records for the reverse-proxy choice, the shared-Postgres decision, and the two-network topology. Add a new one when a similarly consequential architectural choice gets made, don't retrofit one for routine changes. ## Validation @@ -58,9 +69,11 @@ docker compose -f docker-compose.yml -f docker-compose.dev.yml config --quiet rm .env # Lint shell scripts -shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh +shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh scripts/restore.sh scripts/backup-drill.sh ``` +To actually verify the stack boots (not just that the YAML is valid), run `make test` — needs a local Docker daemon. If one isn't available in your environment, push to a branch and let the `validate.yml` smoke-test job do it; don't claim runtime behavior is verified without one or the other actually running. + ## Critical Constraints - `N8N_ENCRYPTION_KEY` — generated once by `setup.sh` via `openssl rand -hex 32`. **Never change it after first run**; doing so breaks all stored n8n credentials. @@ -70,6 +83,8 @@ shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh - `backup.sh` backs up `n8n_data`, `baserow_data`, and `postgres_data` only. `caddy_data` and `caddy_config` are intentionally excluded — certificates are re-issuable from Let's Encrypt for free. - n8n and Baserow start only after Postgres passes its healthcheck (`pg_isready`). If either service fails to start, confirm Postgres is `(healthy)` first: `docker compose ps postgres`. - `setup.sh` uses a `sed_inplace()` helper (macOS vs Linux `sed -i` differences). Any additions to that script that write to `.env` must use the same helper. +- All four images are version-pinned (no `:latest`). Bump via the Dependabot PR, not by hand, unless you have a specific reason to jump ahead of it — and review the linked changelog before merging either way. +- `.env.example` ships with `N8N_ENCRYPTION_KEY` and `BASEROW_SECRET_KEY` blank by design (real users generate them via `setup.sh`). Anything that boots the stack directly from `.env.example` (CI, a quick local test) must generate real values first — Baserow's Django backend won't start with an empty `SECRET_KEY`. See the "Prepare .env" step in `validate.yml`'s smoke-test job for the pattern. ## Docs @@ -78,3 +93,4 @@ shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh - `docs/backup-and-restore.md` — restore procedure + cron automation - `docs/updating-services.md` — safe upgrade process - `docs/troubleshooting.md` — common issues +- `docs/adr/` — architecture decision records (reverse proxy choice, shared Postgres, two-network topology) diff --git a/README.md b/README.md index 683050a..e6db81e 100644 --- a/README.md +++ b/README.md @@ -2,37 +2,42 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) [![Validate](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/validate.yml/badge.svg)](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/validate.yml) +[![Security Scan](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/security.yml/badge.svg)](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/security.yml) +[![Backup Drill](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/backup-drill.yml/badge.svg)](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/backup-drill.yml) A production-ready Docker Compose stack running **n8n**, **Baserow**, and **PostgreSQL** behind **Caddy** with automatic HTTPS. Clone, configure, and deploy a self-hosted automation suite on any VPS in minutes — no manual SSL certificate management, no external dependencies. +Every push boots the stack for real and checks the health endpoints (not just YAML syntax), the four pinned images are scanned weekly for CVEs, and the backup/restore path is proven weekly by actually destroying a volume, restoring it, and verifying the data survived. See [Security & Reliability](#security--reliability) below. + ## Architecture -``` - ┌──────────────────────────────────────────────────┐ - │ VPS (Docker host) │ - │ │ - Internet │ ┌─────────┐ proxy-network │ - ──:80/:443 ────►│ │ Caddy │──► n8n (:5678) │ - │ │ (TLS) │└──► Baserow (:80) │ - │ └─────────┘ │ │ │ - │ │ │ backend-network │ - │ ▼ ▼ │ - │ ┌──────────────────┐ │ - │ │ PostgreSQL │ │ - │ └──────────────────┘ │ - └──────────────────────────────────────────────────┘ +```mermaid +graph TB + Internet(["Internet"]) -->|":80 / :443"| Caddy["Caddy
(TLS termination)"] + + subgraph proxy["proxy-network"] + Caddy -->|reverse_proxy| n8n["n8n
:5678"] + Caddy -->|reverse_proxy| Baserow["Baserow
:80"] + end + + subgraph backend["backend-network"] + n8n --> Postgres[("PostgreSQL")] + Baserow --> Postgres + end ``` -Caddy terminates TLS and proxies to n8n and Baserow over `proxy-network`. Neither Caddy nor the internet can reach PostgreSQL directly — it is only reachable from n8n and Baserow via `backend-network`. +Caddy terminates TLS and proxies to n8n and Baserow over `proxy-network`. Neither Caddy nor the internet can reach PostgreSQL directly — it is only reachable from n8n and Baserow via `backend-network`. See [ADR-0003](docs/adr/0003-two-network-topology.md) for the reasoning. ## Services -| Service | Image | Purpose | Default URL | -|------------|---------------------------|-------------------------------------|----------------------------------| -| n8n | `n8nio/n8n:latest` | Workflow automation | `https://n8n.yourdomain.com` | -| Baserow | `baserow/baserow:latest` | No-code database / spreadsheet UI | `https://baserow.yourdomain.com` | -| PostgreSQL | `postgres:15-alpine` | Shared database backend | Internal only | -| Caddy | `caddy:2-alpine` | Reverse proxy + automatic TLS | Handles 80/443 | +| Service | Image | Purpose | Default URL | +|------------|-----------------------------------|-------------------------------------|-----------------------------------| +| n8n | `n8nio/n8n:2.35.7` | Workflow automation | `https://n8n.yourdomain.com` | +| Baserow | `baserow/baserow:2.3.3` | No-code database / spreadsheet UI | `https://baserow.yourdomain.com` | +| PostgreSQL | `postgres:15.15-alpine3.24` | Shared database backend | Internal only | +| Caddy | `caddy:2.11.4-alpine` | Reverse proxy + automatic TLS | Handles 80/443 | + +All four are version-pinned rather than `:latest` — [Dependabot](.github/dependabot.yml) opens a PR when a new release lands, so upgrades are reviewed, not silent. See [Updating Services](docs/updating-services.md). ## Prerequisites @@ -70,6 +75,7 @@ make down # Stop and remove containers make restart # Restart all services make logs # Follow logs for all services make dev # Start in dev mode (direct ports, no Caddy) +make test # Boot the dev stack and verify both health endpoints respond ``` Or with Docker Compose directly: @@ -80,14 +86,15 @@ docker compose logs n8n --follow docker compose restart n8n ``` -### Backup +### Backup & Restore ```bash -make backup -# or: bash scripts/backup.sh +make backup # Stops the stack, snapshots all volumes to ./backups/, restarts +make restore TIMESTAMP=20260823_140000 # Restore from a specific backup +make backup-test # Full backup -> destroy -> restore -> verify drill (needs a running stack) ``` -Stops the stack, snapshots all volumes to `./backups/`, and restarts. See [Backup & Restore](docs/backup-and-restore.md) for restore instructions and cron automation. +The same drill `make backup-test` runs locally runs weekly in CI — see [Backup & Restore](docs/backup-and-restore.md). ### Update @@ -110,6 +117,20 @@ make dev - n8n: [http://localhost:5678](http://localhost:5678) - Baserow: [http://localhost:8080](http://localhost:8080) +## Security & Reliability + +- **Network isolation** — `proxy-network` (Caddy + apps) and `backend-network` (apps + Postgres) are separate bridge networks; Postgres is unreachable from Caddy or the internet. +- **Container hardening** — every service runs with `no-new-privileges`; Caddy, n8n, and Postgres run with all Linux capabilities dropped except the specific ones each needs (`NET_BIND_SERVICE` for Caddy's privileged ports, five ownership-related caps for Postgres's first-init step). Baserow is intentionally not capability-dropped yet — its all-in-one process supervisor's root requirements aren't confirmed safe to guess at. +- **Resource limits & log rotation** — every service has a memory ceiling and capped, rotated JSON logs, so a runaway execution or import can't take down the whole VPS or fill the disk. +- **HSTS + standard security headers** on both Caddy virtual hosts. +- **Pinned images, tracked by Dependabot** — no `:latest` tags; version bumps arrive as reviewable PRs. +- **Weekly Trivy scan** of all four pinned images for known CVEs, results in the repo's Security tab. +- **CI actually boots the stack** — `validate.yml`'s smoke-test job runs the real dev stack and checks both health endpoints externally, not just `docker compose config`. +- **Backup/restore is proven, not just documented** — the weekly Backup Drill workflow writes a marker row, backs up, destroys the volume, restores from the tarball, and fails if the data isn't back. +- **Third-party GitHub Actions pinned by commit SHA**, not by tag — a direct response to the `aquasecurity/trivy-action` supply-chain compromise (March 2026), where 75 of 76 version tags were force-pushed with malware. Tags are mutable; SHAs aren't. + +See [docs/adr/](docs/adr/) for the reasoning behind the bigger architectural choices. + ## Environment Variables | Variable | Description | Example | @@ -137,11 +158,13 @@ All variables are documented with comments in [`.env.example`](.env.example). ## Scripts -| Script | Description | -|----------------------|----------------------------------------------------------------------| -| `scripts/setup.sh` | Interactive first-run: writes `.env`, pulls images, starts stack | -| `scripts/backup.sh` | Stops stack, snapshots all volumes to `backups/`, restarts stack | -| `scripts/update.sh` | Pulls latest images, redeploys with `--remove-orphans`, prunes old | +| Script | Description | +|-----------------------------|----------------------------------------------------------------------| +| `scripts/setup.sh` | Interactive first-run: writes `.env`, pulls images, starts stack | +| `scripts/backup.sh` | Stops stack, snapshots all volumes to `backups/`, restarts stack | +| `scripts/restore.sh` | Restores all volumes from a timestamped backup | +| `scripts/backup-drill.sh` | Full backup → destroy → restore → verify cycle (local or CI) | +| `scripts/update.sh` | Pulls latest images, redeploys with `--remove-orphans`, prunes old | ## Documentation @@ -150,6 +173,7 @@ All variables are documented with comments in [`.env.example`](.env.example). - [Backup & Restore](docs/backup-and-restore.md) — Backup volumes and restore procedures - [Updating Services](docs/updating-services.md) — Safely updating n8n, Baserow, and Postgres - [Troubleshooting](docs/troubleshooting.md) — Common issues and solutions +- [Architecture Decision Records](docs/adr/) — Why Caddy, why a shared Postgres instance, why two networks ## License diff --git a/docs/adr/0001-caddy-as-reverse-proxy.md b/docs/adr/0001-caddy-as-reverse-proxy.md new file mode 100644 index 0000000..bd983ac --- /dev/null +++ b/docs/adr/0001-caddy-as-reverse-proxy.md @@ -0,0 +1,18 @@ +# ADR-0001: Caddy as the reverse proxy + +## Context + +The stack needs a reverse proxy in front of n8n and Baserow that terminates TLS for two subdomains on a single VPS, without requiring the operator to manage certificates by hand. The realistic alternatives were nginx + Certbot, or Traefik. + +## Decision + +Use Caddy 2. + +Caddy issues and renews Let's Encrypt certificates automatically from a declarative `Caddyfile` — no separate Certbot process, no cron job for renewal, no ACME client to wire up. The entire routing + TLS config for this stack is the ~25-line `Caddyfile` in the repo root. nginx would need Certbot bolted on as a second moving part (and a renewal cron job); Traefik's Docker-label-driven config is a better fit for a large, dynamically-changing set of services than for two static virtual hosts known upfront. + +## Consequences + +- One fewer service, one fewer moving part than nginx + Certbot. +- Config is a plain Caddyfile, not Docker labels — easy to read and diff, but means adding a new subdomain requires a Caddyfile edit + restart rather than being automatic from compose. +- Caddy is less battle-tested at extreme scale than nginx, which is irrelevant here — this stack fronts two low-traffic internal tools, not a public CDN edge. +- Automatic HTTPS depends on ports 80 and 443 being reachable from the internet for the ACME HTTP-01 challenge; documented in [SSL Configuration](../ssl-configuration.md). diff --git a/docs/adr/0002-shared-postgres-instance.md b/docs/adr/0002-shared-postgres-instance.md new file mode 100644 index 0000000..b79e23e --- /dev/null +++ b/docs/adr/0002-shared-postgres-instance.md @@ -0,0 +1,18 @@ +# ADR-0002: One shared PostgreSQL instance, separate databases per app + +## Context + +Both n8n and Baserow need a PostgreSQL database. They could each run their own `postgres` container, or share one instance with separate logical databases. + +## Decision + +Run a single `postgres` service; n8n and Baserow each get their own database inside it (`N8N_DB_NAME`, `BASEROW_DB_NAME`), created automatically by each app on first connection. `POSTGRES_DB` only exists because the official Postgres image requires a default database at init — neither app uses it directly. + +For a stack sized for a single small-to-mid VPS, running two separate Postgres containers means two sets of memory overhead, two volumes to back up, and two processes to keep healthy for no functional gain — n8n and Baserow never query each other's data, so there's no coupling risk from sharing the instance. + +## Consequences + +- One `postgres_data` volume, one healthcheck, one memory budget to size instead of two. +- `scripts/backup.sh` backs up a single volume that contains both apps' data — simpler, but a restore of that volume restores both apps' state together, not independently. +- A Postgres major-version upgrade (e.g. 15 → 16) affects both apps at once and needs `pg_dumpall`, documented in [Updating Services](../updating-services.md) — this is the real cost of sharing the instance, and worth re-evaluating if either app's data volume grows large enough that independent upgrade cadences become valuable. +- Credentials are shared at the PostgreSQL-user level (both apps connect as `POSTGRES_USER`) but data is isolated at the database level — acceptable here since both apps are operated by the same person on the same VPS, not a multi-tenant setup. diff --git a/docs/adr/0003-two-network-topology.md b/docs/adr/0003-two-network-topology.md new file mode 100644 index 0000000..3c839d0 --- /dev/null +++ b/docs/adr/0003-two-network-topology.md @@ -0,0 +1,21 @@ +# ADR-0003: Two Docker networks instead of one + +## Context + +The original version of this stack put all four services (Caddy, n8n, Baserow, Postgres) on a single bridge network. That's the simplest possible setup, but it means Caddy — the one service directly exposed to the internet — is on the same network as Postgres, which has no business being reachable from the proxy layer at all. + +## Decision + +Split into two bridge networks: + +- `proxy-network`: Caddy, n8n, Baserow — Caddy routes to the two apps. +- `backend-network`: n8n, Baserow, Postgres — the apps talk to the database. + +n8n and Baserow sit on both networks (they need to reach Caddy's routing and the database); Caddy and Postgres are each on exactly one, and never share a network with each other. + +## Consequences + +- If Caddy were ever compromised (a config error, a vulnerable module, a bad upstream dependency), it has no network path to Postgres at all — not "credentials would stop it," but "there is no route." That's a meaningfully stronger guarantee than relying on auth alone. +- Two `networks:` blocks and two entries in most services' `networks:` list instead of one — a small readability cost for a real isolation gain. +- No performance cost: Docker bridge networks are cheap, and n8n/Baserow already needed to be on two networks' worth of routes regardless of how many network objects that's expressed as. +- This is the kind of thing that's easy to skip when a stack is stood up quickly and easy to forget to add later — worth stating explicitly here so it doesn't regress back to one flat network in a future edit. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 98ac4c4..d06852f 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -85,6 +85,28 @@ docker compose restart baserow --- +## Container Won't Start After a Capability Change + +**Symptom:** A service that was working fine now exits immediately, and the logs mention `permission denied`, `operation not permitted`, or a failure to bind a port — after `docker-compose.yml`'s `cap_drop`/`cap_add`/`security_opt` settings changed. + +Every service runs with `no-new-privileges` and most run with `cap_drop: [ALL]` plus only the specific capabilities they need (see [Security & Reliability](../README.md#security--reliability) in the README, and [ADR-0003](adr/0003-two-network-topology.md) for the isolation reasoning). If a service needs a capability that isn't in its `cap_add` list, it fails at the exact operation that needs it — often only on first run against a fresh volume, not on every start. + +**Step 1:** Confirm the failure is capability-related, not something else: + +```bash +docker compose logs --tail=50 +``` + +Look for `EPERM`, `operation not permitted`, `permission denied`, or (for Caddy specifically) a failure to bind port 80/443. + +**Step 2:** Temporarily comment out that service's `cap_drop`/`cap_add` block in `docker-compose.yml` and restart it. If it starts cleanly with capabilities unrestricted, you've confirmed the diagnosis — the service needs a capability that isn't currently granted. + +**Step 3:** Add the missing capability to that service's `cap_add:` list rather than removing `cap_drop: [ALL]` entirely — narrow the fix to what's actually needed. [Docker's capabilities reference](https://docs.docker.com/engine/containers/run/#runtime-privilege-and-linux-capabilities) lists what each capability controls. + +**Step 4:** Restore the hardening once the fix is confirmed — don't leave a service running capability-unrestricted after debugging. + +--- + ## SSL Not Issuing See the full guide in [ssl-configuration.md](ssl-configuration.md). diff --git a/docs/updating-services.md b/docs/updating-services.md index f168194..5f9e675 100644 --- a/docs/updating-services.md +++ b/docs/updating-services.md @@ -22,22 +22,18 @@ bash scripts/backup.sh bash scripts/update.sh ``` -## Pinning to Specific Versions +## Version Pins and Dependabot -By default, `docker-compose.yml` uses `:latest` tags. For production stability, pin to specific versions to control exactly when updates happen. +`docker-compose.yml` pins all four images to specific versions rather than `:latest` — an update is a deliberate, reviewable change, not something that silently happens on the next `docker compose pull`. -Edit `docker-compose.yml`: +[Dependabot](../.github/dependabot.yml) watches those pinned tags (`package-ecosystem: docker`, GA support for `docker-compose.yml` since Feb 2025) and opens a PR whenever n8n, Baserow, Postgres, or Caddy cut a new release. **Review the linked changelog before merging** — Dependabot bumps the tag, it doesn't read the release notes for you. For n8n specifically, check the breaking-changes guidance below before merging a major-version bump. + +To bump a version by hand instead of waiting for Dependabot, edit the `image:` line for that service in `docker-compose.yml` directly: ```yaml services: n8n: - image: n8nio/n8n:1.89.1 - - baserow: - image: baserow/baserow:1.32.5 - - postgres: - image: postgres:15.7-alpine + image: n8nio/n8n:2.36.0 # example — check the current release first ``` Check available versions: @@ -45,7 +41,7 @@ Check available versions: - **Baserow:** https://hub.docker.com/r/baserow/baserow/tags - **PostgreSQL:** https://hub.docker.com/_/postgres/tags -After editing, apply: +After editing (or merging a Dependabot PR), apply: ```bash docker compose pull From c62cb19fc467ba7ec0e59d6ce7bb700c55a853e2 Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 11:43:21 +0300 Subject: [PATCH 07/17] test: temporarily trigger backup-drill on pull_request for PR validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Will be reverted before merge — see comment in the workflow file. --- .github/workflows/backup-drill.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/backup-drill.yml b/.github/workflows/backup-drill.yml index ce8526b..4eada29 100644 --- a/.github/workflows/backup-drill.yml +++ b/.github/workflows/backup-drill.yml @@ -4,6 +4,9 @@ on: schedule: - cron: "0 7 * * 1" # weekly workflow_dispatch: + pull_request: # TEMPORARY — validates this workflow in the PR itself, since + # workflow_dispatch only works for files already on the default branch. + # Remove this trigger before merging. # Proves the restore path actually works, not just that backup.sh runs # without error — see scripts/backup-drill.sh for the drill logic itself From 21d84c2bbc681cd8007f74fd7c49ec2e9144e3a3 Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 11:50:45 +0300 Subject: [PATCH 08/17] fix: correct nonexistent postgres image tag, guard SARIF upload on scan failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit postgres:15.15-alpine3.24 was never a published tag — 15.15 only ever paired up to alpine3.23; alpine3.24 variants start at 15.18. Verified the real current tag against the Docker Hub API directly rather than trusting a summarized fetch (which is exactly how the wrong tag got in here). Correct tag is postgres:15.19-alpine3.24, confirmed to exist. Also guard the Trivy SARIF upload step on the output file actually existing — it was crashing the whole job with a bare "Path does not exist" whenever the scan step failed for any reason (this bug, or a future transient one), instead of failing informatively or being skipped cleanly. Caught by the smoke-test and backup-drill jobs actually trying to boot the stack — exactly the kind of failure `docker compose config --quiet` alone would never catch. --- .github/workflows/security.yml | 7 ++++++- Makefile | 2 +- README.md | 2 +- docker-compose.yml | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index b91ed19..718522c 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -29,7 +29,7 @@ jobs: - name: baserow image: baserow/baserow:2.3.3 - name: postgres - image: postgres:15.15-alpine3.24 + image: postgres:15.19-alpine3.24 - name: caddy image: caddy:2.11.4-alpine @@ -50,7 +50,12 @@ jobs: ignore-unfixed: true continue-on-error: true # visibility, not a merge gate — some CVEs are unfixable from this repo + # Guarded on the SARIF file actually existing: the scan step above can + # fail (bad tag, registry hiccup, transient error) without producing + # output, and continue-on-error would otherwise let this step crash + # the whole job on a bare "Path does not exist". - name: Upload SARIF to GitHub code scanning + if: always() && hashFiles('trivy-results.sarif') != '' uses: github/codeql-action/upload-sarif@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 with: sarif_file: trivy-results.sarif diff --git a/Makefile b/Makefile index b5358f6..bd8ca42 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ backup-test: bash scripts/backup-drill.sh security-scan: - @for img in n8nio/n8n:2.35.7 baserow/baserow:2.3.3 postgres:15.15-alpine3.24 caddy:2.11.4-alpine; do \ + @for img in n8nio/n8n:2.35.7 baserow/baserow:2.3.3 postgres:15.19-alpine3.24 caddy:2.11.4-alpine; do \ echo "--- $$img ---"; \ trivy image --severity CRITICAL,HIGH --ignore-unfixed "$$img"; \ done diff --git a/README.md b/README.md index e6db81e..a3c841e 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Caddy terminates TLS and proxies to n8n and Baserow over `proxy-network`. Neithe |------------|-----------------------------------|-------------------------------------|-----------------------------------| | n8n | `n8nio/n8n:2.35.7` | Workflow automation | `https://n8n.yourdomain.com` | | Baserow | `baserow/baserow:2.3.3` | No-code database / spreadsheet UI | `https://baserow.yourdomain.com` | -| PostgreSQL | `postgres:15.15-alpine3.24` | Shared database backend | Internal only | +| PostgreSQL | `postgres:15.19-alpine3.24` | Shared database backend | Internal only | | Caddy | `caddy:2.11.4-alpine` | Reverse proxy + automatic TLS | Handles 80/443 | All four are version-pinned rather than `:latest` — [Dependabot](.github/dependabot.yml) opens a PR when a new release lands, so upgrades are reviewed, not silent. See [Updating Services](docs/updating-services.md). diff --git a/docker-compose.yml b/docker-compose.yml index e801ef0..df5ce03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: postgres: - image: postgres:15.15-alpine3.24 + image: postgres:15.19-alpine3.24 restart: unless-stopped environment: POSTGRES_USER: ${POSTGRES_USER} From da0b9cf16dafa406503b9b57f6824c9e953e9293 Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 11:57:18 +0300 Subject: [PATCH 09/17] =?UTF-8?q?fix:=20create=20n8n=20and=20Baserow=20dat?= =?UTF-8?q?abases=20on=20Postgres=20init=20=E2=80=94=20they=20never=20exis?= =?UTF-8?q?ted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real bug the smoke-test job caught on its first actual boot of the stack: POSTGRES_DB only creates one database (the default, "automation") at initdb time. Neither n8n's pg client nor Baserow's Django backend creates its own database on connect — both just fail hard with "database does not exist". This means N8N_DB_NAME/BASEROW_DB_NAME never worked; the stack had apparently never been booted for real before this CI job existed. Fix: postgres-init/init-app-databases.sh, mounted into Postgres's docker-entrypoint-initdb.d/, creates both databases explicitly using the official image's first-init hook. Only runs once, against an empty postgres_data volume — documented in docs/troubleshooting.md and CLAUDE.md, including the manual remediation for anyone already running an already-initialized volume from before this fix. Corrected the same false "auto-created by each app" claim in .env.example and CLAUDE.md, which had been asserting this incorrectly since the original plan. --- .env.example | 4 +++- CLAUDE.md | 3 ++- docker-compose.yml | 5 +++++ docs/troubleshooting.md | 17 +++++++++++++---- postgres-init/init-app-databases.sh | 17 +++++++++++++++++ 5 files changed, 40 insertions(+), 6 deletions(-) create mode 100755 postgres-init/init-app-databases.sh diff --git a/.env.example b/.env.example index e9dfe76..fb53791 100644 --- a/.env.example +++ b/.env.example @@ -21,7 +21,9 @@ POSTGRES_PASSWORD=changeme_use_strong_password # Default DB created at init (not used directly by apps) POSTGRES_DB=automation -# Database names for each service (auto-created by each app on first run) +# Database names for each service — created by postgres-init/init-app-databases.sh +# on first container init (empty data directory only). Neither app creates +# its own database on connect. N8N_DB_NAME=n8n BASEROW_DB_NAME=baserow diff --git a/CLAUDE.md b/CLAUDE.md index dedb10d..1c490c9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,6 +47,7 @@ Internet → Caddy (80/443) ──proxy-network──► n8n (5678) - **`docker-compose.yml`** — production definition; Caddy is the only public-facing service - **`docker-compose.dev.yml`** — override that ports n8n and Baserow directly and puts Caddy behind a `production` profile (so it doesn't start) +- **`postgres-init/init-app-databases.sh`** — mounted into Postgres's `docker-entrypoint-initdb.d/`; creates the `n8n` and `baserow` databases on first init of an empty volume. See the Critical Constraints note below — this doesn't run again against an already-initialized volume. - **`Caddyfile`** — reads domain/subdomain from env vars at runtime via `{$VAR}` syntax; security headers include HSTS (`max-age=31536000; includeSubDomains`), X-Frame-Options, X-Content-Type-Options, Referrer-Policy; gzip on both virtual hosts - **`.github/workflows/validate.yml`** — CI runs on every push to main and every PR: validates both compose files, shellchecks all scripts in `scripts/`, then actually boots the dev stack (`--wait`) and curls both health endpoints externally — this is real runtime verification, not just YAML syntax checking - **`.github/workflows/security.yml`** — weekly + on-push Trivy CVE scan of the four pinned images (not the compose file — Trivy has no native docker-compose misconfig scanner), results land in the repo's Security tab. Non-blocking (`continue-on-error`). @@ -78,7 +79,7 @@ To actually verify the stack boots (not just that the YAML is valid), run `make - `N8N_ENCRYPTION_KEY` — generated once by `setup.sh` via `openssl rand -hex 32`. **Never change it after first run**; doing so breaks all stored n8n credentials. - `BASEROW_SECRET_KEY` — Django secret key; same rule applies. -- Both databases (`N8N_DB_NAME`, `BASEROW_DB_NAME`) are created automatically by each app on first run inside the shared Postgres container. The `POSTGRES_DB` var is just the default DB created at container init and isn't used by n8n or Baserow directly. +- `POSTGRES_DB` is just the default DB created at container init and isn't used by n8n or Baserow directly. **Neither app creates its own database on connect** (discovered via the smoke-test job actually booting the stack, 2026-08-23 — the repo had never been booted for real before that). `postgres-init/init-app-databases.sh` creates `N8N_DB_NAME` and `BASEROW_DB_NAME` explicitly via Postgres's `docker-entrypoint-initdb.d` mechanism, which only runs once, against an empty `postgres_data` volume. An existing volume from before this fix needs the databases created manually — see `docs/troubleshooting.md` § Can't Connect to PostgreSQL. - Volume names in `backup.sh` are detected dynamically via `docker compose config` — no hardcoded prefix. The script requires `.env` to exist so compose config can resolve variable substitutions. - `backup.sh` backs up `n8n_data`, `baserow_data`, and `postgres_data` only. `caddy_data` and `caddy_config` are intentionally excluded — certificates are re-issuable from Let's Encrypt for free. - n8n and Baserow start only after Postgres passes its healthcheck (`pg_isready`). If either service fails to start, confirm Postgres is `(healthy)` first: `docker compose ps postgres`. diff --git a/docker-compose.yml b/docker-compose.yml index df5ce03..f5b5f79 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,13 @@ services: POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: ${POSTGRES_DB} + # Read by postgres-init/init-app-databases.sh — POSTGRES_DB only gets + # one database created for free; n8n and Baserow need their own. + N8N_DB_NAME: ${N8N_DB_NAME} + BASEROW_DB_NAME: ${BASEROW_DB_NAME} volumes: - postgres_data:/var/lib/postgresql/data + - ./postgres-init/init-app-databases.sh:/docker-entrypoint-initdb.d/init-app-databases.sh:ro healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"] interval: 10s diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d06852f..3b7d1f1 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -74,15 +74,24 @@ docker compose exec postgres psql -U $POSTGRES_USER -c "ALTER USER $POSTGRES_USE Then update `.env` to match and restart: `docker compose restart n8n baserow` -**Check 3: Database doesn't exist yet** +**Check 3: Database doesn't exist** -n8n and Baserow create their own databases on first startup. If they crash before the DB is created, restart them after PostgreSQL is healthy: +`postgres-init/init-app-databases.sh` creates the `n8n` and `baserow` databases automatically — but **only** the first time Postgres initializes against an empty `postgres_data` volume. Neither app creates its own database on connect; if the database is missing, n8n/Baserow will keep crash-looping with `database "..." does not exist` no matter how many times you restart them. + +This happens if: +- You're restoring/upgrading an existing volume that was created before this init script existed. +- The volume was manually created without ever running Postgres's first-init sequence against it. + +Fix — create the missing database directly: ```bash -docker compose restart n8n -docker compose restart baserow +docker compose exec postgres psql -U $POSTGRES_USER -c "CREATE DATABASE n8n;" +docker compose exec postgres psql -U $POSTGRES_USER -c "CREATE DATABASE baserow;" +docker compose restart n8n baserow ``` +(Substitute your actual `N8N_DB_NAME`/`BASEROW_DB_NAME` if you changed them from the defaults.) + --- ## Container Won't Start After a Capability Change diff --git a/postgres-init/init-app-databases.sh b/postgres-init/init-app-databases.sh new file mode 100755 index 0000000..e19abc4 --- /dev/null +++ b/postgres-init/init-app-databases.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -e + +# Runs automatically on first container init (only when the data directory +# is empty) — this is the official postgres image's docker-entrypoint-initdb.d +# mechanism, not custom logic. +# +# POSTGRES_DB only gets one database created for free at initdb time. n8n and +# Baserow each need their own database on this shared instance, and neither +# app creates its own database on connect — n8n's pg client and Baserow's +# Django/psycopg both just fail with "database does not exist" if it isn't +# already there. This script creates both explicitly, once, at first boot. + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE DATABASE "$N8N_DB_NAME"; + CREATE DATABASE "$BASEROW_DB_NAME"; +EOSQL From 6e354a2bd213c8ca61c03cb6fffcd9972bc9f183 Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 12:03:25 +0300 Subject: [PATCH 10/17] fix: widen Baserow healthcheck window and --wait-timeout for first-boot migrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both n8n and Baserow actually came up correctly after the database fix — confirmed from the raw logs: Baserow logged a real 200 on /api/_health/ and finished all Django migrations. It still got marked unhealthy because the old healthcheck window (start_period 45s + 5 retries * 30s interval ≈ 195s) was too tight: first-boot on an empty volume also runs Baserow's built-in template sync (importing dozens of bundled templates via a background worker), which took ~150s before the health endpoint reliably responded on a GitHub Actions runner. Widened to start_period 150s / interval 15s / retries 10 (~300s ceiling, checks more often once past start_period so it doesn't overshoot success by much). Bumped --wait-timeout from 180 to 320 everywhere docker compose up --wait is used, so compose's own wait doesn't time out before the healthcheck's retry budget is exhausted. --- .github/workflows/backup-drill.yml | 2 +- .github/workflows/validate.yml | 2 +- Makefile | 2 +- docker-compose.yml | 12 +++++++++--- scripts/backup-drill.sh | 2 +- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/backup-drill.yml b/.github/workflows/backup-drill.yml index 4eada29..0c6bebd 100644 --- a/.github/workflows/backup-drill.yml +++ b/.github/workflows/backup-drill.yml @@ -33,7 +33,7 @@ jobs: sed -i "s|^BASEROW_SECRET_KEY=.*|BASEROW_SECRET_KEY=$(openssl rand -hex 32)|" .env - name: Boot the stack - run: docker compose up -d --wait --wait-timeout 180 + run: docker compose up -d --wait --wait-timeout 320 - name: Run the backup/restore drill run: bash scripts/backup-drill.sh diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 5986de3..e400576 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -44,7 +44,7 @@ jobs: sed -i "s|^BASEROW_SECRET_KEY=.*|BASEROW_SECRET_KEY=$(openssl rand -hex 32)|" .env - name: Boot dev stack and wait for healthchecks - run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait --wait-timeout 180 + run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait --wait-timeout 320 - name: Check n8n health endpoint (external, through the mapped port) run: curl -sf http://localhost:5678/healthz diff --git a/Makefile b/Makefile index bd8ca42..8984a7e 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ update: bash scripts/update.sh test: - docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait --wait-timeout 180 + docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait --wait-timeout 320 curl -sf http://localhost:5678/healthz && echo "n8n: healthy" curl -sf http://localhost:8080/api/_health/ && echo "baserow: healthy" diff --git a/docker-compose.yml b/docker-compose.yml index f5b5f79..3baf42f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,10 +104,16 @@ services: - baserow_data:/baserow/data healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost/api/_health/"] - interval: 30s + # First boot on an empty volume runs Django migrations *and* Baserow's + # built-in template sync (importing dozens of bundled templates via a + # background worker) before the health endpoint reliably responds — + # observed taking ~150s on a GitHub Actions runner. start_period + + # retries*interval must comfortably exceed that or docker compose + # up --wait marks the container unhealthy mid-migration. + interval: 15s timeout: 10s - retries: 5 - start_period: 45s + retries: 10 + start_period: 150s security_opt: - no-new-privileges:true # No cap_drop here yet: Baserow's all-in-one image runs nginx + diff --git a/scripts/backup-drill.sh b/scripts/backup-drill.sh index e4f477f..defc664 100755 --- a/scripts/backup-drill.sh +++ b/scripts/backup-drill.sh @@ -55,7 +55,7 @@ echo "Restoring from backup timestamped: $TIMESTAMP" bash "$SCRIPT_DIR/restore.sh" "$TIMESTAMP" --force echo "Waiting for the restored stack to become healthy..." -docker compose up -d --wait --wait-timeout 180 +docker compose up -d --wait --wait-timeout 320 RESULT=$(docker compose exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -t -c \ "SELECT 1 FROM backup_drill WHERE marker = '${MARKER}';" | tr -d '[:space:]') From db00ab2c45c4ebbb67883a5d320fcb500e56408b Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 12:13:35 +0300 Subject: [PATCH 11/17] fix: replace docker compose up --wait with a tolerant health-poll script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --wait fails permanently on the first "unhealthy" transition, even if the container recovers seconds later. Confirmed from raw logs: Baserow's first-boot migration + template sync briefly reported unhealthy under CI resource contention, then answered 200 on every single subsequent check for over two minutes — but --wait had already killed the job on the first blip. scripts/wait-for-healthy.sh polls `docker inspect` directly in a loop and only fails on a genuine timeout with nothing ever reporting healthy, which is what a smoke test should actually care about. Used by validate.yml's smoke-test job, backup-drill.yml's boot step, backup-drill.sh's post-restore wait, and `make test`. --- .github/workflows/backup-drill.yml | 5 +++- .github/workflows/validate.yml | 9 ++++-- CLAUDE.md | 5 ++-- Makefile | 3 +- README.md | 1 + scripts/backup-drill.sh | 2 +- scripts/wait-for-healthy.sh | 48 ++++++++++++++++++++++++++++++ 7 files changed, 65 insertions(+), 8 deletions(-) create mode 100755 scripts/wait-for-healthy.sh diff --git a/.github/workflows/backup-drill.yml b/.github/workflows/backup-drill.yml index 0c6bebd..caeafe9 100644 --- a/.github/workflows/backup-drill.yml +++ b/.github/workflows/backup-drill.yml @@ -33,7 +33,10 @@ jobs: sed -i "s|^BASEROW_SECRET_KEY=.*|BASEROW_SECRET_KEY=$(openssl rand -hex 32)|" .env - name: Boot the stack - run: docker compose up -d --wait --wait-timeout 320 + run: docker compose up -d + + - name: Wait for healthchecks + run: bash scripts/wait-for-healthy.sh postgres n8n baserow - name: Run the backup/restore drill run: bash scripts/backup-drill.sh diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e400576..b0a9bbc 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -24,7 +24,7 @@ jobs: - name: Lint shell scripts run: | sudo apt-get install -y shellcheck - shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh scripts/restore.sh scripts/backup-drill.sh + shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh scripts/restore.sh scripts/backup-drill.sh scripts/wait-for-healthy.sh smoke-test: runs-on: ubuntu-latest @@ -43,8 +43,11 @@ jobs: sed -i "s|^N8N_ENCRYPTION_KEY=.*|N8N_ENCRYPTION_KEY=$(openssl rand -hex 32)|" .env sed -i "s|^BASEROW_SECRET_KEY=.*|BASEROW_SECRET_KEY=$(openssl rand -hex 32)|" .env - - name: Boot dev stack and wait for healthchecks - run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait --wait-timeout 320 + - name: Boot dev stack + run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d + + - name: Wait for healthchecks + run: bash scripts/wait-for-healthy.sh postgres n8n baserow - name: Check n8n health endpoint (external, through the mapped port) run: curl -sf http://localhost:5678/healthz diff --git a/CLAUDE.md b/CLAUDE.md index 1c490c9..06537ac 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,7 +49,8 @@ Internet → Caddy (80/443) ──proxy-network──► n8n (5678) - **`docker-compose.dev.yml`** — override that ports n8n and Baserow directly and puts Caddy behind a `production` profile (so it doesn't start) - **`postgres-init/init-app-databases.sh`** — mounted into Postgres's `docker-entrypoint-initdb.d/`; creates the `n8n` and `baserow` databases on first init of an empty volume. See the Critical Constraints note below — this doesn't run again against an already-initialized volume. - **`Caddyfile`** — reads domain/subdomain from env vars at runtime via `{$VAR}` syntax; security headers include HSTS (`max-age=31536000; includeSubDomains`), X-Frame-Options, X-Content-Type-Options, Referrer-Policy; gzip on both virtual hosts -- **`.github/workflows/validate.yml`** — CI runs on every push to main and every PR: validates both compose files, shellchecks all scripts in `scripts/`, then actually boots the dev stack (`--wait`) and curls both health endpoints externally — this is real runtime verification, not just YAML syntax checking +- **`.github/workflows/validate.yml`** — CI runs on every push to main and every PR: validates both compose files, shellchecks all scripts in `scripts/`, then actually boots the dev stack and curls both health endpoints externally — this is real runtime verification, not just YAML syntax checking +- **Don't use `docker compose up --wait`** for boot verification — it fails permanently on the *first* "unhealthy" transition, even if the container recovers seconds later. Baserow's first-boot migration + template sync briefly reports unhealthy under CI resource contention, then starts passing every subsequent check — `--wait` killed the job anyway (discovered 2026-08-23). Use `scripts/wait-for-healthy.sh ` instead: it polls `docker inspect` directly and only fails on a genuine timeout with nothing ever reporting healthy. Used by `validate.yml`, `backup-drill.yml`, `scripts/backup-drill.sh`, and `make test`. - **`.github/workflows/security.yml`** — weekly + on-push Trivy CVE scan of the four pinned images (not the compose file — Trivy has no native docker-compose misconfig scanner), results land in the repo's Security tab. Non-blocking (`continue-on-error`). - **`.github/workflows/backup-drill.yml`** — weekly + on-demand: writes a marker row, backs up, destroys the volumes, restores, and fails if the marker isn't back. Proves the restore path works instead of just documenting it. Logic lives in `scripts/backup-drill.sh` (also runnable locally via `make backup-test`), shared between CI and local use so they can't drift apart. - **`.github/dependabot.yml`** — tracks the pinned image tags in `docker-compose.yml` (`docker` ecosystem) and third-party Action versions (`github-actions` ecosystem) weekly. @@ -70,7 +71,7 @@ docker compose -f docker-compose.yml -f docker-compose.dev.yml config --quiet rm .env # Lint shell scripts -shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh scripts/restore.sh scripts/backup-drill.sh +shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh scripts/restore.sh scripts/backup-drill.sh scripts/wait-for-healthy.sh ``` To actually verify the stack boots (not just that the YAML is valid), run `make test` — needs a local Docker daemon. If one isn't available in your environment, push to a branch and let the `validate.yml` smoke-test job do it; don't claim runtime behavior is verified without one or the other actually running. diff --git a/Makefile b/Makefile index 8984a7e..067f7c5 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,8 @@ update: bash scripts/update.sh test: - docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --wait --wait-timeout 320 + docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d + bash scripts/wait-for-healthy.sh postgres n8n baserow curl -sf http://localhost:5678/healthz && echo "n8n: healthy" curl -sf http://localhost:8080/api/_health/ && echo "baserow: healthy" diff --git a/README.md b/README.md index a3c841e..e229d80 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ All variables are documented with comments in [`.env.example`](.env.example). | `scripts/backup.sh` | Stops stack, snapshots all volumes to `backups/`, restarts stack | | `scripts/restore.sh` | Restores all volumes from a timestamped backup | | `scripts/backup-drill.sh` | Full backup → destroy → restore → verify cycle (local or CI) | +| `scripts/wait-for-healthy.sh` | Polls services until Docker reports them healthy; tolerates transient blips instead of failing on the first one | | `scripts/update.sh` | Pulls latest images, redeploys with `--remove-orphans`, prunes old | ## Documentation diff --git a/scripts/backup-drill.sh b/scripts/backup-drill.sh index defc664..d5c6fa1 100755 --- a/scripts/backup-drill.sh +++ b/scripts/backup-drill.sh @@ -55,7 +55,7 @@ echo "Restoring from backup timestamped: $TIMESTAMP" bash "$SCRIPT_DIR/restore.sh" "$TIMESTAMP" --force echo "Waiting for the restored stack to become healthy..." -docker compose up -d --wait --wait-timeout 320 +bash "$SCRIPT_DIR/wait-for-healthy.sh" postgres n8n baserow RESULT=$(docker compose exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -t -c \ "SELECT 1 FROM backup_drill WHERE marker = '${MARKER}';" | tr -d '[:space:]') diff --git a/scripts/wait-for-healthy.sh b/scripts/wait-for-healthy.sh new file mode 100755 index 0000000..adb12bb --- /dev/null +++ b/scripts/wait-for-healthy.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -e + +# ------------------------------------------------------- +# wait-for-healthy.sh — Poll docker compose services until Docker reports +# them healthy. +# +# Deliberately NOT `docker compose up --wait`: that command fails +# permanently on the first "unhealthy" transition, even if the container +# recovers a few seconds later. Observed happening to Baserow in CI — +# its first-boot migration + template sync left it briefly unhealthy under +# runner resource contention, then it started responding 200 on every +# subsequent check, but `--wait` had already given up. This script tolerates +# that and only fails on a genuine timeout with nothing ever reporting +# healthy. +# +# Usage: ./scripts/wait-for-healthy.sh [ ...] +# Env: WAIT_ATTEMPTS (default 20), WAIT_INTERVAL_SECS (default 20) +# ------------------------------------------------------- + +ATTEMPTS="${WAIT_ATTEMPTS:-20}" +INTERVAL="${WAIT_INTERVAL_SECS:-20}" + +for service in "$@"; do + echo "Waiting for $service to become healthy (up to $((ATTEMPTS * INTERVAL))s)..." + healthy="" + for i in $(seq 1 "$ATTEMPTS"); do + cid=$(docker compose ps -q "$service") + status="no container" + if [ -n "$cid" ]; then + status=$(docker inspect --format='{{.State.Health.Status}}' "$cid" 2>/dev/null || echo "none") + fi + if [ "$status" = "healthy" ]; then + echo " $service is healthy" + healthy="yes" + break + fi + echo " [$i/$ATTEMPTS] $service: $status" + sleep "$INTERVAL" + done + if [ -z "$healthy" ]; then + echo "ERROR: $service did not become healthy within $((ATTEMPTS * INTERVAL))s" + docker compose logs "$service" --tail=100 + exit 1 + fi +done + +echo "All services healthy: $*" From 71a968222b433adcae882b55aaed00ae18640e4f Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 12:24:28 +0300 Subject: [PATCH 12/17] fix: bypass Docker's Health.Status entirely, exec the check command directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Still failing after the previous fix, with a stranger symptom: Baserow's /api/_health/ endpoint returned 200 on every single logged request for over two minutes straight, yet docker inspect .State.Health.Status never once reported "healthy" — confirmed from raw logs (six consecutive 200s, script still timed out at 400s). Docker's own HEALTHCHECK state tracking appears to get stuck under GitHub Actions runner resource contention independent of the actual endpoint's behavior. wait-for-healthy.sh now runs each service's check command directly via `docker compose exec` and retries until THAT succeeds, never consulting Docker's Health.Status at all. This is the same command the container's own HEALTHCHECK uses, just executed and retried outside of Docker's internal bookkeeping — immune to whatever was causing that bookkeeping to get stuck. docker-compose.yml's Baserow healthcheck config is left in place (still useful for `docker compose ps` / operator visibility on a real deploy) but its comment now says plainly that it isn't load-bearing for anything in this repo anymore. --- docker-compose.yml | 15 +++++---- scripts/wait-for-healthy.sh | 64 +++++++++++++++++++++---------------- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3baf42f..c55e849 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,12 +104,15 @@ services: - baserow_data:/baserow/data healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost/api/_health/"] - # First boot on an empty volume runs Django migrations *and* Baserow's - # built-in template sync (importing dozens of bundled templates via a - # background worker) before the health endpoint reliably responds — - # observed taking ~150s on a GitHub Actions runner. start_period + - # retries*interval must comfortably exceed that or docker compose - # up --wait marks the container unhealthy mid-migration. + # For docker compose ps / operator visibility only — CI's actual + # readiness gate is scripts/wait-for-healthy.sh, which polls this same + # check directly rather than trusting Docker's own Health.Status. That + # status was observed getting stuck on "unhealthy" under CI resource + # contention even while every request to this endpoint was returning + # 200, so it's deliberately not load-bearing for anything anymore. + # First boot on an empty volume runs Django migrations *and* + # Baserow's built-in template sync before this responds reliably — + # observed taking 150s+ on a GitHub Actions runner. interval: 15s timeout: 10s retries: 10 diff --git a/scripts/wait-for-healthy.sh b/scripts/wait-for-healthy.sh index adb12bb..a167ece 100755 --- a/scripts/wait-for-healthy.sh +++ b/scripts/wait-for-healthy.sh @@ -2,47 +2,57 @@ set -e # ------------------------------------------------------- -# wait-for-healthy.sh — Poll docker compose services until Docker reports -# them healthy. +# wait-for-healthy.sh — Poll a service's own healthcheck command via +# `docker compose exec` until it succeeds. # -# Deliberately NOT `docker compose up --wait`: that command fails -# permanently on the first "unhealthy" transition, even if the container -# recovers a few seconds later. Observed happening to Baserow in CI — -# its first-boot migration + template sync left it briefly unhealthy under -# runner resource contention, then it started responding 200 on every -# subsequent check, but `--wait` had already given up. This script tolerates -# that and only fails on a genuine timeout with nothing ever reporting -# healthy. +# Deliberately does NOT rely on `docker compose up --wait` or `docker +# inspect .State.Health.Status`: both depend on Docker's own HEALTHCHECK +# state machine, which was observed getting stuck reporting "unhealthy" for +# Baserow even while its /api/_health/ endpoint was returning 200 on every +# single check for minutes straight, under GitHub Actions runner resource +# contention during first-boot migrations. Running the exact same check +# command directly and retrying until IT succeeds sidesteps Docker's health +# bookkeeping entirely — what matters is "does the service respond right +# now", not "what does Docker's internal state say". # # Usage: ./scripts/wait-for-healthy.sh [ ...] -# Env: WAIT_ATTEMPTS (default 20), WAIT_INTERVAL_SECS (default 20) +# Known services: postgres, n8n, baserow +# Env: WAIT_ATTEMPTS (default 40), WAIT_INTERVAL_SECS (default 10) # ------------------------------------------------------- -ATTEMPTS="${WAIT_ATTEMPTS:-20}" -INTERVAL="${WAIT_INTERVAL_SECS:-20}" +ATTEMPTS="${WAIT_ATTEMPTS:-40}" +INTERVAL="${WAIT_INTERVAL_SECS:-10}" + +check_cmd() { + case "$1" in + postgres) echo 'pg_isready -U "$POSTGRES_USER"' ;; + n8n) echo 'wget -qO- http://localhost:5678/healthz' ;; + baserow) echo 'wget -qO- http://localhost/api/_health/' ;; + *) + echo "Unknown service: $1 (known: postgres, n8n, baserow)" >&2 + exit 1 + ;; + esac +} for service in "$@"; do - echo "Waiting for $service to become healthy (up to $((ATTEMPTS * INTERVAL))s)..." - healthy="" + cmd=$(check_cmd "$service") + echo "Waiting for $service (up to $((ATTEMPTS * INTERVAL))s)..." + ok="" for i in $(seq 1 "$ATTEMPTS"); do - cid=$(docker compose ps -q "$service") - status="no container" - if [ -n "$cid" ]; then - status=$(docker inspect --format='{{.State.Health.Status}}' "$cid" 2>/dev/null || echo "none") - fi - if [ "$status" = "healthy" ]; then - echo " $service is healthy" - healthy="yes" + if docker compose exec -T "$service" sh -c "$cmd" >/dev/null 2>&1; then + echo " $service is responding" + ok="yes" break fi - echo " [$i/$ATTEMPTS] $service: $status" + echo " [$i/$ATTEMPTS] $service not ready yet" sleep "$INTERVAL" done - if [ -z "$healthy" ]; then - echo "ERROR: $service did not become healthy within $((ATTEMPTS * INTERVAL))s" + if [ -z "$ok" ]; then + echo "ERROR: $service did not respond within $((ATTEMPTS * INTERVAL))s" docker compose logs "$service" --tail=100 exit 1 fi done -echo "All services healthy: $*" +echo "All services responding: $*" From 130362c188335f993f52f35139d088d48476a903 Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 18:21:34 +0300 Subject: [PATCH 13/17] =?UTF-8?q?fix:=20raise=20Baserow's=20memory=20limit?= =?UTF-8?q?=20=E2=80=94=201g=20was=20OOM-killing=20a=20worker=20on=20first?= =?UTF-8?q?=20boot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the persistent boot failure, finally isolated from the raw logs: "Worker exited prematurely: signal 9 (SIGKILL)" in Baserow's celery export worker — a classic OOM kill, not a healthcheck or timing issue at all. Baserow runs gunicorn + a celery worker + celery beat concurrently, and first boot on an empty volume adds Django migrations and template-sync import on top of that. 1g covers steady-state but not that burst. Raised to 2g. Also corrected docs/initial-setup.md's VPS sizing guidance — "2GB RAM, 4GB recommended" understated it; the real worst-case ceiling across all four services' limits is ~3.6GB, and Baserow specifically needs its full 2g during first boot, not just at steady state. --- docker-compose.yml | 8 +++++++- docs/initial-setup.md | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c55e849..43e25b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -123,10 +123,16 @@ services: # gunicorn/daphne + celery via an internal process supervisor whose root # requirements aren't confirmed. Drop capabilities in a follow-up pass # once this base hardening is proven in CI, not blindly in this one. + # 2g, not 1g: Baserow runs gunicorn + celery worker + celery beat + # concurrently, and first boot on an empty volume adds Django migrations + # + template-sync on top of that. Observed a celery worker getting + # OOM-killed (SIGKILL) under a 1g cap during exactly this burst on a + # GitHub Actions runner — steady-state usage is lower, but the limit has + # to cover the worst moment, not the average one. deploy: resources: limits: - memory: 1g + memory: 2g logging: driver: json-file options: diff --git a/docs/initial-setup.md b/docs/initial-setup.md index 3454f60..a793ee2 100644 --- a/docs/initial-setup.md +++ b/docs/initial-setup.md @@ -6,7 +6,7 @@ This guide walks you through deploying the stack on a fresh Ubuntu VPS from scra You need a VPS with: - Ubuntu 20.04 LTS or later -- At least 2 GB RAM (4 GB recommended for running both n8n and Baserow) +- At least 4 GB RAM. `docker-compose.yml`'s per-service memory limits (postgres 512m, n8n 1g, baserow 2g, caddy 128m) sum to ~3.6 GB at worst case — Baserow alone needs the full 2g during its first-boot migration + template sync, observed OOM-killing a celery worker under 1g. 2 GB will work for steady-state use once past first boot, but risks an OOM on the very first `docker compose up`. - Ports 80 and 443 open in the firewall ### Open Firewall Ports (UFW) From 9f7ce15453ffdec20d6c2611868b3a2e9f0bf54f Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 18:32:14 +0300 Subject: [PATCH 14/17] debug: capture and print the actual exec failure instead of swallowing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Still timing out on Baserow after fixing the OOM (1g -> 2g) — but this time the app log showed 28 consecutive 200s on /api/_health/ spanning the full 4.5 minutes the script was polling, while the script itself reported "not ready yet" on every single attempt. Those 200s are Docker's own internal HEALTHCHECK directive (still defined in docker-compose.yml, running on its own schedule) — a completely separate code path from this script's own `docker compose exec` calls, which apparently were failing for a reason the script was actively discarding (>/dev/null 2>&1, exit code unchecked beyond zero/nonzero). No new fix yet — this commit only makes the next failure diagnosable instead of guessing again from indirect evidence. --- scripts/wait-for-healthy.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/wait-for-healthy.sh b/scripts/wait-for-healthy.sh index a167ece..4de9ab8 100755 --- a/scripts/wait-for-healthy.sh +++ b/scripts/wait-for-healthy.sh @@ -40,12 +40,21 @@ for service in "$@"; do echo "Waiting for $service (up to $((ATTEMPTS * INTERVAL))s)..." ok="" for i in $(seq 1 "$ATTEMPTS"); do - if docker compose exec -T "$service" sh -c "$cmd" >/dev/null 2>&1; then + # if/else around the assignment (not `out=$(...) ; rc=$?`) so a nonzero + # exit from the exec doesn't trip `set -e` on the assignment itself. + if out=$(docker compose exec -T "$service" sh -c "$cmd" 2>&1); then + rc=0 + else + rc=$? + fi + if [ "$rc" -eq 0 ]; then echo " $service is responding" ok="yes" break fi - echo " [$i/$ATTEMPTS] $service not ready yet" + # Print what actually failed — swallowing this is exactly what turned + # earlier failures into multiple rounds of guessing instead of one. + echo " [$i/$ATTEMPTS] $service not ready (exec exit $rc): $(printf '%s' "$out" | tr '\n' ' ' | cut -c1-300)" sleep "$INTERVAL" done if [ -z "$ok" ]; then From 90878e58c56105664a2ccca2c82c9fdb48d294fc Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 18:42:17 +0300 Subject: [PATCH 15/17] fix: exec wget directly for Baserow instead of wrapping in sh -c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause, finally confirmed from the debug instrumentation added last commit: "sh: 1: wget: not found" on every single attempt. Baserow's own CMD-array healthcheck (["CMD", "wget", ...], no shell involved) resolves wget fine — whatever sets up Baserow's PATH for its actual entrypoint process isn't present in a fresh `docker compose exec ... sh` session. n8n and postgres both worked correctly through sh -c the whole time (n8n had one transient "connection refused" that self-resolved) — this was specific to Baserow. Fixed by execing wget directly for Baserow, matching how the CMD-array form resolves it, while leaving postgres/n8n on sh -c since they need it (env var expansion) and it already works for them. --- scripts/wait-for-healthy.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/scripts/wait-for-healthy.sh b/scripts/wait-for-healthy.sh index 4de9ab8..67916bb 100755 --- a/scripts/wait-for-healthy.sh +++ b/scripts/wait-for-healthy.sh @@ -15,6 +15,13 @@ set -e # bookkeeping entirely — what matters is "does the service respond right # now", not "what does Docker's internal state say". # +# Baserow's check execs `wget` directly (no `sh -c` wrapper), matching how +# docker-compose.yml's own `["CMD", "wget", ...]` array-form healthcheck +# resolves it. Confirmed via `sh -c 'wget ...'` failing with +# "wget: not found" while the CMD-array form (no shell involved) succeeds — +# whatever sets Baserow's PATH for its main process apparently isn't applied +# to an ad-hoc `docker exec ... sh` session. +# # Usage: ./scripts/wait-for-healthy.sh [ ...] # Known services: postgres, n8n, baserow # Env: WAIT_ATTEMPTS (default 40), WAIT_INTERVAL_SECS (default 10) @@ -23,26 +30,25 @@ set -e ATTEMPTS="${WAIT_ATTEMPTS:-40}" INTERVAL="${WAIT_INTERVAL_SECS:-10}" -check_cmd() { +run_check() { case "$1" in - postgres) echo 'pg_isready -U "$POSTGRES_USER"' ;; - n8n) echo 'wget -qO- http://localhost:5678/healthz' ;; - baserow) echo 'wget -qO- http://localhost/api/_health/' ;; + postgres) docker compose exec -T postgres sh -c 'pg_isready -U "$POSTGRES_USER"' ;; + n8n) docker compose exec -T n8n sh -c 'wget -qO- http://localhost:5678/healthz' ;; + baserow) docker compose exec -T baserow wget -qO- http://localhost/api/_health/ ;; *) echo "Unknown service: $1 (known: postgres, n8n, baserow)" >&2 - exit 1 + return 2 ;; esac } for service in "$@"; do - cmd=$(check_cmd "$service") echo "Waiting for $service (up to $((ATTEMPTS * INTERVAL))s)..." ok="" for i in $(seq 1 "$ATTEMPTS"); do # if/else around the assignment (not `out=$(...) ; rc=$?`) so a nonzero # exit from the exec doesn't trip `set -e` on the assignment itself. - if out=$(docker compose exec -T "$service" sh -c "$cmd" 2>&1); then + if out=$(run_check "$service" 2>&1); then rc=0 else rc=$? From 6c4a99f5e3e879a9fb4da1791cd2bca870e00091 Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 18:52:17 +0300 Subject: [PATCH 16/17] fix: curl the host-mapped port instead of execing wget inside Baserow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirmed definitively: baserow/baserow:2.3.3 has no wget binary anywhere in $PATH — OCI runtime itself reports "exec: wget: executable file not found in \$PATH" trying to run it directly (no shell involved at all this time). docker-compose.yml's own wget-based HEALTHCHECK for Baserow has apparently never actually been able to run in this image; the 200s seen in earlier debugging were Baserow's own internal startup watcher process hitting its own endpoint, unrelated to Docker's HEALTHCHECK or this script. Stopped trying to find something inside each app's container that can make the HTTP call. Both current call sites already run with the dev compose override active (host ports 5678/8080 mapped), so this now just curls those directly from the caller's machine — sidesteps every container-image assumption entirely. Postgres keeps `docker compose exec` + pg_isready, which has worked correctly in every run so far and has no host-reachable alternative (no port mapping, by design). --- scripts/wait-for-healthy.sh | 38 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/scripts/wait-for-healthy.sh b/scripts/wait-for-healthy.sh index 67916bb..b2beeb7 100755 --- a/scripts/wait-for-healthy.sh +++ b/scripts/wait-for-healthy.sh @@ -2,25 +2,25 @@ set -e # ------------------------------------------------------- -# wait-for-healthy.sh — Poll a service's own healthcheck command via -# `docker compose exec` until it succeeds. +# wait-for-healthy.sh — Poll each service directly until it responds. # # Deliberately does NOT rely on `docker compose up --wait` or `docker # inspect .State.Health.Status`: both depend on Docker's own HEALTHCHECK # state machine, which was observed getting stuck reporting "unhealthy" for -# Baserow even while its /api/_health/ endpoint was returning 200 on every -# single check for minutes straight, under GitHub Actions runner resource -# contention during first-boot migrations. Running the exact same check -# command directly and retrying until IT succeeds sidesteps Docker's health -# bookkeeping entirely — what matters is "does the service respond right -# now", not "what does Docker's internal state say". +# Baserow even while it was actually up, under GitHub Actions runner +# resource contention during first-boot migrations. # -# Baserow's check execs `wget` directly (no `sh -c` wrapper), matching how -# docker-compose.yml's own `["CMD", "wget", ...]` array-form healthcheck -# resolves it. Confirmed via `sh -c 'wget ...'` failing with -# "wget: not found" while the CMD-array form (no shell involved) succeeds — -# whatever sets Baserow's PATH for its main process apparently isn't applied -# to an ad-hoc `docker exec ... sh` session. +# Also deliberately does NOT `docker compose exec` into n8n/Baserow to run +# their own healthcheck command: baserow/baserow:2.3.3 has no `wget` binary +# anywhere in $PATH at all (confirmed at the OCI runtime level, not a shell +# quirk) — docker-compose.yml's own `wget`-based HEALTHCHECK for that +# service has never actually been able to run. Instead this curls each +# HTTP service directly on its host-mapped port from the caller's machine. +# +# PRECONDITION: the caller must already have the dev compose override +# active (host ports 5678/8080 mapped) — e.g. via `-f docker-compose.dev.yml` +# or `COMPOSE_FILE` including it. Both current call sites (validate.yml's +# smoke-test, backup-drill.yml) already do this. # # Usage: ./scripts/wait-for-healthy.sh [ ...] # Known services: postgres, n8n, baserow @@ -33,8 +33,8 @@ INTERVAL="${WAIT_INTERVAL_SECS:-10}" run_check() { case "$1" in postgres) docker compose exec -T postgres sh -c 'pg_isready -U "$POSTGRES_USER"' ;; - n8n) docker compose exec -T n8n sh -c 'wget -qO- http://localhost:5678/healthz' ;; - baserow) docker compose exec -T baserow wget -qO- http://localhost/api/_health/ ;; + n8n) curl -sf http://localhost:5678/healthz ;; + baserow) curl -sf http://localhost:8080/api/_health/ ;; *) echo "Unknown service: $1 (known: postgres, n8n, baserow)" >&2 return 2 @@ -47,7 +47,7 @@ for service in "$@"; do ok="" for i in $(seq 1 "$ATTEMPTS"); do # if/else around the assignment (not `out=$(...) ; rc=$?`) so a nonzero - # exit from the exec doesn't trip `set -e` on the assignment itself. + # exit doesn't trip `set -e` on the assignment itself. if out=$(run_check "$service" 2>&1); then rc=0 else @@ -58,9 +58,7 @@ for service in "$@"; do ok="yes" break fi - # Print what actually failed — swallowing this is exactly what turned - # earlier failures into multiple rounds of guessing instead of one. - echo " [$i/$ATTEMPTS] $service not ready (exec exit $rc): $(printf '%s' "$out" | tr '\n' ' ' | cut -c1-300)" + echo " [$i/$ATTEMPTS] $service not ready (exit $rc): $(printf '%s' "$out" | tr '\n' ' ' | cut -c1-300)" sleep "$INTERVAL" done if [ -z "$ok" ]; then From bf68b213585b3c0e72ad0de1d395209695039f58 Mon Sep 17 00:00:00 2001 From: Theo Date: Sun, 23 Aug 2026 18:57:18 +0300 Subject: [PATCH 17/17] chore: remove temporary pull_request trigger from backup-drill.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Was only there to validate the workflow inside this PR before merge, since workflow_dispatch requires the file to already exist on the default branch. Confirmed working (backup-restore-verify passed). Back to schedule + workflow_dispatch only, as designed — it will run for real once merged to main and can be triggered on-demand via gh workflow run after that. --- .github/workflows/backup-drill.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/backup-drill.yml b/.github/workflows/backup-drill.yml index caeafe9..7b073bc 100644 --- a/.github/workflows/backup-drill.yml +++ b/.github/workflows/backup-drill.yml @@ -4,9 +4,6 @@ on: schedule: - cron: "0 7 * * 1" # weekly workflow_dispatch: - pull_request: # TEMPORARY — validates this workflow in the PR itself, since - # workflow_dispatch only works for files already on the default branch. - # Remove this trigger before merging. # Proves the restore path actually works, not just that backup.sh runs # without error — see scripts/backup-drill.sh for the drill logic itself