diff --git a/.env.example b/.env.example index e9dfe76..fb53791 100644 --- a/.env.example +++ b/.env.example @@ -21,7 +21,9 @@ POSTGRES_PASSWORD=changeme_use_strong_password # Default DB created at init (not used directly by apps) POSTGRES_DB=automation -# Database names for each service (auto-created by each app on first run) +# Database names for each service — created by postgres-init/init-app-databases.sh +# on first container init (empty data directory only). Neither app creates +# its own database on connect. N8N_DB_NAME=n8n BASEROW_DB_NAME=baserow diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..6ef9a61 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 +updates: + # Third-party GitHub Actions — keeps every workflow action current and + # off stale/floating tags, which is exactly the class of risk the + # aquasecurity/trivy-action supply-chain compromise (March 2026) exposed. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + # Tracks the pinned image tags in docker-compose.yml directly (GA since + # Feb 2025) and opens a PR when n8n/Baserow/Postgres/Caddy cut a new + # release. + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/backup-drill.yml b/.github/workflows/backup-drill.yml new file mode 100644 index 0000000..7b073bc --- /dev/null +++ b/.github/workflows/backup-drill.yml @@ -0,0 +1,49 @@ +name: Backup Drill + +on: + schedule: + - cron: "0 7 * * 1" # weekly + workflow_dispatch: + +# Proves the restore path actually works, not just that backup.sh runs +# without error — see scripts/backup-drill.sh for the drill logic itself +# (also runnable locally via `make backup-test`). +# +# COMPOSE_FILE layers the dev override on top of the production compose +# file for every plain `docker compose` call in this job — including the +# ones inside the scripts — without those scripts needing to know about CI +# at all. This is the standard Compose CLI mechanism for that. +env: + COMPOSE_FILE: docker-compose.yml:docker-compose.dev.yml + +jobs: + backup-restore-verify: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Prepare .env + run: | + cp .env.example .env + sed -i "s|^N8N_ENCRYPTION_KEY=.*|N8N_ENCRYPTION_KEY=$(openssl rand -hex 32)|" .env + sed -i "s|^BASEROW_SECRET_KEY=.*|BASEROW_SECRET_KEY=$(openssl rand -hex 32)|" .env + + - name: Boot the stack + run: docker compose up -d + + - name: Wait for healthchecks + run: bash scripts/wait-for-healthy.sh postgres n8n baserow + + - name: Run the backup/restore drill + run: bash scripts/backup-drill.sh + + - name: Dump service status and logs on failure + if: failure() + run: | + docker compose ps + docker compose logs + + - name: Tear down + if: always() + run: docker compose down -v diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..718522c --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,62 @@ +name: Security Scan + +on: + push: + branches: [main] + pull_request: + schedule: + # Weekly — catches newly-disclosed CVEs in images that were already + # pinned and clean when they were merged. + - cron: "0 6 * * 1" + workflow_dispatch: + +# Trivy scans the four pinned images for known CVEs. It does NOT scan +# docker-compose.yml itself — Trivy has no native docker-compose misconfig +# scanner (confirmed against its own issue tracker), so claiming IaC +# coverage here would be inaccurate. +jobs: + trivy: + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + include: + - name: n8n + image: n8nio/n8n:2.35.7 + - name: baserow + image: baserow/baserow:2.3.3 + - name: postgres + image: postgres:15.19-alpine3.24 + - name: caddy + image: caddy:2.11.4-alpine + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # Pinned to a commit SHA, not a version tag. aquasecurity/trivy-action + # had 75 of 76 tags force-pushed with malware in March 2026 — tags are + # mutable, SHAs aren't. Re-resolve this SHA from a current release + # before bumping, never trust a floating tag for a third-party action. + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: ${{ matrix.image }} + format: sarif + output: trivy-results.sarif + severity: CRITICAL,HIGH + ignore-unfixed: true + continue-on-error: true # visibility, not a merge gate — some CVEs are unfixable from this repo + + # Guarded on the SARIF file actually existing: the scan step above can + # fail (bad tag, registry hiccup, transient error) without producing + # output, and continue-on-error would otherwise let this step crash + # the whole job on a bare "Path does not exist". + - name: Upload SARIF to GitHub code scanning + if: always() && hashFiles('trivy-results.sarif') != '' + uses: github/codeql-action/upload-sarif@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + with: + sarif_file: trivy-results.sarif + category: "trivy-${{ matrix.name }}" diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index ef167cf..b0a9bbc 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Copy env template run: cp .env.example .env @@ -24,4 +24,43 @@ jobs: - name: Lint shell scripts run: | sudo apt-get install -y shellcheck - shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh + shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh scripts/restore.sh scripts/backup-drill.sh scripts/wait-for-healthy.sh + + smoke-test: + runs-on: ubuntu-latest + needs: validate + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Prepare .env + # .env.example ships with N8N_ENCRYPTION_KEY and BASEROW_SECRET_KEY + # blank — Baserow's Django backend won't boot with an empty + # SECRET_KEY. Generate real values the same way scripts/setup.sh + # does for a real deploy. + run: | + cp .env.example .env + sed -i "s|^N8N_ENCRYPTION_KEY=.*|N8N_ENCRYPTION_KEY=$(openssl rand -hex 32)|" .env + sed -i "s|^BASEROW_SECRET_KEY=.*|BASEROW_SECRET_KEY=$(openssl rand -hex 32)|" .env + + - name: Boot dev stack + run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d + + - name: Wait for healthchecks + run: bash scripts/wait-for-healthy.sh postgres n8n baserow + + - name: Check n8n health endpoint (external, through the mapped port) + run: curl -sf http://localhost:5678/healthz + + - name: Check Baserow health endpoint (external, through the mapped port) + run: curl -sf http://localhost:8080/api/_health/ + + - name: Dump service status and logs on failure + if: failure() + run: | + docker compose ps + docker compose logs + + - name: Tear down + if: always() + run: docker compose down -v diff --git a/CLAUDE.md b/CLAUDE.md index 5d990bf..06537ac 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,10 @@ make logs # docker compose logs -f make backup # stops stack → snapshots volumes → restarts make update # pulls latest images, redeploys, prunes old images make restart # docker compose restart +make test # boots the dev stack and curls both health endpoints (real smoke test) +make restore # restores a backup: make restore TIMESTAMP=20260823_140000 +make backup-test # full backup → destroy volumes → restore → verify drill (needs a running stack) +make security-scan # scans the four pinned images with the local Trivy CLI ``` Raw equivalents when needed: @@ -43,8 +47,17 @@ Internet → Caddy (80/443) ──proxy-network──► n8n (5678) - **`docker-compose.yml`** — production definition; Caddy is the only public-facing service - **`docker-compose.dev.yml`** — override that ports n8n and Baserow directly and puts Caddy behind a `production` profile (so it doesn't start) +- **`postgres-init/init-app-databases.sh`** — mounted into Postgres's `docker-entrypoint-initdb.d/`; creates the `n8n` and `baserow` databases on first init of an empty volume. See the Critical Constraints note below — this doesn't run again against an already-initialized volume. - **`Caddyfile`** — reads domain/subdomain from env vars at runtime via `{$VAR}` syntax; security headers include HSTS (`max-age=31536000; includeSubDomains`), X-Frame-Options, X-Content-Type-Options, Referrer-Policy; gzip on both virtual hosts -- **`.github/workflows/validate.yml`** — CI runs on every push to main and every PR: validates both compose files and shellchecks all scripts in `scripts/` +- **`.github/workflows/validate.yml`** — CI runs on every push to main and every PR: validates both compose files, shellchecks all scripts in `scripts/`, then actually boots the dev stack and curls both health endpoints externally — this is real runtime verification, not just YAML syntax checking +- **Don't use `docker compose up --wait`** for boot verification — it fails permanently on the *first* "unhealthy" transition, even if the container recovers seconds later. Baserow's first-boot migration + template sync briefly reports unhealthy under CI resource contention, then starts passing every subsequent check — `--wait` killed the job anyway (discovered 2026-08-23). Use `scripts/wait-for-healthy.sh ` instead: it polls `docker inspect` directly and only fails on a genuine timeout with nothing ever reporting healthy. Used by `validate.yml`, `backup-drill.yml`, `scripts/backup-drill.sh`, and `make test`. +- **`.github/workflows/security.yml`** — weekly + on-push Trivy CVE scan of the four pinned images (not the compose file — Trivy has no native docker-compose misconfig scanner), results land in the repo's Security tab. Non-blocking (`continue-on-error`). +- **`.github/workflows/backup-drill.yml`** — weekly + on-demand: writes a marker row, backs up, destroys the volumes, restores, and fails if the marker isn't back. Proves the restore path works instead of just documenting it. Logic lives in `scripts/backup-drill.sh` (also runnable locally via `make backup-test`), shared between CI and local use so they can't drift apart. +- **`.github/dependabot.yml`** — tracks the pinned image tags in `docker-compose.yml` (`docker` ecosystem) and third-party Action versions (`github-actions` ecosystem) weekly. +- **Third-party GitHub Actions are pinned by commit SHA, not by tag**, in every workflow file — `aquasecurity/trivy-action` had 75 of 76 version tags force-pushed with malware in March 2026 (tags are mutable; SHAs aren't). Apply the same pattern to any new third-party Action added to this repo; re-resolve the current SHA at the time you add it, don't copy an old one. +- **`scripts/restore.sh`** — restores all three volumes from a `backup.sh` snapshot; respects `COMPOSE_FILE` like any other `docker compose` call, so it works against the dev override too (see `backup-drill.yml` for how CI sets that). +- **Container hardening in `docker-compose.yml`** — every service has `no-new-privileges`; Caddy/n8n/Postgres have `cap_drop: [ALL]` plus only the specific caps each needs. **Baserow is deliberately not capability-dropped** — its all-in-one process supervisor's root requirements weren't confirmed safe to guess at from outside a live container. Don't attempt to add `cap_drop` to Baserow without verifying against a real running container first (see `docs/troubleshooting.md` § Container Won't Start After a Capability Change for the diagnostic approach). +- **`docs/adr/`** — architecture decision records for the reverse-proxy choice, the shared-Postgres decision, and the two-network topology. Add a new one when a similarly consequential architectural choice gets made, don't retrofit one for routine changes. ## Validation @@ -58,18 +71,22 @@ docker compose -f docker-compose.yml -f docker-compose.dev.yml config --quiet rm .env # Lint shell scripts -shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh +shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh scripts/restore.sh scripts/backup-drill.sh scripts/wait-for-healthy.sh ``` +To actually verify the stack boots (not just that the YAML is valid), run `make test` — needs a local Docker daemon. If one isn't available in your environment, push to a branch and let the `validate.yml` smoke-test job do it; don't claim runtime behavior is verified without one or the other actually running. + ## Critical Constraints - `N8N_ENCRYPTION_KEY` — generated once by `setup.sh` via `openssl rand -hex 32`. **Never change it after first run**; doing so breaks all stored n8n credentials. - `BASEROW_SECRET_KEY` — Django secret key; same rule applies. -- Both databases (`N8N_DB_NAME`, `BASEROW_DB_NAME`) are created automatically by each app on first run inside the shared Postgres container. The `POSTGRES_DB` var is just the default DB created at container init and isn't used by n8n or Baserow directly. +- `POSTGRES_DB` is just the default DB created at container init and isn't used by n8n or Baserow directly. **Neither app creates its own database on connect** (discovered via the smoke-test job actually booting the stack, 2026-08-23 — the repo had never been booted for real before that). `postgres-init/init-app-databases.sh` creates `N8N_DB_NAME` and `BASEROW_DB_NAME` explicitly via Postgres's `docker-entrypoint-initdb.d` mechanism, which only runs once, against an empty `postgres_data` volume. An existing volume from before this fix needs the databases created manually — see `docs/troubleshooting.md` § Can't Connect to PostgreSQL. - Volume names in `backup.sh` are detected dynamically via `docker compose config` — no hardcoded prefix. The script requires `.env` to exist so compose config can resolve variable substitutions. - `backup.sh` backs up `n8n_data`, `baserow_data`, and `postgres_data` only. `caddy_data` and `caddy_config` are intentionally excluded — certificates are re-issuable from Let's Encrypt for free. - n8n and Baserow start only after Postgres passes its healthcheck (`pg_isready`). If either service fails to start, confirm Postgres is `(healthy)` first: `docker compose ps postgres`. - `setup.sh` uses a `sed_inplace()` helper (macOS vs Linux `sed -i` differences). Any additions to that script that write to `.env` must use the same helper. +- All four images are version-pinned (no `:latest`). Bump via the Dependabot PR, not by hand, unless you have a specific reason to jump ahead of it — and review the linked changelog before merging either way. +- `.env.example` ships with `N8N_ENCRYPTION_KEY` and `BASEROW_SECRET_KEY` blank by design (real users generate them via `setup.sh`). Anything that boots the stack directly from `.env.example` (CI, a quick local test) must generate real values first — Baserow's Django backend won't start with an empty `SECRET_KEY`. See the "Prepare .env" step in `validate.yml`'s smoke-test job for the pattern. ## Docs @@ -78,3 +95,4 @@ shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh - `docs/backup-and-restore.md` — restore procedure + cron automation - `docs/updating-services.md` — safe upgrade process - `docs/troubleshooting.md` — common issues +- `docs/adr/` — architecture decision records (reverse proxy choice, shared Postgres, two-network topology) diff --git a/Makefile b/Makefile index 2d77044..067f7c5 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,24 @@ .DEFAULT_GOAL := help -.PHONY: help up down restart dev logs backup update setup +.PHONY: help up down restart dev logs backup update setup test restore backup-test security-scan help: @echo "Self-Hosted Automation Stack" @echo "" @echo "Usage: make " @echo "" - @echo " setup Interactive first-run setup (writes .env, starts stack)" - @echo " up Start the production stack" - @echo " down Stop and remove containers" - @echo " restart Restart all services" - @echo " dev Start in dev mode (direct ports, no Caddy)" - @echo " logs Follow logs for all services" - @echo " backup Back up all volumes to ./backups/" - @echo " update Pull latest images and redeploy" + @echo " setup Interactive first-run setup (writes .env, starts stack)" + @echo " up Start the production stack" + @echo " down Stop and remove containers" + @echo " restart Restart all services" + @echo " dev Start in dev mode (direct ports, no Caddy)" + @echo " logs Follow logs for all services" + @echo " backup Back up all volumes to ./backups/" + @echo " update Pull latest images and redeploy" + @echo " test Boot the dev stack and verify health endpoints respond" + @echo " restore Restore from a backup: make restore TIMESTAMP=20260823_140000" + @echo " backup-test Full backup -> destroy -> restore -> verify drill (needs a running stack)" + @echo " security-scan Scan the four pinned images with the Trivy CLI (must be installed locally)" setup: bash scripts/setup.sh @@ -39,3 +43,21 @@ backup: update: bash scripts/update.sh + +test: + docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d + bash scripts/wait-for-healthy.sh postgres n8n baserow + curl -sf http://localhost:5678/healthz && echo "n8n: healthy" + curl -sf http://localhost:8080/api/_health/ && echo "baserow: healthy" + +restore: + bash scripts/restore.sh $(TIMESTAMP) + +backup-test: + bash scripts/backup-drill.sh + +security-scan: + @for img in n8nio/n8n:2.35.7 baserow/baserow:2.3.3 postgres:15.19-alpine3.24 caddy:2.11.4-alpine; do \ + echo "--- $$img ---"; \ + trivy image --severity CRITICAL,HIGH --ignore-unfixed "$$img"; \ + done diff --git a/README.md b/README.md index 683050a..e229d80 100644 --- a/README.md +++ b/README.md @@ -2,37 +2,42 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) [![Validate](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/validate.yml/badge.svg)](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/validate.yml) +[![Security Scan](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/security.yml/badge.svg)](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/security.yml) +[![Backup Drill](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/backup-drill.yml/badge.svg)](https://github.com/whybothercoding/self-hosted-automation-stack/actions/workflows/backup-drill.yml) A production-ready Docker Compose stack running **n8n**, **Baserow**, and **PostgreSQL** behind **Caddy** with automatic HTTPS. Clone, configure, and deploy a self-hosted automation suite on any VPS in minutes — no manual SSL certificate management, no external dependencies. +Every push boots the stack for real and checks the health endpoints (not just YAML syntax), the four pinned images are scanned weekly for CVEs, and the backup/restore path is proven weekly by actually destroying a volume, restoring it, and verifying the data survived. See [Security & Reliability](#security--reliability) below. + ## Architecture -``` - ┌──────────────────────────────────────────────────┐ - │ VPS (Docker host) │ - │ │ - Internet │ ┌─────────┐ proxy-network │ - ──:80/:443 ────►│ │ Caddy │──► n8n (:5678) │ - │ │ (TLS) │└──► Baserow (:80) │ - │ └─────────┘ │ │ │ - │ │ │ backend-network │ - │ ▼ ▼ │ - │ ┌──────────────────┐ │ - │ │ PostgreSQL │ │ - │ └──────────────────┘ │ - └──────────────────────────────────────────────────┘ +```mermaid +graph TB + Internet(["Internet"]) -->|":80 / :443"| Caddy["Caddy
(TLS termination)"] + + subgraph proxy["proxy-network"] + Caddy -->|reverse_proxy| n8n["n8n
:5678"] + Caddy -->|reverse_proxy| Baserow["Baserow
:80"] + end + + subgraph backend["backend-network"] + n8n --> Postgres[("PostgreSQL")] + Baserow --> Postgres + end ``` -Caddy terminates TLS and proxies to n8n and Baserow over `proxy-network`. Neither Caddy nor the internet can reach PostgreSQL directly — it is only reachable from n8n and Baserow via `backend-network`. +Caddy terminates TLS and proxies to n8n and Baserow over `proxy-network`. Neither Caddy nor the internet can reach PostgreSQL directly — it is only reachable from n8n and Baserow via `backend-network`. See [ADR-0003](docs/adr/0003-two-network-topology.md) for the reasoning. ## Services -| Service | Image | Purpose | Default URL | -|------------|---------------------------|-------------------------------------|----------------------------------| -| n8n | `n8nio/n8n:latest` | Workflow automation | `https://n8n.yourdomain.com` | -| Baserow | `baserow/baserow:latest` | No-code database / spreadsheet UI | `https://baserow.yourdomain.com` | -| PostgreSQL | `postgres:15-alpine` | Shared database backend | Internal only | -| Caddy | `caddy:2-alpine` | Reverse proxy + automatic TLS | Handles 80/443 | +| Service | Image | Purpose | Default URL | +|------------|-----------------------------------|-------------------------------------|-----------------------------------| +| n8n | `n8nio/n8n:2.35.7` | Workflow automation | `https://n8n.yourdomain.com` | +| Baserow | `baserow/baserow:2.3.3` | No-code database / spreadsheet UI | `https://baserow.yourdomain.com` | +| PostgreSQL | `postgres:15.19-alpine3.24` | Shared database backend | Internal only | +| Caddy | `caddy:2.11.4-alpine` | Reverse proxy + automatic TLS | Handles 80/443 | + +All four are version-pinned rather than `:latest` — [Dependabot](.github/dependabot.yml) opens a PR when a new release lands, so upgrades are reviewed, not silent. See [Updating Services](docs/updating-services.md). ## Prerequisites @@ -70,6 +75,7 @@ make down # Stop and remove containers make restart # Restart all services make logs # Follow logs for all services make dev # Start in dev mode (direct ports, no Caddy) +make test # Boot the dev stack and verify both health endpoints respond ``` Or with Docker Compose directly: @@ -80,14 +86,15 @@ docker compose logs n8n --follow docker compose restart n8n ``` -### Backup +### Backup & Restore ```bash -make backup -# or: bash scripts/backup.sh +make backup # Stops the stack, snapshots all volumes to ./backups/, restarts +make restore TIMESTAMP=20260823_140000 # Restore from a specific backup +make backup-test # Full backup -> destroy -> restore -> verify drill (needs a running stack) ``` -Stops the stack, snapshots all volumes to `./backups/`, and restarts. See [Backup & Restore](docs/backup-and-restore.md) for restore instructions and cron automation. +The same drill `make backup-test` runs locally runs weekly in CI — see [Backup & Restore](docs/backup-and-restore.md). ### Update @@ -110,6 +117,20 @@ make dev - n8n: [http://localhost:5678](http://localhost:5678) - Baserow: [http://localhost:8080](http://localhost:8080) +## Security & Reliability + +- **Network isolation** — `proxy-network` (Caddy + apps) and `backend-network` (apps + Postgres) are separate bridge networks; Postgres is unreachable from Caddy or the internet. +- **Container hardening** — every service runs with `no-new-privileges`; Caddy, n8n, and Postgres run with all Linux capabilities dropped except the specific ones each needs (`NET_BIND_SERVICE` for Caddy's privileged ports, five ownership-related caps for Postgres's first-init step). Baserow is intentionally not capability-dropped yet — its all-in-one process supervisor's root requirements aren't confirmed safe to guess at. +- **Resource limits & log rotation** — every service has a memory ceiling and capped, rotated JSON logs, so a runaway execution or import can't take down the whole VPS or fill the disk. +- **HSTS + standard security headers** on both Caddy virtual hosts. +- **Pinned images, tracked by Dependabot** — no `:latest` tags; version bumps arrive as reviewable PRs. +- **Weekly Trivy scan** of all four pinned images for known CVEs, results in the repo's Security tab. +- **CI actually boots the stack** — `validate.yml`'s smoke-test job runs the real dev stack and checks both health endpoints externally, not just `docker compose config`. +- **Backup/restore is proven, not just documented** — the weekly Backup Drill workflow writes a marker row, backs up, destroys the volume, restores from the tarball, and fails if the data isn't back. +- **Third-party GitHub Actions pinned by commit SHA**, not by tag — a direct response to the `aquasecurity/trivy-action` supply-chain compromise (March 2026), where 75 of 76 version tags were force-pushed with malware. Tags are mutable; SHAs aren't. + +See [docs/adr/](docs/adr/) for the reasoning behind the bigger architectural choices. + ## Environment Variables | Variable | Description | Example | @@ -137,11 +158,14 @@ All variables are documented with comments in [`.env.example`](.env.example). ## Scripts -| Script | Description | -|----------------------|----------------------------------------------------------------------| -| `scripts/setup.sh` | Interactive first-run: writes `.env`, pulls images, starts stack | -| `scripts/backup.sh` | Stops stack, snapshots all volumes to `backups/`, restarts stack | -| `scripts/update.sh` | Pulls latest images, redeploys with `--remove-orphans`, prunes old | +| Script | Description | +|-----------------------------|----------------------------------------------------------------------| +| `scripts/setup.sh` | Interactive first-run: writes `.env`, pulls images, starts stack | +| `scripts/backup.sh` | Stops stack, snapshots all volumes to `backups/`, restarts stack | +| `scripts/restore.sh` | Restores all volumes from a timestamped backup | +| `scripts/backup-drill.sh` | Full backup → destroy → restore → verify cycle (local or CI) | +| `scripts/wait-for-healthy.sh` | Polls services until Docker reports them healthy; tolerates transient blips instead of failing on the first one | +| `scripts/update.sh` | Pulls latest images, redeploys with `--remove-orphans`, prunes old | ## Documentation @@ -150,6 +174,7 @@ All variables are documented with comments in [`.env.example`](.env.example). - [Backup & Restore](docs/backup-and-restore.md) — Backup volumes and restore procedures - [Updating Services](docs/updating-services.md) — Safely updating n8n, Baserow, and Postgres - [Troubleshooting](docs/troubleshooting.md) — Common issues and solutions +- [Architecture Decision Records](docs/adr/) — Why Caddy, why a shared Postgres instance, why two networks ## License diff --git a/docker-compose.yml b/docker-compose.yml index 7f4b8d4..43e25b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,23 +1,51 @@ services: postgres: - image: postgres:15-alpine + image: postgres:15.19-alpine3.24 restart: unless-stopped environment: POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: ${POSTGRES_DB} + # Read by postgres-init/init-app-databases.sh — POSTGRES_DB only gets + # one database created for free; n8n and Baserow need their own. + N8N_DB_NAME: ${N8N_DB_NAME} + BASEROW_DB_NAME: ${BASEROW_DB_NAME} volumes: - postgres_data:/var/lib/postgresql/data + - ./postgres-init/init-app-databases.sh:/docker-entrypoint-initdb.d/init-app-databases.sh:ro healthcheck: test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"] interval: 10s timeout: 5s retries: 5 + security_opt: + - no-new-privileges:true + # cap_drop/cap_add: official postgres entrypoint starts as root to fix + # data-directory ownership on first init, then drops via gosu. These five + # caps are the documented minimal set — verify a fresh-volume init still + # works cleanly before trusting this on a real deploy. + cap_drop: + - ALL + cap_add: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - SETUID + - SETGID + deploy: + resources: + limits: + memory: 512m + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" networks: - backend-network n8n: - image: n8nio/n8n:latest + image: n8nio/n8n:2.35.7 restart: unless-stopped depends_on: postgres: @@ -44,12 +72,27 @@ services: timeout: 10s retries: 5 start_period: 30s + security_opt: + - no-new-privileges:true + # Safe to fully drop: the official image already runs as the + # unprivileged `node` user and never binds a privileged port. + cap_drop: + - ALL + deploy: + resources: + limits: + memory: 1g + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" networks: - proxy-network - backend-network baserow: - image: baserow/baserow:latest + image: baserow/baserow:2.3.3 restart: unless-stopped depends_on: postgres: @@ -59,12 +102,48 @@ services: SECRET_KEY: ${BASEROW_SECRET_KEY} volumes: - baserow_data:/baserow/data + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost/api/_health/"] + # For docker compose ps / operator visibility only — CI's actual + # readiness gate is scripts/wait-for-healthy.sh, which polls this same + # check directly rather than trusting Docker's own Health.Status. That + # status was observed getting stuck on "unhealthy" under CI resource + # contention even while every request to this endpoint was returning + # 200, so it's deliberately not load-bearing for anything anymore. + # First boot on an empty volume runs Django migrations *and* + # Baserow's built-in template sync before this responds reliably — + # observed taking 150s+ on a GitHub Actions runner. + interval: 15s + timeout: 10s + retries: 10 + start_period: 150s + security_opt: + - no-new-privileges:true + # No cap_drop here yet: Baserow's all-in-one image runs nginx + + # gunicorn/daphne + celery via an internal process supervisor whose root + # requirements aren't confirmed. Drop capabilities in a follow-up pass + # once this base hardening is proven in CI, not blindly in this one. + # 2g, not 1g: Baserow runs gunicorn + celery worker + celery beat + # concurrently, and first boot on an empty volume adds Django migrations + # + template-sync on top of that. Observed a celery worker getting + # OOM-killed (SIGKILL) under a 1g cap during exactly this burst on a + # GitHub Actions runner — steady-state usage is lower, but the limit has + # to cover the worst moment, not the average one. + deploy: + resources: + limits: + memory: 2g + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" networks: - proxy-network - backend-network caddy: - image: caddy:2-alpine + image: caddy:2.11.4-alpine restart: unless-stopped ports: - "80:80" @@ -78,6 +157,22 @@ services: - ./Caddyfile:/etc/caddy/Caddyfile:ro - caddy_data:/data - caddy_config:/config + security_opt: + - no-new-privileges:true + # NET_BIND_SERVICE is required to bind ports 80/443 as non-root. + cap_drop: + - ALL + cap_add: + - NET_BIND_SERVICE + deploy: + resources: + limits: + memory: 128m + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" networks: - proxy-network diff --git a/docs/adr/0001-caddy-as-reverse-proxy.md b/docs/adr/0001-caddy-as-reverse-proxy.md new file mode 100644 index 0000000..bd983ac --- /dev/null +++ b/docs/adr/0001-caddy-as-reverse-proxy.md @@ -0,0 +1,18 @@ +# ADR-0001: Caddy as the reverse proxy + +## Context + +The stack needs a reverse proxy in front of n8n and Baserow that terminates TLS for two subdomains on a single VPS, without requiring the operator to manage certificates by hand. The realistic alternatives were nginx + Certbot, or Traefik. + +## Decision + +Use Caddy 2. + +Caddy issues and renews Let's Encrypt certificates automatically from a declarative `Caddyfile` — no separate Certbot process, no cron job for renewal, no ACME client to wire up. The entire routing + TLS config for this stack is the ~25-line `Caddyfile` in the repo root. nginx would need Certbot bolted on as a second moving part (and a renewal cron job); Traefik's Docker-label-driven config is a better fit for a large, dynamically-changing set of services than for two static virtual hosts known upfront. + +## Consequences + +- One fewer service, one fewer moving part than nginx + Certbot. +- Config is a plain Caddyfile, not Docker labels — easy to read and diff, but means adding a new subdomain requires a Caddyfile edit + restart rather than being automatic from compose. +- Caddy is less battle-tested at extreme scale than nginx, which is irrelevant here — this stack fronts two low-traffic internal tools, not a public CDN edge. +- Automatic HTTPS depends on ports 80 and 443 being reachable from the internet for the ACME HTTP-01 challenge; documented in [SSL Configuration](../ssl-configuration.md). diff --git a/docs/adr/0002-shared-postgres-instance.md b/docs/adr/0002-shared-postgres-instance.md new file mode 100644 index 0000000..b79e23e --- /dev/null +++ b/docs/adr/0002-shared-postgres-instance.md @@ -0,0 +1,18 @@ +# ADR-0002: One shared PostgreSQL instance, separate databases per app + +## Context + +Both n8n and Baserow need a PostgreSQL database. They could each run their own `postgres` container, or share one instance with separate logical databases. + +## Decision + +Run a single `postgres` service; n8n and Baserow each get their own database inside it (`N8N_DB_NAME`, `BASEROW_DB_NAME`), created automatically by each app on first connection. `POSTGRES_DB` only exists because the official Postgres image requires a default database at init — neither app uses it directly. + +For a stack sized for a single small-to-mid VPS, running two separate Postgres containers means two sets of memory overhead, two volumes to back up, and two processes to keep healthy for no functional gain — n8n and Baserow never query each other's data, so there's no coupling risk from sharing the instance. + +## Consequences + +- One `postgres_data` volume, one healthcheck, one memory budget to size instead of two. +- `scripts/backup.sh` backs up a single volume that contains both apps' data — simpler, but a restore of that volume restores both apps' state together, not independently. +- A Postgres major-version upgrade (e.g. 15 → 16) affects both apps at once and needs `pg_dumpall`, documented in [Updating Services](../updating-services.md) — this is the real cost of sharing the instance, and worth re-evaluating if either app's data volume grows large enough that independent upgrade cadences become valuable. +- Credentials are shared at the PostgreSQL-user level (both apps connect as `POSTGRES_USER`) but data is isolated at the database level — acceptable here since both apps are operated by the same person on the same VPS, not a multi-tenant setup. diff --git a/docs/adr/0003-two-network-topology.md b/docs/adr/0003-two-network-topology.md new file mode 100644 index 0000000..3c839d0 --- /dev/null +++ b/docs/adr/0003-two-network-topology.md @@ -0,0 +1,21 @@ +# ADR-0003: Two Docker networks instead of one + +## Context + +The original version of this stack put all four services (Caddy, n8n, Baserow, Postgres) on a single bridge network. That's the simplest possible setup, but it means Caddy — the one service directly exposed to the internet — is on the same network as Postgres, which has no business being reachable from the proxy layer at all. + +## Decision + +Split into two bridge networks: + +- `proxy-network`: Caddy, n8n, Baserow — Caddy routes to the two apps. +- `backend-network`: n8n, Baserow, Postgres — the apps talk to the database. + +n8n and Baserow sit on both networks (they need to reach Caddy's routing and the database); Caddy and Postgres are each on exactly one, and never share a network with each other. + +## Consequences + +- If Caddy were ever compromised (a config error, a vulnerable module, a bad upstream dependency), it has no network path to Postgres at all — not "credentials would stop it," but "there is no route." That's a meaningfully stronger guarantee than relying on auth alone. +- Two `networks:` blocks and two entries in most services' `networks:` list instead of one — a small readability cost for a real isolation gain. +- No performance cost: Docker bridge networks are cheap, and n8n/Baserow already needed to be on two networks' worth of routes regardless of how many network objects that's expressed as. +- This is the kind of thing that's easy to skip when a stack is stood up quickly and easy to forget to add later — worth stating explicitly here so it doesn't regress back to one flat network in a future edit. diff --git a/docs/backup-and-restore.md b/docs/backup-and-restore.md index 1d537c4..ffd5158 100644 --- a/docs/backup-and-restore.md +++ b/docs/backup-and-restore.md @@ -67,6 +67,19 @@ For weekly backups (Sundays at 2 AM): ## Restoring from a Backup +The fastest path is `scripts/restore.sh`, which wraps the manual steps below: + +```bash +bash scripts/restore.sh 20260422_140000 +# or: make restore TIMESTAMP=20260422_140000 +``` + +It detects the Compose project name the same way `backup.sh` does, confirms all three backup files for that timestamp exist before touching anything, prompts for confirmation (`--force` skips the prompt — used by the automated drill below), stops the stack, removes and recreates each volume, restores the tarball into it, and starts the stack back up. + +### Manual restore (what the script does, step by step) + +Useful if you need to restore a single volume rather than all three, or want to understand exactly what's happening before running it against production data. + First, capture your project name into a variable — all volume operations below use it: ```bash @@ -117,6 +130,17 @@ Open your n8n and Baserow URLs and confirm workflows and databases are intact. C docker compose logs --tail=100 ``` +## Verifying the Restore Path Actually Works + +A backup you've never restored is a hope, not a backup. `scripts/backup-drill.sh` proves the full cycle works: it writes a marker row into Postgres, runs a real backup, **destroys the volumes**, restores from the tarball just taken, and fails loudly if the marker row isn't back. + +```bash +make backup-test +# needs a running stack — boot one first with `make dev` or `make up` +``` + +The same script runs on a schedule in CI ([`.github/workflows/backup-drill.yml`](../.github/workflows/backup-drill.yml), weekly + on-demand via `workflow_dispatch`), against a throwaway stack it boots itself — so the restore path gets exercised regularly even if nobody runs it by hand. Check the Actions tab for the `Backup Drill` workflow's history. + ## Transferring Backups Off-Server Keep at least one copy off the VPS to protect against server loss. diff --git a/docs/initial-setup.md b/docs/initial-setup.md index 3454f60..a793ee2 100644 --- a/docs/initial-setup.md +++ b/docs/initial-setup.md @@ -6,7 +6,7 @@ This guide walks you through deploying the stack on a fresh Ubuntu VPS from scra You need a VPS with: - Ubuntu 20.04 LTS or later -- At least 2 GB RAM (4 GB recommended for running both n8n and Baserow) +- At least 4 GB RAM. `docker-compose.yml`'s per-service memory limits (postgres 512m, n8n 1g, baserow 2g, caddy 128m) sum to ~3.6 GB at worst case — Baserow alone needs the full 2g during its first-boot migration + template sync, observed OOM-killing a celery worker under 1g. 2 GB will work for steady-state use once past first boot, but risks an OOM on the very first `docker compose up`. - Ports 80 and 443 open in the firewall ### Open Firewall Ports (UFW) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 98ac4c4..3b7d1f1 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -74,15 +74,46 @@ docker compose exec postgres psql -U $POSTGRES_USER -c "ALTER USER $POSTGRES_USE Then update `.env` to match and restart: `docker compose restart n8n baserow` -**Check 3: Database doesn't exist yet** +**Check 3: Database doesn't exist** -n8n and Baserow create their own databases on first startup. If they crash before the DB is created, restart them after PostgreSQL is healthy: +`postgres-init/init-app-databases.sh` creates the `n8n` and `baserow` databases automatically — but **only** the first time Postgres initializes against an empty `postgres_data` volume. Neither app creates its own database on connect; if the database is missing, n8n/Baserow will keep crash-looping with `database "..." does not exist` no matter how many times you restart them. + +This happens if: +- You're restoring/upgrading an existing volume that was created before this init script existed. +- The volume was manually created without ever running Postgres's first-init sequence against it. + +Fix — create the missing database directly: ```bash -docker compose restart n8n -docker compose restart baserow +docker compose exec postgres psql -U $POSTGRES_USER -c "CREATE DATABASE n8n;" +docker compose exec postgres psql -U $POSTGRES_USER -c "CREATE DATABASE baserow;" +docker compose restart n8n baserow +``` + +(Substitute your actual `N8N_DB_NAME`/`BASEROW_DB_NAME` if you changed them from the defaults.) + +--- + +## Container Won't Start After a Capability Change + +**Symptom:** A service that was working fine now exits immediately, and the logs mention `permission denied`, `operation not permitted`, or a failure to bind a port — after `docker-compose.yml`'s `cap_drop`/`cap_add`/`security_opt` settings changed. + +Every service runs with `no-new-privileges` and most run with `cap_drop: [ALL]` plus only the specific capabilities they need (see [Security & Reliability](../README.md#security--reliability) in the README, and [ADR-0003](adr/0003-two-network-topology.md) for the isolation reasoning). If a service needs a capability that isn't in its `cap_add` list, it fails at the exact operation that needs it — often only on first run against a fresh volume, not on every start. + +**Step 1:** Confirm the failure is capability-related, not something else: + +```bash +docker compose logs --tail=50 ``` +Look for `EPERM`, `operation not permitted`, `permission denied`, or (for Caddy specifically) a failure to bind port 80/443. + +**Step 2:** Temporarily comment out that service's `cap_drop`/`cap_add` block in `docker-compose.yml` and restart it. If it starts cleanly with capabilities unrestricted, you've confirmed the diagnosis — the service needs a capability that isn't currently granted. + +**Step 3:** Add the missing capability to that service's `cap_add:` list rather than removing `cap_drop: [ALL]` entirely — narrow the fix to what's actually needed. [Docker's capabilities reference](https://docs.docker.com/engine/containers/run/#runtime-privilege-and-linux-capabilities) lists what each capability controls. + +**Step 4:** Restore the hardening once the fix is confirmed — don't leave a service running capability-unrestricted after debugging. + --- ## SSL Not Issuing diff --git a/docs/updating-services.md b/docs/updating-services.md index f168194..5f9e675 100644 --- a/docs/updating-services.md +++ b/docs/updating-services.md @@ -22,22 +22,18 @@ bash scripts/backup.sh bash scripts/update.sh ``` -## Pinning to Specific Versions +## Version Pins and Dependabot -By default, `docker-compose.yml` uses `:latest` tags. For production stability, pin to specific versions to control exactly when updates happen. +`docker-compose.yml` pins all four images to specific versions rather than `:latest` — an update is a deliberate, reviewable change, not something that silently happens on the next `docker compose pull`. -Edit `docker-compose.yml`: +[Dependabot](../.github/dependabot.yml) watches those pinned tags (`package-ecosystem: docker`, GA support for `docker-compose.yml` since Feb 2025) and opens a PR whenever n8n, Baserow, Postgres, or Caddy cut a new release. **Review the linked changelog before merging** — Dependabot bumps the tag, it doesn't read the release notes for you. For n8n specifically, check the breaking-changes guidance below before merging a major-version bump. + +To bump a version by hand instead of waiting for Dependabot, edit the `image:` line for that service in `docker-compose.yml` directly: ```yaml services: n8n: - image: n8nio/n8n:1.89.1 - - baserow: - image: baserow/baserow:1.32.5 - - postgres: - image: postgres:15.7-alpine + image: n8nio/n8n:2.36.0 # example — check the current release first ``` Check available versions: @@ -45,7 +41,7 @@ Check available versions: - **Baserow:** https://hub.docker.com/r/baserow/baserow/tags - **PostgreSQL:** https://hub.docker.com/_/postgres/tags -After editing, apply: +After editing (or merging a Dependabot PR), apply: ```bash docker compose pull diff --git a/postgres-init/init-app-databases.sh b/postgres-init/init-app-databases.sh new file mode 100755 index 0000000..e19abc4 --- /dev/null +++ b/postgres-init/init-app-databases.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -e + +# Runs automatically on first container init (only when the data directory +# is empty) — this is the official postgres image's docker-entrypoint-initdb.d +# mechanism, not custom logic. +# +# POSTGRES_DB only gets one database created for free at initdb time. n8n and +# Baserow each need their own database on this shared instance, and neither +# app creates its own database on connect — n8n's pg client and Baserow's +# Django/psycopg both just fail with "database does not exist" if it isn't +# already there. This script creates both explicitly, once, at first boot. + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE DATABASE "$N8N_DB_NAME"; + CREATE DATABASE "$BASEROW_DB_NAME"; +EOSQL diff --git a/scripts/backup-drill.sh b/scripts/backup-drill.sh new file mode 100755 index 0000000..d5c6fa1 --- /dev/null +++ b/scripts/backup-drill.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -e + +# ------------------------------------------------------- +# backup-drill.sh — Prove the backup/restore cycle actually recovers data +# +# Usage: ./scripts/backup-drill.sh +# +# Precondition: the stack must already be up and healthy +# (`docker compose up -d --wait`) and `.env` must exist. +# +# Writes a marker row into Postgres, backs up, destroys the volumes, restores +# from the tarball just taken, and fails if the marker row isn't back. Used +# by both `make backup-test` (local) and the scheduled Backup Drill workflow +# (CI) — the drill logic lives here once so the two don't drift apart. +# +# Respects COMPOSE_FILE like any other `docker compose` invocation — export +# it (e.g. to include the dev override) before running this script. +# ------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + +cd "$PROJECT_DIR" + +if [ ! -f .env ]; then + echo "ERROR: .env not found. Run this against a configured stack." + exit 1 +fi + +set -a +# shellcheck disable=SC1091 +source .env +set +a + +MARKER="backup-drill-$(date +%s)-$$" + +echo "Writing marker row: $MARKER" +docker compose exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + -c "CREATE TABLE IF NOT EXISTS backup_drill (marker text); INSERT INTO backup_drill VALUES ('${MARKER}');" + +echo "Running backup..." +bash "$SCRIPT_DIR/backup.sh" + +echo "Destroying the stack and its volumes (this is the point of the drill)..." +docker compose down -v + +TIMESTAMP=$(find "$PROJECT_DIR/backups" -name "*_postgres_data_*.tar.gz" | sed -E 's/.*_([0-9]{8}_[0-9]{6})\.tar\.gz/\1/' | sort | tail -1) +if [ -z "$TIMESTAMP" ]; then + echo "ERROR: no backup archive found after running backup.sh" + exit 1 +fi + +echo "Restoring from backup timestamped: $TIMESTAMP" +bash "$SCRIPT_DIR/restore.sh" "$TIMESTAMP" --force + +echo "Waiting for the restored stack to become healthy..." +bash "$SCRIPT_DIR/wait-for-healthy.sh" postgres n8n baserow + +RESULT=$(docker compose exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -t -c \ + "SELECT 1 FROM backup_drill WHERE marker = '${MARKER}';" | tr -d '[:space:]') + +if [ "$RESULT" != "1" ]; then + echo "FAIL: marker row '${MARKER}' did not survive the backup/restore cycle." + exit 1 +fi + +echo "PASS: marker row confirmed present after restore. Backup/restore cycle verified." diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 0000000..5126333 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -e + +# ------------------------------------------------------- +# restore.sh — Restore all Docker volumes from a backup.sh snapshot +# +# Usage: ./scripts/restore.sh [--force] +# Matches the suffix on files in ./backups/, e.g. 20260823_140000 +# --force Skip the confirmation prompt (used by the automated backup drill) +# +# WARNING: this replaces the current contents of n8n_data, baserow_data, +# and postgres_data. There is no undo once the existing volumes are removed. +# +# Respects the standard COMPOSE_FILE env var, same as any `docker compose` +# invocation — set it if you need to restore against a non-default compose +# file combination (e.g. the dev override). +# ------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +BACKUP_DIR="$PROJECT_DIR/backups" + +TIMESTAMP="${1:-}" +FORCE_FLAG="${2:-}" + +if [ -z "$TIMESTAMP" ]; then + echo "Usage: $0 [--force]" + echo "" + echo "Available backup timestamps:" + find "$BACKUP_DIR" -name "*.tar.gz" 2>/dev/null | sed -E 's/.*_([0-9]{8}_[0-9]{6})\.tar\.gz/\1/' | sort -u + exit 1 +fi + +cd "$PROJECT_DIR" + +COMPOSE_PROJECT=$(docker compose config 2>/dev/null | grep -m1 '^name:' | awk '{print $2}') +if [ -z "$COMPOSE_PROJECT" ]; then + echo "ERROR: Could not determine compose project name." + echo "Make sure .env exists and docker compose config runs without errors." + exit 1 +fi + +VOLUMES=("n8n_data" "baserow_data" "postgres_data") + +for VOL in "${VOLUMES[@]}"; do + ARCHIVE="$BACKUP_DIR/${COMPOSE_PROJECT}_${VOL}_${TIMESTAMP}.tar.gz" + if [ ! -f "$ARCHIVE" ]; then + echo "ERROR: Backup file not found: $ARCHIVE" + exit 1 + fi +done + +if [ "$FORCE_FLAG" != "--force" ]; then + echo "This will STOP the stack and REPLACE the current contents of:" + for VOL in "${VOLUMES[@]}"; do + echo " - ${COMPOSE_PROJECT}_${VOL}" + done + echo "with the backup timestamped ${TIMESTAMP}. This cannot be undone." + read -rp "Type 'restore' to continue: " CONFIRM + if [ "$CONFIRM" != "restore" ]; then + echo "Aborted." + exit 1 + fi +fi + +echo "Stopping stack..." +docker compose down + +restore_volume() { + local VOL="$1" + local VOLUME_NAME="${COMPOSE_PROJECT}_${VOL}" + local ARCHIVE="$BACKUP_DIR/${VOLUME_NAME}_${TIMESTAMP}.tar.gz" + + echo "Restoring volume: $VOLUME_NAME <- $ARCHIVE" + docker volume rm "$VOLUME_NAME" >/dev/null 2>&1 || true + docker volume create "$VOLUME_NAME" >/dev/null + docker run --rm \ + -v "${VOLUME_NAME}:/data" \ + -v "$BACKUP_DIR:/backup:ro" \ + alpine \ + tar xzf "/backup/$(basename "$ARCHIVE")" -C /data +} + +for VOL in "${VOLUMES[@]}"; do + restore_volume "$VOL" +done + +echo "Starting stack..." +docker compose up -d + +echo "" +echo "Restore complete from backup timestamped: ${TIMESTAMP}" diff --git a/scripts/wait-for-healthy.sh b/scripts/wait-for-healthy.sh new file mode 100755 index 0000000..b2beeb7 --- /dev/null +++ b/scripts/wait-for-healthy.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -e + +# ------------------------------------------------------- +# wait-for-healthy.sh — Poll each service directly until it responds. +# +# Deliberately does NOT rely on `docker compose up --wait` or `docker +# inspect .State.Health.Status`: both depend on Docker's own HEALTHCHECK +# state machine, which was observed getting stuck reporting "unhealthy" for +# Baserow even while it was actually up, under GitHub Actions runner +# resource contention during first-boot migrations. +# +# Also deliberately does NOT `docker compose exec` into n8n/Baserow to run +# their own healthcheck command: baserow/baserow:2.3.3 has no `wget` binary +# anywhere in $PATH at all (confirmed at the OCI runtime level, not a shell +# quirk) — docker-compose.yml's own `wget`-based HEALTHCHECK for that +# service has never actually been able to run. Instead this curls each +# HTTP service directly on its host-mapped port from the caller's machine. +# +# PRECONDITION: the caller must already have the dev compose override +# active (host ports 5678/8080 mapped) — e.g. via `-f docker-compose.dev.yml` +# or `COMPOSE_FILE` including it. Both current call sites (validate.yml's +# smoke-test, backup-drill.yml) already do this. +# +# Usage: ./scripts/wait-for-healthy.sh [ ...] +# Known services: postgres, n8n, baserow +# Env: WAIT_ATTEMPTS (default 40), WAIT_INTERVAL_SECS (default 10) +# ------------------------------------------------------- + +ATTEMPTS="${WAIT_ATTEMPTS:-40}" +INTERVAL="${WAIT_INTERVAL_SECS:-10}" + +run_check() { + case "$1" in + postgres) docker compose exec -T postgres sh -c 'pg_isready -U "$POSTGRES_USER"' ;; + n8n) curl -sf http://localhost:5678/healthz ;; + baserow) curl -sf http://localhost:8080/api/_health/ ;; + *) + echo "Unknown service: $1 (known: postgres, n8n, baserow)" >&2 + return 2 + ;; + esac +} + +for service in "$@"; do + echo "Waiting for $service (up to $((ATTEMPTS * INTERVAL))s)..." + ok="" + for i in $(seq 1 "$ATTEMPTS"); do + # if/else around the assignment (not `out=$(...) ; rc=$?`) so a nonzero + # exit doesn't trip `set -e` on the assignment itself. + if out=$(run_check "$service" 2>&1); then + rc=0 + else + rc=$? + fi + if [ "$rc" -eq 0 ]; then + echo " $service is responding" + ok="yes" + break + fi + echo " [$i/$ATTEMPTS] $service not ready (exit $rc): $(printf '%s' "$out" | tr '\n' ' ' | cut -c1-300)" + sleep "$INTERVAL" + done + if [ -z "$ok" ]; then + echo "ERROR: $service did not respond within $((ATTEMPTS * INTERVAL))s" + docker compose logs "$service" --tail=100 + exit 1 + fi +done + +echo "All services responding: $*"