Portfolio technical elevation: real CI verification, hardening, verified backups

.env.example

@@ -21,7 +21,9 @@ POSTGRES_PASSWORD=changeme_use_strong_password
 # Default DB created at init (not used directly by apps)
 POSTGRES_DB=automation
 
-# Database names for each service (auto-created by each app on first run)
+# Database names for each service — created by postgres-init/init-app-databases.sh
+# on first container init (empty data directory only). Neither app creates
+# its own database on connect.
 N8N_DB_NAME=n8n
 BASEROW_DB_NAME=baserow
 

.github/dependabot.yml

@@ -0,0 +1,17 @@
+version: 2
+updates:
+  # Third-party GitHub Actions — keeps every workflow action current and
+  # off stale/floating tags, which is exactly the class of risk the
+  # aquasecurity/trivy-action supply-chain compromise (March 2026) exposed.
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  # Tracks the pinned image tags in docker-compose.yml directly (GA since
+  # Feb 2025) and opens a PR when n8n/Baserow/Postgres/Caddy cut a new
+  # release.
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/backup-drill.yml

@@ -0,0 +1,49 @@
+name: Backup Drill
+
+on:
+  schedule:
+    - cron: "0 7 * * 1" # weekly
+  workflow_dispatch:
+
+# Proves the restore path actually works, not just that backup.sh runs
+# without error — see scripts/backup-drill.sh for the drill logic itself
+# (also runnable locally via `make backup-test`).
+#
+# COMPOSE_FILE layers the dev override on top of the production compose
+# file for every plain `docker compose` call in this job — including the
+# ones inside the scripts — without those scripts needing to know about CI
+# at all. This is the standard Compose CLI mechanism for that.
+env:
+  COMPOSE_FILE: docker-compose.yml:docker-compose.dev.yml
+
+jobs:
+  backup-restore-verify:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Prepare .env
+        run: |
+          cp .env.example .env
+          sed -i "s|^N8N_ENCRYPTION_KEY=.*|N8N_ENCRYPTION_KEY=$(openssl rand -hex 32)|" .env
+          sed -i "s|^BASEROW_SECRET_KEY=.*|BASEROW_SECRET_KEY=$(openssl rand -hex 32)|" .env
+
+      - name: Boot the stack
+        run: docker compose up -d
+
+      - name: Wait for healthchecks
+        run: bash scripts/wait-for-healthy.sh postgres n8n baserow
+
+      - name: Run the backup/restore drill
+        run: bash scripts/backup-drill.sh
+
+      - name: Dump service status and logs on failure
+        if: failure()
+        run: |
+          docker compose ps
+          docker compose logs
+
+      - name: Tear down
+        if: always()
+        run: docker compose down -v

.github/workflows/security.yml

@@ -0,0 +1,62 @@
+name: Security Scan
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  schedule:
+    # Weekly — catches newly-disclosed CVEs in images that were already
+    # pinned and clean when they were merged.
+    - cron: "0 6 * * 1"
+  workflow_dispatch:
+
+# Trivy scans the four pinned images for known CVEs. It does NOT scan
+# docker-compose.yml itself — Trivy has no native docker-compose misconfig
+# scanner (confirmed against its own issue tracker), so claiming IaC
+# coverage here would be inaccurate.
+jobs:
+  trivy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: n8n
+            image: n8nio/n8n:2.35.7
+          - name: baserow
+            image: baserow/baserow:2.3.3
+          - name: postgres
+            image: postgres:15.19-alpine3.24
+          - name: caddy
+            image: caddy:2.11.4-alpine
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      # Pinned to a commit SHA, not a version tag. aquasecurity/trivy-action
+      # had 75 of 76 tags force-pushed with malware in March 2026 — tags are
+      # mutable, SHAs aren't. Re-resolve this SHA from a current release
+      # before bumping, never trust a floating tag for a third-party action.
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
+        with:
+          image-ref: ${{ matrix.image }}
+          format: sarif
+          output: trivy-results.sarif
+          severity: CRITICAL,HIGH
+          ignore-unfixed: true
+        continue-on-error: true # visibility, not a merge gate — some CVEs are unfixable from this repo
+
+      # Guarded on the SARIF file actually existing: the scan step above can
+      # fail (bad tag, registry hiccup, transient error) without producing
+      # output, and continue-on-error would otherwise let this step crash
+      # the whole job on a bare "Path does not exist".
+      - name: Upload SARIF to GitHub code scanning
+        if: always() && hashFiles('trivy-results.sarif') != ''
+        uses: github/codeql-action/upload-sarif@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8
+        with:
+          sarif_file: trivy-results.sarif
+          category: "trivy-${{ matrix.name }}"

.github/workflows/validate.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - name: Copy env template
         run: cp .env.example .env
@@ -24,4 +24,43 @@ jobs:
       - name: Lint shell scripts
         run: |
           sudo apt-get install -y shellcheck
-          shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh
+          shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh scripts/restore.sh scripts/backup-drill.sh scripts/wait-for-healthy.sh
+
+  smoke-test:
+    runs-on: ubuntu-latest
+    needs: validate
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Prepare .env
+        # .env.example ships with N8N_ENCRYPTION_KEY and BASEROW_SECRET_KEY
+        # blank — Baserow's Django backend won't boot with an empty
+        # SECRET_KEY. Generate real values the same way scripts/setup.sh
+        # does for a real deploy.
+        run: |
+          cp .env.example .env
+          sed -i "s|^N8N_ENCRYPTION_KEY=.*|N8N_ENCRYPTION_KEY=$(openssl rand -hex 32)|" .env
+          sed -i "s|^BASEROW_SECRET_KEY=.*|BASEROW_SECRET_KEY=$(openssl rand -hex 32)|" .env
+
+      - name: Boot dev stack
+        run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
+
+      - name: Wait for healthchecks
+        run: bash scripts/wait-for-healthy.sh postgres n8n baserow
+
+      - name: Check n8n health endpoint (external, through the mapped port)
+        run: curl -sf http://localhost:5678/healthz
+
+      - name: Check Baserow health endpoint (external, through the mapped port)
+        run: curl -sf http://localhost:8080/api/_health/
+
+      - name: Dump service status and logs on failure
+        if: failure()
+        run: |
+          docker compose ps
+          docker compose logs
+
+      - name: Tear down
+        if: always()
+        run: docker compose down -v

CLAUDE.md

@@ -19,6 +19,10 @@ make logs     # docker compose logs -f
 make backup   # stops stack → snapshots volumes → restarts
 make update   # pulls latest images, redeploys, prunes old images
 make restart  # docker compose restart
+make test     # boots the dev stack and curls both health endpoints (real smoke test)
+make restore  # restores a backup: make restore TIMESTAMP=20260823_140000
+make backup-test    # full backup → destroy volumes → restore → verify drill (needs a running stack)
+make security-scan  # scans the four pinned images with the local Trivy CLI
 ```
 
 Raw equivalents when needed:
@@ -43,8 +47,17 @@ Internet → Caddy (80/443) ──proxy-network──► n8n (5678)
 
 - **`docker-compose.yml`** — production definition; Caddy is the only public-facing service
 - **`docker-compose.dev.yml`** — override that ports n8n and Baserow directly and puts Caddy behind a `production` profile (so it doesn't start)
+- **`postgres-init/init-app-databases.sh`** — mounted into Postgres's `docker-entrypoint-initdb.d/`; creates the `n8n` and `baserow` databases on first init of an empty volume. See the Critical Constraints note below — this doesn't run again against an already-initialized volume.
 - **`Caddyfile`** — reads domain/subdomain from env vars at runtime via `{$VAR}` syntax; security headers include HSTS (`max-age=31536000; includeSubDomains`), X-Frame-Options, X-Content-Type-Options, Referrer-Policy; gzip on both virtual hosts
-- **`.github/workflows/validate.yml`** — CI runs on every push to main and every PR: validates both compose files and shellchecks all scripts in `scripts/`
+- **`.github/workflows/validate.yml`** — CI runs on every push to main and every PR: validates both compose files, shellchecks all scripts in `scripts/`, then actually boots the dev stack and curls both health endpoints externally — this is real runtime verification, not just YAML syntax checking
+- **Don't use `docker compose up --wait`** for boot verification — it fails permanently on the *first* "unhealthy" transition, even if the container recovers seconds later. Baserow's first-boot migration + template sync briefly reports unhealthy under CI resource contention, then starts passing every subsequent check — `--wait` killed the job anyway (discovered 2026-08-23). Use `scripts/wait-for-healthy.sh <service...>` instead: it polls `docker inspect` directly and only fails on a genuine timeout with nothing ever reporting healthy. Used by `validate.yml`, `backup-drill.yml`, `scripts/backup-drill.sh`, and `make test`.
+- **`.github/workflows/security.yml`** — weekly + on-push Trivy CVE scan of the four pinned images (not the compose file — Trivy has no native docker-compose misconfig scanner), results land in the repo's Security tab. Non-blocking (`continue-on-error`).
+- **`.github/workflows/backup-drill.yml`** — weekly + on-demand: writes a marker row, backs up, destroys the volumes, restores, and fails if the marker isn't back. Proves the restore path works instead of just documenting it. Logic lives in `scripts/backup-drill.sh` (also runnable locally via `make backup-test`), shared between CI and local use so they can't drift apart.
+- **`.github/dependabot.yml`** — tracks the pinned image tags in `docker-compose.yml` (`docker` ecosystem) and third-party Action versions (`github-actions` ecosystem) weekly.
+- **Third-party GitHub Actions are pinned by commit SHA, not by tag**, in every workflow file — `aquasecurity/trivy-action` had 75 of 76 version tags force-pushed with malware in March 2026 (tags are mutable; SHAs aren't). Apply the same pattern to any new third-party Action added to this repo; re-resolve the current SHA at the time you add it, don't copy an old one.
+- **`scripts/restore.sh`** — restores all three volumes from a `backup.sh` snapshot; respects `COMPOSE_FILE` like any other `docker compose` call, so it works against the dev override too (see `backup-drill.yml` for how CI sets that).
+- **Container hardening in `docker-compose.yml`** — every service has `no-new-privileges`; Caddy/n8n/Postgres have `cap_drop: [ALL]` plus only the specific caps each needs. **Baserow is deliberately not capability-dropped** — its all-in-one process supervisor's root requirements weren't confirmed safe to guess at from outside a live container. Don't attempt to add `cap_drop` to Baserow without verifying against a real running container first (see `docs/troubleshooting.md` § Container Won't Start After a Capability Change for the diagnostic approach).
+- **`docs/adr/`** — architecture decision records for the reverse-proxy choice, the shared-Postgres decision, and the two-network topology. Add a new one when a similarly consequential architectural choice gets made, don't retrofit one for routine changes.
 
 ## Validation
 
@@ -58,18 +71,22 @@ docker compose -f docker-compose.yml -f docker-compose.dev.yml config --quiet
 rm .env
 
 # Lint shell scripts
-shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh
+shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh scripts/restore.sh scripts/backup-drill.sh scripts/wait-for-healthy.sh
 ```
 
+To actually verify the stack boots (not just that the YAML is valid), run `make test` — needs a local Docker daemon. If one isn't available in your environment, push to a branch and let the `validate.yml` smoke-test job do it; don't claim runtime behavior is verified without one or the other actually running.
+
 ## Critical Constraints
 
 - `N8N_ENCRYPTION_KEY` — generated once by `setup.sh` via `openssl rand -hex 32`. **Never change it after first run**; doing so breaks all stored n8n credentials.
 - `BASEROW_SECRET_KEY` — Django secret key; same rule applies.
-- Both databases (`N8N_DB_NAME`, `BASEROW_DB_NAME`) are created automatically by each app on first run inside the shared Postgres container. The `POSTGRES_DB` var is just the default DB created at container init and isn't used by n8n or Baserow directly.
+- `POSTGRES_DB` is just the default DB created at container init and isn't used by n8n or Baserow directly. **Neither app creates its own database on connect** (discovered via the smoke-test job actually booting the stack, 2026-08-23 — the repo had never been booted for real before that). `postgres-init/init-app-databases.sh` creates `N8N_DB_NAME` and `BASEROW_DB_NAME` explicitly via Postgres's `docker-entrypoint-initdb.d` mechanism, which only runs once, against an empty `postgres_data` volume. An existing volume from before this fix needs the databases created manually — see `docs/troubleshooting.md` § Can't Connect to PostgreSQL.
 - Volume names in `backup.sh` are detected dynamically via `docker compose config` — no hardcoded prefix. The script requires `.env` to exist so compose config can resolve variable substitutions.
 - `backup.sh` backs up `n8n_data`, `baserow_data`, and `postgres_data` only. `caddy_data` and `caddy_config` are intentionally excluded — certificates are re-issuable from Let's Encrypt for free.
 - n8n and Baserow start only after Postgres passes its healthcheck (`pg_isready`). If either service fails to start, confirm Postgres is `(healthy)` first: `docker compose ps postgres`.
 - `setup.sh` uses a `sed_inplace()` helper (macOS vs Linux `sed -i` differences). Any additions to that script that write to `.env` must use the same helper.
+- All four images are version-pinned (no `:latest`). Bump via the Dependabot PR, not by hand, unless you have a specific reason to jump ahead of it — and review the linked changelog before merging either way.
+- `.env.example` ships with `N8N_ENCRYPTION_KEY` and `BASEROW_SECRET_KEY` blank by design (real users generate them via `setup.sh`). Anything that boots the stack directly from `.env.example` (CI, a quick local test) must generate real values first — Baserow's Django backend won't start with an empty `SECRET_KEY`. See the "Prepare .env" step in `validate.yml`'s smoke-test job for the pattern.
 
 ## Docs
 
@@ -78,3 +95,4 @@ shellcheck --severity=error scripts/setup.sh scripts/backup.sh scripts/update.sh
 - `docs/backup-and-restore.md` — restore procedure + cron automation
 - `docs/updating-services.md` — safe upgrade process
 - `docs/troubleshooting.md` — common issues
+- `docs/adr/` — architecture decision records (reverse proxy choice, shared Postgres, two-network topology)

Makefile

@@ -1,20 +1,24 @@
 .DEFAULT_GOAL := help
 
-.PHONY: help up down restart dev logs backup update setup
+.PHONY: help up down restart dev logs backup update setup test restore backup-test security-scan
 
 help:
 	@echo "Self-Hosted Automation Stack"
 	@echo ""
 	@echo "Usage: make <target>"
 	@echo ""
-	@echo "  setup     Interactive first-run setup (writes .env, starts stack)"
-	@echo "  up        Start the production stack"
-	@echo "  down      Stop and remove containers"
-	@echo "  restart   Restart all services"
-	@echo "  dev       Start in dev mode (direct ports, no Caddy)"
-	@echo "  logs      Follow logs for all services"
-	@echo "  backup    Back up all volumes to ./backups/"
-	@echo "  update    Pull latest images and redeploy"
+	@echo "  setup          Interactive first-run setup (writes .env, starts stack)"
+	@echo "  up             Start the production stack"
+	@echo "  down           Stop and remove containers"
+	@echo "  restart        Restart all services"
+	@echo "  dev            Start in dev mode (direct ports, no Caddy)"
+	@echo "  logs           Follow logs for all services"
+	@echo "  backup         Back up all volumes to ./backups/"
+	@echo "  update         Pull latest images and redeploy"
+	@echo "  test           Boot the dev stack and verify health endpoints respond"
+	@echo "  restore        Restore from a backup: make restore TIMESTAMP=20260823_140000"
+	@echo "  backup-test    Full backup -> destroy -> restore -> verify drill (needs a running stack)"
+	@echo "  security-scan  Scan the four pinned images with the Trivy CLI (must be installed locally)"
 
 setup:
 	bash scripts/setup.sh
@@ -39,3 +43,21 @@ backup:
 
 update:
 	bash scripts/update.sh
+
+test:
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
+	bash scripts/wait-for-healthy.sh postgres n8n baserow
+	curl -sf http://localhost:5678/healthz && echo "n8n: healthy"
+	curl -sf http://localhost:8080/api/_health/ && echo "baserow: healthy"
+
+restore:
+	bash scripts/restore.sh $(TIMESTAMP)
+
+backup-test:
+	bash scripts/backup-drill.sh
+
+security-scan:
+	@for img in n8nio/n8n:2.35.7 baserow/baserow:2.3.3 postgres:15.19-alpine3.24 caddy:2.11.4-alpine; do \
+		echo "--- $$img ---"; \
+		trivy image --severity CRITICAL,HIGH --ignore-unfixed "$$img"; \
+	done