Migrate writable data families from SQLite to Supabase Postgres

.env.example

@@ -133,18 +133,21 @@ HTTPS_ONLY=true
 # LOG_FORMAT: text (default, human-readable) or json (Railway / structured aggregators)
 # LOG_FORMAT=text
 
-# ── Database Paths ────────────────────────────────────────────────────────────
-# Every DB has a DB_<name>_PATH override. Defaults sit under data/ alongside
+# ── Postgres (writable families: users / parses / census / zones / raids) ────
+# One database, five schemas (db/migrations/). The app resolves the DSN as
+# DATABASE_URL → SUPABASE_DB_URL → POSTGRES_CONNECTION_STRING. Production
+# uses the Supabase SESSION pooler (:5432, sslmode=require) — NOT the
+# transaction pooler (:6543), which breaks prepared statements and the
+# per-checkout search_path. Migrations apply at app startup.
+# DATABASE_URL=postgresql://postgres.<ref>:<password>@<region>.pooler.supabase.com:5432/postgres?sslmode=require
+#
+# Local test suite target (tests/fixtures/pg.py; created automatically):
+# TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/eq2lexicon_test
+
+# ── SQLite catalogue paths (read-only Census mirrors) ────────────────────────
+# The remaining file-backed catalogues. Defaults sit under data/ alongside
 # the application; in Railway, point each one at the persistent volume mount.
-# The DB_ prefix keeps them grouped alphabetically in your env-var list.
 #
-#   DB_USERS_PATH       Discord logins, claims, api_tokens, roles, notifications.
-#                       Backed up via litestream → R2.
-#   DB_PARSES_PATH      Normalised ACT parse uploads (the rankings + DPS boards).
-#                       Backed up via litestream → R2.
-#   DB_CENSUS_PATH      Persistent best-known cache of Census character/guild
-#                       lookups (web read path never blocks on the Census API).
-#                       Lives on the volume so it survives redeploys.
 #   DB_ITEMS_PATH       Census item cache (item tooltips, stats, set bonuses).
 #                       Local-generated via scripts/download_items.py.
 #   DB_SPELLS_PATH      Census spell cache (icons, tiers, given_by). Local-only.
@@ -155,20 +158,11 @@ HTTPS_ONLY=true
 #   DB_AAS_PATH         AA tree + node catalogue built from data/AAs/trees via
 #                       scripts/build_aas_db.py. Committed pre-populated (like
 #                       classes.db) so no upload is needed.
-#   DB_ZONES_PATH       Zone + raid-boss roster reference. Built locally from
-#                       scripts/dev/eq2_raid_bosses.review.txt — manual upload.
-#   DB_RAIDS_PATH       Raid strategies + ACT trigger packs (human-edited content).
-#                       Manual upload — accumulates edits, do not clobber.
 #   DB_ACT_EXPORT_PATH  Path to ACT's OWN export.db that parses/act_reader.py reads
 #                       from at ingest time (dev-only; not used in production).
 #
-# DB_USERS_PATH=/data/users.db
-# DB_PARSES_PATH=/data/parses/parses.db
-# DB_CENSUS_PATH=/data/census/census.db
 # DB_ITEMS_PATH=/data/items/items.db
 # DB_SPELLS_PATH=/data/spells/spells.db
 # DB_RECIPES_PATH=/data/recipes/recipes.db
 # DB_AAS_PATH=/data/AAs/aas.db
-# DB_ZONES_PATH=/data/zones/zones.db
-# DB_RAIDS_PATH=/data/raids/raids.db
 # DB_ACT_EXPORT_PATH=/data/parses/act_export.db

.github/workflows/ci.yml

@@ -15,6 +15,24 @@ jobs:
   python:
     name: Python (lint + types + tests)
     runs-on: ubuntu-latest
+    services:
+      # Users-family tests run against real PostgreSQL (tests/fixtures/pg.py
+      # provisions schemas from db/migrations/); SQLite families still use
+      # tmpdir files. Match the local dev install's major version.
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: eq2lexicon_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/eq2lexicon_test
     steps:
       - uses: actions/checkout@v7
 

.github/workflows/pg-backup.yml

@@ -0,0 +1,58 @@
+name: Nightly Postgres backup
+
+# pg_dump of the five family schemas → Cloudflare R2 (the bucket litestream
+# used pre-migration, repurposed). Restore drill:
+#   pg_restore -d "$DATABASE_URL" --clean --if-exists eq2lexicon-<ts>.dump
+# Secrets: SUPABASE_DB_URL (session pooler, sslmode=require), R2_ENDPOINT,
+# R2_BUCKET, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY.
+
+on:
+  schedule:
+    - cron: "17 4 * * *"  # 04:17 UTC daily
+  workflow_dispatch:
+
+jobs:
+  dump:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install PostgreSQL 17 client
+        run: |
+          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+          wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc > /dev/null
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq postgresql-client-17
+
+      - name: pg_dump (family schemas only)
+        env:
+          DATABASE_URL: ${{ secrets.SUPABASE_DB_URL }}
+        run: |
+          ts=$(date -u +%Y%m%dT%H%M%SZ)
+          dump="eq2lexicon-$ts.dump"
+          # Only our schemas — Supabase-internal schemas (auth/storage/…)
+          # are not ours to dump and would fail on permissions.
+          pg_dump -Fc -d "$DATABASE_URL" \
+            -n users -n parses -n census -n zones -n raids -n public \
+            -f "$dump"
+          ls -la "$dump"
+          echo "DUMP=$dump" >> "$GITHUB_ENV"
+
+      - name: Upload to R2 and prune dumps older than 30 days
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
+          R2_BUCKET: ${{ secrets.R2_BUCKET }}
+        run: |
+          aws s3 cp "$DUMP" "s3://$R2_BUCKET/pgdump/$DUMP" --endpoint-url "$R2_ENDPOINT"
+          cutoff=$(date -u -d '30 days ago' +%Y%m%dT%H%M%SZ)
+          aws s3 ls "s3://$R2_BUCKET/pgdump/" --endpoint-url "$R2_ENDPOINT" \
+            | awk '{print $4}' \
+            | while read -r f; do
+                [ -n "$f" ] || continue
+                tspart=$(printf '%s' "$f" | sed -E 's/^eq2lexicon-([0-9TZ]+)\.dump$/\1/')
+                if [ "$tspart" != "$f" ] && [ "$tspart" \< "$cutoff" ]; then
+                  echo "pruning $f"
+                  aws s3 rm "s3://$R2_BUCKET/pgdump/$f" --endpoint-url "$R2_ENDPOINT"
+                fi
+              done

.gitignore

@@ -346,3 +346,12 @@ data/census/census.db-shm
 backend/**/data/
 scripts/dev/*_class_refs*/
 scripts/dev/.epics_wikitext_cache.json
+
+# Local MCP server config (Claude Code) — carries the Supabase personal
+# access token. Machine-local only, never committed.
+.mcp.json
+
+# Cutover artifacts: restored prod SQLite replicas + copy-script audit
+# reports (both carry real user data) — never committed.
+/restore/
+/reports/