From 29ab1d0e09181b869ec1179f6f15a4bf9516bd8f Mon Sep 17 00:00:00 2001 From: Alistair Young Date: Sat, 10 Oct 2026 09:02:39 -0500 Subject: [PATCH] Add local Docker restore testing (phase 2); release 0.2.0 scripts/restore-test.sh restore-tests backup sets on any machine with Docker, never on the cluster: for each target it starts a throwaway scratch server matching the recorded server version (postgres:, mariadb:, mongo: as a single-node replica set) in its own compose project, restores the set into it, checks the result, and removes everything. Backups are mounted read-only and read as their owner; scratch data can live in RAM (--tmpfs); --keep leaves a scratch server up for inspection; scratch startup is retried, since MariaDB's first boot can overrun its entrypoint's limit on busy hosts. New subcommands, needing no config file: - dbbackup inspect PATH: describe a set (text, json, or shell-quoted env). - dbbackup restore-test PATH: verify, restore into a scratch server, and compare it with the set's inventory. Missing databases, objects, users or roles are errors; row-count differences are warnings. Row counts: the restored side is always counted exactly. InnoDB's TABLE_ROWS estimate can be off by orders of magnitude, so it's no longer recorded; Aria/MyISAM counts, pg reltuples and Mongo's metadata counts still are. The new exact_counts option records COUNT(*)s at backup time for exact comparison. 0.1.x MariaDB sets have their counts skipped. make restore-test-check (also in CI) backs up the compose stack and runs the script against the result. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 1 + Makefile | 9 +- README.md | 8 ++ compose.restore-test.yaml | 92 ++++++++++++++ deploy/k8s/config.yaml | 3 + deploy/k8s/kustomization.yaml | 2 +- docs/restore.md | 58 ++++++++- pyproject.toml | 2 +- scripts/restore-test.sh | 180 +++++++++++++++++++++++++++ src/dbbackup/__init__.py | 2 +- src/dbbackup/cli.py | 137 ++++++++++++++++---- src/dbbackup/config.py | 12 +- src/dbbackup/engines/base.py | 8 +- src/dbbackup/engines/mariadb.py | 20 ++- src/dbbackup/engines/mongodb.py | 10 +- src/dbbackup/engines/postgres.py | 23 +++- src/dbbackup/restoretest.py | 135 ++++++++++++++++++++ src/dbbackup/runner.py | 2 +- src/dbbackup/storage.py | 12 ++ tests/integration/config.yaml | 1 + tests/integration/test_end_to_end.py | 5 + tests/test_config.py | 12 ++ tests/test_restoretest.py | 151 ++++++++++++++++++++++ 23 files changed, 835 insertions(+), 50 deletions(-) create mode 100644 compose.restore-test.yaml create mode 100755 scripts/restore-test.sh create mode 100644 src/dbbackup/restoretest.py create mode 100644 tests/test_restoretest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e0884b..3e15078 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: make integration + - run: make restore-test-check - if: failure() run: docker compose logs --no-color - if: always() diff --git a/Makefile b/Makefile index 12ba735..4c035ac 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= .venv/bin/python IMAGE ?= ghcr.io/arkane-systems/db-backup TAG ?= dev -.PHONY: venv lint format test integration stack-up stack-down image +.PHONY: venv lint format test integration restore-test-check stack-up stack-down image venv: python3 -m venv .venv @@ -25,6 +25,13 @@ test: integration: stack-up docker compose --profile tool run --rm --build tests +# Phase-2 restore testing, end to end: back up the compose.yaml servers, then run +# scripts/restore-test.sh on the result (scratch servers, restore, inventory check). +restore-test-check: stack-up + rm -rf .backups && mkdir -p .backups + env UID=$$(id -u) GID=$$(id -g) docker compose --profile tool run --rm --build dbbackup backup pg maria mongo mongo-filtered + scripts/restore-test.sh --build .backups + stack-up: mkdir -p .backups docker compose up -d --wait diff --git a/README.md b/README.md index de1dbda..857a8a7 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ A containerized database backup solution: logical backups of any number of **Mar - **Isolates failures.** One server's failure never stops the others. A target is pruned only after one of its own backups succeeds, and the newest good set is always kept. - **Reports to MQTT,** with Home Assistant discovery. Failures, and backups that have *stopped happening*, can raise [Alert Redux](https://github.com/arkane-systems/ha-alert-redux) alerts; see [docs/home-assistant.md](docs/home-assistant.md). - **Restores with one command,** `dbbackup restore`. Every set can also be restored with stock client tools; see [docs/restore.md](docs/restore.md). +- **Tests restores locally.** `scripts/restore-test.sh` restores sets into throwaway Docker servers of the matching version, and checks every database, table, collection, user and role against what the backup recorded. Supported servers: MariaDB 11.4+, PostgreSQL 18+, MongoDB 8.3+ (as a replica set, for `--oplog`). Older versions may well work: the tool logs a warning and carries on. The image ships the PostgreSQL 18 client, the MariaDB 11.8 client and MongoDB Database Tools 100.19. It is built for linux/amd64 only, because MongoDB publishes its Debian tools packages only for x86-64. @@ -64,6 +65,7 @@ Secrets never go in the file. Each one is named by an environment variable (`pas | `uri_env`/`uri_file` | mongodb target | (none) | A full connection URI, instead of host/port/username. | | `uri_options` | mongodb target | (none) | Extra URI options when connecting by host, e.g. `replicaSet: rs0`. | | `dump_args` | target | (none) | Extra arguments for each per-database dump command. | +| `exact_counts` | `defaults` or target | `false` | Record exact `COUNT(*)` row counts in each set's inventory, for restore tests to check. This scans every table at backup time. Otherwise the cheap estimates are recorded; MariaDB's InnoDB estimates are too unreliable, so they're left out. | | `mqtt` | top level | (none) | `host`, `port`, `username`, `password_env`/`password_file`, `tls`, `topic_prefix` (`dbbackup`), `ha_discovery` (`false`), `discovery_prefix` (`homeassistant`). | For MongoDB, `include` or `exclude` changes how the backup is made. `mongodump` can't leave databases out of a whole-instance dump, so such a target gets one archive per database instead. Those dumps can't use `--oplog`, so they are consistent only per collection, and the manifest records a warning saying so. Leave both unset for point-in-time consistency. @@ -129,8 +131,13 @@ dbbackup verify [TARGET ...] [--set S | --all] re-check checksums and dump in dbbackup prune [TARGET ...] [--dry-run] apply retention without backing up dbbackup restore TARGET [--set S] [-d DB ...] [--host H --port P --username U --password-env VAR | --uri-env VAR] [--no-globals] [--force] +dbbackup inspect PATH [--set S] [--format text|env|json] describe a set (PATH: a set or target directory; no config needed) +dbbackup restore-test PATH [--set S] (--host H --username U --password-env VAR | --uri-env VAR) [--tolerance F] + restore a set into a scratch server and check it against the backup's inventory ``` +To test restores routinely, without touching real servers or the cluster, run `scripts/restore-test.sh /path/to/backups` on any machine with Docker. It restores each target's latest set into a throwaway server of the matching version, checks the result, and cleans up. See [docs/restore.md](docs/restore.md#testing-restores). + Exit status: 0 on success, 1 if anything failed, 2 for configuration or usage errors. ## Development @@ -139,6 +146,7 @@ Exit status: 0 on success, 1 if anything failed, 2 for configuration or usage er make venv # .venv with the package, pytest and ruff make lint test # ruff, and unit tests (no servers needed) make integration # start compose.yaml's servers, then back up and restore them for real +make restore-test-check # back up those servers, then run scripts/restore-test.sh on the result make stack-down # remove the test servers ``` diff --git a/compose.restore-test.yaml b/compose.restore-test.yaml new file mode 100644 index 0000000..a750123 --- /dev/null +++ b/compose.restore-test.yaml @@ -0,0 +1,92 @@ +# Scratch servers for restore testing; driven by scripts/restore-test.sh, which +# starts one scratch server per target in its own throwaway compose project, +# restores into it, checks the result, and removes everything (`down -v`). +# +# Set by the script: +# BACKUP_ROOT the backup share (mounted read-only) +# SCRATCH_IMAGE the scratch server's image, matching the backed-up server's version +# SCRATCH_STORAGE "volume" (default: an anonymous volume) or "tmpfs" (RAM: faster, but the +# restored data must fit in memory) +# DBBACKUP_IMAGE the db-backup image to test with +# RUN_AS uid:gid to read the backups as (the backup root's owner) +# +# Nothing is published on the host: the scratch servers are reachable only on +# the project's own network. + +x-scratch-env: &scratch-env + # Throwaway credentials for throwaway servers. + SCRATCH_PASSWORD: restore-test + SCRATCH_MONGO_URI: mongodb://root:restore-test@scratch-mongodb:27017/?authSource=admin&directConnection=true + +services: + scratch-postgres: + profiles: [postgres] + image: ${SCRATCH_IMAGE:-postgres:18} + environment: + POSTGRES_PASSWORD: restore-test + volumes: + - type: ${SCRATCH_STORAGE:-volume} + target: /var/lib/postgresql + healthcheck: + test: [CMD, pg_isready, -U, postgres] + interval: 2s + retries: 60 + + scratch-mariadb: + profiles: [mariadb] + image: ${SCRATCH_IMAGE:-mariadb:11.4} + environment: + MARIADB_ROOT_PASSWORD: restore-test + volumes: + - type: ${SCRATCH_STORAGE:-volume} + target: /var/lib/mysql + healthcheck: + test: [CMD, healthcheck.sh, --connect, --innodb_initialized] + interval: 2s + retries: 60 + + scratch-mongodb: + profiles: [mongodb] + image: ${SCRATCH_IMAGE:-mongo:8.3} + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: restore-test + # A single-node replica set, so --oplogReplay restores behave as in production. + # Replica sets with auth need a keyfile; make a throwaway one. + entrypoint: + - bash + - -c + - | + openssl rand -base64 756 > /tmp/keyfile && chmod 400 /tmp/keyfile && chown mongodb:mongodb /tmp/keyfile + exec docker-entrypoint.sh "$$@" + - -- + command: [mongod, --replSet, rs0, --keyFile, /tmp/keyfile, --bind_ip_all] + volumes: + - type: ${SCRATCH_STORAGE:-volume} + target: /data/db + - type: ${SCRATCH_STORAGE:-volume} + target: /data/configdb + healthcheck: + test: + - CMD + - mongosh + - --quiet + - -u + - root + - -p + - restore-test + - --eval + - "try { rs.status().myState === 1 ? 0 : quit(1) } catch (e) { rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'scratch-mongodb:27017'}]}); quit(1) }" + interval: 3s + retries: 60 + + # The tool, with the backups mounted read-only; always run with --no-deps. + restore-test: + profiles: [tool] + image: ${DBBACKUP_IMAGE:-ghcr.io/arkane-systems/db-backup:latest} + user: ${RUN_AS:-1004:1004} + read_only: true + tmpfs: [/tmp] + environment: *scratch-env + volumes: + - ${BACKUP_ROOT:?BACKUP_ROOT must be set}:/backups:ro diff --git a/deploy/k8s/config.yaml b/deploy/k8s/config.yaml index ecc026a..46f0c6d 100644 --- a/deploy/k8s/config.yaml +++ b/deploy/k8s/config.yaml @@ -11,6 +11,9 @@ defaults: retry_delay_s: 60 stale_partial_hours: 24 # abandoned .partial sets older than this are removed lock_stale_hours: 24 # a lock older than this is assumed abandoned + # Record exact row counts (a COUNT(*) of every table, each night) for restore + # tests to check, rather than cheap estimates. MariaDB/InnoDB has no usable estimate. + exact_counts: false mqtt: host: mqtt.example.lan # CHANGE ME diff --git a/deploy/k8s/kustomization.yaml b/deploy/k8s/kustomization.yaml index d26e4f2..42b2893 100644 --- a/deploy/k8s/kustomization.yaml +++ b/deploy/k8s/kustomization.yaml @@ -23,7 +23,7 @@ configMapGenerator: images: - name: ghcr.io/arkane-systems/db-backup - newTag: "0.1.1" + newTag: "0.2.0" labels: - pairs: diff --git a/docs/restore.md b/docs/restore.md index 3e75a98..c9f20bc 100644 --- a/docs/restore.md +++ b/docs/restore.md @@ -5,7 +5,7 @@ This is the disaster-recovery runbook. It covers two cases: - **With the tool**: `dbbackup restore`, the quick path. - **With stock tools only**: `psql`, `pg_restore`, `mariadb`, `mongorestore` and `zstd`, for when the tool or its image isn't available. Nothing in a backup set needs this project to read it. -Test restores before you need them. A backup that has never been restored is only a hope. +Test restores before you need them. A backup that has never been restored is only a hope. [Testing restores](#testing-restores) shows how to do it routinely, on a workstation. ## Finding a backup set @@ -68,7 +68,7 @@ spec: securityContext: {runAsNonRoot: true, runAsUser: 1004, runAsGroup: 1004} containers: - name: restore - image: ghcr.io/arkane-systems/db-backup:0.1.1 + image: ghcr.io/arkane-systems/db-backup:0.2.0 args: [restore, postgres-main, --host, new-postgres.example.lan, --username, postgres, --password-env, ADMIN_PASSWORD] env: - name: ADMIN_PASSWORD @@ -85,7 +85,7 @@ spec: ```bash docker run --rm -v /mnt/db-backups:/backups:ro -v $PWD/config.yaml:/etc/dbbackup/config.yaml:ro \ - -e ADMIN_PASSWORD ghcr.io/arkane-systems/db-backup:0.1.1 \ + -e ADMIN_PASSWORD ghcr.io/arkane-systems/db-backup:0.2.0 \ restore mariadb-main --host new-mariadb.example.lan --username root --password-env ADMIN_PASSWORD ``` @@ -169,3 +169,55 @@ zstd -dc admin.archive.zst | mongorestore --uri '...' --archive - **Point applications at the new server,** and check they can log in. Restored application accounts keep their old passwords. - **Check the restore against the backup's inventory.** The manifest's `inventory` lists every table and collection with an approximate row count, plus every user and role, as they were at backup time. It's a quick sanity check that nothing went missing. - **Update the backup config** if the server's address changed, so tonight's backup covers the new server. + +## Testing restores + +`scripts/restore-test.sh` proves a backup can actually be restored, without touching any real server and without using the cluster. It runs on any machine with Docker and the backup share mounted (or a copy of it): + +```bash +scripts/restore-test.sh /mnt/db-backups # the latest set of every target +scripts/restore-test.sh /mnt/db-backups postgres-main # just one target +scripts/restore-test.sh --set 20261009T021500Z /mnt/db-backups mariadb-main +``` + +For each target, the script: + +1. Reads the set's manifest, using the db-backup image, and starts a throwaway scratch server matching the backed-up server's version: `postgres:`, `mariadb:` or `mongo:`. MongoDB runs as a single-node replica set, so the oplog is replayed as it would be in production. Each scratch server has its own compose project and network, and nothing is published on the host. +2. Runs `dbbackup restore-test`, which: + - re-checks every file's checksum, and the dumps' integrity + - restores the set exactly as `dbbackup restore` would, users and roles included + - compares the restored server with the inventory recorded at backup time +3. Removes the scratch server and its data (unless you pass `--keep`), and goes on to the next target. + +The comparison treats these as **errors**: + +- a database missing from the restored server +- a missing table, view, materialized view or collection +- a missing user or role +- an object restored as a different kind (say, a view that came back as a table) + +Row-count differences are only **warnings**. The restored server is always counted exactly; what it's compared with depends on the target's `exact_counts` setting: + +- **`exact_counts: true`.** The set recorded a `COUNT(*)` of every table, just before the dump, and the counts must match exactly. Writes between the count and the dump's snapshot show up as small differences. +- **The default.** The set recorded the servers' cheap estimates (`pg_class.reltuples`, `estimatedDocumentCount`, and Aria/MyISAM row counts), and a difference of more than 25% (and more than 100 rows) is reported. InnoDB's estimate (`TABLE_ROWS`) can be off by orders of magnitude, for example straight after a bulk load, so MariaDB InnoDB tables have no recorded count: only their presence is checked, unless you set `exact_counts`. Sets made by 0.1.x recorded those InnoDB estimates anyway; their MariaDB counts are skipped, with a note saying why. + +The run ends with a summary table, including how long each restore took, which is a rough figure for how long recovery would take. The script exits non-zero if any target fails. + +| Option | Effect | +|---|---| +| `-s`, `--set SET` | Test this set instead of the latest. | +| `-i`, `--image IMAGE` | The db-backup image to use (default `$DBBACKUP_IMAGE`, else `ghcr.io/arkane-systems/db-backup:latest`). | +| `-b`, `--build` | Build the image from the checkout instead. | +| `-t`, `--tmpfs` | Keep the scratch servers' data in RAM. Faster, but the restored data must fit in memory. | +| `-k`, `--keep` | Leave each scratch server running afterwards, to look around in. The script prints how to remove it. | + +To use a scratch image other than the one matching the recorded version, set `SCRATCH_IMAGE_POSTGRES`, `SCRATCH_IMAGE_MARIADB` or `SCRATCH_IMAGE_MONGODB`. For example, `SCRATCH_IMAGE_POSTGRES=postgres:19` rehearses an upgrade. + +The backups are mounted read-only, and are read as the backup root's owner (`pg_dump`'s directory archives are private to their owner). + +To restore-test one set by hand against a server you've started yourself: + +```bash +dbbackup inspect /backups/postgres-main # what's in the latest set +dbbackup restore-test /backups/postgres-main --host scratch --username postgres --password-env SCRATCH_PASSWORD +``` diff --git a/pyproject.toml b/pyproject.toml index 8bff90d..64f40b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dbbackup" -version = "0.1.1" +version = "0.2.0" description = "Containerized logical backups of MariaDB, PostgreSQL and MongoDB servers, with GFS retention" readme = "README.md" license = "MIT" diff --git a/scripts/restore-test.sh b/scripts/restore-test.sh new file mode 100755 index 0000000..9883450 --- /dev/null +++ b/scripts/restore-test.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# Restore-test backup sets in throwaway local database servers. +# +# For each target, starts a scratch server matching the backed-up server's version +# (in its own compose project), restores the chosen set into it with +# `dbbackup restore-test`, checks the result against the set's recorded inventory, +# and removes the scratch server and its data. Needs only Docker on the host. +# +# scripts/restore-test.sh [options] BACKUP_ROOT [TARGET...] +# +# BACKUP_ROOT is the backup share (mounted locally) or a copy of it; it is mounted +# read-only. With no TARGETs, every target directory with a finished set is tested. +# +# Options: +# -s, --set SET set to test: "latest" (default) or a timestamp like 20261009T021500Z +# -i, --image IMAGE db-backup image to test with (default: $DBBACKUP_IMAGE, else +# ghcr.io/arkane-systems/db-backup:latest) +# -b, --build build the image from this checkout instead +# -t, --tmpfs keep the scratch servers' data in RAM: faster, but it must fit +# -k, --keep leave each scratch server running afterwards, for inspection +# -h, --help show this help +# +# Scratch images follow the recorded server version (postgres:, +# mariadb:, mongo:); override them with +# SCRATCH_IMAGE_POSTGRES, SCRATCH_IMAGE_MARIADB or SCRATCH_IMAGE_MONGODB. +# +# Exits non-zero if any target fails. + +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +compose_file="$here/compose.restore-test.yaml" + +usage() { sed -n '2,/^$/{s/^# \{0,1\}//;p}' "${BASH_SOURCE[0]}"; } +die() { echo "restore-test: $*" >&2; exit 2; } + +set_name=latest +image=${DBBACKUP_IMAGE:-ghcr.io/arkane-systems/db-backup:latest} +build=false +storage=volume +keep=false +positional=() +while (($#)); do + case $1 in + -s|--set) set_name=${2:?--set needs a value}; shift 2 ;; + -i|--image) image=${2:?--image needs a value}; shift 2 ;; + -b|--build) build=true; shift ;; + -t|--tmpfs) storage=tmpfs; shift ;; + -k|--keep) keep=true; shift ;; + -h|--help) usage; exit 0 ;; + --) shift; positional+=("$@"); break ;; + -*) die "unknown option $1 (see --help)" ;; + *) positional+=("$1"); shift ;; + esac +done +((${#positional[@]})) || { usage >&2; exit 2; } + +command -v docker >/dev/null || die "docker is not installed" +root=$(cd "${positional[0]}" 2>/dev/null && pwd) || die "${positional[0]}: no such directory" +targets=("${positional[@]:1}") + +# Read the backups as their owner (pg_dump's directory archives are mode 0700). +run_as=$(stat -c %u:%g "$root" 2>/dev/null || echo "$(id -u):$(id -g)") + +if $build; then + image=dbbackup:restore-test + echo "==> building $image from $here" + if ! build_log=$(docker build --load --quiet --target runtime -t "$image" "$here" 2>&1); then + echo "$build_log" >&2 + die "image build failed" + fi +fi + +if ((${#targets[@]} == 0)); then + for dir in "$root"/*/; do + name=$(basename "$dir") + compgen -G "$dir[0-9]*T[0-9]*Z/manifest.json" >/dev/null && targets+=("$name") + done + ((${#targets[@]})) || die "no target directories with finished sets in $root" +fi + +export BACKUP_ROOT=$root DBBACKUP_IMAGE=$image SCRATCH_STORAGE=$storage RUN_AS=$run_as + +current_project="" +cleanup() { + if [[ -n $current_project ]] && ! $keep; then + docker compose -p "$current_project" -f "$compose_file" --profile '*' down -v --remove-orphans >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT +trap 'echo; echo "restore-test: interrupted" >&2; exit 130' INT TERM + +results=() +failed=0 +for target in "${targets[@]}"; do + project="dbbackup-rt-$(tr -c 'a-z0-9_\n-' '_' <<<"${target,,}")" + dc=(docker compose --progress quiet -p "$project" -f "$compose_file") + current_project=$project + echo + echo "==> $target" + + if ! info=$("${dc[@]}" run --rm --no-deps -T restore-test inspect "/backups/$target" --set "$set_name" --format env); then + results+=("$target|-|-|ERROR (no usable set)|-") + failed=1 + cleanup + continue + fi + DBB_TYPE="" DBB_SET="" DBB_SERVER_VERSION="" + while IFS= read -r line; do + [[ $line =~ ^DBB_[A-Z_]+= ]] && eval "$line" # values are shell-quoted by dbbackup + done <<<"$info" + + version=${DBB_SERVER_VERSION%%-*} # 11.4.13-MariaDB-ubu2404 -> 11.4.13 + major=${version%%.*} + minor=${version#*.}; minor=${minor%%.*} + case $DBB_TYPE in + postgres) + scratch_image=${SCRATCH_IMAGE_POSTGRES:-postgres:$major} + conn=(--host scratch-postgres --username postgres --password-env SCRATCH_PASSWORD) ;; + mariadb) + scratch_image=${SCRATCH_IMAGE_MARIADB:-mariadb:$major.$minor} + conn=(--host scratch-mariadb --username root --password-env SCRATCH_PASSWORD) ;; + mongodb) + scratch_image=${SCRATCH_IMAGE_MONGODB:-mongo:$major.$minor} + conn=(--uri-env SCRATCH_MONGO_URI) ;; + *) die "$target: unknown server type '$DBB_TYPE'" ;; + esac + echo " set $DBB_SET ($DBB_TYPE $DBB_SERVER_VERSION), scratch server $scratch_image" + + # Some images' first-boot initialization (notably MariaDB's, on slow or busy + # hosts) can overrun their entrypoint's own startup limit, so try a few times. + export SCRATCH_IMAGE=$scratch_image + started=false + for attempt in 1 2 3; do + if "${dc[@]}" --profile "$DBB_TYPE" up -d --quiet-pull --wait "scratch-$DBB_TYPE" >/dev/null 2>&1; then + started=true + break + fi + echo " scratch server didn't start (attempt $attempt of 3)" >&2 + if ((attempt == 3)); then + echo " its log:" >&2 + "${dc[@]}" --profile "$DBB_TYPE" logs --tail 30 "scratch-$DBB_TYPE" >&2 || true + fi + "${dc[@]}" --profile '*' down -v >/dev/null 2>&1 || true + done + if ! $started; then + results+=("$target|$DBB_SET|$scratch_image|ERROR (scratch server didn't start)|-") + failed=1 + cleanup + continue + fi + + log=$(mktemp) + start=$SECONDS + if "${dc[@]}" run --rm --no-deps -T restore-test restore-test "/backups/$target/$DBB_SET" "${conn[@]}" 2>&1 | tee "$log"; then + verdict=PASSED + else + verdict=FAILED + failed=1 + fi + restore_time=$(grep -o 'restored in [0-9.]*s' "$log" | tail -1 | cut -d' ' -f3 || true) + rm -f "$log" + results+=("$target|$DBB_SET|$scratch_image|$verdict|${restore_time:-$((SECONDS - start))s total}") + + if $keep; then + echo " kept: SCRATCH_IMAGE=$scratch_image docker compose -p $project -f $compose_file --profile '*' down -v # to remove it" + current_project="" + else + cleanup + current_project="" + fi +done + +echo +printf '%-24s %-17s %-16s %-36s %s\n' TARGET SET SCRATCH RESULT RESTORE +for row in "${results[@]}"; do + IFS='|' read -r t s i r d <<<"$row" + printf '%-24s %-17s %-16s %-36s %s\n' "$t" "$s" "$i" "$r" "$d" +done +exit $failed diff --git a/src/dbbackup/__init__.py b/src/dbbackup/__init__.py index 668c39e..c49409c 100644 --- a/src/dbbackup/__init__.py +++ b/src/dbbackup/__init__.py @@ -1,3 +1,3 @@ """Logical backups of MariaDB, PostgreSQL and MongoDB servers, with GFS retention.""" -__version__ = "0.1.1" +__version__ = "0.2.0" diff --git a/src/dbbackup/cli.py b/src/dbbackup/cli.py index 72e8e11..fa5f735 100644 --- a/src/dbbackup/cli.py +++ b/src/dbbackup/cli.py @@ -1,20 +1,24 @@ -"""Command-line interface: ``dbbackup backup|list|verify|prune|restore``.""" +"""Command-line interface: ``dbbackup backup|list|verify|prune|restore|inspect|restore-test``.""" from __future__ import annotations import argparse +import json import logging import os +import shlex import sys from datetime import timedelta +from pathlib import Path from . import __version__ from .config import ConfigError, load_config from .engines import Connection, EngineError, engine_for from .notify import publish_statuses from .proc import CommandError +from .restoretest import DEFAULT_TOLERANCE, run_restore_test, target_for_set from .runner import backup_targets, human_size -from .storage import StorageError, Store, verify_checksums +from .storage import StorageError, Store, open_set, verify_checksums log = logging.getLogger("dbbackup") @@ -84,6 +88,28 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--force", action="store_true", help="replace databases that already exist") p.set_defaults(func=cmd_restore) + p = sub.add_parser("inspect", help="describe a backup set (no config needed)") + p.add_argument("path", help="a set directory, or a target directory (then --set picks the set)") + p.add_argument("--set", default="latest", help="with a target directory: 'latest' (default) or a timestamp") + p.add_argument("--format", choices=["text", "env", "json"], default="text", help="env: shell-quoted KEY=value lines") + p.set_defaults(func=cmd_inspect) + + p = sub.add_parser("restore-test", help="restore a set into a scratch server and check it against the backup's inventory") + p.add_argument("path", help="a set directory, or a target directory (then --set picks the set)") + p.add_argument("--set", default="latest", help="with a target directory: 'latest' (default) or a timestamp") + p.add_argument("--host", help="the scratch server") + p.add_argument("--port", type=int, help="its port (default: the server type's)") + p.add_argument("--username", help="an admin user on the scratch server") + p.add_argument("--password-env", metavar="VAR", help="environment variable holding that user's password") + p.add_argument("--uri-env", metavar="VAR", help="MongoDB: environment variable holding the scratch server's URI") + p.add_argument( + "--tolerance", + type=float, + default=DEFAULT_TOLERANCE, + help=f"allowed relative difference in estimated row counts (default {DEFAULT_TOLERANCE})", + ) + p.set_defaults(func=cmd_restore_test) + return parser @@ -162,29 +188,8 @@ def cmd_restore(args) -> int: store = Store(config.backup_root) bset = store.find(target.name, args.set) - conn = Connection.for_target(target) - if overriding: - password = None - if args.password_env: - if args.password_env not in os.environ: - raise ConfigError(f"--password-env: environment variable {args.password_env} is not set") - password = os.environ[args.password_env] - uri = None - if args.uri_env: - if target.type != "mongodb": - raise ConfigError("--uri-env is only for mongodb targets") - if args.uri_env not in os.environ: - raise ConfigError(f"--uri-env: environment variable {args.uri_env} is not set") - uri = os.environ[args.uri_env] - conn = Connection( - # A URI says where it points; don't leave the target's host around to mislead the logs. - host=None if uri else (args.host or target.host), - port=args.port or target.port, - username=args.username or target.username, - password=password, - uri=uri, - ) - elif args.port: + conn = connection_from_args(args, target) if overriding else Connection.for_target(target) + if args.port and not overriding: conn = conn.overridden(port=args.port) problems = verify_checksums(bset) @@ -213,5 +218,87 @@ def cmd_restore(args) -> int: return EXIT_OK +def connection_from_args(args, target) -> Connection: + """The connection given by --host/--port/--username/--password-env/--uri-env, defaulting to the target's.""" + password = None + if args.password_env: + if args.password_env not in os.environ: + raise ConfigError(f"--password-env: environment variable {args.password_env} is not set") + password = os.environ[args.password_env] + uri = None + if args.uri_env: + if target.type != "mongodb": + raise ConfigError("--uri-env is only for mongodb targets") + if args.uri_env not in os.environ: + raise ConfigError(f"--uri-env: environment variable {args.uri_env} is not set") + uri = os.environ[args.uri_env] + return Connection( + # A URI says where it points; don't leave the target's host around to mislead the logs. + host=None if uri else (args.host or target.host), + port=args.port or target.port, + username=args.username or target.username, + password=password, + uri=uri, + ) + + +def cmd_inspect(args) -> int: + bset = open_set(Path(args.path), args.set) + if not bset.ok: + raise StorageError(f"{bset.path} has no valid manifest") + m = bset.manifest + if args.format == "json": + print(json.dumps({k: v for k, v in m.items() if k != "files"}, indent=2, sort_keys=True)) + elif args.format == "env": + values = { + "DBB_TARGET": m["target"], + "DBB_TYPE": m["type"], + "DBB_SET": bset.name, + "DBB_SERVER_VERSION": m.get("server_version", ""), + "DBB_DATABASES": str(len(m.get("databases", []))), + "DBB_SIZE_BYTES": str(m.get("size_bytes", "")), + } + for key, value in values.items(): + print(f"{key}={shlex.quote(value)}") + else: + print(f"target: {m['target']} ({m['type']} {m.get('server_version', '?')}, {m.get('host') or 'URI'})") + print(f"set: {bset.name} finished {m.get('finished')} ({human_size(m.get('size_bytes'))}, {m.get('duration_s')}s)") + print(f"contents: {', '.join(m.get('databases', [])) or '(no databases)'}") + for warning in m.get("warnings", []): + print(f"warning: {warning}") + return EXIT_OK + + +def cmd_restore_test(args) -> int: + bset = open_set(Path(args.path), args.set) + if not bset.ok: + raise StorageError(f"{bset.path} has no valid manifest") + target = target_for_set(bset) + if not (args.host or args.uri_env): + raise ConfigError("restore-test needs the scratch server: --host (with --username/--password-env), or --uri-env") + report = run_restore_test(bset, connection_from_args(args, target), tolerance=args.tolerance) + + name = f"{report.target}/{report.set}" + for warning in report.restore_warnings: + log.warning("%s: restore: %s", name, warning) + for note in report.check.notes: + log.info("%s: %s", name, note) + for warning in report.check.warnings: + log.warning("%s: %s", name, warning) + for error in report.check.errors: + log.error("%s: %s", name, error) + c = report.check + summary = ( + f"{len(bset.manifest['databases'])} database(s) restored in {report.restore_seconds:.1f}s; " + f"checked {c.objects_checked} object(s), {c.counts_checked} row count(s), {c.users_checked} user(s)/role(s): " + f"{len(c.errors)} error(s), {len(c.warnings) + len(report.restore_warnings)} warning(s)" + ) + if report.passed: + log.info("%s: PASSED: %s", name, summary) + return EXIT_OK + log.error("%s: FAILED: %s", name, summary) + return EXIT_FAILED + + if __name__ == "__main__": sys.exit(main()) diff --git a/src/dbbackup/config.py b/src/dbbackup/config.py index ee926c2..7246595 100644 --- a/src/dbbackup/config.py +++ b/src/dbbackup/config.py @@ -74,6 +74,7 @@ class Target: ssl: bool | None # mariadb: None = client default, True = --ssl, False = --skip-ssl uri_options: dict[str, str] # mongodb: extra URI query options when built from host/port dump_args: tuple[str, ...] # extra args appended to each per-database dump command + exact_counts: bool = False # record COUNT(*)s in the inventory (scans every table) instead of estimates @dataclass(frozen=True) @@ -133,7 +134,7 @@ def parse_config(raw: Any, environ: dict[str, str], *, secrets_for: Collection[s defaults = raw.get("defaults") or {} _check_keys( defaults, - {"retention", "compression_level", "retries", "retry_delay_s", "stale_partial_hours", "lock_stale_hours", "jobs"}, + {"retention", "compression_level", "retries", "retry_delay_s", "stale_partial_hours", "lock_stale_hours", "jobs", "exact_counts"}, "defaults", ) base_retention = Retention().merged(defaults.get("retention"), "defaults.retention") @@ -191,6 +192,7 @@ def _parse_target( "ssl", "uri_options", "dump_args", + "exact_counts", }, where, ) @@ -255,6 +257,7 @@ def _parse_target( ssl=ssl, uri_options={str(k): str(v) for k, v in uri_options.items()}, dump_args=_str_list(rt, "dump_args", where), + exact_counts=_bool(rt, "exact_counts", _bool(defaults, "exact_counts", False, "defaults"), where), ) @@ -329,6 +332,13 @@ def _int(section: dict, key: str, default: int, where: str, minimum: int | None return value +def _bool(section: dict, key: str, default: bool, where: str) -> bool: + value = section.get(key, default) + if not isinstance(value, bool): + raise ConfigError(f"{where}.{key}: must be true or false") + return value + + def _str_list(section: dict, key: str, where: str) -> tuple[str, ...]: value = section.get(key) or [] if not isinstance(value, list) or not all(isinstance(v, str) for v in value): diff --git a/src/dbbackup/engines/base.py b/src/dbbackup/engines/base.py index c161968..95780bf 100644 --- a/src/dbbackup/engines/base.py +++ b/src/dbbackup/engines/base.py @@ -60,10 +60,12 @@ def list_databases(self) -> list[str]: """Every user database on the server, system databases excluded.""" @abstractmethod - def inventory(self, databases: list[str]) -> dict: - """Objects, approximate row counts and users, recorded so a restore can be checked against them. + def inventory(self, databases: list[str], *, exact: bool = False) -> dict: + """Objects, row counts and users, recorded so a restore can be checked against them. - Shape: ``{"databases": {db: {"objects": {name: {"kind": str, "rows": int | None}}}}, "users": [str] | None}``. + Shape: ``{"counts": "exact" | "estimated", "databases": {db: {"objects": {name: {"kind": str, "rows": int | None}}}}, + "users": [str] | None}``. With ``exact``, row counts are COUNT(*)s (a scan of every table); otherwise they're + the server's cheap estimates, or None where those are too unreliable to record. """ @abstractmethod diff --git a/src/dbbackup/engines/mariadb.py b/src/dbbackup/engines/mariadb.py index 9154a85..f60e863 100644 --- a/src/dbbackup/engines/mariadb.py +++ b/src/dbbackup/engines/mariadb.py @@ -138,15 +138,27 @@ def _tables(self, databases: list[str]) -> list[list[str | None]]: f"WHERE TABLE_SCHEMA IN ({schemas})" ) - def inventory(self, databases: list[str]) -> dict: + def inventory(self, databases: list[str], *, exact: bool = False) -> dict: result = {db: {"objects": {}} for db in databases} - for schema, name, ttype, _engine, rows in self._tables(databases): + countable = [] + for schema, name, ttype, engine, rows in self._tables(databases): kind = TABLE_KINDS.get(ttype, ttype.lower()) - result[schema]["objects"][name] = {"kind": kind, "rows": int(rows) if rows is not None and kind == "table" else None} + # InnoDB's TABLE_ROWS is a statistics estimate that can be off by orders of + # magnitude (e.g. right after a bulk load), so don't record it; other + # engines' (Aria, MyISAM) are exact. + estimate = int(rows) if kind == "table" and rows is not None and engine != "InnoDB" else None + result[schema]["objects"][name] = {"kind": kind, "rows": None if exact else estimate} + if exact and kind == "table": + countable.append((schema, name)) + if countable: + sql = " UNION ALL ".join(f"SELECT {i}, COUNT(*) FROM {quote_ident(s)}.{quote_ident(n)}" for i, (s, n) in enumerate(countable)) + for i, count in self.query(sql): + schema, name = countable[int(i)] + result[schema]["objects"][name]["rows"] = int(count) users = sorted( name if is_role == "Y" else f"{name}@{host}" for name, host, is_role in self.query("SELECT User, Host, is_role FROM mysql.user") ) - return {"databases": result, "users": users} + return {"counts": "exact" if exact else "estimated", "databases": result, "users": users} # -- backup ---------------------------------------------------------------- diff --git a/src/dbbackup/engines/mongodb.py b/src/dbbackup/engines/mongodb.py index d6610b7..236e76b 100644 --- a/src/dbbackup/engines/mongodb.py +++ b/src/dbbackup/engines/mongodb.py @@ -69,7 +69,7 @@ def list_databases(self) -> list[str]: with self._client() as client: return sorted(d for d in client.list_database_names() if d not in SYSTEM_DATABASES) - def inventory(self, databases: list[str]) -> dict: + def inventory(self, databases: list[str], *, exact: bool = False) -> dict: from pymongo.errors import OperationFailure result = {} @@ -80,7 +80,11 @@ def inventory(self, databases: list[str]) -> dict: name, ctype = coll["name"], coll.get("type", "collection") if name.startswith("system."): continue - rows = client[db][name].estimated_document_count() if ctype == "collection" else None + rows = None + if ctype == "collection": + # The estimate is collection metadata, normally accurate; exact means a scan. + collection = client[db][name] + rows = collection.count_documents({}) if exact else collection.estimated_document_count() objects[name] = {"kind": ctype, "rows": rows} result[db] = {"objects": objects} try: @@ -89,7 +93,7 @@ def inventory(self, databases: list[str]) -> dict: except OperationFailure as e: log.warning("%s: can't list users (%s); restore checks will skip them", self.target.name, e) users = None - return {"databases": result, "users": users} + return {"counts": "exact" if exact else "estimated", "databases": result, "users": users} # -- backup ---------------------------------------------------------------- diff --git a/src/dbbackup/engines/postgres.py b/src/dbbackup/engines/postgres.py index 1a78828..f5699ab 100644 --- a/src/dbbackup/engines/postgres.py +++ b/src/dbbackup/engines/postgres.py @@ -105,11 +105,12 @@ def can_read_passwords(self) -> bool: """Superusers, and members of pg_read_all_data, can read pg_authid and so dump role passwords.""" return self.query("SELECT has_table_privilege('pg_catalog.pg_authid', 'SELECT')")[0][0] == "t" - def inventory(self, databases: list[str]) -> dict: + def inventory(self, databases: list[str], *, exact: bool = False) -> dict: result = {} for db in databases: rows = self.query( - r"""SELECT n.nspname, c.relname, c.relkind, c.reltuples::bigint + r"""SELECT n.nspname, c.relname, c.relkind, c.reltuples::bigint, c.relispopulated, + quote_ident(n.nspname) || '.' || quote_ident(c.relname) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f') AND n.nspname NOT IN ('pg_catalog', 'information_schema') @@ -117,12 +118,22 @@ def inventory(self, databases: list[str]) -> dict: db=db, ) objects = {} - for schema, name, kind, tuples in rows: - rows_est = int(tuples) if kind in ("r", "p", "m") and int(tuples) >= 0 else None - objects[f"{schema}.{name}"] = {"kind": RELKINDS[kind], "rows": rows_est} + countable = [] + for schema, name, kind, tuples, populated, qualified in rows: + key = f"{schema}.{name}" + has_rows = kind in ("r", "p", "m") and populated == "t" + # reltuples is -1 until the table is first vacuumed or analyzed. + estimate = int(tuples) if has_rows and int(tuples) >= 0 else None + objects[key] = {"kind": RELKINDS[kind], "rows": None if exact else estimate} + if exact and has_rows: + countable.append((key, qualified)) + if countable: + sql = " UNION ALL ".join(f"SELECT {i}, count(*) FROM {qualified}" for i, (_, qualified) in enumerate(countable)) + for i, count in self.query(sql, db=db): + objects[countable[int(i)][0]]["rows"] = int(count) result[db] = {"objects": objects} users = [r[0] for r in self.query("SELECT rolname FROM pg_roles WHERE rolname !~ '^pg_' ORDER BY rolname")] - return {"databases": result, "users": users} + return {"counts": "exact" if exact else "estimated", "databases": result, "users": users} # -- backup ---------------------------------------------------------------- diff --git a/src/dbbackup/restoretest.py b/src/dbbackup/restoretest.py new file mode 100644 index 0000000..e6c35f0 --- /dev/null +++ b/src/dbbackup/restoretest.py @@ -0,0 +1,135 @@ +"""Restore testing: restore a set into a scratch server, then check the result +against the inventory its manifest recorded at backup time. + +Anything missing (a database, table, view, collection, user or role) is an error. +Row-count differences are only warnings. The restored side is always counted +exactly (it's a scratch server); what it's compared with depends on the backup: + +- ``counts: exact`` (the target's ``exact_counts``): COUNT(*)s taken just before + the dump, compared exactly. Writes between the count and the dump's snapshot + show up as small differences. +- ``counts: estimated``: the servers' cheap estimates (pg_class.reltuples, + estimatedDocumentCount; Aria/MyISAM row counts), compared within a tolerance. +- No ``counts`` key (backups made by 0.1.x): as estimated, except MariaDB, whose + sets recorded InnoDB's TABLE_ROWS, an estimate too unreliable to compare. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass, field + +from .config import Target, parse_config +from .engines import Connection, engine_for +from .storage import BackupSet, StorageError, verify_checksums + +log = logging.getLogger(__name__) + +DEFAULT_TOLERANCE = 0.25 +# Below this many rows' difference, estimates are never worth a warning. +COUNT_SLACK = 100 + + +@dataclass +class CheckResult: + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + objects_checked: int = 0 + counts_checked: int = 0 + users_checked: int = 0 + notes: list[str] = field(default_factory=list) + + +def counts_match(expected: int, actual: int, tolerance: float, slack: int = COUNT_SLACK) -> bool: + return abs(expected - actual) <= max(tolerance * max(expected, actual), slack) + + +def compare_inventory( + expected: dict, actual: dict, present: set[str], *, tolerance: float = DEFAULT_TOLERANCE, server_type: str | None = None +) -> CheckResult: + """Check a restored server's inventory (``actual``, counted exactly) against the manifest's (``expected``). + + ``present`` is the set of databases that exist on the restored server. + """ + result = CheckResult() + mode = expected.get("counts") + exact = mode == "exact" + compare_counts = not (mode is None and server_type == "mariadb") + if not compare_counts: + result.notes.append("row counts not compared: this backup recorded InnoDB's TABLE_ROWS estimates, which are too unreliable") + for db, info in sorted(expected.get("databases", {}).items()): + if db not in present: + result.errors.append(f"database {db} is missing") + continue + restored = actual.get("databases", {}).get(db, {}).get("objects", {}) + for name, obj in sorted(info.get("objects", {}).items()): + result.objects_checked += 1 + got = restored.get(name) + if got is None: + result.errors.append(f"{db}: {obj['kind']} {name} is missing") + continue + if got["kind"] != obj["kind"]: + result.errors.append(f"{db}: {name} was a {obj['kind']} but was restored as a {got['kind']}") + continue + if compare_counts and obj.get("rows") is not None and got.get("rows") is not None: + result.counts_checked += 1 + if exact and got["rows"] != obj["rows"]: + result.warnings.append(f"{db}: {name} has {got['rows']} rows, but {obj['rows']} were counted at backup time") + elif not exact and not counts_match(obj["rows"], got["rows"], tolerance): + result.warnings.append(f"{db}: {name} has {got['rows']} rows, but the backup estimated about {obj['rows']}") + + expected_users = expected.get("users") + actual_users = actual.get("users") + if expected_users is not None and actual_users is not None: + result.users_checked = len(expected_users) + for user in sorted(set(expected_users) - set(actual_users)): + result.errors.append(f"user/role {user} is missing") + elif expected_users is not None: + result.warnings.append("couldn't list the restored server's users, so they weren't checked") + return result + + +def target_for_set(bset: BackupSet) -> Target: + """A Target with default settings for the set's server type: restore testing needs no config file.""" + manifest = bset.manifest + raw = {"targets": [{"name": manifest["target"], "type": manifest["type"], "host": "unused"}]} + return parse_config(raw, {}).targets[0] + + +@dataclass +class RestoreTestReport: + target: str + set: str + restore_seconds: float + check: CheckResult + restore_warnings: list[str] + + @property + def passed(self) -> bool: + return not self.check.errors + + +def run_restore_test(bset: BackupSet, conn: Connection, *, tolerance: float = DEFAULT_TOLERANCE) -> RestoreTestReport: + """Verify, restore and check one set against a scratch server reached through ``conn``.""" + if not bset.ok: + raise StorageError(f"{bset.path} has no valid manifest") + problems = verify_checksums(bset) + if problems: + raise StorageError(f"{bset.target}/{bset.name} failed its checksum check: {'; '.join(problems)}") + + target = target_for_set(bset) + engine = engine_for(target, conn) + engine.verify(bset.path, bset.manifest) + databases = list(bset.manifest["databases"]) + + log.info("%s/%s: restoring %d database(s) into the scratch server", bset.target, bset.name, len(databases)) + started = time.monotonic() + restore_warnings = engine.restore(bset.path, bset.manifest, databases, force=False, include_globals=True) + restore_seconds = time.monotonic() - started + + log.info("%s/%s: checking the restored server against the backup's inventory", bset.target, bset.name) + present = set(engine.list_databases()) + actual = engine.inventory([d for d in databases if d in present], exact=True) + check = compare_inventory(bset.manifest.get("inventory", {}), actual, present, tolerance=tolerance, server_type=target.type) + return RestoreTestReport(bset.target, bset.name, restore_seconds, check, restore_warnings) diff --git a/src/dbbackup/runner.py b/src/dbbackup/runner.py index 9fbbda6..bfd07f5 100644 --- a/src/dbbackup/runner.py +++ b/src/dbbackup/runner.py @@ -99,7 +99,7 @@ def backup_once(store: Store, target: Target) -> BackupSet: if not databases: warnings.append("no databases selected; only server-level objects (users/roles) were backed up") log.info("%s: server %s, %d database(s): %s", target.name, version, len(databases), ", ".join(databases)) - inventory = engine.inventory(databases) + inventory = engine.inventory(databases, exact=target.exact_counts) partial = store.begin(target.name, started) try: diff --git a/src/dbbackup/storage.py b/src/dbbackup/storage.py index c7dc118..57561a5 100644 --- a/src/dbbackup/storage.py +++ b/src/dbbackup/storage.py @@ -205,6 +205,18 @@ def prune( return doomed +def open_set(path: Path, which: str = "latest") -> BackupSet: + """A set given by its directory, or by its target's directory (then ``which``: 'latest' or a set name).""" + path = Path(path) + if (path / MANIFEST).is_file(): + if not TS_RE.match(path.name): + raise StorageError(f"{path} is not named like a backup set ({TS_FORMAT})") + return BackupSet(path.parent.name, parse_ts(path.name), path, _load_manifest(path)) + if path.is_dir(): + return Store(path.parent).find(path.name, which) + raise StorageError(f"{path} is neither a backup set nor a target directory") + + def checksum_tree(root: Path) -> dict[str, dict]: files = {} for path in sorted(root.rglob("*")): diff --git a/tests/integration/config.yaml b/tests/integration/config.yaml index 61f2b80..bd954c5 100644 --- a/tests/integration/config.yaml +++ b/tests/integration/config.yaml @@ -26,6 +26,7 @@ targets: username: dbbackup password_file: /run/secrets/dbbackup/mariadb-password exclude: [scratch] + exact_counts: true # Whole-instance dump with --oplog. - name: mongo diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index 5e062d9..0199ecc 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -145,6 +145,7 @@ def test_postgres_manifest(): m = manifest("pg") assert m["databases"] == ["app1", "odd.name/db", "postgres"] # scratch excluded assert m["contents"]["role_passwords"] is True # pg_read_all_data can read pg_authid + assert m["inventory"]["counts"] == "estimated" assert m["server_version"].startswith("18.") objects = m["inventory"]["databases"]["app1"]["objects"] assert objects["billing.invoices"] == {"kind": "table", "rows": 20000} @@ -157,6 +158,10 @@ def test_mariadb_manifest(): m = manifest("maria") assert m["databases"] == ["shop", "wiki-db"] assert any("shop.legacy_log uses Aria" in w for w in m["warnings"]) + # exact_counts: real COUNT(*)s, where InnoDB's TABLE_ROWS estimate can be wildly off. + assert m["inventory"]["counts"] == "exact" + assert m["inventory"]["databases"]["shop"]["objects"]["products"] == {"kind": "table", "rows": 3000} + assert m["inventory"]["databases"]["wiki-db"]["objects"]["pages"]["rows"] == 500 assert {"shop_app@%", "wiki_ro@%", "reporting"} <= set(m["inventory"]["users"]) diff --git a/tests/test_config.py b/tests/test_config.py index 6f6862e..93fae7e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -129,3 +129,15 @@ def test_mqtt_defaults(): raw["mqtt"] = {"host": "mq", "tls": True, "topic_prefix": "x/y/"} mqtt = parse_config(raw, ENV).mqtt assert (mqtt.port, mqtt.topic_prefix) == (8883, "x/y") + + +def test_exact_counts_default_and_override(): + raw = minimal() + raw["targets"].append({"name": "two", "type": "mariadb", "host": "h", "exact_counts": True}) + assert [t.exact_counts for t in parse_config(raw, ENV).targets] == [False, True] + raw["defaults"] = {"exact_counts": True} + raw["targets"][1]["exact_counts"] = False + assert [t.exact_counts for t in parse_config(raw, ENV).targets] == [True, False] + raw["targets"][1]["exact_counts"] = "yes" + with pytest.raises(ConfigError, match="must be true or false"): + parse_config(raw, ENV) diff --git a/tests/test_restoretest.py b/tests/test_restoretest.py new file mode 100644 index 0000000..fbc3490 --- /dev/null +++ b/tests/test_restoretest.py @@ -0,0 +1,151 @@ +import shlex +from datetime import UTC, datetime + +import pytest + +from dbbackup.cli import main +from dbbackup.restoretest import compare_inventory, counts_match, target_for_set +from dbbackup.storage import StorageError, Store, open_set + +EXPECTED = { + "databases": { + "app": { + "objects": { + "public.customers": {"kind": "table", "rows": 5000}, + "public.big_customers": {"kind": "view", "rows": None}, + "public.unanalyzed": {"kind": "table", "rows": None}, + } + }, + "logs": {"objects": {"events": {"kind": "collection", "rows": 2000}}}, + }, + "users": ["app_owner", "postgres"], +} + + +def restored(**overrides): + actual = { + "databases": { + "app": { + "objects": { + "public.customers": {"kind": "table", "rows": 5000}, + "public.big_customers": {"kind": "view", "rows": None}, + "public.unanalyzed": {"kind": "table", "rows": 12}, + "public.extra": {"kind": "table", "rows": 1}, + } + }, + "logs": {"objects": {"events": {"kind": "collection", "rows": 2000}}}, + }, + "users": ["app_owner", "postgres", "scratch_admin"], + } + actual.update(overrides) + return actual + + +def test_a_faithful_restore_passes(): + result = compare_inventory(EXPECTED, restored(), {"app", "logs"}) + assert result.errors == [] and result.warnings == [] + assert (result.objects_checked, result.counts_checked, result.users_checked) == (4, 2, 2) + + +def test_missing_database_object_and_user_are_errors(): + actual = restored(users=["postgres"]) + del actual["databases"]["app"]["objects"]["public.big_customers"] + result = compare_inventory(EXPECTED, actual, {"app"}) + assert result.errors == [ + "app: view public.big_customers is missing", + "database logs is missing", + "user/role app_owner is missing", + ] + + +def test_kind_change_is_an_error(): + actual = restored() + actual["databases"]["app"]["objects"]["public.big_customers"] = {"kind": "table", "rows": 0} + assert compare_inventory(EXPECTED, actual, {"app", "logs"}).errors == [ + "app: public.big_customers was a view but was restored as a table" + ] + + +def test_count_mismatch_beyond_tolerance_is_only_a_warning(): + actual = restored() + actual["databases"]["app"]["objects"]["public.customers"]["rows"] = 2000 + result = compare_inventory(EXPECTED, actual, {"app", "logs"}) + assert result.errors == [] + assert result.warnings == ["app: public.customers has 2000 rows, but the backup estimated about 5000"] + + +def test_exact_counts_are_compared_exactly(): + expected = {**EXPECTED, "counts": "exact"} + actual = restored() + actual["databases"]["app"]["objects"]["public.customers"]["rows"] = 4999 + result = compare_inventory(expected, actual, {"app", "logs"}) + assert result.warnings == ["app: public.customers has 4999 rows, but 5000 were counted at backup time"] + + +def test_old_mariadb_manifests_skip_counts(): + actual = restored() + actual["databases"]["app"]["objects"]["public.customers"]["rows"] = 41 + result = compare_inventory(EXPECTED, actual, {"app", "logs"}, server_type="mariadb") + assert result.warnings == [] and result.counts_checked == 0 + assert result.notes and "InnoDB" in result.notes[0] + # ...but a new-style estimated MariaDB manifest is compared (its InnoDB rows are recorded as None). + result = compare_inventory({**EXPECTED, "counts": "estimated"}, actual, {"app", "logs"}, server_type="mariadb") + assert result.counts_checked == 2 and result.warnings + + +@pytest.mark.parametrize( + "expected, actual, ok", + [(5000, 4000, True), (5000, 3000, False), (0, 99, True), (0, 101, False), (1_000_000, 800_001, True)], +) +def test_counts_match(expected, actual, ok): + assert counts_match(expected, actual, 0.25) is ok + + +def test_users_not_listable_on_either_side(): + assert compare_inventory({"databases": {}, "users": None}, {"databases": {}, "users": ["x"]}, set()).users_checked == 0 + result = compare_inventory({"databases": {}, "users": ["a"]}, {"databases": {}, "users": None}, set()) + assert result.errors == [] and result.warnings + + +def make_set(tmp_path, target="pg-main", ttype="postgres"): + store = Store(tmp_path) + when = datetime(2026, 10, 9, 2, 15, tzinfo=UTC) + partial = store.begin(target, when) + (partial / "globals.sql.zst").write_bytes(b"x") + manifest = {"status": "ok", "target": target, "type": ttype, "server_version": "18.6", "databases": ["app", "it's"], "size_bytes": 1} + return store.finalize(partial, manifest) + + +def test_open_set_by_set_or_target_directory(tmp_path): + bset = make_set(tmp_path) + assert open_set(bset.path).path == bset.path + assert open_set(tmp_path / "pg-main").path == bset.path + assert open_set(tmp_path / "pg-main", bset.name).path == bset.path + with pytest.raises(StorageError): + open_set(tmp_path / "nope") + with pytest.raises(StorageError, match="not found"): + open_set(tmp_path / "pg-main", "20200101T000000Z") + + +def test_target_for_set_uses_type_defaults(tmp_path): + target = target_for_set(make_set(tmp_path, "maria", "mariadb")) + assert (target.name, target.type, target.port, target.jobs) == ("maria", "mariadb", 3306, 2) + + +def test_inspect_env_output_is_shell_safe(tmp_path, capsys): + bset = make_set(tmp_path) + assert main(["inspect", str(tmp_path / "pg-main"), "--format", "env"]) == 0 + values = dict(line.split("=", 1) for line in capsys.readouterr().out.splitlines()) + assert {k: shlex.split(v)[0] for k, v in values.items()} == { + "DBB_TARGET": "pg-main", + "DBB_TYPE": "postgres", + "DBB_SET": bset.name, + "DBB_SERVER_VERSION": "18.6", + "DBB_DATABASES": "2", + "DBB_SIZE_BYTES": "1", + } + + +def test_restore_test_needs_a_scratch_server(tmp_path): + make_set(tmp_path) + assert main(["restore-test", str(tmp_path / "pg-main")]) == 2