From a37e40d82e68dd08560d78ab0b91d9ca0808cb75 Mon Sep 17 00:00:00 2001 From: Alistair Young Date: Thu, 8 Oct 2026 14:21:04 -0500 Subject: [PATCH] Add containerized MariaDB/PostgreSQL/MongoDB backup tool A single image, run as a Kubernetes CronJob (or with Docker), that makes logical backups of any number of MariaDB, PostgreSQL and MongoDB servers onto a shared filesystem, with GFS retention and MQTT status reporting. - Enumerates every database per server (with include/exclude), and saves server-level objects: PG roles/tablespaces, MariaDB users/grants, MongoDB users/roles (whole-instance mongodump --oplog). - Sets are written to .partial, verified, checksummed into a manifest and renamed into place; finished sets are never modified, so off-site sync only sees new files. - GFS retention (daily/weekly/monthly/yearly/last), applied only after a target's own backup succeeds; the newest good set is always kept. - Per-target isolation, retries and locking; non-zero exit on any failure. - MQTT retained status with Home Assistant discovery, for Alert Redux failure and staleness (dead man's switch) alerts. - dbbackup restore, which leaves accounts that already exist on the destination untouched; plus a stock-tools DR runbook. - Secrets only via env vars or files, never on argv. - Kustomize base (CronJob, NFS PV/PVC, ConfigMap), compose test stack, unit and integration tests, and GitHub Actions CI with GHCR publishing. Co-Authored-By: Claude Opus 5.5 --- .dockerignore | 9 + .github/workflows/ci.yml | 67 ++++ .gitignore | 8 + Dockerfile | 53 +++ Makefile | 36 +++ README.md | 155 ++++++++- compose.yaml | 165 ++++++++++ deploy/k8s/config.yaml | 47 +++ deploy/k8s/cronjob.yaml | 71 +++++ deploy/k8s/kustomization.yaml | 31 ++ deploy/k8s/namespace.yaml | 4 + deploy/k8s/nfs-volume.yaml | 32 ++ deploy/k8s/secret.example.yaml | 21 ++ docs/home-assistant.md | 65 ++++ docs/restore.md | 171 ++++++++++ pyproject.toml | 40 +++ src/dbbackup/__init__.py | 3 + src/dbbackup/cli.py | 217 +++++++++++++ src/dbbackup/config.py | 336 ++++++++++++++++++++ src/dbbackup/engines/__init__.py | 18 ++ src/dbbackup/engines/base.py | 107 +++++++ src/dbbackup/engines/mariadb.py | 239 ++++++++++++++ src/dbbackup/engines/mongodb.py | 201 ++++++++++++ src/dbbackup/engines/postgres.py | 231 ++++++++++++++ src/dbbackup/notify.py | 114 +++++++ src/dbbackup/proc.py | 129 ++++++++ src/dbbackup/retention.py | 49 +++ src/dbbackup/runner.py | 140 ++++++++ src/dbbackup/storage.py | 265 +++++++++++++++ tests/integration/config.yaml | 49 +++ tests/integration/secrets/mariadb-password | 1 + tests/integration/seed/mariadb/01-seed.sql | 57 ++++ tests/integration/seed/mongodb/01-seed.js | 16 + tests/integration/seed/postgres/01-seed.sql | 43 +++ tests/integration/test.env | 11 + tests/integration/test_end_to_end.py | 321 +++++++++++++++++++ tests/test_config.py | 131 ++++++++ tests/test_engines.py | 220 +++++++++++++ tests/test_notify.py | 40 +++ tests/test_retention.py | 82 +++++ tests/test_storage.py | 124 ++++++++ 41 files changed, 4118 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 compose.yaml create mode 100644 deploy/k8s/config.yaml create mode 100644 deploy/k8s/cronjob.yaml create mode 100644 deploy/k8s/kustomization.yaml create mode 100644 deploy/k8s/namespace.yaml create mode 100644 deploy/k8s/nfs-volume.yaml create mode 100644 deploy/k8s/secret.example.yaml create mode 100644 docs/home-assistant.md create mode 100644 docs/restore.md create mode 100644 pyproject.toml create mode 100644 src/dbbackup/__init__.py create mode 100644 src/dbbackup/cli.py create mode 100644 src/dbbackup/config.py create mode 100644 src/dbbackup/engines/__init__.py create mode 100644 src/dbbackup/engines/base.py create mode 100644 src/dbbackup/engines/mariadb.py create mode 100644 src/dbbackup/engines/mongodb.py create mode 100644 src/dbbackup/engines/postgres.py create mode 100644 src/dbbackup/notify.py create mode 100644 src/dbbackup/proc.py create mode 100644 src/dbbackup/retention.py create mode 100644 src/dbbackup/runner.py create mode 100644 src/dbbackup/storage.py create mode 100644 tests/integration/config.yaml create mode 100644 tests/integration/secrets/mariadb-password create mode 100644 tests/integration/seed/mariadb/01-seed.sql create mode 100644 tests/integration/seed/mongodb/01-seed.js create mode 100644 tests/integration/seed/postgres/01-seed.sql create mode 100644 tests/integration/test.env create mode 100644 tests/integration/test_end_to_end.py create mode 100644 tests/test_config.py create mode 100644 tests/test_engines.py create mode 100644 tests/test_notify.py create mode 100644 tests/test_retention.py create mode 100644 tests/test_storage.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b4af2e0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.backups +.venv +**/__pycache__ +**/*.egg-info +.pytest_cache +.ruff_cache +deploy +docs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3e0884b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,67 @@ +name: ci + +on: + push: + branches: [master] + tags: ["v*"] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - run: python -m pip install --upgrade pip + - run: python -m pip install -e '.[test]' ruff + - run: make lint + - run: make test PYTHON=python + + integration: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + - run: make integration + - if: failure() + run: docker compose logs --no-color + - if: always() + run: make stack-down + + image: + runs-on: ubuntu-latest + needs: integration + if: github.event_name == 'push' + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + type=raw,value=latest,enable={{is_default_branch}} + - uses: docker/build-push-action@v6 + with: + context: . + target: runtime + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c724a29 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.backups/ +.venv/ +__pycache__/ +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +build/ +dist/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..86b6e1d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +# syntax=docker/dockerfile:1 + +FROM python:3.13-slim-trixie AS runtime + +# Client tools must be at least as new as the servers they back up. +ARG PG_MAJOR=18 +ARG MONGO_TOOLS_VERSION=100.19.1 +ARG TARGETARCH + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH=/usr/lib/postgresql/${PG_MAJOR}/bin:$PATH \ + DBBACKUP_CONFIG=/etc/dbbackup/config.yaml + +RUN set -eux; \ + if [ "${TARGETARCH:-amd64}" != "amd64" ]; then \ + echo "MongoDB Database Tools are only published for Debian on x86_64" >&2; exit 1; \ + fi; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl zstd mariadb-client; \ + install -d /usr/share/postgresql-common/pgdg; \ + curl -fsSL -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc https://www.postgresql.org/media/keys/ACCC4CF8.asc; \ + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt trixie-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends postgresql-client-${PG_MAJOR}; \ + curl -fsSL -o /tmp/mongodb-database-tools.deb \ + "https://fastdl.mongodb.org/tools/db/mongodb-database-tools-debian13-x86_64-${MONGO_TOOLS_VERSION}.deb"; \ + apt-get install -y --no-install-recommends /tmp/mongodb-database-tools.deb; \ + rm /tmp/mongodb-database-tools.deb; \ + apt-get purge -y --auto-remove curl; \ + rm -rf /var/lib/apt/lists/*; \ + pg_dump --version; mariadb-dump --version; mongodump --version; zstd --version + +WORKDIR /app +COPY pyproject.toml README.md LICENSE ./ +COPY src ./src +RUN pip install --no-cache-dir . \ + && useradd --system --uid 10001 --user-group --home-dir /nonexistent --shell /usr/sbin/nologin dbbackup + +USER 10001 +ENTRYPOINT ["dbbackup"] +CMD ["backup"] + + +# Integration-test image: the runtime plus pytest and the tests. +FROM runtime AS test +USER root +RUN pip install --no-cache-dir "pytest>=8" +COPY tests ./tests +USER 10001 +ENTRYPOINT [] +CMD ["python", "-m", "pytest", "-p", "no:cacheprovider", "-m", "integration", "-v", "tests/integration"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..12ba735 --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +PYTHON ?= .venv/bin/python +IMAGE ?= ghcr.io/arkane-systems/db-backup +TAG ?= dev + +.PHONY: venv lint format test integration stack-up stack-down image + +venv: + python3 -m venv .venv + .venv/bin/pip install --upgrade pip + .venv/bin/pip install -e '.[test]' ruff + +lint: + ruff check src tests + ruff format --check src tests + +format: + ruff check --fix src tests + ruff format src tests + +# Unit tests: no servers needed. +test: + $(PYTHON) -m pytest + +# Integration tests: back up and restore real servers in the compose.yaml stack. +integration: stack-up + docker compose --profile tool run --rm --build tests + +stack-up: + mkdir -p .backups + docker compose up -d --wait + +stack-down: + docker compose down -v + +image: + docker build --target runtime -t $(IMAGE):$(TAG) . diff --git a/README.md b/README.md index c183981..de1dbda 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,155 @@ # db-backup -A containerized database backup solution. + +A containerized database backup solution: logical backups of any number of **MariaDB**, **PostgreSQL** and **MongoDB** servers onto a shared filesystem (an NFS share, say), with grandfather-father-son retention and MQTT status reporting. It is built to run as a Kubernetes CronJob, and it can be run with Docker too. + +- **Finds every database itself.** Each server's databases are enumerated on every run, minus any you exclude. It also saves the server-level objects needed to rebuild the server: PostgreSQL roles and tablespaces, MariaDB users and grants, MongoDB users and roles. +- **Makes restorable, consistent dumps:** + - PostgreSQL: parallel directory-format `pg_dump`, plus `pg_dumpall --globals-only`. + - MariaDB: `mariadb-dump --single-transaction`, plus `--system=users`. + - MongoDB: a whole-instance `mongodump --oplog`, consistent to a single point in time. +- **Never leaves a half-finished set looking finished.** Each set is written to a `.partial` directory and verified, then given a manifest with SHA-256 checksums and renamed into place. Finished sets are never modified or renamed again, so an off-site sync of the share only ever uploads new files. +- **Keeps real GFS history:** the newest backup from each of the last N days, ISO weeks, months and years. +- **Isolates failures.** One server's failure never stops the others. A target is pruned only after one of its own backups succeeds, and the newest good set is always kept. +- **Reports to MQTT,** with Home Assistant discovery. Failures, and backups that have *stopped happening*, can raise [Alert Redux](https://github.com/arkane-systems/ha-alert-redux) alerts; see [docs/home-assistant.md](docs/home-assistant.md). +- **Restores with one command,** `dbbackup restore`. Every set can also be restored with stock client tools; see [docs/restore.md](docs/restore.md). + +Supported servers: MariaDB 11.4+, PostgreSQL 18+, MongoDB 8.3+ (as a replica set, for `--oplog`). Older versions may well work: the tool logs a warning and carries on. The image ships the PostgreSQL 18 client, the MariaDB 11.8 client and MongoDB Database Tools 100.19. It is built for linux/amd64 only, because MongoDB publishes its Debian tools packages only for x86-64. + +## How it works + +Each run backs up every target in turn (or just the ones named on the command line): + +1. Take the target's lock on the share. +2. Connect, enumerate the databases, and record an inventory: tables or collections with approximate row counts, plus users and roles. +3. Dump into `//.partial/`. +4. Verify the dumps: + - completion trailers and zstd integrity + - `pg_restore --list` + - the mongodump archive header +5. Write `manifest.json`, fsync, and rename the set to `///`. +6. Apply retention, and clear out abandoned partial sets. + +A failed attempt is retried (`retries`), and failing that the run goes on to the next target. At the end, every target's status is published to MQTT. The process exits non-zero if any target failed, so the Kubernetes Job shows as failed too. + +``` +/backups/ + postgres-main/20261009T021500Z/ + manifest.json globals.sql.zst app1.pgdump/ postgres.pgdump/ + mariadb-main/20261009T021512Z/ + manifest.json system-users.sql.zst shop.sql.zst wiki-db.sql.zst + mongodb-main/20261009T021530Z/ + manifest.json mongodb.archive.zst +``` + +## Configuration + +A YAML file, at `/etc/dbbackup/config.yaml` by default (override with `--config` or `$DBBACKUP_CONFIG`). [deploy/k8s/config.yaml](deploy/k8s/config.yaml) is a complete, commented example. + +Secrets never go in the file. Each one is named by an environment variable (`password_env`, `uri_env`) or a file (`password_file`, `uri_file`), such as a mounted Secret. Secrets are only resolved for the targets a command needs, so a restore needs only its own target's credentials. Secrets are never passed on a command line; they reach the client tools through the environment or `0600` temporary files. + +| Setting | Where | Default | Meaning | +|---|---|---|---| +| `backup_root` | top level | `/backups` | Where sets are written. | +| `retention` | `defaults` or target | `{daily: 7, weekly: 4, monthly: 6}` | GFS policy. Also `yearly`, and `last` (keep the N newest outright). A target's values override the defaults key by key. | +| `compression_level` | `defaults` or target | `3` | zstd level, 1–19. | +| `retries`, `retry_delay_s` | `defaults` or target | `1`, `30` | Per-target retries within a run. | +| `jobs` | `defaults` or postgres target | `2` | Parallel `pg_dump` and `pg_restore` workers. | +| `stale_partial_hours`, `lock_stale_hours` | `defaults` | `24`, `24` | When abandoned partial sets and locks are cleaned up. | +| `name`, `type` | target | (required) | Name: letters, digits, `-`, `_`. Type: `postgres`, `mariadb` or `mongodb`. | +| `host`, `port`, `username`, `password_env`/`password_file` | target | (type's default port) | Connection details. | +| `include`, `exclude` | target | all databases | Limit which databases are backed up. | +| `sslmode` | postgres target | (libpq default) | `PGSSLMODE`. | +| `maintenance_db` | postgres target | `postgres` | Database used for enumeration and globals. | +| `ssl` | mariadb target | (client default: TLS on) | `true` or `false`. | +| `uri_env`/`uri_file` | mongodb target | (none) | A full connection URI, instead of host/port/username. | +| `uri_options` | mongodb target | (none) | Extra URI options when connecting by host, e.g. `replicaSet: rs0`. | +| `dump_args` | target | (none) | Extra arguments for each per-database dump command. | +| `mqtt` | top level | (none) | `host`, `port`, `username`, `password_env`/`password_file`, `tls`, `topic_prefix` (`dbbackup`), `ha_discovery` (`false`), `discovery_prefix` (`homeassistant`). | + +For MongoDB, `include` or `exclude` changes how the backup is made. `mongodump` can't leave databases out of a whole-instance dump, so such a target gets one archive per database instead. Those dumps can't use `--oplog`, so they are consistent only per collection, and the manifest records a warning saying so. Leave both unset for point-in-time consistency. + +## Database accounts + +Each server needs a read-only account for backups. These are the privileges the integration tests run with. + +**PostgreSQL.** Membership in `pg_read_all_data` lets the account read everything, including `pg_authid`. That means role passwords are saved too, without the account being a superuser. Also allow the account in `pg_hba.conf`. + +```sql +CREATE ROLE dbbackup LOGIN PASSWORD '...' IN ROLE pg_read_all_data; +``` + +**MariaDB:** + +```sql +CREATE USER 'dbbackup'@'%' IDENTIFIED BY '...'; +GRANT SELECT, SHOW VIEW, TRIGGER, EVENT, LOCK TABLES, PROCESS, RELOAD, SHOW DATABASES ON *.* TO 'dbbackup'@'%'; +``` + +**MongoDB:** + +```js +db.getSiblingDB("admin").createUser({ user: "dbbackup", pwd: "...", roles: ["backup"] }) +``` + +Restores need an administrative account (`postgres`, `root`, MongoDB `root`), passed with `restore --username/--password-env` or `--uri-env`. + +## Deploying to Kubernetes + +[deploy/k8s](deploy/k8s) is a kustomize base with these pieces: + +- **Namespace.** +- **Static NFS PersistentVolume and claim**, mounted `hard`. +- **ConfigMap** generated from `config.yaml`. +- **CronJob**, which runs daily at 02:15 UTC with: + - `concurrencyPolicy: Forbid` and `backoffLimit: 0` (the tool retries per target itself) + - a read-only root filesystem, non-root, no capabilities + +```bash +# 1. Edit deploy/k8s/config.yaml and deploy/k8s/nfs-volume.yaml for your servers and share, +# and set runAsUser/runAsGroup in cronjob.yaml to the NFS export's owner. +# 2. Create the credentials Secret (see deploy/k8s/secret.example.yaml): +kubectl create namespace db-backup +kubectl -n db-backup create secret generic db-backup-credentials \ + --from-literal=MARIADB_PASSWORD=... --from-literal=POSTGRES_PASSWORD=... \ + --from-literal=MONGODB_PASSWORD=... --from-literal=MQTT_PASSWORD=... +# 3. Deploy, then try a run now rather than waiting for 02:15: +kubectl apply -k deploy/k8s +kubectl -n db-backup create job --from=cronjob/db-backup db-backup-manual +kubectl -n db-backup logs -f job/db-backup-manual +``` + +The schedule is in `cronjob.yaml`. If you change it, adjust the staleness threshold in your Home Assistant alert to match. + +## Commands + +``` +dbbackup backup [TARGET ...] back up (all targets by default), prune, publish status +dbbackup list [TARGET ...] list backup sets +dbbackup verify [TARGET ...] [--set S | --all] re-check checksums and dump integrity +dbbackup prune [TARGET ...] [--dry-run] apply retention without backing up +dbbackup restore TARGET [--set S] [-d DB ...] [--host H --port P --username U --password-env VAR | --uri-env VAR] + [--no-globals] [--force] +``` + +Exit status: 0 on success, 1 if anything failed, 2 for configuration or usage errors. + +## Development + +```bash +make venv # .venv with the package, pytest and ruff +make lint test # ruff, and unit tests (no servers needed) +make integration # start compose.yaml's servers, then back up and restore them for real +make stack-down # remove the test servers +``` + +The integration tests run in the `test` image target against [compose.yaml](compose.yaml). That stack has seeded source servers, empty restore servers with different admin passwords, and an MQTT broker. The tests check that backups are complete, that every manifest is right, and that restores reproduce the data, users, grants and passwords. They also check that restores leave the destination server's own accounts alone, and that failures, locking, pruning and corruption detection behave. + +To run the tool by hand against the test servers: + +```bash +make stack-up +docker compose --profile tool run --rm dbbackup backup # writes to ./.backups +docker compose --profile tool run --rm dbbackup list +``` + +**Slow-to-start MariaDB on WSL.** On some WSL2 kernels, `mariadb:11.4` stalls for 15–40 s during first-boot initialization, and its entrypoint gives up after 30 s ("Unable to start server"). The compose file starts each restore server after its source to avoid this. If it still happens, `docker compose rm -sfv mariadb-restore && make stack-up` retries it. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..e80248b --- /dev/null +++ b/compose.yaml @@ -0,0 +1,165 @@ +# Local test stack: one seeded server of each type (the backup sources), one empty +# server of each type (restore destinations), and an MQTT broker. +# +# docker compose up -d --wait # start the databases +# mkdir -p .backups # (so it isn't created root-owned) +# docker compose --profile tool run --rm dbbackup backup # back them all up into ./.backups +# docker compose --profile tool run --rm --build tests # run the integration tests +# docker compose down -v # throw it all away +# +# Or just `make integration`. +# +# Credentials here are throwaway test values. The *-restore servers deliberately +# have different root passwords from the sources: a restore that overwrote the +# restoring account's own password would then break, visibly. + +name: dbbackup-dev + +x-mongo: &mongo + image: mongo:8.3 + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: mongo-root + # Replica set with auth needs a keyfile; make a throwaway one. + entrypoint: + - bash + - -c + - | + openssl rand -base64 756 > /tmp/keyfile && chmod 400 /tmp/keyfile && chown mongodb:mongodb /tmp/keyfile + exec docker-entrypoint.sh "$$@" + - -- + command: [mongod, --replSet, rs0, --keyFile, /tmp/keyfile, --bind_ip_all] + +services: + postgres: + image: postgres:18 + environment: + POSTGRES_PASSWORD: postgres-root + volumes: + - ./tests/integration/seed/postgres:/docker-entrypoint-initdb.d:ro + healthcheck: + test: [CMD, pg_isready, -U, postgres] + interval: 2s + retries: 30 + + postgres-restore: + image: postgres:18 + environment: + POSTGRES_PASSWORD: restore-root + # Start after the source server: initializing both at once can hit the + # image entrypoints' 30-second startup limit on slow hosts. + depends_on: + postgres: {condition: service_healthy} + healthcheck: + test: [CMD, pg_isready, -U, postgres] + interval: 2s + retries: 30 + + mariadb: + image: mariadb:11.4 + environment: + MARIADB_ROOT_PASSWORD: mariadb-root + volumes: + - ./tests/integration/seed/mariadb:/docker-entrypoint-initdb.d:ro + healthcheck: + test: [CMD, healthcheck.sh, --connect, --innodb_initialized] + interval: 2s + retries: 30 + + mariadb-restore: + image: mariadb:11.4 + environment: + MARIADB_ROOT_PASSWORD: restore-root + # Start after the source server: initializing both at once can hit the + # image entrypoints' 30-second startup limit on slow hosts. + depends_on: + mariadb: {condition: service_healthy} + healthcheck: + test: [CMD, healthcheck.sh, --connect, --innodb_initialized] + interval: 2s + retries: 30 + + mongodb: + <<: *mongo + volumes: + - ./tests/integration/seed/mongodb:/docker-entrypoint-initdb.d:ro + healthcheck: + test: + - CMD + - mongosh + - --quiet + - -u + - root + - -p + - mongo-root + - --eval + - "try { rs.status().myState === 1 ? 0 : quit(1) } catch (e) { rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'mongodb:27017'}]}); quit(1) }" + interval: 3s + retries: 40 + + mongodb-restore: + <<: *mongo + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: restore-root + # Start after the source server: initializing both at once can hit the + # image entrypoints' 30-second startup limit on slow hosts. + depends_on: + mongodb: {condition: service_healthy} + healthcheck: + test: + - CMD + - mongosh + - --quiet + - -u + - root + - -p + - restore-root + - --eval + - "try { rs.status().myState === 1 ? 0 : quit(1) } catch (e) { rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'mongodb-restore:27017'}]}); quit(1) }" + interval: 3s + retries: 40 + + mosquitto: + image: eclipse-mosquitto:2 + command: [mosquitto, -c, /mosquitto-no-auth.conf] + + # The backup tool itself, pointed at the seeded servers. + dbbackup: + profiles: [tool] + build: + context: . + target: runtime + user: "${UID:-1000}:${GID:-1000}" + # As in the CronJob: read-only root filesystem, writable /tmp. + read_only: true + tmpfs: [/tmp] + env_file: tests/integration/test.env + volumes: + - ./tests/integration/config.yaml:/etc/dbbackup/config.yaml:ro + - ./tests/integration/secrets:/run/secrets/dbbackup:ro + - ./.backups:/backups + depends_on: &db-deps + postgres: {condition: service_healthy} + postgres-restore: {condition: service_healthy} + mariadb: {condition: service_healthy} + mariadb-restore: {condition: service_healthy} + mongodb: {condition: service_healthy} + mongodb-restore: {condition: service_healthy} + mosquitto: {condition: service_started} + + # Integration tests, run inside the image so they use its client tools. + tests: + profiles: [tool] + build: + context: . + target: test + env_file: tests/integration/test.env + volumes: + - ./tests/integration/config.yaml:/etc/dbbackup/config.yaml:ro + - ./tests/integration/secrets:/run/secrets/dbbackup:ro + read_only: true + tmpfs: + - /tmp + - /backups:uid=10001,gid=10001 + depends_on: *db-deps diff --git a/deploy/k8s/config.yaml b/deploy/k8s/config.yaml new file mode 100644 index 0000000..ecc026a --- /dev/null +++ b/deploy/k8s/config.yaml @@ -0,0 +1,47 @@ +# db-backup configuration. Secrets are never written here: each *_env names an +# environment variable, filled from the db-backup-credentials Secret. +backup_root: /backups + +defaults: + # GFS: keep the newest backup from each of the last 7 days, 4 ISO weeks and + # 6 months (and optionally `yearly` years, and the `last` N outright). + retention: {daily: 7, weekly: 4, monthly: 6, yearly: 0} + compression_level: 3 # zstd, 1-19 + retries: 1 # per target + retry_delay_s: 60 + stale_partial_hours: 24 # abandoned .partial sets older than this are removed + lock_stale_hours: 24 # a lock older than this is assumed abandoned + +mqtt: + host: mqtt.example.lan # CHANGE ME + port: 1883 + username: dbbackup + password_env: MQTT_PASSWORD + topic_prefix: dbbackup + ha_discovery: true + +targets: + - name: mariadb-main + type: mariadb + host: mariadb.example.lan # CHANGE ME + username: dbbackup + password_env: MARIADB_PASSWORD + # Every database is backed up unless listed here (or limited with `include`). + exclude: [] + + - name: postgres-main + type: postgres + host: postgres.example.lan # CHANGE ME + username: dbbackup + password_env: POSTGRES_PASSWORD + jobs: 2 # parallel pg_dump workers per database + # sslmode: verify-full + + - name: mongodb-main + type: mongodb + host: mongodb.example.lan # CHANGE ME + username: dbbackup + password_env: MONGODB_PASSWORD + uri_options: + replicaSet: rs0 # CHANGE ME (or directConnection: "true") + # readPreference: secondaryPreferred diff --git a/deploy/k8s/cronjob.yaml b/deploy/k8s/cronjob.yaml new file mode 100644 index 0000000..478439a --- /dev/null +++ b/deploy/k8s/cronjob.yaml @@ -0,0 +1,71 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: db-backup +spec: + schedule: "15 2 * * *" + timeZone: Etc/UTC + # Never two runs at once (the tool also takes a per-target lock on the share). + concurrencyPolicy: Forbid + # If the cluster was down at 02:15, still run when it's back, up to an hour late. + startingDeadlineSeconds: 3600 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + # No pod-level retries: they'd re-run every target. The tool retries each + # failed target itself (defaults.retries), and reports failures over MQTT. + backoffLimit: 0 + activeDeadlineSeconds: 14400 + ttlSecondsAfterFinished: 604800 + template: + metadata: + labels: + app.kubernetes.io/name: db-backup + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + # Must be able to write to the NFS export: set these to its owner. + # (fsGroup doesn't help here, as NFS volumes aren't chowned.) + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: db-backup + image: ghcr.io/arkane-systems/db-backup + args: [backup] + envFrom: + - secretRef: + name: db-backup-credentials + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: config + mountPath: /etc/dbbackup + readOnly: true + - name: backups + mountPath: /backups + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: db-backup-config + - name: backups + persistentVolumeClaim: + claimName: db-backup-nfs + - name: tmp + emptyDir: + sizeLimit: 1Gi diff --git a/deploy/k8s/kustomization.yaml b/deploy/k8s/kustomization.yaml new file mode 100644 index 0000000..3168ab4 --- /dev/null +++ b/deploy/k8s/kustomization.yaml @@ -0,0 +1,31 @@ +# Base deployment: kubectl apply -k deploy/k8s +# +# Before applying: +# 1. Edit config.yaml for your servers and MQTT broker. +# 2. Edit nfs-volume.yaml for your NFS server and export. +# 3. Create the credentials Secret (see secret.example.yaml; it isn't applied from here, +# so real passwords never live in this directory). +# 4. Set runAsUser/runAsGroup in cronjob.yaml to the owner of the NFS export. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: db-backup + +resources: + - namespace.yaml + - nfs-volume.yaml + - cronjob.yaml + +configMapGenerator: + - name: db-backup-config + files: + - config.yaml + +images: + - name: ghcr.io/arkane-systems/db-backup + newTag: "0.1.0" + +labels: + - pairs: + app.kubernetes.io/name: db-backup + includeSelectors: false diff --git a/deploy/k8s/namespace.yaml b/deploy/k8s/namespace.yaml new file mode 100644 index 0000000..cf3e55e --- /dev/null +++ b/deploy/k8s/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: db-backup diff --git a/deploy/k8s/nfs-volume.yaml b/deploy/k8s/nfs-volume.yaml new file mode 100644 index 0000000..6aeb271 --- /dev/null +++ b/deploy/k8s/nfs-volume.yaml @@ -0,0 +1,32 @@ +# A static NFS PersistentVolume, bound to this deployment's claim. If your cluster +# has an NFS CSI driver or provisioner, a claim against its StorageClass works too. +apiVersion: v1 +kind: PersistentVolume +metadata: + name: db-backup-nfs +spec: + capacity: + storage: 500Gi + accessModes: [ReadWriteMany] + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + # "hard": a soft mount can turn a server hiccup into a silently short write. + mountOptions: [nfsvers=4.1, hard] + nfs: + server: nfs.example.lan # CHANGE ME + path: /export/db-backups # CHANGE ME + claimRef: + namespace: db-backup + name: db-backup-nfs +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: db-backup-nfs +spec: + accessModes: [ReadWriteMany] + storageClassName: "" + volumeName: db-backup-nfs + resources: + requests: + storage: 500Gi diff --git a/deploy/k8s/secret.example.yaml b/deploy/k8s/secret.example.yaml new file mode 100644 index 0000000..6321abc --- /dev/null +++ b/deploy/k8s/secret.example.yaml @@ -0,0 +1,21 @@ +# Example only; not applied by kustomization.yaml. Create the real Secret with +# your own tooling (sealed-secrets, external-secrets, ...) or directly: +# +# kubectl -n db-backup create secret generic db-backup-credentials \ +# --from-literal=MARIADB_PASSWORD='...' \ +# --from-literal=POSTGRES_PASSWORD='...' \ +# --from-literal=MONGODB_PASSWORD='...' \ +# --from-literal=MQTT_PASSWORD='...' +# +# Each key becomes an environment variable, referenced by a *_env setting in config.yaml. +apiVersion: v1 +kind: Secret +metadata: + name: db-backup-credentials + namespace: db-backup +type: Opaque +stringData: + MARIADB_PASSWORD: change-me + POSTGRES_PASSWORD: change-me + MONGODB_PASSWORD: change-me + MQTT_PASSWORD: change-me diff --git a/docs/home-assistant.md b/docs/home-assistant.md new file mode 100644 index 0000000..d674965 --- /dev/null +++ b/docs/home-assistant.md @@ -0,0 +1,65 @@ +# Home Assistant alerts with Alert Redux + +db-backup publishes each target's status to MQTT as a retained message: + +- `//status`: one per target. +- `/status`: an overall summary. + +With `ha_discovery: true`, it also publishes MQTT discovery configs. These create two sensors per target, on a single **DB Backup** device: + +| Entity | State | Notes | +|---|---|---| +| `sensor.dbbackup__status` | `ok` or `failed` | The status message's fields are its attributes: `target`, `error`, `warnings`, `last_success`, `set`, `size_bytes`, `databases`, `duration_s`. | +| `sensor.dbbackup__last_success` | timestamp | When the newest successful backup finished. | + +In entity IDs, a target name is lowercased and anything other than letters, digits and underscores becomes `_`. For example, `postgres-main` gives `sensor.dbbackup_postgres_main_status`. + +Every target's sensors follow the same pattern, so two [Alert Redux](https://github.com/arkane-systems/ha-alert-redux) **generators** cover all targets, now and later. A new target gets its alerts as soon as its first run publishes its sensors. + +## Generator 1: backup failed + +This fires as soon as a run reports a failure. + +**Settings → Devices & Services → Alert Redux → Add generator → State** + +| Field | Value | +|---|---| +| Name | `Backup failed` | +| Targets → entity ID pattern | `sensor.dbbackup_*_status` | +| Target state | `failed` | +| Priority | Critical | +| On message | `Backup of {{ state_attr(target, 'target') }} failed: {{ state_attr(target, 'error') }}` | + +It stops firing by itself once a later run of that target succeeds. + +## Generator 2: backup stale (the dead man's switch) + +A CronJob that never runs reports nothing, so failure alerts alone can't catch it: nothing ever fails. This generator fires when a target's last success is too old, whatever the reason. The schedule may be broken, the cluster may be down, or a run may have hung. + +**Add generator → Template** + +| Field | Value | +|---|---| +| Name | `Backup stale` | +| Targets → entity ID pattern | `sensor.dbbackup_*_status` | +| Template | see below | +| Priority | Critical | +| On message | `No successful backup of {{ state_attr(target, 'target') }} since {{ state_attr(target, 'last_success') or 'ever' }}` | + +```jinja +{% set last = state_attr(target, 'last_success') %} +{{ last is none or now() - as_datetime(last) > timedelta(hours=26) }} +``` + +The 26 hours suit a daily schedule: a day, plus slack for a long run. Use your interval plus a margin if you change the schedule. A target that has never succeeded counts as stale. + +### Supersession + +Both generators target the same status sensor, so they can be linked. In **Backup failed**'s **Supersession** section, add the **Backup stale** generator: each target's *Backup failed* alert then supersedes its *Backup stale* alert. While a target is failing you hear about the failure; you don't get a second alert a day later saying the same thing. + +The staleness alert still speaks up on its own when backups simply stop happening, since then nothing is failing to supersede it. + +## Notes + +- **Retained messages and broker persistence.** The status messages are retained, so the sensors survive Home Assistant restarts. If your broker doesn't persist retained messages across its own restarts, the sensors become `unknown` after a broker restart until the next backup run. Alert Redux shows that as `no_data` rather than silently not firing. +- **Warnings are not failures.** A run with warnings, such as a MariaDB table that isn't InnoDB or a PostgreSQL role that couldn't read passwords, is still `ok`. The warnings are in the status sensor's `warnings` attribute. To be alerted about them, add a template generator on `{{ state_attr(target, 'warnings') | length > 0 }}` at a lower priority. diff --git a/docs/restore.md b/docs/restore.md new file mode 100644 index 0000000..5a72939 --- /dev/null +++ b/docs/restore.md @@ -0,0 +1,171 @@ +# Restoring from backup + +This is the disaster-recovery runbook. It covers two cases: + +- **With the tool**: `dbbackup restore`, the quick path. +- **With stock tools only**: `psql`, `pg_restore`, `mariadb`, `mongorestore` and `zstd`, for when the tool or its image isn't available. Nothing in a backup set needs this project to read it. + +Test restores before you need them. A backup that has never been restored is only a hope. + +## Finding a backup set + +The NFS share, or the off-site copy of it, looks like this: + +``` +///manifest.json +``` + +- **One directory per target.** A target is one database server from the config. +- **One subdirectory per backup set**, named by its UTC start time. +- **Only sets with a `manifest.json` are complete.** A `*.partial` directory is an interrupted run: ignore it. +- **The manifest lists everything in the set:** the server version, the databases, which file holds which database, an inventory of tables, collections and users, any warnings from the backup, and a SHA-256 for every file. + +Check a set's files before restoring it. With the tool: + +```bash +dbbackup verify --set +``` + +Or with stock tools, from inside the set directory: + +```bash +jq -r '.files | to_entries[] | "\(.value.sha256) \(.key)"' manifest.json | sha256sum -c --quiet +``` + +File names are the database names, percent-encoded. For example, database `odd.name/db` is stored as `odd%2Ename%2Fdb.pgdump`. The manifest's `contents` section maps each database to its file. + +## Restoring with the tool + +```bash +dbbackup restore [--set latest|] [-d ...] \ + [--host --port --username --password-env ] \ + [--uri-env ] # MongoDB: a full connection URI instead of host/port/username + [--no-globals] # skip users/roles + [--force] # replace databases that already exist +``` + +- **Default destination.** Without `--host` or `--uri-env`, it restores to the server the backup came from, using the target's own credentials from the config. +- **Use an admin account for restores.** The backup accounts are deliberately read-only. +- **Existing databases.** The tool refuses to overwrite a database that already exists unless you pass `--force`. +- **Existing accounts are left alone.** Users and roles that already exist on the destination keep their passwords and privileges. That includes the account you restore as, and a fresh server's own `root` and `healthcheck` accounts. Missing users and roles are created with their backed-up passwords and grants. +- **The set is checked first.** The tool re-checks every file's checksum before it restores anything. + +### From Kubernetes + +During a disaster you may not have the tool installed anywhere. Run it as a one-off Job from the same image, mounting the same share and config: + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: db-restore + namespace: db-backup +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + securityContext: {runAsNonRoot: true, runAsUser: 10001, runAsGroup: 10001} + containers: + - name: restore + image: ghcr.io/arkane-systems/db-backup:0.1.0 + args: [restore, postgres-main, --host, new-postgres.example.lan, --username, postgres, --password-env, ADMIN_PASSWORD] + env: + - name: ADMIN_PASSWORD + valueFrom: {secretKeyRef: {name: db-restore-admin, key: password}} + volumeMounts: + - {name: config, mountPath: /etc/dbbackup, readOnly: true} + - {name: backups, mountPath: /backups, readOnly: true} + volumes: + - {name: config, configMap: {name: db-backup-config-}} # kubectl -n db-backup get configmap + - {name: backups, persistentVolumeClaim: {claimName: db-backup-nfs}} +``` + +### With Docker + +```bash +docker run --rm -v /mnt/db-backups:/backups:ro -v $PWD/config.yaml:/etc/dbbackup/config.yaml:ro \ + -e ADMIN_PASSWORD ghcr.io/arkane-systems/db-backup:0.1.0 \ + restore mariadb-main --host new-mariadb.example.lan --username root --password-env ADMIN_PASSWORD +``` + +## Restoring with stock tools + +Use client tools at least as new as the server that was backed up. The manifest's `server_version` field records which version that was. + +### PostgreSQL + +A set contains `globals.sql.zst` (roles and tablespaces) and one `.pgdump/` directory per database. Each directory is a `pg_dump --format=directory` archive. + +1. **Roles first.** Restored objects are owned by these roles and granted to them. + + ```bash + zstd -dc globals.sql.zst | psql -X -h NEW -U postgres -d postgres + ``` + + "Role already exists" errors are harmless. Be aware, though, that this also replays `ALTER ROLE ... PASSWORD` for roles that **already exist** on the new server, including `postgres` itself. The tool skips those statements; by hand, delete the lines for existing roles first if their passwords must not change. + + If the manifest says `"role_passwords": false`, login roles were saved without passwords. Set them again with `ALTER ROLE ... PASSWORD`. + +2. **Each database:** + + ```bash + pg_restore -h NEW -U postgres -d postgres --create --jobs 4 app1.pgdump + ``` + + The `postgres` database already exists on every cluster, so restore it in place instead of creating it: + + ```bash + pg_restore -h NEW -U postgres -d postgres --jobs 4 postgres.pgdump + ``` + +3. **Refresh planner statistics,** which `pg_restore` doesn't do: + + ```bash + vacuumdb -h NEW -U postgres --all --analyze-in-stages + ``` + +### MariaDB + +A set contains `system-users.sql.zst` (users, roles and grants) and one `.sql.zst` per database. Each dump includes its own `CREATE DATABASE` and `USE`. + +1. **Each database first.** Table-level grants can only be applied once their tables exist. + + ```bash + zstd -dc shop.sql.zst | mariadb -h NEW -u root -p --max-allowed-packet=1G + ``` + +2. **Then users and grants.** `--force` carries on past individual statements that fail. + + ```bash + zstd -dc system-users.sql.zst | mariadb -h NEW -u root -p --force + ``` + + As with PostgreSQL, this re-applies the backed-up password of every account in it, including accounts that already exist on the new server, such as `root` or the Docker image's `healthcheck` user. The tool skips those; by hand, remove their lines first if that matters. + +A manifest warning such as `shop.legacy_log uses Aria, not InnoDB` means that table wasn't dumped inside the same transaction as the rest. Check it before relying on it. + +### MongoDB + +**Whole-instance set.** `contents.mode` is `"instance"`, and there's a single `mongodb.archive.zst`: + +```bash +zstd -dc mongodb.archive.zst | mongorestore --uri 'mongodb://root:...@NEW:27017/?authSource=admin' --archive --oplogReplay +``` + +- `--oplogReplay` brings every database to the same point in time. Use it whenever the manifest says `"oplog": true`. +- **Don't add `--drop`.** Besides dropping collections, it replaces existing users, including the one you're restoring as. Drop clashing databases yourself first instead. +- To restore only some databases, add `--nsInclude='shop.*'` (once per database) and leave out `--oplogReplay`, which can't be combined with namespace filters. + +**Per-database set.** `contents.mode` is `"per-database"`. This happens when the target has `include` or `exclude` set. There's one `.archive.zst` per database, plus `admin.archive.zst` for users and roles. Restore each one the same way, without `--oplogReplay`: + +```bash +zstd -dc shop.archive.zst | mongorestore --uri '...' --archive +zstd -dc admin.archive.zst | mongorestore --uri '...' --archive +``` + +## After restoring + +- **Point applications at the new server,** and check they can log in. Restored application accounts keep their old passwords. +- **Check the restore against the backup's inventory.** The manifest's `inventory` lists every table and collection with an approximate row count, plus every user and role, as they were at backup time. It's a quick sanity check that nothing went missing. +- **Update the backup config** if the server's address changed, so tonight's backup covers the new server. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7027c54 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "dbbackup" +version = "0.1.0" +description = "Containerized logical backups of MariaDB, PostgreSQL and MongoDB servers, with GFS retention" +readme = "README.md" +license = "MIT" +requires-python = ">=3.12" +dependencies = [ + "pyyaml>=6.0", + "paho-mqtt>=2.1", + "pymongo>=4.10", +] + +[project.optional-dependencies] +test = [ + "pytest>=8", +] + +[project.scripts] +dbbackup = "dbbackup.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = 140 + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "B", "UP"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "integration: needs the compose test stack (run inside the test image)", +] +addopts = "-m 'not integration'" diff --git a/src/dbbackup/__init__.py b/src/dbbackup/__init__.py new file mode 100644 index 0000000..6f44d38 --- /dev/null +++ b/src/dbbackup/__init__.py @@ -0,0 +1,3 @@ +"""Logical backups of MariaDB, PostgreSQL and MongoDB servers, with GFS retention.""" + +__version__ = "0.1.0" diff --git a/src/dbbackup/cli.py b/src/dbbackup/cli.py new file mode 100644 index 0000000..72e8e11 --- /dev/null +++ b/src/dbbackup/cli.py @@ -0,0 +1,217 @@ +"""Command-line interface: ``dbbackup backup|list|verify|prune|restore``.""" + +from __future__ import annotations + +import argparse +import logging +import os +import sys +from datetime import timedelta + +from . import __version__ +from .config import ConfigError, load_config +from .engines import Connection, EngineError, engine_for +from .notify import publish_statuses +from .proc import CommandError +from .runner import backup_targets, human_size +from .storage import StorageError, Store, verify_checksums + +log = logging.getLogger("dbbackup") + +EXIT_OK, EXIT_FAILED, EXIT_USAGE = 0, 1, 2 + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + logging.basicConfig( + level=getattr(logging, args.log_level.upper()), + format="%(asctime)s %(levelname)-7s %(message)s", + datefmt="%Y-%m-%dT%H:%M:%S%z", + stream=sys.stderr, + ) + try: + return args.func(args) + except ConfigError as e: + log.error("config: %s", e) + return EXIT_USAGE + except (StorageError, EngineError, CommandError) as e: + log.error("%s", e) + return EXIT_FAILED + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="dbbackup", description="Back up and restore MariaDB, PostgreSQL and MongoDB servers.") + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + parser.add_argument( + "-c", + "--config", + default=os.environ.get("DBBACKUP_CONFIG", "/etc/dbbackup/config.yaml"), + help="config file (default: $DBBACKUP_CONFIG or /etc/dbbackup/config.yaml)", + ) + parser.add_argument("--log-level", default=os.environ.get("DBBACKUP_LOG_LEVEL", "info"), choices=["debug", "info", "warning", "error"]) + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("backup", help="back up targets (all of them by default), apply retention, publish status") + p.add_argument("targets", nargs="*", help="target names (default: all)") + p.set_defaults(func=cmd_backup) + + p = sub.add_parser("list", help="list backup sets") + p.add_argument("targets", nargs="*", help="target names (default: all)") + p.set_defaults(func=cmd_list) + + p = sub.add_parser("verify", help="re-check backup sets' checksums and dump integrity") + p.add_argument("targets", nargs="*", help="target names (default: all)") + group = p.add_mutually_exclusive_group() + group.add_argument("--set", default="latest", help="set to verify: 'latest' (default) or a timestamp") + group.add_argument("--all", action="store_true", help="verify every set") + p.set_defaults(func=cmd_verify) + + p = sub.add_parser("prune", help="apply retention without backing up") + p.add_argument("targets", nargs="*", help="target names (default: all)") + p.add_argument("-n", "--dry-run", action="store_true", help="only show what would be deleted") + p.set_defaults(func=cmd_prune) + + p = sub.add_parser("restore", help="restore a backup set to its server or another one") + p.add_argument("target", help="target name") + p.add_argument("--set", default="latest", help="set to restore: 'latest' (default) or a timestamp") + p.add_argument("-d", "--database", action="append", dest="databases", help="restore only this database (repeatable)") + p.add_argument("--host", help="restore to this host instead of the target's") + p.add_argument("--port", type=int, help="restore to this port instead of the target's") + p.add_argument("--username", help="connect as this user instead of the target's") + p.add_argument("--password-env", metavar="VAR", help="environment variable holding the password for --username") + p.add_argument("--uri-env", metavar="VAR", help="MongoDB: environment variable holding a connection URI to restore to") + p.add_argument("--no-globals", action="store_true", help="don't restore users/roles") + p.add_argument("--force", action="store_true", help="replace databases that already exist") + p.set_defaults(func=cmd_restore) + + return parser + + +def cmd_backup(args) -> int: + config = load_config(args.config, secrets_for=args.targets or None) + targets = config.select(args.targets) + statuses = backup_targets(config, targets) + publish_statuses(config.mqtt, statuses) + failed = [s["target"] for s in statuses if s["state"] != "ok"] + if failed: + log.error("backup failed for %d of %d target(s): %s", len(failed), len(statuses), ", ".join(failed)) + return EXIT_FAILED + log.info("all %d target(s) backed up", len(statuses)) + return EXIT_OK + + +def cmd_list(args) -> int: + config = load_config(args.config, secrets_for=(), mqtt_secrets=False) + store = Store(config.backup_root) + print(f"{'TARGET':<20} {'SET':<17} {'STATUS':<7} {'SIZE':>10} {'DBS':>4} {'WARN':>4} SERVER") + for target in config.select(args.targets): + for s in store.sets(target.name): + m = s.manifest or {} + print( + f"{target.name:<20} {s.name:<17} {'ok' if s.ok else 'BAD':<7} {human_size(m.get('size_bytes')):>10} " + f"{len(m.get('databases', [])):>4} {len(m.get('warnings', [])):>4} {m.get('server_version', '-')}" + ) + return EXIT_OK + + +def cmd_verify(args) -> int: + config = load_config(args.config, secrets_for=(), mqtt_secrets=False) + store = Store(config.backup_root) + failures = 0 + for target in config.select(args.targets): + if args.all: + sets = store.sets(target.name) + else: + try: + sets = [store.find(target.name, args.set)] + except StorageError as e: + log.error("%s", e) + failures += 1 + continue + for s in sets: + problems = ["no valid manifest"] if not s.ok else verify_checksums(s) + if not problems: + try: + engine_for(target).verify(s.path, s.manifest) + except (EngineError, CommandError) as e: + problems.append(str(e)) + if problems: + failures += 1 + for problem in problems: + log.error("%s/%s: %s", target.name, s.name, problem) + else: + log.info("%s/%s: ok", target.name, s.name) + return EXIT_FAILED if failures else EXIT_OK + + +def cmd_prune(args) -> int: + config = load_config(args.config, secrets_for=(), mqtt_secrets=False) + store = Store(config.backup_root) + for target in config.select(args.targets): + with store.lock(target.name, timedelta(hours=config.lock_stale_hours)): + store.prune( + target.name, target.retention, stale_partial_after=timedelta(hours=config.stale_partial_hours), dry_run=args.dry_run + ) + return EXIT_OK + + +def cmd_restore(args) -> int: + overriding = bool(args.host or args.uri_env or args.username) + config = load_config(args.config, secrets_for=() if overriding else [args.target], mqtt_secrets=False) + (target,) = config.select([args.target]) + store = Store(config.backup_root) + bset = store.find(target.name, args.set) + + conn = Connection.for_target(target) + if overriding: + password = None + if args.password_env: + if args.password_env not in os.environ: + raise ConfigError(f"--password-env: environment variable {args.password_env} is not set") + password = os.environ[args.password_env] + uri = None + if args.uri_env: + if target.type != "mongodb": + raise ConfigError("--uri-env is only for mongodb targets") + if args.uri_env not in os.environ: + raise ConfigError(f"--uri-env: environment variable {args.uri_env} is not set") + uri = os.environ[args.uri_env] + conn = Connection( + # A URI says where it points; don't leave the target's host around to mislead the logs. + host=None if uri else (args.host or target.host), + port=args.port or target.port, + username=args.username or target.username, + password=password, + uri=uri, + ) + elif args.port: + conn = conn.overridden(port=args.port) + + problems = verify_checksums(bset) + if problems: + raise StorageError(f"{target.name}/{bset.name} failed its checksum check: {'; '.join(problems)}") + + databases = args.databases or list(bset.manifest["databases"]) + unknown = [d for d in databases if d not in bset.manifest["databases"]] + if unknown: + raise ConfigError(f"database(s) not in set {bset.name}: {', '.join(unknown)}") + + engine = engine_for(target, conn) + log.info( + "%s: restoring set %s (%s %s) to %s: %s", + target.name, + bset.name, + target.type, + bset.manifest.get("server_version"), + conn.host or "the server in the given URI", + ", ".join(databases) or "(no databases)", + ) + warnings = engine.restore(bset.path, bset.manifest, databases, force=args.force, include_globals=not args.no_globals) + for warning in warnings: + log.warning("%s: %s", target.name, warning) + log.info("%s: restore complete%s", target.name, f" with {len(warnings)} warning(s)" if warnings else "") + return EXIT_OK + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/dbbackup/config.py b/src/dbbackup/config.py new file mode 100644 index 0000000..ee926c2 --- /dev/null +++ b/src/dbbackup/config.py @@ -0,0 +1,336 @@ +"""Configuration loading and validation. + +The config file is YAML (normally a mounted ConfigMap). Secrets never appear in it +directly: each one is named by an environment variable (``*_env``) or a file path +(``*_file``, e.g. a mounted Secret), and resolved when the config is loaded. +""" + +from __future__ import annotations + +import os +import re +from collections.abc import Collection +from dataclasses import dataclass, field, replace +from pathlib import Path +from typing import Any + +import yaml + +DEFAULT_CONFIG_PATH = "/etc/dbbackup/config.yaml" + +TARGET_TYPES = ("postgres", "mariadb", "mongodb") +DEFAULT_PORTS = {"postgres": 5432, "mariadb": 3306, "mongodb": 27017} + +# Target names become directory names and MQTT topic segments. +TARGET_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,62}$") + + +class ConfigError(Exception): + pass + + +@dataclass(frozen=True) +class Retention: + """GFS retention: keep the newest set in each of the last N periods of each kind.""" + + last: int = 0 + daily: int = 7 + weekly: int = 4 + monthly: int = 6 + yearly: int = 0 + + FIELDS = ("last", "daily", "weekly", "monthly", "yearly") + + def merged(self, overrides: dict[str, Any] | None, where: str) -> Retention: + if not overrides: + return self + _check_keys(overrides, set(self.FIELDS), where) + values = {} + for key, value in overrides.items(): + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ConfigError(f"{where}.{key}: must be a non-negative integer") + values[key] = value + return replace(self, **values) + + +@dataclass(frozen=True) +class Target: + name: str + type: str + host: str | None + port: int + username: str | None + password: str | None = field(repr=False) + uri: str | None = field(repr=False) # MongoDB only; overrides host/port/username/password + include: tuple[str, ...] + exclude: tuple[str, ...] + retention: Retention + compression_level: int + retries: int + retry_delay_s: int + jobs: int # postgres: pg_dump/pg_restore -j + maintenance_db: str # postgres: database to connect to for enumeration and globals + sslmode: str | None # postgres: PGSSLMODE + ssl: bool | None # mariadb: None = client default, True = --ssl, False = --skip-ssl + uri_options: dict[str, str] # mongodb: extra URI query options when built from host/port + dump_args: tuple[str, ...] # extra args appended to each per-database dump command + + +@dataclass(frozen=True) +class MqttConfig: + host: str + port: int + username: str | None + password: str | None = field(repr=False) + tls: bool + topic_prefix: str + ha_discovery: bool + discovery_prefix: str + client_id: str + + +@dataclass(frozen=True) +class Config: + backup_root: Path + targets: tuple[Target, ...] + mqtt: MqttConfig | None + stale_partial_hours: int + lock_stale_hours: int + + def select(self, names: list[str] | None) -> list[Target]: + if not names: + return list(self.targets) + by_name = {t.name: t for t in self.targets} + unknown = [n for n in names if n not in by_name] + if unknown: + raise ConfigError(f"unknown target(s): {', '.join(unknown)}") + return [by_name[n] for n in names] + + +def load_config( + path: str | os.PathLike, environ: dict[str, str] | None = None, *, secrets_for: Collection[str] | None = None, mqtt_secrets: bool = True +) -> Config: + """Load and validate a config file. + + Secrets are resolved only for the targets named in ``secrets_for`` (all of them if + None), so e.g. a restore needs only its own target's credentials to be available. + """ + path = Path(path) + try: + raw = yaml.safe_load(path.read_text()) + except OSError as e: + raise ConfigError(f"cannot read config {path}: {e}") from e + except yaml.YAMLError as e: + raise ConfigError(f"invalid YAML in {path}: {e}") from e + return parse_config(raw, os.environ if environ is None else environ, secrets_for=secrets_for, mqtt_secrets=mqtt_secrets) + + +def parse_config(raw: Any, environ: dict[str, str], *, secrets_for: Collection[str] | None = None, mqtt_secrets: bool = True) -> Config: + if not isinstance(raw, dict): + raise ConfigError("config must be a mapping") + _check_keys(raw, {"backup_root", "defaults", "mqtt", "targets"}, "config") + + defaults = raw.get("defaults") or {} + _check_keys( + defaults, + {"retention", "compression_level", "retries", "retry_delay_s", "stale_partial_hours", "lock_stale_hours", "jobs"}, + "defaults", + ) + base_retention = Retention().merged(defaults.get("retention"), "defaults.retention") + + raw_targets = raw.get("targets") + if not isinstance(raw_targets, list) or not raw_targets: + raise ConfigError("targets: must be a non-empty list") + + targets = [] + seen = set() + for i, rt in enumerate(raw_targets): + target = _parse_target(rt, f"targets[{i}]", defaults, base_retention, environ, secrets_for) + if target.name in seen: + raise ConfigError(f"targets[{i}].name: duplicate target name {target.name!r}") + seen.add(target.name) + targets.append(target) + + mqtt = _parse_mqtt(raw["mqtt"], environ, mqtt_secrets) if raw.get("mqtt") else None + + return Config( + backup_root=Path(raw.get("backup_root", "/backups")), + targets=tuple(targets), + mqtt=mqtt, + stale_partial_hours=_int(defaults, "stale_partial_hours", 24, "defaults", minimum=1), + lock_stale_hours=_int(defaults, "lock_stale_hours", 24, "defaults", minimum=1), + ) + + +def _parse_target( + rt: Any, where: str, defaults: dict, base_retention: Retention, environ: dict[str, str], secrets_for: Collection[str] | None +) -> Target: + if not isinstance(rt, dict): + raise ConfigError(f"{where}: must be a mapping") + _check_keys( + rt, + { + "name", + "type", + "host", + "port", + "username", + "password_env", + "password_file", + "uri_env", + "uri_file", + "include", + "exclude", + "retention", + "compression_level", + "retries", + "retry_delay_s", + "jobs", + "maintenance_db", + "sslmode", + "ssl", + "uri_options", + "dump_args", + }, + where, + ) + + name = rt.get("name") + if not isinstance(name, str) or not TARGET_NAME_RE.match(name): + raise ConfigError(f"{where}.name: must match {TARGET_NAME_RE.pattern}") + where = f"target {name!r}" + + ttype = rt.get("type") + if ttype not in TARGET_TYPES: + raise ConfigError(f"{where}.type: must be one of {', '.join(TARGET_TYPES)}") + + resolve = secrets_for is None or name in secrets_for + has_uri = "uri_env" in rt or "uri_file" in rt + if has_uri and ttype != "mongodb": + raise ConfigError(f"{where}: uri_env/uri_file are only valid for mongodb targets") + uri = resolve_secret(rt, "uri", where, environ) if resolve else None + + host = rt.get("host") + if not has_uri and not (isinstance(host, str) and host): + raise ConfigError(f"{where}.host: required" + (" (or uri_env/uri_file)" if ttype == "mongodb" else "")) + + password = resolve_secret(rt, "password", where, environ) if resolve else None + username = rt.get("username") + if ("password_env" in rt or "password_file" in rt) and not username: + raise ConfigError(f"{where}.username: required when a password is given") + + for key, only_for in ( + ("sslmode", "postgres"), + ("maintenance_db", "postgres"), + ("jobs", "postgres"), + ("ssl", "mariadb"), + ("uri_options", "mongodb"), + ): + if key in rt and ttype != only_for: + raise ConfigError(f"{where}.{key}: only valid for {only_for} targets") + ssl = rt.get("ssl") + if ssl is not None and not isinstance(ssl, bool): + raise ConfigError(f"{where}.ssl: must be true or false") + uri_options = rt.get("uri_options") or {} + if not isinstance(uri_options, dict): + raise ConfigError(f"{where}.uri_options: must be a mapping") + + return Target( + name=name, + type=ttype, + host=host, + port=_int(rt, "port", DEFAULT_PORTS[ttype], where, minimum=1), + username=username, + password=password, + uri=uri, + include=_str_list(rt, "include", where), + exclude=_str_list(rt, "exclude", where), + retention=base_retention.merged(rt.get("retention"), f"{where}.retention"), + compression_level=_int(rt, "compression_level", _int(defaults, "compression_level", 3, "defaults", 1, 19), where, 1, 19), + retries=_int(rt, "retries", _int(defaults, "retries", 1, "defaults", minimum=0), where, minimum=0), + retry_delay_s=_int(rt, "retry_delay_s", _int(defaults, "retry_delay_s", 30, "defaults", minimum=0), where, minimum=0), + jobs=_int(rt, "jobs", _int(defaults, "jobs", 2, "defaults", minimum=1), where, minimum=1), + maintenance_db=rt.get("maintenance_db", "postgres"), + sslmode=rt.get("sslmode"), + ssl=ssl, + uri_options={str(k): str(v) for k, v in uri_options.items()}, + dump_args=_str_list(rt, "dump_args", where), + ) + + +def _parse_mqtt(rm: Any, environ: dict[str, str], resolve: bool) -> MqttConfig: + where = "mqtt" + if not isinstance(rm, dict): + raise ConfigError(f"{where}: must be a mapping") + _check_keys( + rm, + { + "host", + "port", + "username", + "password_env", + "password_file", + "tls", + "topic_prefix", + "ha_discovery", + "discovery_prefix", + "client_id", + }, + where, + ) + if not isinstance(rm.get("host"), str) or not rm["host"]: + raise ConfigError(f"{where}.host: required") + tls = bool(rm.get("tls", False)) + return MqttConfig( + host=rm["host"], + port=_int(rm, "port", 8883 if tls else 1883, where, minimum=1), + username=rm.get("username"), + password=resolve_secret(rm, "password", where, environ) if resolve else None, + tls=tls, + topic_prefix=str(rm.get("topic_prefix", "dbbackup")).rstrip("/"), + ha_discovery=bool(rm.get("ha_discovery", False)), + discovery_prefix=str(rm.get("discovery_prefix", "homeassistant")).rstrip("/"), + client_id=str(rm.get("client_id", "dbbackup")), + ) + + +def resolve_secret(section: dict, base: str, where: str, environ: dict[str, str]) -> str | None: + """Resolve ``_env`` or ``_file`` to its value; None if neither is set.""" + env_key, file_key = f"{base}_env", f"{base}_file" + if env_key in section and file_key in section: + raise ConfigError(f"{where}: set only one of {env_key} and {file_key}") + if env_key in section: + var = section[env_key] + if var not in environ: + raise ConfigError(f"{where}.{env_key}: environment variable {var} is not set") + return environ[var] + if file_key in section: + try: + return Path(section[file_key]).read_text().rstrip("\r\n") + except OSError as e: + raise ConfigError(f"{where}.{file_key}: cannot read {section[file_key]}: {e.strerror}") from e + return None + + +def _check_keys(section: Any, allowed: set[str], where: str) -> None: + if not isinstance(section, dict): + raise ConfigError(f"{where}: must be a mapping") + unknown = sorted(set(section) - allowed) + if unknown: + raise ConfigError(f"{where}: unknown key(s): {', '.join(unknown)}") + + +def _int(section: dict, key: str, default: int, where: str, minimum: int | None = None, maximum: int | None = None) -> int: + value = section.get(key, default) + if not isinstance(value, int) or isinstance(value, bool): + raise ConfigError(f"{where}.{key}: must be an integer") + if (minimum is not None and value < minimum) or (maximum is not None and value > maximum): + raise ConfigError(f"{where}.{key}: must be between {minimum} and {maximum}" if maximum else f"{where}.{key}: must be >= {minimum}") + return value + + +def _str_list(section: dict, key: str, where: str) -> tuple[str, ...]: + value = section.get(key) or [] + if not isinstance(value, list) or not all(isinstance(v, str) for v in value): + raise ConfigError(f"{where}.{key}: must be a list of strings") + return tuple(value) diff --git a/src/dbbackup/engines/__init__.py b/src/dbbackup/engines/__init__.py new file mode 100644 index 0000000..2fd3031 --- /dev/null +++ b/src/dbbackup/engines/__init__.py @@ -0,0 +1,18 @@ +from ..config import Target +from .base import Connection, Engine, EngineError +from .mariadb import MariaDBEngine +from .mongodb import MongoDBEngine +from .postgres import PostgresEngine + +ENGINES: dict[str, type[Engine]] = { + "postgres": PostgresEngine, + "mariadb": MariaDBEngine, + "mongodb": MongoDBEngine, +} + + +def engine_for(target: Target, conn: Connection | None = None) -> Engine: + return ENGINES[target.type](target, conn) + + +__all__ = ["Connection", "Engine", "EngineError", "engine_for", "ENGINES"] diff --git a/src/dbbackup/engines/base.py b/src/dbbackup/engines/base.py new file mode 100644 index 0000000..c161968 --- /dev/null +++ b/src/dbbackup/engines/base.py @@ -0,0 +1,107 @@ +"""The interface every database engine implements.""" + +from __future__ import annotations + +import re +from abc import ABC, abstractmethod +from dataclasses import dataclass, field, replace +from pathlib import Path +from urllib.parse import quote + +from ..config import Target + + +class EngineError(Exception): + pass + + +@dataclass(frozen=True) +class Connection: + """Where to connect. Built from a target, but restore can point it elsewhere.""" + + host: str | None + port: int + username: str | None + password: str | None = field(repr=False) + uri: str | None = field(default=None, repr=False) + + @classmethod + def for_target(cls, target: Target) -> Connection: + return cls(target.host, target.port, target.username, target.password, target.uri) + + def overridden(self, **changes) -> Connection: + return replace(self, **{k: v for k, v in changes.items() if v is not None}) + + +@dataclass +class DumpResult: + """What an engine's backup produced: engine-specific ``contents`` for the manifest, and any warnings.""" + + contents: dict + warnings: list[str] = field(default_factory=list) + + +class Engine(ABC): + type: str + min_version: tuple[int, ...] + # Databases that are never backed up per-database (server internals). + system_databases: frozenset[str] = frozenset() + + def __init__(self, target: Target, conn: Connection | None = None): + self.target = target + self.conn = conn or Connection.for_target(target) + + @abstractmethod + def server_version(self) -> str: + """The server's version string, e.g. ``18.6``.""" + + @abstractmethod + def list_databases(self) -> list[str]: + """Every user database on the server, system databases excluded.""" + + @abstractmethod + def inventory(self, databases: list[str]) -> dict: + """Objects, approximate row counts and users, recorded so a restore can be checked against them. + + Shape: ``{"databases": {db: {"objects": {name: {"kind": str, "rows": int | None}}}}, "users": [str] | None}``. + """ + + @abstractmethod + def backup(self, set_dir: Path, databases: list[str]) -> DumpResult: + """Dump ``databases`` (plus server-level objects such as users) into ``set_dir``.""" + + @abstractmethod + def verify(self, set_dir: Path, manifest: dict) -> None: + """Check the dump files are complete and readable; raise EngineError if not.""" + + @abstractmethod + def restore(self, set_dir: Path, manifest: dict, databases: list[str], *, force: bool, include_globals: bool) -> list[str]: + """Restore ``databases`` (and, if asked, users/roles) from a set. Returns warnings.""" + + def select_databases(self) -> list[str]: + """The databases to back up: all of them, or ``include``, minus ``exclude``.""" + available = self.list_databases() + if self.target.include: + missing = [d for d in self.target.include if d not in available] + if missing: + raise EngineError(f"included database(s) not found on server: {', '.join(missing)}") + chosen = list(self.target.include) + else: + chosen = available + return [d for d in chosen if d not in self.target.exclude] + + def version_warning(self, version: str) -> str | None: + if version_tuple(version) < self.min_version: + minimum = ".".join(map(str, self.min_version)) + return f"server version {version} is older than the minimum supported {minimum}; backups are best-effort" + return None + + +def version_tuple(version: str) -> tuple[int, ...]: + m = re.match(r"\d+(?:\.\d+)*", version.strip()) + return tuple(int(p) for p in m.group(0).split(".")) if m else () + + +def safe_filename(name: str) -> str: + """A filesystem-safe, reversible file name for a database name (which may contain anything).""" + return quote(name, safe="-_").replace(".", "%2E").replace("~", "%7E") diff --git a/src/dbbackup/engines/mariadb.py b/src/dbbackup/engines/mariadb.py new file mode 100644 index 0000000..9154a85 --- /dev/null +++ b/src/dbbackup/engines/mariadb.py @@ -0,0 +1,239 @@ +"""MariaDB: ``mariadb-dump --system=users`` for accounts and grants, plus one +``--single-transaction`` dump per database, each piped through zstd.""" + +from __future__ import annotations + +import contextlib +import logging +import re +from collections.abc import Iterator +from pathlib import Path + +from .. import proc +from .base import DumpResult, Engine, EngineError, safe_filename + +log = logging.getLogger(__name__) + +SYSTEM_DATABASES = frozenset({"information_schema", "performance_schema", "sys", "mysql"}) +USERS_FILE = "system-users.sql.zst" +TRAILER = b"-- Dump completed" +MAX_PACKET = "--max-allowed-packet=1G" + +TABLE_KINDS = {"BASE TABLE": "table", "SYSTEM VERSIONED": "table", "VIEW": "view", "SEQUENCE": "sequence"} +_UNESCAPE = re.compile(r"\\(.)") +_UNESCAPES = {"n": "\n", "t": "\t", "0": "\0", "\\": "\\"} + + +def quote_ident(name: str) -> str: + return "`" + name.replace("`", "``") + "`" + + +def quote_str(value: str) -> str: + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" + + +def option_file_value(value: str) -> str: + return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +# An account name part: `backticked`, 'single-quoted', or bare. +_NAME = r"(?:`(?:[^`]|``)*`|'(?:[^'\\]|\\.|'')*'|[A-Za-z0-9_$.]+)" +_ACCOUNT = rf"(?P{_NAME})(?:@(?P{_NAME}))?" +# Statements in a --system=users dump that create or change one account (user or role). +_ACCOUNT_STATEMENTS = [ + re.compile(rf"^CREATE (?:USER|ROLE) IF NOT EXISTS {_ACCOUNT}(?:\s|;|$)"), + re.compile(rf"^ALTER USER {_ACCOUNT}(?:\s|;|$)"), + re.compile(rf"^SET DEFAULT ROLE \S+ FOR {_ACCOUNT}(?:\s|;|$)"), + re.compile(rf"^GRANT .*? TO {_ACCOUNT}(?:\s|;|$)"), +] +# Version-conditional comments, /*!80001 ... */ and /*M!100005 ... */, wrap many of them. +_CONDITIONAL = re.compile(r"^/\*M?!\d+\s+(.*?)\s*\*/;?$") + + +def _unquote(name: str) -> str: + if name[0] == "`": + return name[1:-1].replace("``", "`") + if name[0] == "'": + return re.sub(r"\\(.)|''", lambda m: m.group(1) or "'", name[1:-1]) + return name + + +def statement_account(line: str) -> tuple[str, str | None] | None: + """The (user, host) a users-dump statement creates or changes; host is None for a role. None if no single account.""" + m = _CONDITIONAL.match(line) + body = m.group(1) if m else line + for pattern in _ACCOUNT_STATEMENTS: + if m := pattern.match(body): + return _unquote(m["user"]), _unquote(m["host"]) if m["host"] else None + return None + + +def without_accounts(sql: str, existing: set[tuple[str, str | None]]) -> tuple[str, set[tuple[str, str | None]]]: + """Drop the statements in a ``--system=users`` dump that would create or change an account in ``existing``. + + Returns the remaining SQL, and the accounts that were left alone. + """ + kept = [] + skipped = set() + for line in sql.splitlines(): + account = statement_account(line) + if account is not None and account in existing: + skipped.add(account) + else: + kept.append(line) + return "\n".join(kept) + "\n", skipped + + +def format_account(account: tuple[str, str | None]) -> str: + user, host = account + return user if host is None else f"{user}@{host}" + + +class MariaDBEngine(Engine): + type = "mariadb" + min_version = (11, 4) + + # -- connection helpers ---------------------------------------------------- + + @contextlib.contextmanager + def _client_args(self) -> Iterator[list[str]]: + """Connection arguments; the credentials go in a 0600 option file, which must be the first argument.""" + lines = ["[client]"] + if self.conn.username: + lines.append(f"user={option_file_value(self.conn.username)}") + if self.conn.password is not None: + lines.append(f"password={option_file_value(self.conn.password)}") + with proc.secret_file("\n".join(lines) + "\n", suffix=".cnf") as path: + args = [f"--defaults-extra-file={path}", "-h", self.conn.host, "-P", str(self.conn.port), "--protocol=TCP"] + if self.target.ssl is True: + args.append("--ssl") + elif self.target.ssl is False: + args.append("--skip-ssl") + yield args + + def query(self, sql: str) -> list[list[str | None]]: + with self._client_args() as args: + out = proc.run(["mariadb", *args, "--batch", "--skip-column-names", "-e", sql]).stdout + rows = [] + for line in out.splitlines(): + rows.append( + [None if f == "NULL" else _UNESCAPE.sub(lambda m: _UNESCAPES.get(m.group(1), m.group(1)), f) for f in line.split("\t")] + ) + return rows + + # -- discovery ------------------------------------------------------------- + + def server_version(self) -> str: + return self.query("SELECT VERSION()")[0][0] + + def list_databases(self) -> list[str]: + return sorted(r[0] for r in self.query("SHOW DATABASES") if r[0] not in SYSTEM_DATABASES) + + def _tables(self, databases: list[str]) -> list[list[str | None]]: + if not databases: + return [] + schemas = ", ".join(quote_str(d) for d in databases) + return self.query( + "SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, ENGINE, TABLE_ROWS FROM information_schema.TABLES " + f"WHERE TABLE_SCHEMA IN ({schemas})" + ) + + def inventory(self, databases: list[str]) -> dict: + result = {db: {"objects": {}} for db in databases} + for schema, name, ttype, _engine, rows in self._tables(databases): + kind = TABLE_KINDS.get(ttype, ttype.lower()) + result[schema]["objects"][name] = {"kind": kind, "rows": int(rows) if rows is not None and kind == "table" else None} + users = sorted( + name if is_role == "Y" else f"{name}@{host}" for name, host, is_role in self.query("SELECT User, Host, is_role FROM mysql.user") + ) + return {"databases": result, "users": users} + + # -- backup ---------------------------------------------------------------- + + def backup(self, set_dir: Path, databases: list[str]) -> DumpResult: + warnings = [] + for schema, name, ttype, engine, _rows in self._tables(databases): + if ttype == "BASE TABLE" and engine and engine != "InnoDB": + warnings.append(f"{schema}.{name} uses {engine}, not InnoDB, so its dump isn't transactionally consistent") + + with self._client_args() as args: + log.info("%s: dumping users and grants", self.target.name) + proc.pipe_to_zstd( + ["mariadb-dump", *args, "--system=users", "--insert-ignore"], set_dir / USERS_FILE, self.target.compression_level + ) + + files = {} + for db in databases: + name = safe_filename(db) + ".sql.zst" + log.info("%s: dumping database %s", self.target.name, db) + cmd = [ + "mariadb-dump", + *args, + "--single-transaction", + "--routines", + "--events", + "--triggers", + "--hex-blob", + MAX_PACKET, + *self.target.dump_args, + "--databases", + db, + ] + stderr = proc.pipe_to_zstd(cmd, set_dir / name, self.target.compression_level) + warnings += [f"{db}: {line.strip()}" for line in stderr.splitlines() if "warning" in line.lower()] + files[db] = name + + return DumpResult({"users": USERS_FILE, "databases": files}, warnings) + + def verify(self, set_dir: Path, manifest: dict) -> None: + contents = manifest["contents"] + for label, name in [("users", contents["users"]), *contents["databases"].items()]: + try: + _, tail = proc.zstd_head_tail(set_dir / name) + except proc.CommandError as e: + raise EngineError(f"{label}: {name} is corrupt: {e}") from e + if TRAILER not in tail: + raise EngineError(f"{label}: {name} is incomplete (no completion trailer)") + + # -- restore --------------------------------------------------------------- + + def restore(self, set_dir: Path, manifest: dict, databases: list[str], *, force: bool, include_globals: bool) -> list[str]: + contents = manifest["contents"] + warnings = [] + + existing = set(self.list_databases()) + clashes = [db for db in databases if db in existing] + if clashes and not force: + raise EngineError(f"database(s) already exist on {self.conn.host}: {', '.join(clashes)} (use --force to replace them)") + + # Databases first: table-level grants can only be applied once their tables exist. + with self._client_args() as args: + for db in databases: + if db in clashes: + log.warning("%s: dropping existing database %s (--force)", self.target.name, db) + self.query(f"DROP DATABASE {quote_ident(db)}") + log.info("%s: restoring database %s", self.target.name, db) + try: + proc.zstd_pipe_from(set_dir / contents["databases"][db], ["mariadb", *args, MAX_PACKET]) + except proc.CommandError as e: + raise EngineError(f"{db}: restore failed: {e}") from e + + if include_globals: + log.info("%s: restoring users and grants", self.target.name) + warnings += self._restore_users(set_dir / contents["users"], args) + return warnings + + def _restore_users(self, path: Path, args: list[str]) -> list[str]: + # Accounts that already exist on the destination (including the one we're + # restoring as, and on a fresh server its own root and healthcheck accounts) + # are left exactly as they are: replaying their GRANT ... IDENTIFIED BY + # PASSWORD would reset their passwords to the backed-up server's. + existing = { + (user, None if is_role == "Y" else host) for user, host, is_role in self.query("SELECT User, Host, is_role FROM mysql.user") + } + sql, skipped = without_accounts(proc.run(["zstd", "-q", "-dc", str(path)]).stdout, existing) + if skipped: + log.info("%s: left existing account(s) as they are: %s", self.target.name, ", ".join(sorted(map(format_account, skipped)))) + # --force: carry on past individual failures (e.g. grants on tables that weren't restored) and report them. + result = proc.run(["mariadb", *args, "--force"], input=sql, check=False) + return [f"users: {line.strip()}" for line in result.stderr.splitlines() if line.startswith("ERROR")] diff --git a/src/dbbackup/engines/mongodb.py b/src/dbbackup/engines/mongodb.py new file mode 100644 index 0000000..d6610b7 --- /dev/null +++ b/src/dbbackup/engines/mongodb.py @@ -0,0 +1,201 @@ +"""MongoDB: by default one whole-instance ``mongodump --oplog --archive``, which +includes users and roles (in ``admin``) and is consistent to a single point in time. + +``mongodump`` can't exclude databases from a whole-instance dump, so a target with +``include`` or ``exclude`` gets one archive per database (plus ``admin``, for users +and roles) instead. Those can't use ``--oplog``, so each is consistent only per +collection; the manifest records a warning saying so. +""" + +from __future__ import annotations + +import json +import logging +import re +from pathlib import Path +from urllib.parse import quote_plus, urlencode + +from .. import proc +from .base import DumpResult, Engine, EngineError, safe_filename + +log = logging.getLogger(__name__) + +SYSTEM_DATABASES = frozenset({"admin", "config", "local"}) +INSTANCE_FILE = "mongodb.archive.zst" +USERS_FILE = "admin.archive.zst" +ARCHIVE_MAGIC = bytes.fromhex("6de29981") # 0x8199e26d, little-endian +AUTH_COLLECTIONS = ("admin.system.users", "admin.system.roles") +FAILED_RE = re.compile(r"(\d+) document\(s\) failed to restore") + + +class MongoDBEngine(Engine): + type = "mongodb" + min_version = (8, 3) + + # -- connection helpers ---------------------------------------------------- + + def uri(self) -> str: + if self.conn.uri: + return self.conn.uri + userinfo = "" + if self.conn.username: + userinfo = quote_plus(self.conn.username) + if self.conn.password is not None: + userinfo += ":" + quote_plus(self.conn.password) + userinfo += "@" + params = {"authSource": "admin", **self.target.uri_options} + return f"mongodb://{userinfo}{self.conn.host}:{self.conn.port}/?{urlencode(params)}" + + def _config_file(self): + """A 0600 mongodump/mongorestore --config file holding the URI (it contains the password).""" + return proc.secret_file(f"uri: {json.dumps(self.uri())}\n", suffix=".yaml") + + def _client(self): + from pymongo import MongoClient + + return MongoClient(self.uri(), serverSelectionTimeoutMS=15000, appname="dbbackup") + + # -- discovery ------------------------------------------------------------- + + def server_version(self) -> str: + with self._client() as client: + return client.server_info()["version"] + + def is_replica_set(self) -> bool: + with self._client() as client: + return bool(client.admin.command("hello").get("setName")) + + def list_databases(self) -> list[str]: + with self._client() as client: + return sorted(d for d in client.list_database_names() if d not in SYSTEM_DATABASES) + + def inventory(self, databases: list[str]) -> dict: + from pymongo.errors import OperationFailure + + result = {} + with self._client() as client: + for db in databases: + objects = {} + for coll in client[db].list_collections(): + name, ctype = coll["name"], coll.get("type", "collection") + if name.startswith("system."): + continue + rows = client[db][name].estimated_document_count() if ctype == "collection" else None + objects[name] = {"kind": ctype, "rows": rows} + result[db] = {"objects": objects} + try: + users = sorted(f"{u['user']}@{u['db']}" for u in client.admin["system.users"].find({}, {"user": 1, "db": 1})) + users += sorted(f"role:{r['role']}@{r['db']}" for r in client.admin["system.roles"].find({}, {"role": 1, "db": 1})) + except OperationFailure as e: + log.warning("%s: can't list users (%s); restore checks will skip them", self.target.name, e) + users = None + return {"databases": result, "users": users} + + # -- backup ---------------------------------------------------------------- + + def backup(self, set_dir: Path, databases: list[str]) -> DumpResult: + warnings = [] + replica_set = self.is_replica_set() + level = self.target.compression_level + + with self._config_file() as cfg: + base = ["mongodump", f"--config={cfg}", "--archive", *self.target.dump_args] + + if not (self.target.include or self.target.exclude): + if not replica_set: + warnings.append("server is not a replica set member, so the dump can't use --oplog and isn't point-in-time consistent") + log.info("%s: dumping whole instance%s", self.target.name, " with oplog" if replica_set else "") + proc.pipe_to_zstd(base + (["--oplog"] if replica_set else []), set_dir / INSTANCE_FILE, level) + return DumpResult({"mode": "instance", "oplog": replica_set, "archive": INSTANCE_FILE}, warnings) + + warnings.append( + "include/exclude is set, so databases are dumped one by one without --oplog: " + "each is consistent per collection only, not across the instance" + ) + log.info("%s: dumping users and roles", self.target.name) + proc.pipe_to_zstd(base + ["--db=admin"], set_dir / USERS_FILE, level) + files = {} + for db in databases: + name = safe_filename(db) + ".archive.zst" + log.info("%s: dumping database %s", self.target.name, db) + proc.pipe_to_zstd(base + [f"--db={db}"], set_dir / name, level) + files[db] = name + return DumpResult({"mode": "per-database", "oplog": False, "users": USERS_FILE, "databases": files}, warnings) + + def verify(self, set_dir: Path, manifest: dict) -> None: + contents = manifest["contents"] + if contents["mode"] == "instance": + archives = [contents["archive"]] + else: + archives = [contents["users"], *contents["databases"].values()] + for name in archives: + try: + head, _ = proc.zstd_head_tail(set_dir / name) + except proc.CommandError as e: + raise EngineError(f"{name} is corrupt: {e}") from e + if head[:4] != ARCHIVE_MAGIC: + raise EngineError(f"{name} is not a mongodump archive") + + # -- restore --------------------------------------------------------------- + + def restore(self, set_dir: Path, manifest: dict, databases: list[str], *, force: bool, include_globals: bool) -> list[str]: + contents = manifest["contents"] + warnings = [] + + existing = set(self.list_databases()) + clashes = [db for db in databases if db in existing] + if clashes and not force: + where = self.conn.host or "the server" + raise EngineError(f"database(s) already exist on {where}: {', '.join(clashes)} (use --force to replace them)") + if clashes: + with self._client() as client: + for db in clashes: + log.warning("%s: dropping existing database %s (--force)", self.target.name, db) + client.drop_database(db) + + with self._config_file() as cfg: + # Never --drop: it would also replace existing users with the backed-up + # server's (resetting e.g. the password of the account we restore as). + # Without it, accounts that already exist are left alone; --force is + # handled above, by dropping the clashing databases first. + base = ["mongorestore", f"--config={cfg}", "--archive"] + + if contents["mode"] == "instance": + everything = set(databases) == set(manifest["databases"]) + cmd = list(base) + if not everything: + cmd += [f"--nsInclude={_ns_escape(db)}.*" for db in databases] + if contents["oplog"]: + warnings.append("restoring only some databases, so the oplog was not replayed") + if include_globals: + warnings.append("restoring only some databases, so users and roles were not restored") + elif include_globals: + if contents["oplog"]: + cmd.append("--oplogReplay") + else: + cmd += [f"--nsExclude={ns}" for ns in AUTH_COLLECTIONS] + if contents["oplog"]: + warnings.append("mongorestore can't replay the oplog while excluding users and roles, so it was not replayed") + log.info("%s: restoring %s", self.target.name, "whole instance" if everything else ", ".join(databases)) + warnings += self._mongorestore(cmd, set_dir / contents["archive"]) + else: + for db in databases: + log.info("%s: restoring database %s", self.target.name, db) + warnings += self._mongorestore(base, set_dir / contents["databases"][db]) + if include_globals: + log.info("%s: restoring users and roles", self.target.name) + warnings += self._mongorestore(base, set_dir / contents["users"]) + return warnings + + def _mongorestore(self, cmd: list[str], archive: Path) -> list[str]: + try: + stderr = proc.zstd_pipe_from(archive, cmd) + except proc.CommandError as e: + raise EngineError(f"mongorestore failed: {e}") from e + failed = sum(int(n) for n in FAILED_RE.findall(stderr)) + return [f"{archive.name}: {failed} document(s) failed to restore"] if failed else [] + + +def _ns_escape(db: str) -> str: + """Escape a database name for a mongorestore namespace pattern.""" + return re.sub(r"([\\*$])", r"\\\1", db) diff --git a/src/dbbackup/engines/postgres.py b/src/dbbackup/engines/postgres.py new file mode 100644 index 0000000..1a78828 --- /dev/null +++ b/src/dbbackup/engines/postgres.py @@ -0,0 +1,231 @@ +"""PostgreSQL: ``pg_dumpall --globals-only`` for roles and tablespaces, plus one +directory-format ``pg_dump`` per database (parallel dump and restore, zstd inside).""" + +from __future__ import annotations + +import logging +import re +from pathlib import Path + +from .. import proc +from .base import DumpResult, Engine, EngineError, safe_filename + +log = logging.getLogger(__name__) + +# psql output separators that can't plausibly appear in identifiers. +FIELD_SEP = "\x1f" +RECORD_SEP = "\x1e" + +GLOBALS_FILE = "globals.sql.zst" +GLOBALS_TRAILER = b"PostgreSQL database cluster dump complete" + +RELKINDS = {"r": "table", "p": "partitioned table", "v": "view", "m": "materialized view", "f": "foreign table"} + + +def conninfo_dbname(db: str) -> str: + """A ``dbname=...`` conninfo string, so database names are never parsed as connection strings.""" + return "dbname='" + db.replace("\\", "\\\\").replace("'", "\\'") + "'" + + +_ROLE_STATEMENT = re.compile(r'^(?:CREATE|ALTER|COMMENT ON) ROLE ("(?:[^"]|"")*"|[^\s;]+)') + + +def without_roles(sql: str, existing: set[str]) -> tuple[str, set[str]]: + """Drop the statements in a globals dump that create or alter a role in ``existing``. + + Returns the remaining SQL, and the roles that were left alone. Memberships + (GRANT role TO role) are kept: they only add. + """ + kept = [] + skipped = set() + for line in sql.splitlines(): + if m := _ROLE_STATEMENT.match(line): + name = m.group(1) + role = name[1:-1].replace('""', '"') if name.startswith('"') else name + if role in existing: + skipped.add(role) + continue + kept.append(line) + return "\n".join(kept) + "\n", skipped + + +class PostgresEngine(Engine): + type = "postgres" + min_version = (18,) + + # -- connection helpers ---------------------------------------------------- + + def _env(self) -> dict[str, str]: + env = {"PGAPPNAME": "dbbackup", "PGCONNECT_TIMEOUT": "15"} + if self.conn.password is not None: + env["PGPASSWORD"] = self.conn.password + if self.target.sslmode: + env["PGSSLMODE"] = self.target.sslmode + return env + + def _conn_args(self) -> list[str]: + args = ["-h", self.conn.host, "-p", str(self.conn.port)] + if self.conn.username: + args += ["-U", self.conn.username] + return args + + def query(self, sql: str, db: str | None = None, **variables: str) -> list[list[str]]: + """Run SQL through psql. ``variables`` are psql variables, interpolated safely as :'name' or :"name".""" + cmd = [ + "psql", + "-X", + "-q", + "-A", + "-t", + "-F", + FIELD_SEP, + "-R", + RECORD_SEP, + "-v", + "ON_ERROR_STOP=1", + *self._conn_args(), + "-d", + conninfo_dbname(db or self.target.maintenance_db), + ] + for name, value in variables.items(): + cmd += ["-v", f"{name}={value}"] + out = proc.run(cmd, env=self._env(), input=sql).stdout + return [record.split(FIELD_SEP) for record in out.rstrip("\n").split(RECORD_SEP) if record] + + # -- discovery ------------------------------------------------------------- + + def server_version(self) -> str: + return self.query("SHOW server_version")[0][0].split()[0] + + def list_databases(self) -> list[str]: + rows = self.query("SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate ORDER BY datname") + return [r[0] for r in rows] + + def can_read_passwords(self) -> bool: + """Superusers, and members of pg_read_all_data, can read pg_authid and so dump role passwords.""" + return self.query("SELECT has_table_privilege('pg_catalog.pg_authid', 'SELECT')")[0][0] == "t" + + def inventory(self, databases: list[str]) -> dict: + result = {} + for db in databases: + rows = self.query( + r"""SELECT n.nspname, c.relname, c.relkind, c.reltuples::bigint + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg\_toast%' AND n.nspname NOT LIKE 'pg\_temp%'""", + db=db, + ) + objects = {} + for schema, name, kind, tuples in rows: + rows_est = int(tuples) if kind in ("r", "p", "m") and int(tuples) >= 0 else None + objects[f"{schema}.{name}"] = {"kind": RELKINDS[kind], "rows": rows_est} + result[db] = {"objects": objects} + users = [r[0] for r in self.query("SELECT rolname FROM pg_roles WHERE rolname !~ '^pg_' ORDER BY rolname")] + return {"databases": result, "users": users} + + # -- backup ---------------------------------------------------------------- + + def backup(self, set_dir: Path, databases: list[str]) -> DumpResult: + warnings = [] + passwords = self.can_read_passwords() + globals_cmd = ["pg_dumpall", *self._conn_args(), "-d", conninfo_dbname(self.target.maintenance_db), "--globals-only"] + if not passwords: + globals_cmd.append("--no-role-passwords") + warnings.append("backup role can't read pg_authid, so role passwords were not saved; grant it pg_read_all_data") + log.info("%s: dumping roles and tablespaces", self.target.name) + proc.pipe_to_zstd(globals_cmd, set_dir / GLOBALS_FILE, self.target.compression_level, env=self._env()) + + files = {} + for db in databases: + name = safe_filename(db) + ".pgdump" + log.info("%s: dumping database %s", self.target.name, db) + cmd = [ + "pg_dump", + *self._conn_args(), + "-d", + conninfo_dbname(db), + "--format=directory", + f"--jobs={self.target.jobs}", + f"--compress=zstd:{self.target.compression_level}", + f"--file={set_dir / name}", + *self.target.dump_args, + ] + stderr = proc.run(cmd, env=self._env()).stderr + warnings += [f"{db}: {line.strip()}" for line in stderr.splitlines() if "warning" in line.lower()] + files[db] = name + + return DumpResult({"globals": GLOBALS_FILE, "role_passwords": passwords, "databases": files}, warnings) + + def verify(self, set_dir: Path, manifest: dict) -> None: + contents = manifest["contents"] + _, tail = proc.zstd_head_tail(set_dir / contents["globals"]) + if GLOBALS_TRAILER not in tail: + raise EngineError(f"{contents['globals']}: dump is incomplete (no completion trailer)") + for db, name in contents["databases"].items(): + if not (set_dir / name / "toc.dat").is_file(): + raise EngineError(f"{db}: {name} has no toc.dat") + try: + proc.run(["pg_restore", "--list", str(set_dir / name)]) + except proc.CommandError as e: + raise EngineError(f"{db}: pg_restore cannot read {name}: {e}") from e + + # -- restore --------------------------------------------------------------- + + def restore(self, set_dir: Path, manifest: dict, databases: list[str], *, force: bool, include_globals: bool) -> list[str]: + contents = manifest["contents"] + warnings = [] + maintenance_db = self.target.maintenance_db + + existing = set(self.list_databases()) + # Every cluster has the maintenance database, so it only clashes if something is in it. + clashes = [db for db in databases if db in existing and (db != maintenance_db or self._has_user_objects(db))] + if clashes and not force: + raise EngineError(f"database(s) already exist on {self.conn.host}: {', '.join(clashes)} (use --force to replace them)") + + if include_globals: + # Roles first: restored objects are owned by, and granted to, them. + log.info("%s: restoring roles and tablespaces", self.target.name) + warnings += self._restore_globals(set_dir / contents["globals"]) + if not contents.get("role_passwords", True): + warnings.append("this backup has no role passwords; restored login roles need their passwords set again") + + for db in databases: + cmd = ["pg_restore", *self._conn_args(), f"--jobs={self.target.jobs}"] + if db == maintenance_db and db in existing: + # We can't drop or create the database we connect through, so restore into it in place. + log.info("%s: restoring database %s in place", self.target.name, db) + cmd += ["-d", conninfo_dbname(db)] + (["--clean", "--if-exists"] if db in clashes else []) + else: + if db in clashes: + log.warning("%s: dropping existing database %s (--force)", self.target.name, db) + self.query('DROP DATABASE :"target_db" WITH (FORCE)', target_db=db) + log.info("%s: restoring database %s", self.target.name, db) + cmd += ["-d", conninfo_dbname(maintenance_db), "--create"] + cmd.append(str(set_dir / contents["databases"][db])) + try: + proc.run(cmd, env=self._env()) + except proc.CommandError as e: + raise EngineError(f"{db}: pg_restore failed: {e}") from e + proc.run(["vacuumdb", *self._conn_args(), "--analyze-only", "--quiet", "-d", conninfo_dbname(db)], env=self._env()) + return warnings + + def _has_user_objects(self, db: str) -> bool: + rows = self.query( + r"""SELECT EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND n.nspname NOT LIKE 'pg\_toast%')""", + db=db, + ) + return rows[0][0] == "t" + + def _restore_globals(self, path: Path) -> list[str]: + # Roles that already exist on the destination (including the one we're + # restoring as) are left exactly as they are: replaying their ALTER ROLE + # ... PASSWORD would reset their passwords to the backed-up server's. + existing = {r[0] for r in self.query("SELECT rolname FROM pg_roles")} + sql, skipped = without_roles(proc.run(["zstd", "-q", "-dc", str(path)]).stdout, existing) + if skipped: + log.info("%s: left existing role(s) as they are: %s", self.target.name, ", ".join(sorted(skipped))) + cmd = ["psql", "-X", "-q", "-v", "ON_ERROR_STOP=0", *self._conn_args(), "-d", conninfo_dbname(self.target.maintenance_db)] + stderr = proc.run(cmd, env=self._env(), input=sql).stderr + return [f"globals: {line.strip()}" for line in stderr.splitlines() if "ERROR:" in line] diff --git a/src/dbbackup/notify.py b/src/dbbackup/notify.py new file mode 100644 index 0000000..f48407c --- /dev/null +++ b/src/dbbackup/notify.py @@ -0,0 +1,114 @@ +"""MQTT status publishing, with optional Home Assistant discovery. + +Every message is retained, so Home Assistant (or anything else) always sees each +target's latest status, and can tell from ``last_success`` when backups have +stopped happening at all, which a failure notification alone never reveals. +""" + +from __future__ import annotations + +import json +import logging +import re + +from . import __version__ +from .config import MqttConfig + +log = logging.getLogger(__name__) + +DEVICE = { + "identifiers": ["dbbackup"], + "name": "DB Backup", + "manufacturer": "Arkane Systems", + "model": "db-backup", + "sw_version": __version__, +} +ORIGIN = {"name": "db-backup", "sw_version": __version__, "support_url": "https://github.com/arkane-systems/db-backup"} + + +def entity_slug(target: str) -> str: + return re.sub(r"[^a-z0-9_]", "_", target.lower()) + + +def status_topic(cfg: MqttConfig, target: str) -> str: + return f"{cfg.topic_prefix}/{target}/status" + + +def discovery_messages(cfg: MqttConfig, target: str) -> list[tuple[str, dict]]: + """HA MQTT discovery configs for a target's two sensors. + + Entity IDs are fixed (``sensor.dbbackup__status`` and ``..._last_success``) + so Alert Redux generators can target them by pattern. + """ + slug = entity_slug(target) + state_topic = status_topic(cfg, target) + sensors = { + "status": { + "name": f"{target} status", + "value_template": "{{ value_json.state }}", + "json_attributes_topic": state_topic, + "icon": "mdi:database-check", + }, + "last_success": { + "name": f"{target} last success", + "value_template": "{{ value_json.last_success }}", + "device_class": "timestamp", + "icon": "mdi:database-clock", + }, + } + messages = [] + for key, extra in sensors.items(): + object_id = f"dbbackup_{slug}_{key}" + payload = { + "unique_id": object_id, + "default_entity_id": f"sensor.{object_id}", + "state_topic": state_topic, + "device": DEVICE, + "origin": ORIGIN, + **extra, + } + messages.append((f"{cfg.discovery_prefix}/sensor/{object_id}/config", payload)) + return messages + + +def publish_statuses(cfg: MqttConfig | None, statuses: list[dict]) -> bool: + """Publish each target's status, and an overall summary, retained. Never raises: returns False on failure.""" + if cfg is None: + return True + + messages: list[tuple[str, dict]] = [] + for status in statuses: + if cfg.ha_discovery: + messages += discovery_messages(cfg, status["target"]) + messages.append((status_topic(cfg, status["target"]), status)) + overall = { + "state": "ok" if all(s["state"] == "ok" for s in statuses) else "failed", + "finished": max((s["finished"] for s in statuses), default=None), + "targets": {s["target"]: s["state"] for s in statuses}, + } + messages.append((f"{cfg.topic_prefix}/status", overall)) + + try: + import paho.mqtt.client as mqtt + + client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, client_id=cfg.client_id) + if cfg.username: + client.username_pw_set(cfg.username, cfg.password) + if cfg.tls: + client.tls_set() + client.connect(cfg.host, cfg.port, keepalive=30) + client.loop_start() + try: + infos = [client.publish(topic, json.dumps(payload), qos=1, retain=True) for topic, payload in messages] + for info in infos: + info.wait_for_publish(timeout=15) + if not info.is_published(): + raise TimeoutError("MQTT broker did not acknowledge a publish in time") + finally: + client.disconnect() + client.loop_stop() + except Exception as e: # never let notification problems fail a backup run + log.error("MQTT: could not publish status to %s:%d: %s", cfg.host, cfg.port, e) + return False + log.info("MQTT: published status for %d target(s)", len(statuses)) + return True diff --git a/src/dbbackup/proc.py b/src/dbbackup/proc.py new file mode 100644 index 0000000..188130e --- /dev/null +++ b/src/dbbackup/proc.py @@ -0,0 +1,129 @@ +"""Subprocess helpers. + +Secrets are passed to child processes only through their environment or through +0600 temporary files (``secret_file``), never on the command line, where any user +on the host could read them from /proc. + +Pipelines check every process in the pipe, as ``set -o pipefail`` would: a dump +that dies halfway leaves a perfectly valid, truncated zstd file behind it, so +zstd's exit code alone proves nothing. +""" + +from __future__ import annotations + +import contextlib +import logging +import os +import subprocess +import tempfile +from collections.abc import Iterator, Sequence +from pathlib import Path + +log = logging.getLogger(__name__) + +STDERR_TAIL = 4000 + + +class CommandError(Exception): + def __init__(self, cmd: Sequence[str], returncode: int, stderr: str): + self.cmd = list(cmd) + self.returncode = returncode + self.stderr = stderr + detail = stderr.strip()[-STDERR_TAIL:] or "(no output)" + super().__init__(f"{cmd[0]} exited with status {returncode}: {detail}") + + +def _env(extra: dict[str, str] | None) -> dict[str, str]: + env = dict(os.environ) + if extra: + env.update(extra) + return env + + +def run( + cmd: Sequence[str], *, env: dict[str, str] | None = None, input: str | None = None, check: bool = True +) -> subprocess.CompletedProcess: + log.debug("run: %s", " ".join(cmd)) + proc = subprocess.run(list(cmd), env=_env(env), input=input, capture_output=True, text=True) + if check and proc.returncode != 0: + raise CommandError(cmd, proc.returncode, proc.stderr) + return proc + + +def pipe_to_zstd(cmd: Sequence[str], out_path: Path, level: int, *, env: dict[str, str] | None = None) -> str: + """Run ``cmd | zstd - -o out_path``; return cmd's stderr (warnings).""" + log.debug("dump: %s > %s", " ".join(cmd), out_path) + zstd_cmd = ["zstd", "-q", "-f", "-T0", f"-{level}", "-o", str(out_path)] + with tempfile.TemporaryFile() as err1, tempfile.TemporaryFile() as err2: + producer = subprocess.Popen(list(cmd), env=_env(env), stdout=subprocess.PIPE, stderr=err1) + try: + consumer = subprocess.Popen(zstd_cmd, stdin=producer.stdout, stderr=err2) + except BaseException: + producer.kill() + producer.wait() + raise + producer.stdout.close() # so the producer gets SIGPIPE if zstd dies + rc_consumer = consumer.wait() + rc_producer = producer.wait() + stderr = _read(err1) + if rc_producer != 0: + raise CommandError(cmd, rc_producer, stderr) + if rc_consumer != 0: + raise CommandError(zstd_cmd, rc_consumer, _read(err2)) + return stderr + + +def zstd_pipe_from(in_path: Path, cmd: Sequence[str], *, env: dict[str, str] | None = None) -> str: + """Run ``zstd -dc in_path | cmd``; return cmd's stderr.""" + log.debug("load: %s < %s", " ".join(cmd), in_path) + zstd_cmd = ["zstd", "-q", "-dc", str(in_path)] + with tempfile.TemporaryFile() as err1, tempfile.TemporaryFile() as err2: + producer = subprocess.Popen(zstd_cmd, stdout=subprocess.PIPE, stderr=err1) + try: + consumer = subprocess.Popen(list(cmd), env=_env(env), stdin=producer.stdout, stdout=subprocess.DEVNULL, stderr=err2) + except BaseException: + producer.kill() + producer.wait() + raise + producer.stdout.close() + rc_consumer = consumer.wait() + rc_producer = producer.wait() + stderr = _read(err2) + if rc_consumer != 0: + raise CommandError(cmd, rc_consumer, stderr) + if rc_producer != 0: + raise CommandError(zstd_cmd, rc_producer, _read(err1)) + return stderr + + +def zstd_head_tail(path: Path, head: int = 64, tail: int = 4096) -> tuple[bytes, bytes]: + """Decompress the whole of a .zst file (proving it intact) and return its first and last bytes.""" + proc = subprocess.Popen(["zstd", "-q", "-dc", str(path)], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + first = b"" + last = b"" + while chunk := proc.stdout.read(1 << 20): + if len(first) < head: + first += chunk[: head - len(first)] + last = (last + chunk)[-tail:] + stderr = proc.stderr.read().decode(errors="replace") + if proc.wait() != 0: + raise CommandError(["zstd", "-dc", str(path)], proc.returncode, stderr) + return first, last + + +@contextlib.contextmanager +def secret_file(content: str, suffix: str = "") -> Iterator[Path]: + """A 0600 temporary file holding ``content``, deleted afterwards.""" + fd, name = tempfile.mkstemp(prefix="dbbackup-", suffix=suffix) + try: + with os.fdopen(fd, "w") as f: + f.write(content) + yield Path(name) + finally: + with contextlib.suppress(FileNotFoundError): + os.unlink(name) + + +def _read(f) -> str: + f.seek(0) + return f.read().decode(errors="replace") diff --git a/src/dbbackup/retention.py b/src/dbbackup/retention.py new file mode 100644 index 0000000..79410a6 --- /dev/null +++ b/src/dbbackup/retention.py @@ -0,0 +1,49 @@ +"""Grandfather-father-son retention. + +Semantics match restic/borg ``--keep-*``: for each kind of period (day, ISO week, +month, year), walk the backups newest first and keep the newest backup in each of +the last N distinct periods that *have* a backup. ``last`` keeps the N newest +backups outright. The union of everything selected is kept, and the newest backup +is always kept, whatever the policy says. + +Periods are computed in UTC, the timezone backup timestamps are recorded in. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from datetime import datetime + +from .config import Retention + +_PERIODS: tuple[tuple[str, Callable[[datetime], tuple]], ...] = ( + ("daily", lambda t: (t.year, t.month, t.day)), + ("weekly", lambda t: tuple(t.isocalendar())[:2]), + ("monthly", lambda t: (t.year, t.month)), + ("yearly", lambda t: (t.year,)), +) + + +def select_keep(timestamps: Iterable[datetime], policy: Retention) -> set[datetime]: + newest_first = sorted(set(timestamps), reverse=True) + if not newest_first: + return set() + + keep = {newest_first[0]} + keep.update(newest_first[: policy.last]) + + for attr, period_of in _PERIODS: + wanted = getattr(policy, attr) + if wanted <= 0: + continue + seen: set[tuple] = set() + for ts in newest_first: + period = period_of(ts) + if period in seen: + continue + seen.add(period) + keep.add(ts) + if len(seen) >= wanted: + break + + return keep diff --git a/src/dbbackup/runner.py b/src/dbbackup/runner.py new file mode 100644 index 0000000..9fbbda6 --- /dev/null +++ b/src/dbbackup/runner.py @@ -0,0 +1,140 @@ +"""Backing up targets: one at a time, each isolated from the others' failures.""" + +from __future__ import annotations + +import logging +import time +from datetime import UTC, datetime, timedelta + +from . import __version__ +from .config import Config, Target +from .engines import engine_for +from .storage import BackupSet, Store + +log = logging.getLogger(__name__) + +MANIFEST_FORMAT = 1 + + +def now_utc() -> datetime: + return datetime.now(UTC).replace(microsecond=0) + + +def backup_targets(config: Config, targets: list[Target]) -> list[dict]: + store = Store(config.backup_root) + return [backup_target(config, store, target) for target in targets] + + +def backup_target(config: Config, store: Store, target: Target) -> dict: + """Back up one target, with retries, then apply retention. Returns its status; never raises.""" + started = now_utc() + status = { + "target": target.name, + "type": target.type, + "state": "failed", + "started": started.isoformat(), + "finished": None, + "duration_s": None, + "set": None, + "size_bytes": None, + "databases": [], + "warnings": [], + "error": None, + "last_success": None, + } + log.info("%s: starting %s backup", target.name, target.type) + try: + with store.lock(target.name, timedelta(hours=config.lock_stale_hours)): + attempt = 0 + while True: + attempt += 1 + try: + bset = backup_once(store, target) + break + except Exception as e: + if attempt > target.retries: + raise + log.warning("%s: attempt %d failed (%s); retrying in %ds", target.name, attempt, e, target.retry_delay_s) + time.sleep(target.retry_delay_s) + + status.update( + state="ok", + set=bset.name, + size_bytes=bset.manifest["size_bytes"], + databases=bset.manifest["databases"], + warnings=list(bset.manifest["warnings"]), + ) + for warning in status["warnings"]: + log.warning("%s: %s", target.name, warning) + log.info("%s: backup set %s complete (%s)", target.name, bset.name, human_size(bset.manifest["size_bytes"])) + + # Only after a success: a failing target keeps every set it has. + try: + store.prune(target.name, target.retention, stale_partial_after=timedelta(hours=config.stale_partial_hours)) + except Exception as e: + log.exception("%s: pruning failed", target.name) + status["warnings"].append(f"pruning old backup sets failed: {e}") + except Exception as e: + log.error("%s: backup failed: %s", target.name, e) + log.debug("%s: traceback", target.name, exc_info=True) + status["error"] = str(e) + + finished = now_utc() + status["finished"] = finished.isoformat() + status["duration_s"] = int((finished - started).total_seconds()) + latest = store.latest_ok(target.name) + status["last_success"] = latest.manifest.get("finished") if latest else None + return status + + +def backup_once(store: Store, target: Target) -> BackupSet: + engine = engine_for(target) + started = now_utc() + + version = engine.server_version() + warnings = [] + if warning := engine.version_warning(version): + warnings.append(warning) + databases = engine.select_databases() + if not databases: + warnings.append("no databases selected; only server-level objects (users/roles) were backed up") + log.info("%s: server %s, %d database(s): %s", target.name, version, len(databases), ", ".join(databases)) + inventory = engine.inventory(databases) + + partial = store.begin(target.name, started) + try: + result = engine.backup(partial, databases) + manifest = { + "format": MANIFEST_FORMAT, + "tool_version": __version__, + "target": target.name, + "type": target.type, + "host": engine.conn.host, + "port": engine.conn.port, + "server_version": version, + "started": started.isoformat(), + "databases": databases, + "contents": result.contents, + "inventory": inventory, + "warnings": warnings + result.warnings, + "status": "ok", + } + engine.verify(partial, manifest) + finished = now_utc() + manifest["finished"] = finished.isoformat() + manifest["duration_s"] = int((finished - started).total_seconds()) + return store.finalize(partial, manifest) + except BaseException: + store.discard(partial) + raise + + +def human_size(n: int | None) -> str: + if n is None: + return "-" + size = float(n) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if size < 1024 or unit == "TiB": + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + raise AssertionError("unreachable") diff --git a/src/dbbackup/storage.py b/src/dbbackup/storage.py new file mode 100644 index 0000000..c7dc118 --- /dev/null +++ b/src/dbbackup/storage.py @@ -0,0 +1,265 @@ +"""On-disk layout of backup sets. + + ///manifest.json a finished, verified set + //.partial/ a set being written + //.lock held while a run is in progress + +A set is written into its ``.partial`` directory, verified, given its manifest +(which includes a SHA-256 for every file), fsynced, and only then renamed into +place. Finished sets are never modified or renamed again, only deleted by +retention, so the off-site sync of the share only ever sees new files. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import logging +import os +import re +import shutil +import socket +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from .config import Retention +from .retention import select_keep + +log = logging.getLogger(__name__) + +TS_FORMAT = "%Y%m%dT%H%M%SZ" +TS_RE = re.compile(r"^\d{8}T\d{6}Z$") +PARTIAL_SUFFIX = ".partial" +MANIFEST = "manifest.json" +LOCK = ".lock" + + +class StorageError(Exception): + pass + + +class LockedError(StorageError): + pass + + +def format_ts(ts: datetime) -> str: + return ts.astimezone(UTC).strftime(TS_FORMAT) + + +def parse_ts(name: str) -> datetime: + return datetime.strptime(name, TS_FORMAT).replace(tzinfo=UTC) + + +@dataclass +class BackupSet: + target: str + timestamp: datetime + path: Path + manifest: dict | None + + @property + def ok(self) -> bool: + return bool(self.manifest) and self.manifest.get("status") == "ok" + + @property + def name(self) -> str: + return self.path.name + + +class Store: + def __init__(self, root: Path): + self.root = Path(root) + + def target_dir(self, target: str) -> Path: + return self.root / target + + # -- reading --------------------------------------------------------------- + + def sets(self, target: str) -> list[BackupSet]: + """Finished sets for ``target``, oldest first.""" + tdir = self.target_dir(target) + if not tdir.is_dir(): + return [] + result = [] + for entry in tdir.iterdir(): + if not (entry.is_dir() and TS_RE.match(entry.name)): + continue + result.append(BackupSet(target, parse_ts(entry.name), entry, _load_manifest(entry))) + result.sort(key=lambda s: s.timestamp) + return result + + def latest_ok(self, target: str) -> BackupSet | None: + ok = [s for s in self.sets(target) if s.ok] + return ok[-1] if ok else None + + def find(self, target: str, which: str = "latest") -> BackupSet: + if which == "latest": + found = self.latest_ok(target) + if found is None: + raise StorageError(f"no successful backup sets for target {target!r} in {self.target_dir(target)}") + return found + for s in self.sets(target): + if s.name == which: + if not s.ok: + raise StorageError(f"backup set {target}/{which} has no valid manifest") + return s + raise StorageError(f"backup set {target}/{which} not found") + + # -- writing --------------------------------------------------------------- + + def begin(self, target: str, ts: datetime) -> Path: + tdir = self.target_dir(target) + tdir.mkdir(parents=True, exist_ok=True) + partial = tdir / (format_ts(ts) + PARTIAL_SUFFIX) + partial.mkdir() + return partial + + def finalize(self, partial: Path, manifest: dict) -> BackupSet: + """Checksum every file, write the manifest, fsync, and rename the set into place.""" + if not partial.name.endswith(PARTIAL_SUFFIX): + raise StorageError(f"not a partial set: {partial}") + manifest = dict(manifest) + manifest["files"] = checksum_tree(partial) + manifest["size_bytes"] = sum(f["size"] for f in manifest["files"].values()) + + tmp = partial / (MANIFEST + ".tmp") + with open(tmp, "w") as f: + json.dump(manifest, f, indent=2, sort_keys=True) + f.write("\n") + f.flush() + os.fsync(f.fileno()) + os.rename(tmp, partial / MANIFEST) + _fsync_tree(partial) + + final = partial.with_name(partial.name.removesuffix(PARTIAL_SUFFIX)) + os.rename(partial, final) + _fsync_dir(final.parent) + return BackupSet(final.parent.name, parse_ts(final.name), final, manifest) + + def discard(self, partial: Path) -> None: + shutil.rmtree(partial, ignore_errors=True) + + # -- locking --------------------------------------------------------------- + + @contextlib.contextmanager + def lock(self, target: str, stale_after: timedelta) -> Iterator[None]: + """Per-target lock file, created with O_EXCL (which NFSv3+ honours).""" + tdir = self.target_dir(target) + tdir.mkdir(parents=True, exist_ok=True) + path = tdir / LOCK + info = json.dumps({"host": socket.gethostname(), "pid": os.getpid(), "started": datetime.now(UTC).isoformat()}) + for attempt in (1, 2): + try: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + except FileExistsError: + try: + age = datetime.now(UTC) - datetime.fromtimestamp(path.stat().st_mtime, UTC) + holder = path.read_text().strip() + except FileNotFoundError: + continue # released while we looked + if attempt == 1 and age > stale_after: + log.warning("%s: removing stale lock (%s old): %s", target, age, holder) + path.unlink(missing_ok=True) + continue + raise LockedError(f"target {target!r} is locked by another run: {holder}") from None + with os.fdopen(fd, "w") as f: + f.write(info + "\n") + break + try: + yield + finally: + path.unlink(missing_ok=True) + + # -- retention ------------------------------------------------------------- + + def prune( + self, target: str, policy: Retention, *, stale_partial_after: timedelta, now: datetime | None = None, dry_run: bool = False + ) -> list[Path]: + """Delete sets the policy no longer keeps, and abandoned partial sets. Returns what was (or would be) deleted.""" + now = now or datetime.now(UTC) + doomed: list[Path] = [] + + sets = self.sets(target) + ok = [s for s in sets if s.ok] + keep = select_keep((s.timestamp for s in ok), policy) + doomed += [s.path for s in ok if s.timestamp not in keep] + for s in sets: + if not s.ok: + log.warning("%s: leaving set %s alone: it has no valid manifest", target, s.name) + + tdir = self.target_dir(target) + if tdir.is_dir(): + for entry in tdir.iterdir(): + if entry.is_dir() and entry.name.endswith(PARTIAL_SUFFIX): + with contextlib.suppress(ValueError): + if now - parse_ts(entry.name.removesuffix(PARTIAL_SUFFIX)) > stale_partial_after: + doomed.append(entry) + + for path in doomed: + log.info("%s: %s %s", target, "would delete" if dry_run else "deleting", path.name) + if not dry_run: + shutil.rmtree(path) + return doomed + + +def checksum_tree(root: Path) -> dict[str, dict]: + files = {} + for path in sorted(root.rglob("*")): + if path.is_file() and path.name not in (MANIFEST, MANIFEST + ".tmp"): + files[path.relative_to(root).as_posix()] = {"size": path.stat().st_size, "sha256": sha256_file(path)} + return files + + +def verify_checksums(bset: BackupSet) -> list[str]: + """Problems found re-checking a set's files against its manifest (empty if none).""" + problems = [] + expected = (bset.manifest or {}).get("files", {}) + actual = {p.relative_to(bset.path).as_posix() for p in bset.path.rglob("*") if p.is_file() and p.name != MANIFEST} + for rel in sorted(set(expected) - actual): + problems.append(f"missing file: {rel}") + for rel in sorted(actual - set(expected)): + problems.append(f"unexpected file: {rel}") + for rel in sorted(set(expected) & actual): + if sha256_file(bset.path / rel) != expected[rel]["sha256"]: + problems.append(f"checksum mismatch: {rel}") + return problems + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as f: + while chunk := f.read(1 << 20): + h.update(chunk) + return h.hexdigest() + + +def _load_manifest(set_dir: Path) -> dict | None: + try: + manifest = json.loads((set_dir / MANIFEST).read_text()) + except (OSError, ValueError): + return None + return manifest if isinstance(manifest, dict) else None + + +def _fsync_tree(root: Path) -> None: + for path in root.rglob("*"): + if path.is_file(): + fd = os.open(path, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + elif path.is_dir(): + _fsync_dir(path) + _fsync_dir(root) + + +def _fsync_dir(path: Path) -> None: + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) diff --git a/tests/integration/config.yaml b/tests/integration/config.yaml new file mode 100644 index 0000000..61f2b80 --- /dev/null +++ b/tests/integration/config.yaml @@ -0,0 +1,49 @@ +# Integration-test config: the compose.yaml servers. +backup_root: /backups + +defaults: + retention: {daily: 3, weekly: 2, monthly: 2} + retries: 0 + retry_delay_s: 0 + +mqtt: + host: mosquitto + topic_prefix: dbbackup-test + ha_discovery: true + +targets: + - name: pg + type: postgres + host: postgres + username: dbbackup + password_env: PG_PASSWORD + exclude: [scratch] + jobs: 2 + + - name: maria + type: mariadb + host: mariadb + username: dbbackup + password_file: /run/secrets/dbbackup/mariadb-password + exclude: [scratch] + + # Whole-instance dump with --oplog. + - name: mongo + type: mongodb + host: mongodb + username: dbbackup + password_env: MONGO_PASSWORD + uri_options: {replicaSet: rs0} + + # Per-database dumps, because of the exclude. + - name: mongo-filtered + type: mongodb + uri_env: MONGO_URI + exclude: [scratch] + + # Unreachable: must fail without affecting the others. + - name: broken + type: postgres + host: no-such-host.invalid + username: nobody + password_env: PG_PASSWORD diff --git a/tests/integration/secrets/mariadb-password b/tests/integration/secrets/mariadb-password new file mode 100644 index 0000000..2fff205 --- /dev/null +++ b/tests/integration/secrets/mariadb-password @@ -0,0 +1 @@ +dbb#ack"up\pw'x diff --git a/tests/integration/seed/mariadb/01-seed.sql b/tests/integration/seed/mariadb/01-seed.sql new file mode 100644 index 0000000..cd08129 --- /dev/null +++ b/tests/integration/seed/mariadb/01-seed.sql @@ -0,0 +1,57 @@ +-- Test fixture for the MariaDB backup source. Runs once, as root. + +SET SESSION max_recursive_iterations = 1000000; + +CREATE DATABASE shop; +CREATE DATABASE `wiki-db`; +CREATE DATABASE scratch; + +USE shop; +CREATE TABLE products ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100) NOT NULL, + price DECIMAL(10, 2), + img BLOB +) ENGINE = InnoDB; +INSERT INTO products (name, price, img) +WITH RECURSIVE seq (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < 3000) +SELECT CONCAT('product ', n), n * 0.99, UNHEX(MD5(n)) FROM seq; +CREATE TABLE orders ( + id INT AUTO_INCREMENT PRIMARY KEY, + product_id INT NOT NULL, + qty INT NOT NULL, + FOREIGN KEY (product_id) REFERENCES products (id) +) ENGINE = InnoDB; +INSERT INTO orders (product_id, qty) +WITH RECURSIVE seq (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < 10000) +SELECT (n MOD 3000) + 1, n MOD 7 FROM seq; +-- Non-InnoDB table: should produce a consistency warning. +CREATE TABLE legacy_log (id INT PRIMARY KEY, msg TEXT) ENGINE = Aria; +INSERT INTO legacy_log VALUES (1, 'hello'), (2, 'world'); +CREATE VIEW cheap_products AS SELECT id, name FROM products WHERE price < 10; +CREATE PROCEDURE order_count(IN p INT) SELECT COUNT(*) FROM orders WHERE product_id = p; +CREATE TRIGGER orders_qty BEFORE INSERT ON orders FOR EACH ROW SET NEW.qty = GREATEST(NEW.qty, 0); +CREATE EVENT nightly_noop ON SCHEDULE EVERY 1 DAY DISABLE DO SELECT 1; + +USE `wiki-db`; +CREATE TABLE pages (id INT PRIMARY KEY, title VARCHAR(200), body MEDIUMTEXT) ENGINE = InnoDB; +INSERT INTO pages +WITH RECURSIVE seq (n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM seq WHERE n < 500) +SELECT n, CONCAT('Page ', n), REPEAT('lorem ipsum ', 50) FROM seq; + +USE scratch; +CREATE TABLE junk (x INT); + +CREATE USER 'shop_app'@'%' IDENTIFIED BY 'shop-pw'; +GRANT SELECT, INSERT, UPDATE, DELETE ON shop.* TO 'shop_app'@'%'; +CREATE USER 'wiki_ro'@'%' IDENTIFIED BY 'wiki-pw'; +-- Table-level grant: only restorable once the table exists. +GRANT SELECT ON `wiki-db`.pages TO 'wiki_ro'@'%'; +CREATE ROLE reporting; +GRANT SELECT ON shop.* TO reporting; +GRANT reporting TO 'shop_app'@'%'; + +-- Least-privilege backup user. +-- Awkward password: exercises the option-file quoting (see tests/integration/secrets). +CREATE USER 'dbbackup'@'%' IDENTIFIED BY 'dbb#ack"up\\pw\'x'; +GRANT SELECT, SHOW VIEW, TRIGGER, EVENT, LOCK TABLES, PROCESS, RELOAD, SHOW DATABASES ON *.* TO 'dbbackup'@'%'; diff --git a/tests/integration/seed/mongodb/01-seed.js b/tests/integration/seed/mongodb/01-seed.js new file mode 100644 index 0000000..93deec4 --- /dev/null +++ b/tests/integration/seed/mongodb/01-seed.js @@ -0,0 +1,16 @@ +// Test fixture for the MongoDB backup source. Runs once, as root. + +const shop = db.getSiblingDB("shop"); +shop.orders.insertMany(Array.from({ length: 5000 }, (_, i) => ({ n: i, item: "item" + (i % 50), qty: i % 7 }))); +shop.orders.createIndex({ item: 1 }); +shop.products.insertMany(Array.from({ length: 50 }, (_, i) => ({ _id: "item" + i, price: i * 1.25 }))); +shop.createView("big_orders", "orders", [{ $match: { qty: { $gt: 5 } } }]); +shop.createUser({ user: "shop_app", pwd: "shop-pw", roles: [{ role: "readWrite", db: "shop" }] }); + +const logs = db.getSiblingDB("logs"); +logs.events.insertMany(Array.from({ length: 2000 }, (_, i) => ({ ts: new Date(1700000000000 + i * 1000), level: i % 3 }))); + +db.getSiblingDB("scratch").junk.insertOne({ a: 1 }); + +// Least-privilege backup user. +db.getSiblingDB("admin").createUser({ user: "dbbackup", pwd: "dbbackup-pw", roles: ["backup"] }); diff --git a/tests/integration/seed/postgres/01-seed.sql b/tests/integration/seed/postgres/01-seed.sql new file mode 100644 index 0000000..8bfcff0 --- /dev/null +++ b/tests/integration/seed/postgres/01-seed.sql @@ -0,0 +1,43 @@ +-- Test fixture for the PostgreSQL backup source. Runs once, as the postgres superuser. + +CREATE ROLE app_owner LOGIN PASSWORD 'app-owner-pw'; +CREATE ROLE app_reader LOGIN PASSWORD 'reader-pw'; +CREATE ROLE reporting NOLOGIN; +GRANT reporting TO app_reader; + +-- Least-privilege backup role: reads all data, but is not a superuser, so it +-- can't read role passwords (exercising the --no-role-passwords fallback). +CREATE ROLE dbbackup LOGIN PASSWORD 'dbbackup-pw' IN ROLE pg_read_all_data; + +CREATE DATABASE app1 OWNER app_owner; +CREATE DATABASE "odd.name/db" OWNER app_owner; +CREATE DATABASE scratch; + +\connect app1 +SET ROLE app_owner; +CREATE SCHEMA billing; +CREATE TABLE public.customers (id serial PRIMARY KEY, name text NOT NULL, created timestamptz DEFAULT now()); +INSERT INTO public.customers (name) SELECT 'customer ' || g FROM generate_series(1, 5000) g; +CREATE TABLE billing.invoices ( + id bigserial PRIMARY KEY, + customer_id int REFERENCES public.customers (id), + amount numeric(10, 2) +); +INSERT INTO billing.invoices (customer_id, amount) SELECT (g % 5000) + 1, g * 1.5 FROM generate_series(1, 20000) g; +CREATE INDEX invoices_customer ON billing.invoices (customer_id); +CREATE VIEW billing.big_invoices AS SELECT * FROM billing.invoices WHERE amount > 10000; +CREATE MATERIALIZED VIEW billing.totals AS SELECT customer_id, sum(amount) AS total FROM billing.invoices GROUP BY customer_id; +CREATE FUNCTION billing.invoice_count(c int) RETURNS bigint LANGUAGE sql AS $$ SELECT count(*) FROM billing.invoices WHERE customer_id = c $$; +GRANT USAGE ON SCHEMA billing TO reporting; +GRANT SELECT ON ALL TABLES IN SCHEMA billing TO reporting; +RESET ROLE; +ALTER DATABASE app1 SET work_mem = '8MB'; +ANALYZE; + +\connect "odd.name/db" +CREATE TABLE things (id int PRIMARY KEY, payload bytea); +INSERT INTO things SELECT g, decode(md5(g::text), 'hex') FROM generate_series(1, 1000) g; +ANALYZE; + +\connect scratch +CREATE TABLE junk (x int); diff --git a/tests/integration/test.env b/tests/integration/test.env new file mode 100644 index 0000000..b911c06 --- /dev/null +++ b/tests/integration/test.env @@ -0,0 +1,11 @@ +# Throwaway credentials for the compose.yaml test stack. +PG_PASSWORD=dbbackup-pw +MONGO_PASSWORD=dbbackup-pw +MONGO_URI=mongodb://dbbackup:dbbackup-pw@mongodb:27017/?authSource=admin&replicaSet=rs0 +# Source-server superusers, for the tests' own checks. +SOURCE_ROOT_PASSWORD_PG=postgres-root +SOURCE_ROOT_PASSWORD_MARIADB=mariadb-root +SOURCE_ROOT_PASSWORD_MONGO=mongo-root +# Restore-server superusers. +RESTORE_PASSWORD=restore-root +RESTORE_MONGO_URI=mongodb://root:restore-root@mongodb-restore:27017/?authSource=admin&replicaSet=rs0 diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py new file mode 100644 index 0000000..5e062d9 --- /dev/null +++ b/tests/integration/test_end_to_end.py @@ -0,0 +1,321 @@ +"""End-to-end tests against the compose.yaml stack. Run inside the test image: + + docker compose --profile tool run --rm --build tests + +The tests run in file order and build on each other: one full backup run, then +checks of what it wrote, then restores into the empty *-restore servers. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import time +from pathlib import Path + +import pytest +from pymongo import MongoClient + +pytestmark = pytest.mark.integration + +CONFIG = os.environ.get("DBBACKUP_CONFIG", "/etc/dbbackup/config.yaml") +BACKUPS = Path("/backups") +GOOD_TARGETS = ["pg", "maria", "mongo", "mongo-filtered"] + +SOURCE_PG = ("postgres", os.environ.get("SOURCE_ROOT_PASSWORD_PG", "")) +RESTORE_PG = ("postgres-restore", os.environ.get("RESTORE_PASSWORD", "")) +SOURCE_MARIA = ("mariadb", os.environ.get("SOURCE_ROOT_PASSWORD_MARIADB", "")) +RESTORE_MARIA = ("mariadb-restore", os.environ.get("RESTORE_PASSWORD", "")) +_SOURCE_MONGO_PASSWORD = os.environ.get("SOURCE_ROOT_PASSWORD_MONGO", "") +SOURCE_MONGO_URI = f"mongodb://root:{_SOURCE_MONGO_PASSWORD}@mongodb:27017/?authSource=admin&directConnection=true" +RESTORE_MONGO_URI = os.environ.get("RESTORE_MONGO_URI", "") + + +# -- helpers ------------------------------------------------------------------ + + +def dbbackup(*args: str) -> subprocess.CompletedProcess: + result = subprocess.run(["dbbackup", "--config", CONFIG, *args], capture_output=True, text=True) + print(f"$ dbbackup {' '.join(args)} -> {result.returncode}\n{result.stderr}") + return result + + +def pg(server: tuple[str, str], sql: str, db: str = "postgres", user: str = "postgres", password: str | None = None) -> list[str]: + host, root_password = server + env = {**os.environ, "PGPASSWORD": password or root_password} + cmd = ["psql", "-X", "-q", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-h", host, "-U", user, "-d", f"dbname='{db}'", "-c", sql] + return subprocess.run(cmd, env=env, capture_output=True, text=True, check=True).stdout.splitlines() + + +def maria(server: tuple[str, str], sql: str, user: str = "root", password: str | None = None) -> list[list[str]]: + host, root_password = server + env = {**os.environ, "MYSQL_PWD": password or root_password} + cmd = ["mariadb", "-h", host, "-u", user, "--batch", "--skip-column-names", "-e", sql] + out = subprocess.run(cmd, env=env, capture_output=True, text=True, check=True).stdout + return [line.split("\t") for line in out.splitlines()] + + +def sets(target: str) -> list[Path]: + tdir = BACKUPS / target + return sorted(p for p in tdir.iterdir() if p.is_dir() and not p.name.endswith(".partial")) if tdir.exists() else [] + + +def manifest(target: str) -> dict: + return json.loads((sets(target)[-1] / "manifest.json").read_text()) + + +def retained_messages(topic_filter: str, wait: float = 2.0) -> dict[str, dict]: + import paho.mqtt.client as mqtt + + messages = {} + client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2) + client.on_message = lambda _c, _u, msg: messages.__setitem__(msg.topic, json.loads(msg.payload)) + client.connect("mosquitto") + client.subscribe(topic_filter) + client.loop_start() + time.sleep(wait) + client.loop_stop() + client.disconnect() + return messages + + +def reset_restore_servers() -> None: + """Empty the restore servers, so the suite can be re-run against a long-lived stack.""" + for db in pg(RESTORE_PG, "SELECT datname FROM pg_database WHERE NOT datistemplate AND datname <> 'postgres'"): + pg(RESTORE_PG, f'DROP DATABASE "{db}" WITH (FORCE)') + pg(RESTORE_PG, "DROP ROLE IF EXISTS app_owner, app_reader, reporting, dbbackup") + maria(RESTORE_MARIA, "DROP DATABASE IF EXISTS shop; DROP DATABASE IF EXISTS `wiki-db`;") + maria(RESTORE_MARIA, "DROP USER IF EXISTS 'shop_app'@'%', 'wiki_ro'@'%', 'dbbackup'@'%'; DROP ROLE IF EXISTS reporting;") + with MongoClient(RESTORE_MONGO_URI) as client: + for db in client.list_database_names(): + if db not in ("admin", "config", "local"): + client.drop_database(db) + for user in client.admin["system.users"].find({"user": {"$ne": "root"}}): + client[user["db"]].command("dropUser", user["user"]) + + +def table_counts_pg(server, db: str) -> dict[str, int]: + tables = pg(server, "SELECT quote_ident(schemaname) || '.' || quote_ident(relname) FROM pg_stat_user_tables", db=db) + return {t: int(pg(server, f"SELECT count(*) FROM {t}", db=db)[0]) for t in tables} + + +def table_counts_maria(server, db: str) -> dict[str, int]: + tables = [ + r[0] + for r in maria(server, f"SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA='{db}' AND TABLE_TYPE='BASE TABLE'") + ] + return {t: int(maria(server, f"SELECT COUNT(*) FROM `{db}`.`{t}`")[0][0]) for t in tables} + + +def collection_counts(uri: str, dbs: list[str]) -> dict[str, int]: + with MongoClient(uri) as client: + return { + f"{db}.{c['name']}": client[db][c["name"]].count_documents({}) + for db in dbs + for c in client[db].list_collections() + if c.get("type", "collection") == "collection" and not c["name"].startswith("system.") + } + + +# -- the backup run ----------------------------------------------------------- + + +@pytest.fixture(scope="module", autouse=True) +def backup_run(): + reset_restore_servers() + return dbbackup("backup") + + +def test_failed_target_fails_the_run_but_not_the_others(backup_run): + assert backup_run.returncode == 1 + assert "backup failed for 1 of 5 target(s): broken" in backup_run.stderr + assert sets("broken") == [] + for target in GOOD_TARGETS: + assert len(sets(target)) == 1, target + assert manifest(target)["status"] == "ok" + + +def test_no_partial_sets_left_behind(): + assert not list(BACKUPS.glob("*/*.partial")) + assert not list(BACKUPS.glob("*/.lock")) + + +def test_postgres_manifest(): + m = manifest("pg") + assert m["databases"] == ["app1", "odd.name/db", "postgres"] # scratch excluded + assert m["contents"]["role_passwords"] is True # pg_read_all_data can read pg_authid + assert m["server_version"].startswith("18.") + objects = m["inventory"]["databases"]["app1"]["objects"] + assert objects["billing.invoices"] == {"kind": "table", "rows": 20000} + assert objects["billing.big_invoices"]["kind"] == "view" + assert {"app_owner", "app_reader", "reporting"} <= set(m["inventory"]["users"]) + assert m["warnings"] == [] + + +def test_mariadb_manifest(): + m = manifest("maria") + assert m["databases"] == ["shop", "wiki-db"] + assert any("shop.legacy_log uses Aria" in w for w in m["warnings"]) + assert {"shop_app@%", "wiki_ro@%", "reporting"} <= set(m["inventory"]["users"]) + + +def test_mongodb_manifests(): + m = manifest("mongo") + assert m["contents"] == {"mode": "instance", "oplog": True, "archive": "mongodb.archive.zst"} + assert m["databases"] == ["logs", "scratch", "shop"] + assert m["inventory"]["databases"]["shop"]["objects"]["big_orders"]["kind"] == "view" + assert "shop_app@shop" in m["inventory"]["users"] + f = manifest("mongo-filtered") + assert f["contents"]["mode"] == "per-database" + assert f["databases"] == ["logs", "shop"] + assert any("without --oplog" in w for w in f["warnings"]) + + +def test_verify_command(): + assert dbbackup("verify", *GOOD_TARGETS).returncode == 0 + + +def test_list_command(): + result = dbbackup("list") + assert result.returncode == 0 + for target in GOOD_TARGETS: + assert target in result.stdout + + +def test_mqtt_status_and_discovery(): + statuses = retained_messages("dbbackup-test/#") + pg_status = statuses["dbbackup-test/pg/status"] + assert pg_status["state"] == "ok" + assert pg_status["last_success"] == manifest("pg")["finished"] + broken = statuses["dbbackup-test/broken/status"] + assert broken["state"] == "failed" + assert "no-such-host.invalid" in broken["error"] + assert broken["last_success"] is None + assert statuses["dbbackup-test/status"]["state"] == "failed" + + discovery = retained_messages("homeassistant/sensor/+/config") + config = discovery["homeassistant/sensor/dbbackup_mongo_filtered_status/config"] + assert config["default_entity_id"] == "sensor.dbbackup_mongo_filtered_status" + assert config["state_topic"] == "dbbackup-test/mongo-filtered/status" + + +# -- restores ----------------------------------------------------------------- + + +def test_restore_postgres(): + result = dbbackup("restore", "pg", "--host", "postgres-restore", "--username", "postgres", "--password-env", "RESTORE_PASSWORD") + assert result.returncode == 0 + assert "left existing role(s) as they are: postgres" in result.stderr + for db in ("app1", "odd.name/db"): + assert table_counts_pg(RESTORE_PG, db) == table_counts_pg(SOURCE_PG, db) + # Role passwords, memberships, functions, materialized views and database settings came back. + assert pg(RESTORE_PG, "SELECT billing.invoice_count(7)", db="app1", user="app_owner", password="app-owner-pw") == ["4"] + assert pg( + RESTORE_PG, "SET ROLE reporting; SELECT count(*) FROM billing.totals", db="app1", user="app_reader", password="reader-pw" + ) == ["5000"] + assert pg(RESTORE_PG, "SHOW work_mem", db="app1") == ["8MB"] + # ...and the restoring account's own password was left alone (pg() connects with it). + assert pg(RESTORE_PG, "SELECT 1") == ["1"] + + +def test_restore_refuses_to_overwrite_without_force(): + result = dbbackup("restore", "pg", "--host", "postgres-restore", "--username", "postgres", "--password-env", "RESTORE_PASSWORD") + assert result.returncode == 1 + assert "already exist on postgres-restore: app1, odd.name/db" in result.stderr + + +def test_restore_force_replaces_one_database(): + pg(RESTORE_PG, "DELETE FROM things", db="odd.name/db") + result = dbbackup( + "restore", + "pg", + "-d", + "odd.name/db", + "--force", + "--host", + "postgres-restore", + "--username", + "postgres", + "--password-env", + "RESTORE_PASSWORD", + ) + assert result.returncode == 0 + assert pg(RESTORE_PG, "SELECT count(*) FROM things", db="odd.name/db") == ["1000"] + + +def test_restore_mariadb(): + result = dbbackup("restore", "maria", "--host", "mariadb-restore", "--username", "root", "--password-env", "RESTORE_PASSWORD") + assert result.returncode == 0 + for db in ("shop", "wiki-db"): + assert table_counts_maria(RESTORE_MARIA, db) == table_counts_maria(SOURCE_MARIA, db) + assert maria(RESTORE_MARIA, "SELECT COUNT(*) FROM shop.products WHERE img IS NOT NULL") == [["3000"]] + assert maria(RESTORE_MARIA, "SELECT ROUTINE_NAME FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA='shop'") == [["order_count"]] + assert maria(RESTORE_MARIA, "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA='shop'") == [["orders_qty"]] + assert maria(RESTORE_MARIA, "SELECT EVENT_NAME FROM information_schema.EVENTS WHERE EVENT_SCHEMA='shop'") == [["nightly_noop"]] + # Users come back with their passwords, roles and (table-level) grants... + assert maria(RESTORE_MARIA, "SET ROLE reporting; SELECT COUNT(*) FROM shop.cheap_products", user="shop_app", password="shop-pw") == [ + ["10"] + ] + assert maria(RESTORE_MARIA, "SELECT COUNT(*) FROM `wiki-db`.pages", user="wiki_ro", password="wiki-pw") == [["500"]] + # ...while the server's own existing accounts keep theirs (maria() connects as root). + assert maria(RESTORE_MARIA, "SELECT 1") == [["1"]] + assert "root@localhost" in result.stderr and "healthcheck@localhost" in result.stderr + + +def test_restore_mongodb_whole_instance(): + result = dbbackup("restore", "mongo", "--uri-env", "RESTORE_MONGO_URI") + assert result.returncode == 0 + dbs = ["logs", "scratch", "shop"] + assert collection_counts(RESTORE_MONGO_URI, dbs) == collection_counts(SOURCE_MONGO_URI, dbs) + with MongoClient(RESTORE_MONGO_URI) as client: + assert [c["name"] for c in client.shop.list_collections(filter={"type": "view"})] == ["big_orders"] + assert "item_1" in client.shop.orders.index_information() + with MongoClient("mongodb://shop_app:shop-pw@mongodb-restore:27017/shop?authSource=shop&directConnection=true") as client: + assert client.shop.orders.estimated_document_count() == 5000 + # The restore server's root password is untouched (RESTORE_MONGO_URI still works). + with MongoClient(RESTORE_MONGO_URI) as client: + client.admin.command("ping") + + +def test_restore_mongodb_per_database_with_force(): + with MongoClient(RESTORE_MONGO_URI) as client: + client.logs.events.delete_many({}) + result = dbbackup("restore", "mongo-filtered", "-d", "logs", "--force", "--uri-env", "RESTORE_MONGO_URI") + assert result.returncode == 0 + with MongoClient(RESTORE_MONGO_URI) as client: + assert client.logs.events.count_documents({}) == 2000 + + +# -- retention, locking, corruption --------------------------------------------- + + +def test_second_backup_same_day_replaces_the_first(): + first = sets("pg") + time.sleep(1) # set names have one-second resolution + assert dbbackup("backup", "pg").returncode == 0 + after = sets("pg") + # GFS keeps the newest set per day, so the earlier one from today is pruned. + assert len(after) == 1 and after != first + + +def test_locked_target_is_not_backed_up(): + lock = BACKUPS / "pg" / ".lock" + lock.write_text('{"host": "elsewhere", "pid": 1}') + try: + result = dbbackup("backup", "pg") + assert result.returncode == 1 + assert "locked by another run" in result.stderr + assert lock.exists() + finally: + lock.unlink() + + +def test_verify_detects_corruption(): + victim = next((sets("maria")[-1]).glob("*.sql.zst")) + data = bytearray(victim.read_bytes()) + data[len(data) // 2] ^= 0xFF + victim.write_bytes(bytes(data)) + result = dbbackup("verify", "maria") + assert result.returncode == 1 + assert f"checksum mismatch: {victim.name}" in result.stderr diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..6f6862e --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,131 @@ +import pytest + +from dbbackup.config import ConfigError, Retention, parse_config + + +def minimal(**target_overrides): + target = {"name": "pg", "type": "postgres", "host": "db", "username": "u", "password_env": "PW"} + target.update(target_overrides) + return {"targets": [target]} + + +ENV = {"PW": "s3cret", "URI": "mongodb://u:p@h/"} + + +def test_minimal_config_gets_defaults(): + config = parse_config(minimal(), ENV) + (t,) = config.targets + assert str(config.backup_root) == "/backups" + assert t.port == 5432 + assert t.password == "s3cret" + assert t.retention == Retention(last=0, daily=7, weekly=4, monthly=6, yearly=0) + assert t.compression_level == 3 + assert t.retries == 1 + assert config.mqtt is None + + +def test_default_ports_by_type(): + raw = { + "targets": [ + {"name": "a", "type": "postgres", "host": "h"}, + {"name": "b", "type": "mariadb", "host": "h"}, + {"name": "c", "type": "mongodb", "host": "h"}, + ] + } + assert [t.port for t in parse_config(raw, {}).targets] == [5432, 3306, 27017] + + +def test_retention_merges_defaults_then_target(): + raw = minimal(retention={"daily": 14}) + raw["defaults"] = {"retention": {"monthly": 12, "yearly": 3}} + (t,) = parse_config(raw, ENV).targets + assert t.retention == Retention(last=0, daily=14, weekly=4, monthly=12, yearly=3) + + +def test_password_from_file_strips_trailing_newline(tmp_path): + secret = tmp_path / "pw" + secret.write_text("from-file\n") + raw = minimal() + del raw["targets"][0]["password_env"] + raw["targets"][0]["password_file"] = str(secret) + (t,) = parse_config(raw, ENV).targets + assert t.password == "from-file" + + +def test_secret_never_appears_in_repr(): + (t,) = parse_config(minimal(), ENV).targets + assert "s3cret" not in repr(t) + + +@pytest.mark.parametrize( + "raw, message", + [ + ({"targets": []}, "non-empty list"), + (minimal(type="oracle"), "type: must be one of"), + (minimal(name="bad name"), "name: must match"), + (minimal(password_env="MISSING"), "MISSING is not set"), + (minimal(host=None), "host: required"), + (minimal(colour="blue"), "unknown key(s): colour"), + (minimal(ssl=True), "ssl: only valid for mariadb"), + (minimal(uri_env="URI"), "only valid for mongodb"), + (minimal(retention={"daily": -1}), "non-negative integer"), + (minimal(retention={"hourly": 1}), "unknown key(s): hourly"), + (minimal(compression_level=25), "between 1 and 19"), + (minimal(username=None), "username: required"), + ({"targets": [{"name": "x", "type": "mongodb"}]}, "host: required (or uri_env/uri_file)"), + ], +) +def test_invalid_configs(raw, message): + with pytest.raises(ConfigError, match=None) as exc: + parse_config(raw, ENV) + assert message in str(exc.value) + + +def test_duplicate_target_names(): + raw = minimal() + raw["targets"].append(dict(raw["targets"][0])) + with pytest.raises(ConfigError, match="duplicate target name"): + parse_config(raw, ENV) + + +def test_both_env_and_file_is_an_error(): + with pytest.raises(ConfigError, match="set only one"): + parse_config(minimal(password_file="/x"), ENV) + + +def test_mongodb_uri_target(): + raw = {"targets": [{"name": "m", "type": "mongodb", "uri_env": "URI"}]} + (t,) = parse_config(raw, ENV).targets + assert t.uri == "mongodb://u:p@h/" + assert t.host is None + + +def test_secrets_resolved_only_for_named_targets(): + raw = minimal() + raw["targets"].append({"name": "other", "type": "mariadb", "host": "h", "username": "u", "password_env": "UNSET"}) + raw["mqtt"] = {"host": "mq", "username": "u", "password_env": "ALSO_UNSET"} + with pytest.raises(ConfigError, match="UNSET"): + parse_config(raw, ENV) + config = parse_config(raw, ENV, secrets_for=["pg"], mqtt_secrets=False) + assert [t.password for t in config.targets] == ["s3cret", None] + assert config.mqtt.password is None + + +def test_select_targets(): + raw = minimal() + raw["targets"].append({"name": "two", "type": "mariadb", "host": "h"}) + config = parse_config(raw, ENV) + assert [t.name for t in config.select(None)] == ["pg", "two"] + assert [t.name for t in config.select(["two"])] == ["two"] + with pytest.raises(ConfigError, match="unknown target"): + config.select(["nope"]) + + +def test_mqtt_defaults(): + raw = minimal() + raw["mqtt"] = {"host": "mq"} + mqtt = parse_config(raw, ENV).mqtt + assert (mqtt.port, mqtt.topic_prefix, mqtt.ha_discovery, mqtt.discovery_prefix) == (1883, "dbbackup", False, "homeassistant") + raw["mqtt"] = {"host": "mq", "tls": True, "topic_prefix": "x/y/"} + mqtt = parse_config(raw, ENV).mqtt + assert (mqtt.port, mqtt.topic_prefix) == (8883, "x/y") diff --git a/tests/test_engines.py b/tests/test_engines.py new file mode 100644 index 0000000..37b9e6e --- /dev/null +++ b/tests/test_engines.py @@ -0,0 +1,220 @@ +"""Engine logic that doesn't need a server: command construction (no secrets on +argv), dump-filtering for restores, and naming.""" + +import subprocess +from pathlib import Path + +import pytest + +from dbbackup import proc +from dbbackup.config import parse_config +from dbbackup.engines import engine_for +from dbbackup.engines.base import safe_filename, version_tuple +from dbbackup.engines.mariadb import option_file_value, statement_account, without_accounts +from dbbackup.engines.mongodb import _ns_escape +from dbbackup.engines.postgres import conninfo_dbname, without_roles + +SECRET = "pa ss'w\"o#rd\\" + + +def target(ttype, **extra): + raw = {"targets": [{"name": "t", "type": ttype, "host": "db.example", "username": "backup", "password_env": "PW", **extra}]} + return parse_config(raw, {"PW": SECRET}).targets[0] + + +class Recorder: + """Stands in for the proc helpers: records every command and its environment.""" + + def __init__(self, monkeypatch, stdout=""): + self.calls = [] + self.files = {} + self.stdout = stdout + + def run(cmd, *, env=None, input=None, check=True): + self.calls.append((list(cmd), env or {})) + self._capture_files(cmd) + return subprocess.CompletedProcess(cmd, 0, self.stdout, "") + + def pipe_to_zstd(cmd, out_path, level, *, env=None): + self.calls.append((list(cmd), env or {})) + self._capture_files(cmd) + return "" + + monkeypatch.setattr(proc, "run", run) + monkeypatch.setattr(proc, "pipe_to_zstd", pipe_to_zstd) + + def _capture_files(self, cmd): + for arg in cmd: + for prefix in ("--defaults-extra-file=", "--config="): + if arg.startswith(prefix): + self.files[arg] = Path(arg.removeprefix(prefix)).read_text() + + def assert_no_secret_on_argv(self): + for cmd, _env in self.calls: + assert not any(SECRET in arg for arg in cmd), cmd + + +def test_postgres_backup_commands(monkeypatch, tmp_path): + rec = Recorder(monkeypatch, stdout="t\x1e") + engine = engine_for(target("postgres", jobs=3, dump_args=["--no-comments"])) + engine.backup(tmp_path, ["app", "odd'db"]) + rec.assert_no_secret_on_argv() + assert all(env.get("PGPASSWORD") == SECRET for _, env in rec.calls) + + dumps = [cmd for cmd, _ in rec.calls if cmd[0] == "pg_dump"] + assert len(dumps) == 2 + assert "--format=directory" in dumps[0] and "--jobs=3" in dumps[0] and "--compress=zstd:3" in dumps[0] + assert dumps[0][-1] == "--no-comments" + assert "dbname='odd\\'db'" in dumps[1] + globals_cmd = next(cmd for cmd, _ in rec.calls if cmd[0] == "pg_dumpall") + assert "--globals-only" in globals_cmd and "--no-role-passwords" not in globals_cmd + + +def test_postgres_without_pg_authid_access_skips_passwords(monkeypatch, tmp_path): + rec = Recorder(monkeypatch, stdout="f\x1e") + result = engine_for(target("postgres")).backup(tmp_path, []) + globals_cmd = next(cmd for cmd, _ in rec.calls if cmd[0] == "pg_dumpall") + assert "--no-role-passwords" in globals_cmd + assert result.contents["role_passwords"] is False + assert any("passwords were not saved" in w for w in result.warnings) + + +def test_mariadb_credentials_go_in_an_option_file(monkeypatch, tmp_path): + rec = Recorder(monkeypatch) + engine_for(target("mariadb", ssl=False)).backup(tmp_path, ["shop"]) + rec.assert_no_secret_on_argv() + for cmd, _ in rec.calls: + assert cmd[1].startswith("--defaults-extra-file="), "must be the first option" + assert "--skip-ssl" in cmd + (option_file,) = set(rec.files.values()) + assert option_file == f"[client]\nuser={option_file_value('backup')}\npassword={option_file_value(SECRET)}\n" + dump = next(cmd for cmd, _ in rec.calls if "--databases" in cmd) + assert "--single-transaction" in dump and dump[-2:] == ["--databases", "shop"] + assert not any(Path(arg.split("=", 1)[1]).exists() for arg in rec.files), "option file must be deleted" + + +def test_option_file_value_escaping(): + assert option_file_value('a"b\\c') == '"a\\"b\\\\c"' + + +def test_mongodb_uri_goes_in_a_config_file(monkeypatch, tmp_path): + rec = Recorder(monkeypatch) + engine = engine_for(target("mongodb", uri_options={"replicaSet": "rs0"})) + monkeypatch.setattr(type(engine), "is_replica_set", lambda self: True) + engine.backup(tmp_path, ["shop"]) + rec.assert_no_secret_on_argv() + (cmd,) = [c for c, _ in rec.calls] + assert "--oplog" in cmd and "--archive" in cmd + (config,) = rec.files.values() + assert config.startswith('uri: "mongodb://backup:') + assert "replicaSet=rs0" in config and "authSource=admin" in config + + +def test_mongodb_uri_quotes_credentials(): + uri = engine_for(target("mongodb")).uri() + assert uri.startswith("mongodb://backup:pa+ss%27w%22o%23rd%5C@db.example:27017/?") + + +def test_mongodb_include_exclude_dumps_per_database(monkeypatch, tmp_path): + rec = Recorder(monkeypatch) + engine = engine_for(target("mongodb", exclude=["scratch"])) + monkeypatch.setattr(type(engine), "is_replica_set", lambda self: True) + result = engine.backup(tmp_path, ["logs", "shop"]) + dbs = [next(a for a in cmd if a.startswith("--db=")) for cmd, _ in rec.calls] + assert dbs == ["--db=admin", "--db=logs", "--db=shop"] + assert not any("--oplog" in cmd for cmd, _ in rec.calls) + assert result.contents["mode"] == "per-database" + assert result.warnings + + +@pytest.mark.parametrize( + "line, account", + [ + ("CREATE USER IF NOT EXISTS `shop_app`@`%` IDENTIFIED BY PASSWORD '*2E68';", ("shop_app", "%")), + ("/*M!100005 CREATE ROLE IF NOT EXISTS 'reporting' WITH ADMIN mariadb_dump_import_role */;", ("reporting", None)), + ("GRANT `reporting` TO `root`@`localhost` WITH ADMIN OPTION;", ("root", "localhost")), + ("GRANT PROXY ON ``@`%` TO `root`@`%` WITH GRANT OPTION;", ("root", "%")), + ("GRANT USAGE ON *.* TO `reporting`;", ("reporting", None)), + ("/*!80001 ALTER USER 'root'@'%' DEFAULT ROLE NONE */;", ("root", "%")), + ("/*M!100005 SET DEFAULT ROLE NONE FOR 'mariadb.sys'@'localhost' */;", ("mariadb.sys", "localhost")), + ("CREATE USER IF NOT EXISTS `we``ird`@`h` IDENTIFIED BY PASSWORD 'x';", ("we`ird", "h")), + ("GRANT mariadb_dump_import_role TO CURRENT_USER();", None), + ("SET ROLE mariadb_dump_import_role;", None), + ], +) +def test_mariadb_statement_account(line, account): + assert statement_account(line) == account + + +def test_mariadb_without_accounts_leaves_existing_alone(): + sql = "\n".join( + [ + "CREATE USER IF NOT EXISTS `root`@`%` IDENTIFIED BY PASSWORD '*AAAA';", + "CREATE USER IF NOT EXISTS `app`@`%` IDENTIFIED BY PASSWORD '*BBBB';", + "GRANT ALL PRIVILEGES ON *.* TO `root`@`%` IDENTIFIED BY PASSWORD '*AAAA' WITH GRANT OPTION;", + "GRANT SELECT ON `shop`.* TO `app`@`%`;", + "SET ROLE NONE;", + ] + ) + kept, skipped = without_accounts(sql, {("root", "%")}) + assert skipped == {("root", "%")} + assert "root" not in kept + assert kept.count("\n") == 3 and "`app`@`%`" in kept and "SET ROLE NONE;" in kept + + +def test_postgres_without_roles(): + sql = "\n".join( + [ + "CREATE ROLE postgres;", + "ALTER ROLE postgres WITH SUPERUSER LOGIN PASSWORD 'SCRAM';", + 'CREATE ROLE "Odd ""Role""";', + 'ALTER ROLE "Odd ""Role""" WITH LOGIN;', + "CREATE ROLE app;", + "ALTER ROLE app WITH LOGIN PASSWORD 'x';", + "GRANT reporting TO postgres;", + ] + ) + kept, skipped = without_roles(sql, {"postgres", 'Odd "Role"'}) + assert skipped == {"postgres", 'Odd "Role"'} + assert kept == "CREATE ROLE app;\nALTER ROLE app WITH LOGIN PASSWORD 'x';\nGRANT reporting TO postgres;\n" + + +@pytest.mark.parametrize( + "name, expected", + [("app1", "app1"), ("odd.name/db", "odd%2Ename%2Fdb"), ("..", "%2E%2E"), ("wiki-db", "wiki-db"), ("ünï", "%C3%BCn%C3%AF")], +) +def test_safe_filename(name, expected): + assert safe_filename(name) == expected + + +def test_conninfo_dbname(): + assert conninfo_dbname("a b") == "dbname='a b'" + assert conninfo_dbname("x'\\y") == "dbname='x\\'\\\\y'" + + +def test_version_tuple(): + assert version_tuple("11.4.13-MariaDB-ubu2404") == (11, 4, 13) + assert version_tuple("18.6") == (18, 6) + assert version_tuple("8.3.11") >= (8, 3) + assert version_tuple("garbage") == () + + +def test_version_warning(): + engine = engine_for(target("mariadb")) + assert engine.version_warning("11.4.0-MariaDB") is None + assert "older than the minimum" in engine.version_warning("10.11.9-MariaDB") + + +def test_select_databases(monkeypatch): + engine = engine_for(target("postgres", exclude=["scratch"])) + monkeypatch.setattr(type(engine), "list_databases", lambda self: ["app", "postgres", "scratch"]) + assert engine.select_databases() == ["app", "postgres"] + engine = engine_for(target("postgres", include=["app", "scratch"], exclude=["scratch"])) + assert engine.select_databases() == ["app"] + engine = engine_for(target("postgres", include=["missing"])) + with pytest.raises(Exception, match="not found on server: missing"): + engine.select_databases() + + +def test_ns_escape(): + assert _ns_escape("a*b$c") == "a\\*b\\$c" diff --git a/tests/test_notify.py b/tests/test_notify.py new file mode 100644 index 0000000..0a5fed8 --- /dev/null +++ b/tests/test_notify.py @@ -0,0 +1,40 @@ +from dbbackup.config import parse_config +from dbbackup.notify import discovery_messages, entity_slug, publish_statuses + + +def mqtt_config(**extra): + raw = {"targets": [{"name": "t", "type": "postgres", "host": "h"}], "mqtt": {"host": "mq", "ha_discovery": True, **extra}} + return parse_config(raw, {}).mqtt + + +def test_entity_slug(): + assert entity_slug("PG-Main_1") == "pg_main_1" + + +def test_discovery_uses_predictable_entity_ids(): + (status_topic, status), (last_topic, last) = discovery_messages(mqtt_config(), "pg-main") + assert status_topic == "homeassistant/sensor/dbbackup_pg_main_status/config" + assert status["default_entity_id"] == "sensor.dbbackup_pg_main_status" + assert status["state_topic"] == "dbbackup/pg-main/status" + assert status["json_attributes_topic"] == "dbbackup/pg-main/status" + assert last_topic == "homeassistant/sensor/dbbackup_pg_main_last_success/config" + assert last["default_entity_id"] == "sensor.dbbackup_pg_main_last_success" + assert last["device_class"] == "timestamp" + assert status["device"] == last["device"] + + +def test_custom_prefixes(): + ((topic, payload), _) = discovery_messages(mqtt_config(topic_prefix="site/backups", discovery_prefix="ha"), "x") + assert topic.startswith("ha/sensor/") + assert payload["state_topic"] == "site/backups/x/status" + + +def test_publish_failure_is_reported_not_raised(): + # Nothing listens on port 1 of localhost. + cfg = mqtt_config(host="127.0.0.1", port=1) + status = {"target": "t", "state": "ok", "finished": "2026-01-01T00:00:00+00:00"} + assert publish_statuses(cfg, [status]) is False + + +def test_no_mqtt_config_is_a_no_op(): + assert publish_statuses(None, [{"target": "t", "state": "ok", "finished": None}]) is True diff --git a/tests/test_retention.py b/tests/test_retention.py new file mode 100644 index 0000000..2a2745a --- /dev/null +++ b/tests/test_retention.py @@ -0,0 +1,82 @@ +from datetime import UTC, datetime, timedelta + +from dbbackup.config import Retention +from dbbackup.retention import select_keep + + +def ts(*args) -> datetime: + return datetime(*args, tzinfo=UTC) + + +def daily_backups(start: datetime, days: int, hour: int = 2) -> list[datetime]: + return [(start + timedelta(days=i)).replace(hour=hour) for i in range(days)] + + +def test_empty(): + assert select_keep([], Retention()) == set() + + +def test_newest_is_always_kept_even_with_an_empty_policy(): + backups = daily_backups(ts(2026, 1, 1), 5) + assert select_keep(backups, Retention(last=0, daily=0, weekly=0, monthly=0, yearly=0)) == {backups[-1]} + + +def test_daily_keeps_newest_per_day(): + morning, evening = ts(2026, 3, 10, 2), ts(2026, 3, 10, 20) + previous = ts(2026, 3, 9, 2) + keep = select_keep([previous, morning, evening], Retention(daily=2, weekly=0, monthly=0)) + assert keep == {evening, previous} + + +def test_last_keeps_n_newest_regardless_of_period(): + backups = [ts(2026, 3, 10, h) for h in (1, 2, 3, 4)] + assert select_keep(backups, Retention(last=3, daily=0, weekly=0, monthly=0)) == set(backups[1:]) + + +def test_gfs_over_a_year_of_dailies(): + backups = daily_backups(ts(2025, 1, 1), 400) # 2025-01-01 .. 2026-02-04 + keep = select_keep(backups, Retention(daily=7, weekly=4, monthly=6, yearly=2)) + + newest = backups[-1] + dailies = {b for b in backups if newest - b < timedelta(days=7)} + assert dailies <= keep + + # Weekly: the newest backup of each of the 4 newest ISO weeks. + weeks = {} + for b in backups: + weeks[tuple(b.isocalendar())[:2]] = b + newest_weeks = sorted(weeks.values())[-4:] + assert set(newest_weeks) <= keep + + # Monthly: the last backup of each of the 6 newest months (the newest month's is the newest backup). + months = {} + for b in backups: + months[(b.year, b.month)] = b + assert set(sorted(months.values())[-6:]) <= keep + assert ts(2025, 12, 31, 2) in keep + assert ts(2025, 9, 30, 2) in keep + + # Yearly: 2026's newest, and 2025's last. + assert ts(2025, 12, 31, 2) in keep + + # And nothing else. + expected = dailies | set(newest_weeks) | set(sorted(months.values())[-6:]) | {ts(2025, 12, 31, 2), newest} + assert keep == expected + assert len(keep) < 20 + + +def test_periods_with_no_backup_dont_count(): + # A two-month gap: "monthly=3" still keeps three backups, reaching further back. + backups = [ts(2026, 1, 15), ts(2026, 4, 15), ts(2026, 5, 15)] + assert select_keep(backups, Retention(daily=0, weekly=0, monthly=3)) == set(backups) + + +def test_iso_weeks_span_year_boundaries(): + # 2025-12-29 .. 2026-01-04 is ISO week 2026-W01: one week, so weekly=1 keeps only the newest. + backups = daily_backups(ts(2025, 12, 29), 7) + assert select_keep(backups, Retention(daily=0, weekly=1, monthly=0)) == {backups[-1]} + + +def test_duplicates_are_harmless(): + b = ts(2026, 1, 1) + assert select_keep([b, b], Retention()) == {b} diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..798d2d6 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,124 @@ +import json +import os +import time +from datetime import UTC, datetime, timedelta + +import pytest + +from dbbackup.config import Retention +from dbbackup.storage import LockedError, StorageError, Store, format_ts, verify_checksums + +T0 = datetime(2026, 10, 1, 2, 0, tzinfo=UTC) + + +def make_set(store: Store, target: str, when: datetime, files=None, manifest_extra=None): + partial = store.begin(target, when) + for rel, data in (files or {"dump.sql.zst": b"data"}).items(): + (partial / rel).parent.mkdir(parents=True, exist_ok=True) + (partial / rel).write_bytes(data) + return store.finalize(partial, {"status": "ok", "finished": when.isoformat(), **(manifest_extra or {})}) + + +def test_finalize_writes_manifest_and_renames(tmp_path): + store = Store(tmp_path) + bset = make_set(store, "pg", T0, {"a.sql.zst": b"aaa", "db.pgdump/toc.dat": b"toc"}) + assert bset.path == tmp_path / "pg" / "20261001T020000Z" + assert not (tmp_path / "pg" / "20261001T020000Z.partial").exists() + manifest = json.loads((bset.path / "manifest.json").read_text()) + assert set(manifest["files"]) == {"a.sql.zst", "db.pgdump/toc.dat"} + assert manifest["size_bytes"] == 6 + assert store.sets("pg")[0].ok + assert verify_checksums(store.sets("pg")[0]) == [] + + +def test_verify_checksums_detects_tampering(tmp_path): + store = Store(tmp_path) + bset = make_set(store, "pg", T0, {"a": b"one", "b": b"two"}) + (bset.path / "a").write_bytes(b"ONE") + (bset.path / "b").unlink() + (bset.path / "c").write_bytes(b"new") + assert verify_checksums(store.sets("pg")[0]) == ["missing file: b", "unexpected file: c", "checksum mismatch: a"] + + +def test_partial_and_broken_sets_are_not_ok(tmp_path): + store = Store(tmp_path) + store.begin("pg", T0) # never finalized + broken = tmp_path / "pg" / format_ts(T0 + timedelta(days=1)) + broken.mkdir() + (broken / "manifest.json").write_text("{not json") + sets = store.sets("pg") + assert [s.ok for s in sets] == [False] # the .partial isn't listed at all + assert store.latest_ok("pg") is None + with pytest.raises(StorageError, match="no successful backup sets"): + store.find("pg") + + +def test_find_latest_and_by_name(tmp_path): + store = Store(tmp_path) + make_set(store, "pg", T0) + newest = make_set(store, "pg", T0 + timedelta(days=1)) + assert store.find("pg").path == newest.path + assert store.find("pg", "20261001T020000Z").timestamp == T0 + with pytest.raises(StorageError, match="not found"): + store.find("pg", "20200101T000000Z") + + +def test_lock_is_exclusive_and_released(tmp_path): + store = Store(tmp_path) + with store.lock("pg", timedelta(hours=1)): + with pytest.raises(LockedError, match="locked by another run"): + with store.lock("pg", timedelta(hours=1)): + pass + assert not (tmp_path / "pg" / ".lock").exists() + with store.lock("pg", timedelta(hours=1)): + pass + + +def test_stale_lock_is_broken(tmp_path): + store = Store(tmp_path) + lock = tmp_path / "pg" / ".lock" + lock.parent.mkdir(parents=True) + lock.write_text('{"host": "gone"}') + old = time.time() - 3 * 3600 + os.utime(lock, (old, old)) + with store.lock("pg", timedelta(hours=2)): + assert "gone" not in lock.read_text() + + +def test_prune_applies_retention(tmp_path): + store = Store(tmp_path) + for day in range(10): + make_set(store, "pg", T0 + timedelta(days=day)) + deleted = store.prune("pg", Retention(daily=3, weekly=0, monthly=0), stale_partial_after=timedelta(hours=24)) + assert len(deleted) == 7 + assert [s.timestamp.day for s in store.sets("pg")] == [8, 9, 10] + + +def test_prune_dry_run_deletes_nothing(tmp_path): + store = Store(tmp_path) + for day in range(5): + make_set(store, "pg", T0 + timedelta(days=day)) + assert len(store.prune("pg", Retention(daily=1, weekly=0, monthly=0), stale_partial_after=timedelta(hours=1), dry_run=True)) == 4 + assert len(store.sets("pg")) == 5 + + +def test_prune_leaves_unrecognised_sets_and_removes_stale_partials(tmp_path): + store = Store(tmp_path) + make_set(store, "pg", T0) + bad = tmp_path / "pg" / format_ts(T0 - timedelta(days=30)) + bad.mkdir() + old_partial = store.begin("pg", T0 - timedelta(days=2)) + fresh_partial = store.begin("pg", T0 + timedelta(hours=1)) + store.prune("pg", Retention(daily=1, weekly=0, monthly=0), stale_partial_after=timedelta(hours=24), now=T0 + timedelta(hours=2)) + assert bad.exists() + assert not old_partial.exists() + assert fresh_partial.exists() + + +def test_targets_are_independent(tmp_path): + store = Store(tmp_path) + make_set(store, "a", T0) + make_set(store, "b", T0) + store.prune("a", Retention(daily=0, weekly=0, monthly=0), stale_partial_after=timedelta(hours=1)) + assert len(store.sets("a")) == 1 # newest always kept + assert len(store.sets("b")) == 1