Allow databases to be excluded

README.md

@@ -39,6 +39,7 @@ You can also pass the following environment variables to override the defaults:
     -e RESTIC_KEEP_WEEKLY='4'
     -e RESTIC_KEEP_MONTHLY='12'
     -e RESTIC_KEEP_YEARLY='7'
+    -e EXCLUDED_DATABASES='db_to_skip,another_db'  # Global exclusions for all clusters
 
 You can backup 5 different database clusters with `PG*_[1..5]`, and assign an arbitrary hostname with `HOSTNAME_[1..5]` (if `PGHOST` is not a fully qualified domain name) environment variables.
 
@@ -47,6 +48,9 @@ You can backup 5 different database clusters with `PG*_[1..5]`, and assign an ar
     -e PGPASSWORD_2='...'
     -e PGPORT_2='5432'
     -e PGUSER_2='...'
+    -e EXCLUDED_DATABASES_2='db_to_skip_for_cluster_2'  # Overrides EXCLUDED_DATABASES for cluster 2
+
+If both `EXCLUDED_DATABASES` and `EXCLUDED_DATABASES_N` are set for cluster `N`, `EXCLUDED_DATABASES_N` takes precedence for that cluster.
 
 A `docker-compose.yml` file is provided for convenience.
 

bin/backup.sh

@@ -2,6 +2,26 @@
 
 set -e
 
+PG_DUMP_DIR="${PG_DUMP_DIR:-/pg_dump}"
+
+trim_whitespace() {
+	local value="$1"
+	value="${value#"${value%%[![:space:]]*}"}"
+	value="${value%"${value##*[![:space:]]}"}"
+	printf '%s' "$value"
+}
+
+sql_quote_literal() {
+	local value="$1"
+	value="${value//\'/\'\'}"
+	printf "'%s'" "$value"
+}
+
+cleanup_dump_files() {
+	[[ -d "$PG_DUMP_DIR" ]] || return 0
+	find "$PG_DUMP_DIR" -maxdepth 1 -type f -name '*.sql' -delete
+}
+
 setup.sh
 
 for i in {1..5}; do
@@ -40,25 +60,57 @@ for i in {1..5}; do
 		echo "Waited $COUNT seconds."
 	fi
 
-	mkdir -p "/pg_dump"
+	mkdir -p "$PG_DUMP_DIR"
 
 	# Dump individual databases directly to restic repository.
-	DBLIST=$(psql -d postgres -q -t -c "SELECT datname FROM pg_database WHERE datname NOT IN ('postgres', 'rdsadmin', 'template0', 'template1')")
-	for dbname in $DBLIST; do
-		echo "Dumping database '$dbname'"
-		pg_dump --file="/pg_dump/$dbname.sql" --no-owner --no-privileges --dbname="$dbname" || true  # Ignore failures
+	export EXCLUDED_DATABASES_VAR="EXCLUDED_DATABASES_$i"
+	cluster_excluded_raw="$(trim_whitespace "${!EXCLUDED_DATABASES_VAR:-}")"
+	if [[ -n "$cluster_excluded_raw" ]]; then
+		effective_excluded_raw="$cluster_excluded_raw"
+	else
+		effective_excluded_raw="${EXCLUDED_DATABASES:-}"
+	fi
+
+	exclusions=(postgres rdsadmin template0 template1)
+	IFS=',' read -r -a extra_exclusions <<< "$effective_excluded_raw"
+	for raw_name in "${extra_exclusions[@]}"; do
+		trimmed_name="$(trim_whitespace "$raw_name")"
+		[[ -z "$trimmed_name" ]] && continue
+		exclusions+=("$trimmed_name")
 	done
+	declare -A excluded_lookup=()
+	for name in "${exclusions[@]}"; do
+		excluded_lookup["$name"]=1
+	done
+
+	sql_not_in_list=""
+	for name in "${exclusions[@]}"; do
+		quoted_name="$(sql_quote_literal "$name")"
+		if [[ -n "$sql_not_in_list" ]]; then
+			sql_not_in_list+=", "
+		fi
+		sql_not_in_list+="$quoted_name"
+	done
+
+	query="SELECT datname FROM pg_database WHERE datname NOT IN ($sql_not_in_list)"
+	DBLIST=$(psql -d postgres -A -q -t -c "$query")
+	while IFS= read -r dbname; do
+		[[ -z "$dbname" ]] && continue
+		[[ -n "${excluded_lookup[$dbname]:-}" ]] && continue
+		echo "Dumping database '$dbname'"
+		pg_dump --file="$PG_DUMP_DIR/$dbname.sql" --no-owner --no-privileges --dbname="$dbname" || true  # Ignore failures
+	done <<< "$DBLIST"
 
 	# echo "Dumping global objects for '$PGHOST'"
 	# pg_dumpall --file="/pg_dump/!globals.sql" --globals-only
 
 	echo "Sending database dumps to S3"
-	while ! restic backup --host "$HOST" "/pg_dump"; do
+	while ! restic backup --host "$HOST" "$PG_DUMP_DIR"; do
 		echo "Sleeping for 10 seconds before retry..."
 		sleep 10
 	done
 
 	echo 'Finished sending database dumps to S3'
 
-	rm -rf "/pg_dump"
+	cleanup_dump_files
 done

bin/tests/backup-excluded-databases.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+TMP_BASE_DIR="$ROOT_DIR/.superpowers/sdd"
+mkdir -p "$TMP_BASE_DIR"
+TMP_DIR="$TMP_BASE_DIR/backup-excluded-databases-$$-$RANDOM"
+mkdir -p "$TMP_DIR"
+trap 'rm -rf "$TMP_DIR"' EXIT
+
+cat > "$TMP_DIR/setup.sh" <<'EOF'
+#!/usr/bin/env bash
+:
+EOF
+chmod +x "$TMP_DIR/setup.sh"
+
+cat > "$TMP_DIR/psql" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+if [[ "${1:-}" == "-l" ]]; then
+  exit 0
+fi
+if [[ "${1:-}" == "-d" && "${2:-}" == "postgres" ]]; then
+  printf '%s\n' "$*" > "${TEST_QUERY_FILE:?}"
+  printf 'app_keep\napp_exclude\n'
+  exit 0
+fi
+exit 1
+EOF
+chmod +x "$TMP_DIR/psql"
+
+cat > "$TMP_DIR/pg_dump" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+dump_file=""
+for arg in "$@"; do
+  if [[ "$arg" == --file=* ]]; then
+    dump_file="${arg#--file=}"
+  fi
+  if [[ "$arg" == --dbname=* ]]; then
+    printf '%s\n' "${arg#--dbname=}" >> "${TEST_DUMP_FILE:?}"
+  fi
+done
+if [[ -n "$dump_file" ]]; then
+  printf 'dump\n' > "$dump_file"
+fi
+exit 0
+EOF
+chmod +x "$TMP_DIR/pg_dump"
+
+cat > "$TMP_DIR/restic" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+exit 0
+EOF
+chmod +x "$TMP_DIR/restic"
+
+export TEST_QUERY_FILE="$TMP_DIR/query.txt"
+export TEST_DUMP_FILE="$TMP_DIR/dumped.txt"
+touch "$TEST_DUMP_FILE"
+
+# Ensure tests run in this environment by using a writable dump directory
+export PG_DUMP_DIR="$TMP_DIR/pg_dump_dir"
+mkdir -p "$PG_DUMP_DIR"
+printf 'keep\n' > "$PG_DUMP_DIR/keep.me"
+
+(
+  cd "$ROOT_DIR"
+  PATH="$TMP_DIR:$PATH" \
+  PGHOST_1="db.example.internal" \
+  PGPASSWORD_1="secret" \
+  PGUSER_1="postgres" \
+  EXCLUDED_DATABASES="app_exclude" \
+  bash bin/backup.sh >/dev/null 2>&1
+)
+
+if grep -qx 'app_exclude' "$TEST_DUMP_FILE"; then
+  echo "FAIL: expected app_exclude to be skipped"
+  exit 1
+fi
+if ! grep -qx 'app_keep' "$TEST_DUMP_FILE"; then
+  echo "FAIL: expected app_keep to be dumped"
+  exit 1
+fi
+if [[ ! -f "$PG_DUMP_DIR/keep.me" ]]; then
+  echo "FAIL: expected sentinel file to survive cleanup"
+  exit 1
+fi
+if find "$PG_DUMP_DIR" -maxdepth 1 -type f -name '*.sql' | grep -q .; then
+  echo "FAIL: expected generated dump files to be cleaned up"
+  exit 1
+fi
+echo "PASS: global exclusions verified"
+
+: > "$TEST_DUMP_FILE"
+(
+  cd "$ROOT_DIR"
+  PATH="$TMP_DIR:$PATH" \
+  PGHOST_1="db.example.internal" \
+  PGPASSWORD_1="secret" \
+  PGUSER_1="postgres" \
+  EXCLUDED_DATABASES="app_keep" \
+  EXCLUDED_DATABASES_1="app_exclude" \
+  bash bin/backup.sh >/dev/null 2>&1
+)
+
+if grep -qx 'app_exclude' "$TEST_DUMP_FILE"; then
+  echo "FAIL: expected app_exclude to be skipped by EXCLUDED_DATABASES_1"
+  exit 1
+fi
+if ! grep -qx 'app_keep' "$TEST_DUMP_FILE"; then
+  echo "FAIL: expected app_keep to be dumped when EXCLUDED_DATABASES_1 overrides global"
+  exit 1
+fi
+if [[ ! -f "$PG_DUMP_DIR/keep.me" ]]; then
+  echo "FAIL: expected sentinel file to survive cleanup"
+  exit 1
+fi
+if find "$PG_DUMP_DIR" -maxdepth 1 -type f -name '*.sql' | grep -q .; then
+  echo "FAIL: expected generated dump files to be cleaned up"
+  exit 1
+fi
+echo "PASS: per-cluster override verified"