From 8c21f9194ac061d0413a913d94cb1f1a78dc86a3 Mon Sep 17 00:00:00 2001 From: Cameron Simpson Date: Tue, 18 May 2021 09:09:04 +1000 Subject: [PATCH 01/10] setup.sh: report all the missing variables before aborting --- bin/setup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/setup.sh b/bin/setup.sh index 45ab270..636ab1e 100755 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -2,12 +2,14 @@ set -e +ok=1 for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY RESTIC_PASSWORD RESTIC_REPOSITORY; do eval [[ -z \${$var+1} ]] && { >&2 echo "ERROR: Missing required environment variable: $var" - exit 1 + ok= } done +[ $ok ] || exit 1 if ! restic unlock; then restic init From 292491d8dabd461e9a2aba32ae138e1fcfaae661 Mon Sep 17 00:00:00 2001 From: Cameron Simpson Date: Tue, 18 May 2021 09:15:30 +1000 Subject: [PATCH 02/10] backup.sh: downcase unexported variables, put a limit on the psql wait time --- bin/backup.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/bin/backup.sh b/bin/backup.sh index 7bac852..d627304 100755 --- a/bin/backup.sh +++ b/bin/backup.sh @@ -4,6 +4,8 @@ set -e setup.sh +max_pg_wait_count=120 + for i in {1..5}; do export HOSTNAME_VAR="HOSTNAME_$i" export PGHOST_VAR="PGHOST_$i" @@ -28,23 +30,28 @@ for i in {1..5}; do echo "Dumping database cluster $i: $PGUSER@$PGHOST:$PGPORT" # Wait for PostgreSQL to become available. - COUNT=0 + count=0 until psql -l > /dev/null 2>&1; do - if [[ "$COUNT" == 0 ]]; then + if [[ "$count" == 0 ]]; then echo "Waiting for PostgreSQL to become available..." fi - (( COUNT += 1 )) + (( count += 1 )) + [ $count -lt $max_pg_wait_count ] || break sleep 1 done - if (( COUNT > 0 )); then - echo "Waited $COUNT seconds." + if (( count > 0 )); then + echo "Waited $count seconds." + psql -l > /dev/null 2>&1 || { + echo "PostgreSQL still not available, trying next backup." + continue + } fi mkdir -p "/pg_dump" # Dump individual databases directly to restic repository. - DBLIST=$(psql -d postgres -q -t -c "SELECT datname FROM pg_database WHERE datname NOT IN ('postgres', 'rdsadmin', 'template0', 'template1')") - for dbname in $DBLIST; do + dblist=$(psql -d postgres -q -t -c "SELECT datname FROM pg_database WHERE datname NOT IN ('postgres', 'rdsadmin', 'template0', 'template1')") + for dbname in $dblist; do echo "Dumping database '$dbname'" pg_dump --file="/pg_dump/$dbname.sql" --no-owner --no-privileges --dbname="$dbname" || true # Ignore failures done From 1d590943f8e475af1f1cb13d785f02aaeb189207 Mon Sep 17 00:00:00 2001 From: Cameron Simpson Date: Tue, 18 May 2021 09:21:16 +1000 Subject: [PATCH 03/10] backup.sh: drop -p from mkdir (is prone to races), abort script if it fails --- bin/backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/backup.sh b/bin/backup.sh index d627304..6e38ee5 100755 --- a/bin/backup.sh +++ b/bin/backup.sh @@ -47,7 +47,7 @@ for i in {1..5}; do } fi - mkdir -p "/pg_dump" + mkdir "/pg_dump" || exit 1 # Dump individual databases directly to restic repository. dblist=$(psql -d postgres -q -t -c "SELECT datname FROM pg_database WHERE datname NOT IN ('postgres', 'rdsadmin', 'template0', 'template1')") From 2fcb593e8d6624a523c87d7108047f707a785a62 Mon Sep 17 00:00:00 2001 From: Cameron Simpson Date: Tue, 18 May 2021 09:23:55 +1000 Subject: [PATCH 04/10] backup.sh: honour $PGDUMP_BACKUP_AREA to specify a scratch area for backups instead of /pg_dump --- bin/backup.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/backup.sh b/bin/backup.sh index 6e38ee5..ab89139 100755 --- a/bin/backup.sh +++ b/bin/backup.sh @@ -5,6 +5,7 @@ set -e setup.sh max_pg_wait_count=120 +work_area=${PGDUMP_BACKUP_AREA:-/pg_dump} for i in {1..5}; do export HOSTNAME_VAR="HOSTNAME_$i" @@ -47,25 +48,25 @@ for i in {1..5}; do } fi - mkdir "/pg_dump" || exit 1 + mkdir "$work_area" || exit 1 # Dump individual databases directly to restic repository. dblist=$(psql -d postgres -q -t -c "SELECT datname FROM pg_database WHERE datname NOT IN ('postgres', 'rdsadmin', 'template0', 'template1')") for dbname in $dblist; do echo "Dumping database '$dbname'" - pg_dump --file="/pg_dump/$dbname.sql" --no-owner --no-privileges --dbname="$dbname" || true # Ignore failures + pg_dump --file="$work_area/$dbname.sql" --no-owner --no-privileges --dbname="$dbname" || true # Ignore failures done # echo "Dumping global objects for '$PGHOST'" - # pg_dumpall --file="/pg_dump/!globals.sql" --globals-only + # pg_dumpall --file="$work_area/!globals.sql" --globals-only echo "Sending database dumps to S3" - while ! restic backup --host "$HOSTNAME" "/pg_dump"; do + while ! restic backup --host "$HOSTNAME" "$work_area"; do echo "Sleeping for 10 seconds before retry..." sleep 10 done echo 'Finished sending database dumps to S3' - rm -rf "/pg_dump" + rm -rf "$work_area" done From 7c779cecaace688b0652840b39ba2ede5d35d5b2 Mon Sep 17 00:00:00 2001 From: Cameron Simpson Date: Tue, 18 May 2021 12:00:20 +1000 Subject: [PATCH 05/10] backup.sh: make psql wait time configurable via $PGDUMP_BACKUP_WAIT_TIME --- bin/backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/backup.sh b/bin/backup.sh index ab89139..a464c18 100755 --- a/bin/backup.sh +++ b/bin/backup.sh @@ -4,7 +4,7 @@ set -e setup.sh -max_pg_wait_count=120 +max_pg_wait_count=${PGDUMP_BACKUP_WAIT_TIME:-120} work_area=${PGDUMP_BACKUP_AREA:-/pg_dump} for i in {1..5}; do From dfbd82b0cbff7c2b9aa19feb1bf4b0ea27111aa1 Mon Sep 17 00:00:00 2001 From: Cameron Simpson Date: Tue, 18 May 2021 12:06:27 +1000 Subject: [PATCH 06/10] backup.sh: change compact [ ] into if-[[-then for consistency --- bin/backup.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/backup.sh b/bin/backup.sh index a464c18..c00deb5 100755 --- a/bin/backup.sh +++ b/bin/backup.sh @@ -36,9 +36,12 @@ for i in {1..5}; do if [[ "$count" == 0 ]]; then echo "Waiting for PostgreSQL to become available..." fi - (( count += 1 )) - [ $count -lt $max_pg_wait_count ] || break + if [[ $count -ge $max_pg_wait_count ]] + then + break + fi sleep 1 + (( count += 1 )) done if (( count > 0 )); then echo "Waited $count seconds." From 2262680dc6ee565bbde4bbf00db2e848fcf257aa Mon Sep 17 00:00:00 2001 From: Alastair Weakley Date: Fri, 25 Feb 2022 09:16:09 +1100 Subject: [PATCH 07/10] Get postgresql14-client for newer db compatibility --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bff0f57..87c8ea8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,10 @@ RUN apk update \ && apk upgrade \ && apk add \ bash \ - postgresql-client \ tini \ && apk add --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main \ util-linux \ + postgresql-client \ && rm -rf /var/cache/apk/* ENV DOCKERIZE_VERSION=0.5.0 From bf50d5785f2d2ad2a8edeca22181f930c770960c Mon Sep 17 00:00:00 2001 From: Alastair Weakley Date: Fri, 25 Feb 2022 09:17:15 +1100 Subject: [PATCH 08/10] Upgrade restic version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bff0f57..4c37eea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM restic/restic:0.12.0 +FROM restic/restic:0.12.1 RUN apk update \ && apk upgrade \ From 49b9986748dff5fe1922d4bb31148053a996f90f Mon Sep 17 00:00:00 2001 From: rushil Date: Mon, 11 Apr 2022 09:43:08 +1000 Subject: [PATCH 09/10] backup format changed - pg_dump --- README.md | 10 ++++++++++ bin/backup.sh | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2458913..fa60bea 100644 --- a/README.md +++ b/README.md @@ -97,3 +97,13 @@ Then, access the latest snapshot from another terminal: [direnv]: https://direnv.net/ [Homebrew]: https://brew.sh/ [restic]: https://restic.net/ + +# PG Restore + +If DB needs to be to dropped and setup again from backup file. + + pg_restore --clean --if-exists --create -d postgres --format="directory" --jobs={MAX_POSSIBLE_NUM} {PG_DUMP_FOLDER} + +If DB is already setup with privs and access - + + pg_restore -d {DB_NAME} --format="directory" --jobs={MAX_POSSIBLE_NUM} {PG_DUMP_FOLDER} \ No newline at end of file diff --git a/bin/backup.sh b/bin/backup.sh index c00deb5..3bd9382 100755 --- a/bin/backup.sh +++ b/bin/backup.sh @@ -7,6 +7,8 @@ setup.sh max_pg_wait_count=${PGDUMP_BACKUP_WAIT_TIME:-120} work_area=${PGDUMP_BACKUP_AREA:-/pg_dump} +pg_backup_jobs=${PGDUMP_BACKUP_JOBS:-1} + for i in {1..5}; do export HOSTNAME_VAR="HOSTNAME_$i" export PGHOST_VAR="PGHOST_$i" @@ -57,7 +59,8 @@ for i in {1..5}; do dblist=$(psql -d postgres -q -t -c "SELECT datname FROM pg_database WHERE datname NOT IN ('postgres', 'rdsadmin', 'template0', 'template1')") for dbname in $dblist; do echo "Dumping database '$dbname'" - pg_dump --file="$work_area/$dbname.sql" --no-owner --no-privileges --dbname="$dbname" || true # Ignore failures + # pg_dump working at default compression (6) + pg_dump --file="$work_area/$dbname" --format="directory" --no-owner --no-privileges --dbname="$dbname" --jobs=$pg_backup_jobs || true # Ignore failures done # echo "Dumping global objects for '$PGHOST'" From e265f4563d1ba754798c1ffb33cb4282a1448578 Mon Sep 17 00:00:00 2001 From: Rushil Date: Mon, 8 Aug 2022 17:47:59 +1000 Subject: [PATCH 10/10] restic lockfile bug --- crontab.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crontab.tmpl b/crontab.tmpl index cb02573..0ecbf6f 100644 --- a/crontab.tmpl +++ b/crontab.tmpl @@ -1,5 +1,5 @@ -{{ default .Env.BACKUP_SCHEDULE "0 * * * *" }} flock -n /opt/restic-pg-dump/backup.lockfile backup.sh +{{ default .Env.BACKUP_SCHEDULE "0 * * * *" }} flock -n /opt/restic-pg-dump/restic_running.lockfile backup.sh {{ if default .Env.PRUNE_SCHEDULE "0 14 * * 0" }} -{{ default .Env.PRUNE_SCHEDULE "0 14 * * 0" }} flock -n /opt/restic-pg-dump/prune.lockfile prune.sh +{{ default .Env.PRUNE_SCHEDULE "0 14 * * 0" }} flock -n /opt/restic-pg-dump/restic_running.lockfile prune.sh {{ end }}