Skip to content

minimize i/o load during hash calculation #4021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
benderl merged 2 commits into from
Oct 5, 2026
Merged

minimize i/o load during hash calculation #4021

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b3cb310
minimize i/o load during hash calculation
benderl Sep 28, 2026
3a94ae8
abort on error
benderl Sep 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 44 additions & 29 deletions runs/backup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
#!/bin/bash
set -Ee -o pipefail

OPENWBBASEDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
OPENWBDIRNAME=${OPENWBBASEDIR##*/}
OPENWBDIRNAME=${OPENWBDIRNAME:-/}
TARBASEDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
BACKUPDIR="$OPENWBBASEDIR/data/backup"
RAMDISKDIR="$OPENWBBASEDIR/ramdisk"
TEMPDIR=$(mktemp -d --tmpdir openwb_backup_XXXXXX)
TEMPDIR=""
LOGDIR="$OPENWBBASEDIR/data/log"
LOGFILE="$LOGDIR/backup.log"
HOMEDIR="/home/openwb"
KEYFILE="backup.key"
VAR_LIB="/var/lib"
BACKUPFILE=""

# Mosquitto DB files to monitor
DB_FILES=(
Expand Down Expand Up @@ -200,11 +203,13 @@ create_archive() {

# JSON-Dateien im clients-Ordner sammeln
json_files=()
find "$TARBASEDIR/$OPENWBDIRNAME/data/clients" -maxdepth 1 -type f -name '*.json' -print0 >"$TEMPDIR/client_json_files"
while IFS= read -r -d '' file; do
json_files+=("${file#$TARBASEDIR/}")
done < <(find "$TARBASEDIR/$OPENWBDIRNAME/data/clients" -maxdepth 1 -type f -name '*.json' -print0)
done <"$TEMPDIR/client_json_files"
rm "$TEMPDIR/client_json_files"

sudo tar --verbose --create \
sudo tar --create \
--file="$BACKUPFILE" \
--exclude=".gitignore" \
--directory="$TEMPDIR/" \
Expand All @@ -226,7 +231,7 @@ create_archive() {

if [ -f "$VAR_LIB/mosquitto/dynamic-security.json" ]; then
echo "adding mosquitto/dynamic-security.json"
sudo tar --verbose --append \
sudo tar --append \
--file="$BACKUPFILE" \
--directory="$VAR_LIB/" \
"mosquitto/dynamic-security.json"
Expand All @@ -235,7 +240,7 @@ create_archive() {
fi
if [ -f "$HOMEDIR/.config/mosquitto_ctrl" ]; then
echo "adding mosquitto_ctrl file"
sudo tar --verbose --append \
sudo tar --append \
--file="$BACKUPFILE" \
--directory="$HOMEDIR/.config/" \
"mosquitto_ctrl"
Expand All @@ -246,25 +251,22 @@ create_archive() {

calculate_checksums() {
echo "calculating checksums"
IFS=$'\n'
mapfile -t file_list < <(tar -tf "$BACKUPFILE")
# process each file
for file in "${file_list[@]}"; do
# skip directories
if [[ $file =~ /$ ]]; then
echo "skipping directory $file"
continue
fi
# extract the file
tar -xf "$BACKUPFILE" -C "$TEMPDIR" "$file"
# calculate the checksum
sha256sum "$TEMPDIR/$file" | sed -n "s|$TEMPDIR/||p" >> "$TEMPDIR/SHA256SUM"
# remove the file
rm -f "$TEMPDIR/$file"
done
# Stream each regular file directly into sha256sum. This reads the archive
# once and avoids temporary copies of large log files.
# shellcheck disable=SC2016
if ! tar --extract \
--file="$BACKUPFILE" \
--to-command='if [ "$TAR_FILETYPE" = "f" ]; then
checksum=$(sha256sum) || exit 1
printf "%s %s\n" "${checksum%% *}" "$TAR_FILENAME"
fi' \
Comment thread
benderl marked this conversation as resolved.
>"$TEMPDIR/SHA256SUM"; then
echo "failed to calculate checksums"
return 1
fi
Comment thread
benderl marked this conversation as resolved.

echo "adding checksum file to archive"
sudo tar --verbose --append \
sudo tar --append \
--file="$BACKUPFILE" \
--directory="$TEMPDIR/" \
"SHA256SUM"
Expand All @@ -279,7 +281,7 @@ create_archive() {
--directory="$LOGDIR/" \
"backup.log"
echo "zipping archive"
gzip --verbose --suffix "$FILENAMESUFFIX" "$BACKUPFILE"
gzip --suffix "$FILENAMESUFFIX" "$BACKUPFILE"
}

encrypt_backup() {
Expand Down Expand Up @@ -309,16 +311,29 @@ create_archive() {
fix_permissions
}

handle_error() {
local exit_status=$?
local failed_line=${BASH_LINENO[0]:-unknown}
trap - ERR
echo "ERROR: backup failed at line $failed_line (exit status $exit_status)"
if [[ -n "$TEMPDIR" && -d "$TEMPDIR" ]]; then
rm -rf -- "$TEMPDIR" || echo "ERROR: failed to remove temporary directory '$TEMPDIR'"
fi
if [[ -n "$BACKUPFILE" ]]; then
rm -f -- "$BACKUPFILE" "$BACKUPFILE$FILENAMESUFFIX" "$BACKUPFILE$FILENAMESUFFIX.gpg" ||
echo "ERROR: failed to remove incomplete backup files"
Comment thread
benderl marked this conversation as resolved.
fi
exit "$exit_status"
}

{
trap handle_error ERR
TEMPDIR=$(mktemp -d --tmpdir openwb_backup_XXXXXX)
generate_filename
log_environment
remove_old_backups
if collect_git_info; then
echo "git information collected successfully"
else
echo "error: failed to collect git information"
exit 1
fi
collect_git_info
echo "git information collected successfully"
force_mosquitto_write
create_archive
echo "backup finished"
Expand Down
Loading