From 36d19beafa4f4c8b6bff72d69f6e22f41a0d49b8 Mon Sep 17 00:00:00 2001 From: Saikumar VS Date: Tue, 22 Sep 2026 13:42:30 +0530 Subject: [PATCH] Replace MinIO with SeaweedFS in xbstream_fifo_test.sh, fix keyring_file bugs - Swap start_minio() for start_seaweedfs(): runs SeaweedFS's S3 gateway via docker (chrislusf/seaweedfs), publishing container port 8333 as host 9000 so all existing xbcloud --s3-endpoint=http://localhost:9000 calls are unchanged. Generates an S3 identity config so the admin/password creds xbcloud already uses keep working. Readiness check polls the S3 port directly since SeaweedFS has no MinIO-style /minio/health/ready endpoint. cleanup_exit() now stops the seaweedfs container on script exit. - Restore test_scripts/pxb/kmip_helper.sh (present in test_scripts/pxb/old/ but never re-pathed after a prior reorg), which init_datadir() sources for the keyring_kmip scenarios. - Fix full_backup_and_restore(): $COMPRESS_OPTIONS was set by the compressed-backup scenario but never passed to xtrabackup --backup, so compression never actually happened. - Fix init_datadir(): component_keyring_file.cnf used the wrong JSON key (component_keyring_file_data instead of path), so the keyring component silently failed to initialize. - Fix incremental_backup_and_restore(): the xtrabackup --backup calls never passed keyring config, so encrypted backups failed to init the keyring component during backup (prepare already passed it correctly). Verified full/incremental/compressed/partition-table backups against SeaweedFS end to end. Encrypted (keyring_file) backup now gets past both keyring bugs but hits an unrelated InnoDB assertion crash inside xtrabackup itself (fil0fil.cc:10943) during incremental backup of an encrypted tablespace - a separate PXB issue to track, not a script bug. Co-Authored-By: Claude Sonnet 5 --- test_scripts/pxb/kmip_helper.sh | 418 +++++++++++++++++++++++++ test_scripts/pxb/xbstream_fifo_test.sh | 99 ++++-- 2 files changed, 486 insertions(+), 31 deletions(-) create mode 100755 test_scripts/pxb/kmip_helper.sh diff --git a/test_scripts/pxb/kmip_helper.sh b/test_scripts/pxb/kmip_helper.sh new file mode 100755 index 0000000..83bbff2 --- /dev/null +++ b/test_scripts/pxb/kmip_helper.sh @@ -0,0 +1,418 @@ +#!/bin/bash + +# KMIP Helper Library +# Usage: source kmip_helper.sh +# +# This library provides functions for managing KMIP servers (PyKMIP, HashiCorp, etc.) +# Required: Docker must be installed and running + +# set -euo pipefail + +# Global variables +declare -ga KMIP_CONTAINER_NAMES +declare -gA KMIP_CONFIGS_DEFAULTS=( + #[pykmip]="addr=127.0.0.1,image=satyapercona/kmip:latest,port=5696,name=kmip_pykmip" + [hashicorp]="addr=127.0.0.1,port=5696,name=kmip_hashicorp,setup_script=hashicorp-kmip-setup.py" + [fortanix]="addr=216.180.120.88,port=5696,name=kmip_fortanix,setup_script=fortanix_kmip_setup.py" + #[ciphertrust]="addr=127.0.0.1,port=5696,name=kmip_ciphertrust,setup_script=setup_kmip_api.py" +) + +# Initialize default configurations if not already set +init_kmip_configs() { + # If KMIP_CONFIGS not set in main script, initialize with defaults + if [[ -z "${KMIP_CONFIGS[*]-}" ]]; then + declare -gA KMIP_CONFIGS=() + fi + + # Apply defaults for all keys defined in main script if not set + for key in "${!KMIP_CONFIGS[@]}"; do + if [[ -z "${KMIP_CONFIGS[$key]}" ]]; then + KMIP_CONFIGS[$key]="${KMIP_CONFIGS_DEFAULTS[$key]}" + fi + done + + echo "KMIP configurations initialized from Defaults" >&2 +} + +# Cleanup existing Docker container +cleanup_existing_container() { + local container_name="$1" + local container_id=$(sudo docker ps -aq --filter "name=$container_name") + + [ -z "$container_id" ] && return 0 + + if ! sudo docker rm -f "$container_id" >/dev/null 2>&1; then + return 1 + fi + sleep 5 # Allow port to be released + return 0 +} + +# Validate if a port is available +validate_port_available() { + local port="$1" + local max_attempts=10 + + if [[ -z "$port" ]]; then + echo "Error: No port specified" + return 1 + fi + + for i in $(seq 1 $max_attempts); do + local port_in_use=false + + # Method 1: Fast bash TCP check (works without external tools) + if timeout 1 bash -c "exec 3<>/dev/tcp/127.0.0.1/$port" 2>/dev/null; then + port_in_use=true + fi + + # Fallback methods only if bash TCP check failed + if [[ "$port_in_use" == false ]]; then + # Prefer ss over netstat (faster and more modern) + if command -v ss >/dev/null 2>&1; then + ss -tuln | grep -qE ":(${port})\s+" && port_in_use=true + elif command -v netstat >/dev/null 2>&1; then + netstat -tuln 2>/dev/null | grep -qE ":(${port})\s+" && port_in_use=true + fi + fi + + if [[ "$port_in_use" == false ]]; then + return 0 + fi + + echo -n "." + echo + if [[ $i -lt $max_attempts ]]; then + sleep 2 + fi + done + + return 1 +} +validate_environment() { + local type="${1:-}" # Use :- to handle empty input safely + + [[ -z "$type" ]] && { + echo "ERROR: No KMIP type specified" >&2 + return 1 + } + + # Safely check if key exists + [[ -n "${KMIP_CONFIGS[$type]+x}" ]] || { + echo "ERROR: Invalid type '$type'. Available types:" >&2 + printf " - %s\n" "${!KMIP_CONFIGS[@]}" >&2 + return 1 + } + + return 0 +} + +# Get all KMIP container names +get_kmip_container_names() { + KMIP_CONTAINER_NAMES=() + for type in "${!KMIP_CONFIGS[@]}"; do + IFS=',' read -ra pairs <<< "${KMIP_CONFIGS[$type]-}" # Use - for safety + for pair in "${pairs[@]}"; do + IFS='=' read -r key value <<< "${pair-}" + [[ "${key-}" == "name" ]] && KMIP_CONTAINER_NAMES+=("${value-}") && break + done + done +} + +# Parse configuration for a specific type +parse_config() { + local type=$1 + # Clear the existing config array + unset kmip_config + declare -gA kmip_config # Global associative array + + IFS=',' read -ra pairs <<< "${KMIP_CONFIGS[$type]}" + for pair in "${pairs[@]}"; do + IFS='=' read -r key value <<< "$pair" + kmip_config["$key"]="$value" + done + + # Set defaults if not specified + kmip_config["type"]="$type" + [[ -z "${kmip_config[name]}" ]] && kmip_config["name"]="kmip_${type}" + [[ -z "${kmip_config[addr]}" ]] && kmip_config["addr"]="127.0.0.1" + [[ -z "${kmip_config[port]}" ]] && kmip_config["port"]="5696" + [[ -z "${kmip_config[cert_dir]}" ]] && kmip_config["cert_dir"]="kmip_certs_${kmip_config[type]}" +} + +# Generate KMIP configuration file +generate_kmip_config() { + local type="$1" + local addr="$2" + local port="$3" + local cert_dir="$4" + local config_file="${cert_dir}/component_keyring_kmip.cnf" + echo "Generating KMIP config for: ${type}" + + sudo tee "$config_file" > /dev/null </dev/null + fi + + mkdir -p "$cert_dir" || { + echo "ERROR: Failed to create certificate directory: $cert_dir" >&2 + return 1 + } + chmod 700 "$cert_dir" # Restrict access to owner only + + # 1. Cleanup existing resources + echo "Cleaning up existing container... " + if cleanup_existing_container "$container_name"; then + echo "Done" + else + echo "Failed" + return 1 + fi + + # 2. Verify port availability + echo "Checking port $port availability... " + if validate_port_available "$port"; then + echo "Available" + else + echo "Unavailable" + echo "Port $port is in use by:" + lsof -i :"$port" + # Do container at global level, from what we know. + get_kmip_container_names + for kmip_name in "${KMIP_CONTAINER_NAMES[@]}"; do + cleanup_existing_container "$kmip_name" + done + if ! validate_port_available "$port"; then + echo "Still unavailable $port, please check and clean up port $port and retry" + exit 1; + fi + fi + + # 3. Start container + echo "Starting container... " + if ! sudo docker run -d \ + --name "$container_name" \ + --security-opt seccomp=unconfined \ + --cap-add=NET_ADMIN \ + -p "$port:5696" \ + "$image" >/dev/null 2>&1; then + echo "Failed" + return 1 + fi + echo "Started (ID: $(sudo docker inspect --format '{{.Id}}' "$container_name"))" + + sleep 10 + + sudo docker cp "$container_name":/opt/certs/root_certificate.pem $cert_dir/root_certificate.pem >/dev/null 2>&1 + sudo docker cp "$container_name":/opt/certs/client_key_jane_doe.pem $cert_dir/client_key.pem >/dev/null 2>&1 + sudo docker cp "$container_name":/opt/certs/client_certificate_jane_doe.pem $cert_dir/client_certificate.pem >/dev/null 2>&1 + + # Fix ownership of copied files so they're accessible to the user + sudo chown -R "$USER:$USER" "$cert_dir" 2>/dev/null || true + + # Post-startup configuration + echo "Generating KMIP configuration..." + generate_kmip_config "$type" "$addr" "$port" "$cert_dir" || { + echo "Failed to generate KMIP config" + return 1 + } + + echo "PyKMIP server started successfully on address $addr and port $port" + return 0 +} + +# Setup HashiCorp Vault KMIP server +setup_hashicorp() { + local type="hashicorp" + local container_name="${kmip_config[name]}" + local addr="${kmip_config[addr]}" + local port="${kmip_config[port]}" + local image="${kmip_config[image]}" + local setup_script="${kmip_config[setup_script]}" + local cert_dir="${HOME}/${kmip_config[cert_dir]}" + + + echo "Cleaning up existing container... " + if cleanup_existing_container "$container_name"; then + echo "Done" + else + echo "Failed" + return 1 + fi + + echo "Checking port $port availability... " + if validate_port_available "$port"; then + echo "Available" + else + echo "Unavailable" + echo "Port $port is in use by:" + lsof -i :"$port" + # Do container at global level, from what we know. + get_kmip_container_names + for name in "${KMIP_CONTAINER_NAMES[@]}"; do + cleanup_existing_container "$name" + done + if ! validate_port_available "$port"; then + echo "Still unavailable $port, please check and clean up port $port and retry" + exit 1; + fi + fi + + # Download first, then execute the hashicorp setup + script=$(curl -fsSL --retry 5 --retry-delay 2 --retry-connrefused \ + --connect-timeout 5 --max-time 30 \ + https://raw.githubusercontent.com/Percona-QA/percona-qa/master/"$setup_script") + + curl_exit_code=$? + + + if [ "${curl_exit_code:-1}" -ne 0 ] || [ -z "$script" ]; then + echo "Failed to download script after retries (curl exit code: $curl_exit_code)" + exit 1 + fi + + + if [ -d "$cert_dir" ]; then + echo "Cleaning existing certificate directory: $cert_dir" + rm -rf "$cert_dir"/* 2>/dev/null + fi + + mkdir -p "$cert_dir" || { + echo "ERROR: Failed to create certificate directory: $cert_dir" >&2 + return 1 + } + + # Get license file from environment variable + local license_file="${HASHICORP_LICENSE:-}" + if [[ -z "$license_file" ]]; then + echo "ERROR: HASHICORP_LICENSE environment variable must be set for HashiCorp KMIP Provider!!" >&2 + echo "Please set it to the path of your HashiCorp Vault Enterprise license file:" >&2 + echo " export HASHICORP_LICENSE=/path/to/vault.hclic" >&2 + exit 1 + fi + + # Check if license file exists + if [[ ! -f "$license_file" ]]; then + echo "ERROR: License file not found at: $license_file" >&2 + exit 1 + fi + + # Execute the script + # Execute the Python script from a variable + echo "$script" | python3 - --cert-dir="$cert_dir" --license="$license_file" + exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Failed to execute script $setup_script, (exit code: $exit_code)" >&2 + return 1 + fi + + generate_kmip_config "$type" "$addr" "$port" "$cert_dir" || { + echo "Failed to generate KMIP config" >&2; return 1; } + + echo "Hashicorp server started successfully on address $addr and port $port" + return 0 +} + +setup_fortanix() { + local type="fortanix" + local container_name="${kmip_config[name]}" + local addr="${kmip_config[addr]}" + local port="${kmip_config[port]}" + local email="${FORTANIX_EMAIL:-}" + local password="${FORTANIX_PASSWORD:-}" + local setup_script="${kmip_config[setup_script]}" + local cert_dir="${HOME}/${kmip_config[cert_dir]}" + + # Check if both environment variables are set and not empty + if [[ -z "$email" || -z "$password" ]]; then + echo "ERROR: Both FORTANIX_EMAIL and FORTANIX_PASSWORD environment variables must be set for Fortanix KMIP Provider!!" >&2 + echo "Please set them to your Fortanix credentials:" >&2 + echo " export FORTANIX_EMAIL=your-email@example.com" >&2 + echo " export FORTANIX_PASSWORD=your-password" >&2 + exit 1 + fi + + echo "Checking port availability... " + if validate_port_available "$port"; then + echo "Available" + else + echo "Unavailable" + echo "Port $port is in use by:" + lsof -i :"$port" + return 1 + fi + + echo "Starting Fortanix KMIP server in (script method): $setup_script" + # Download first, then execute the fortanix setup script + script=$(wget -qO- https://raw.githubusercontent.com/Percona-QA/percona-qa/8ab34a4da257070518825fcdf8ae547f99705597/"$setup_script") + + # To-Do Remove B4 Merge + # script=$(wget -qO- https://raw.githubusercontent.com/Percona-QA/percona-qa/refs/heads/master/"$setup_script") + wget_exit_code=$? + + if [ $wget_exit_code -ne 0 ]; then + echo "Failed to download script (wget exit code: $wget_exit_code)" + exit 1 + fi + + if [ -z "$script" ]; then + echo "Downloaded script is empty" + exit 1 + fi + + mkdir -p "$cert_dir" || true + + # Execute the Python script from a variable + echo "$script" | python3 - --cert-dir="$cert_dir" --email="$email" --password="$password" + exit_code=$? + + generate_kmip_config "$type" "$addr" "$port" "$cert_dir" || { + echo "Failed to generate KMIP config"; exit 1; } + + echo "Fortanix server started successfully on address $addr and port $port" + return 0 +} + +# Placeholder for CipherTrust setup +setup_cipher_api() { + echo "CipherTrust setup not implemented yet" + return 1 +} + +# Main function to start KMIP server +start_kmip_server() { + local type="$1" + validate_environment "$type" || return 1 + parse_config "$type" + echo "Starting ${type^^} KMIP Server on port ${kmip_config[port]}" + + case "$type" in + pykmip) setup_pykmip ;; + hashicorp) setup_hashicorp ;; + fortanix) setup_fortanix ;; + ciphertrust) setup_cipher_api ;; + *) echo "Unsupported KMIP Type: $type"; return 1 ;; + esac +} diff --git a/test_scripts/pxb/xbstream_fifo_test.sh b/test_scripts/pxb/xbstream_fifo_test.sh index 9899d71..3cfcf47 100755 --- a/test_scripts/pxb/xbstream_fifo_test.sh +++ b/test_scripts/pxb/xbstream_fifo_test.sh @@ -89,50 +89,78 @@ cleanup_exit() { if [ -f "$PS_DIR/bin/mysqld.my" ]; then rm -f "$PS_DIR/bin/mysqld.my" fi + + if docker ps --filter "name=seaweedfs" --filter "status=running" | grep -q seaweedfs; then + echo "Stopping SeaweedFS container..." + docker stop seaweedfs > /dev/null 2>&1 + fi } trap cleanup_exit EXIT INT TERM -start_minio() { - # Check if MinIO is already running - if docker ps --filter "name=minio" --filter "status=running" | grep -q minio; then - echo "MinIO is already running." +start_seaweedfs() { + # Check if SeaweedFS is already running + if docker ps --filter "name=seaweedfs" --filter "status=running" | grep -q seaweedfs; then + echo "SeaweedFS is already running." else # Check if a stopped container exists - if docker ps -a --filter "name=minio" | grep -q minio; then - echo "Found stopped MinIO container. Starting it..." - docker start minio + if docker ps -a --filter "name=seaweedfs" | grep -q seaweedfs; then + echo "Found stopped SeaweedFS container. Starting it..." + docker start seaweedfs else - if [ -d "$HOME/minio/data" ]; then - rm -rf "$HOME/minio/data"/* + if [ -d "$HOME/seaweedfs/data" ]; then + rm -rf "$HOME/seaweedfs/data"/* else - mkdir -p "$HOME/minio/data" + mkdir -p "$HOME/seaweedfs/data" fi - echo "No MinIO container found. Creating and starting one..." + + # S3 identity config: keeps the same admin/password credentials xbcloud already uses + cat > "$HOME/seaweedfs/s3.json" <<-EOFS + { + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": "admin", + "secretKey": "password" + } + ], + "actions": [ + "Admin", + "Read", + "Write" + ] + } + ] + } +EOFS + + echo "No SeaweedFS container found. Creating and starting one..." docker run -d \ - -p 9000:9000 \ - -p 9001:9001 \ - --name minio \ - -v ~/minio/data:/data \ - -e "MINIO_ROOT_USER=admin" \ - -e "MINIO_ROOT_PASSWORD=password" \ - minio/minio:latest server /data --console-address ":9001" + -p 9000:8333 \ + --name seaweedfs \ + -v ~/seaweedfs/data:/data \ + -v ~/seaweedfs/s3.json:/etc/seaweedfs/s3.json \ + chrislusf/seaweedfs:latest \ + server -s3 -s3.port=8333 -s3.config=/etc/seaweedfs/s3.json -dir=/data fi fi - # Poll the health endpoint - echo -n "Waiting for MinIO to become ready" + # Poll the S3 gateway port (SeaweedFS has no /minio/health/ready-style endpoint, + # so just wait until it accepts HTTP connections) + echo -n "Waiting for SeaweedFS to become ready" for i in {1..20}; do - if curl -s -o /dev/null -w "%{http_code}" http://localhost:9000/minio/health/ready | grep -q 200; then - echo -e "\n MinIO is ready!\n" + if [ "$(curl -s -o /dev/null -w "%{http_code}" http://localhost:9000/ 2>/dev/null)" != "000" ]; then + echo -e "\n SeaweedFS is ready!\n" return fi echo -n "." sleep 1 done - echo -n "\n MinIO failed to become ready in time." - docker logs minio + echo -n "\n SeaweedFS failed to become ready in time." + docker logs seaweedfs exit 1 } @@ -182,7 +210,7 @@ init_datadir() { cat > "$PS_DIR/lib/plugin/component_keyring_file.cnf" <<-EOFL { - "component_keyring_file_data": "${PS_DIR}/keyring", + "path": "${PS_DIR}/keyring", "read_only": false } EOFL @@ -266,7 +294,7 @@ pstress_run_load() { full_backup_and_restore() { echo "=>Taking Backup" -$XTRABACKUP_DIR/bin/xtrabackup --user=root --password='' --datadir=$DATADIR -S $SOCKET --backup $ENCRYPT $ENCRYPT_KEY --parallel=64 --fifo-streams=$FIFO_STREAM --fifo-dir=$FIFO_DIR --core-file > $LOGDIR/backup.log 2>&1 & +$XTRABACKUP_DIR/bin/xtrabackup --user=root --password='' --datadir=$DATADIR -S $SOCKET --backup $ENCRYPT $ENCRYPT_KEY $COMPRESS_OPTIONS --parallel=64 --fifo-streams=$FIFO_STREAM --fifo-dir=$FIFO_DIR --core-file > $LOGDIR/backup.log 2>&1 & xbcloud_put full_backup if [ $(cat $LOGDIR/upload.log | grep "Upload failed" | wc -l) -eq 1 ]; then @@ -312,6 +340,15 @@ echo "..Prepare successful" incremental_backup_and_restore() { local keyring_type=$1 +local keyring_backup_opts="" +if [ $ENCRYPTION -eq 1 ]; then + if [ "$keyring_type" = "keyring_kmip" ]; then + keyring_filename="$PS_DIR/lib/plugin/component_keyring_kmip.cnf" + elif [ "$keyring_type" = "keyring_file" ]; then + keyring_filename="$PS_DIR/lib/plugin/component_keyring_file.cnf" + fi + keyring_backup_opts="--xtrabackup-plugin-dir=$XTRABACKUP_DIR/lib/plugin --component-keyring-config=$keyring_filename" +fi echo "=>Taking Full Backup" if [ ! -d $HOME/lsn/full ]; then mkdir -p $HOME/lsn/full @@ -319,7 +356,7 @@ else rm -rf $HOME/lsn/full mkdir -p $HOME/lsn/full fi -$XTRABACKUP_DIR/bin/xtrabackup --backup --user=root -S $SOCKET --datadir=$DATADIR --extra-lsndir=$HOME/lsn/full --fifo-streams=$FIFO_STREAM --fifo-dir=$FIFO_DIR --core-file >> $LOGDIR/backup_inc.log 2>&1 & +$XTRABACKUP_DIR/bin/xtrabackup --backup --user=root -S $SOCKET --datadir=$DATADIR --extra-lsndir=$HOME/lsn/full $keyring_backup_opts --fifo-streams=$FIFO_STREAM --fifo-dir=$FIFO_DIR --core-file >> $LOGDIR/backup_inc.log 2>&1 & xbcloud_put full echo "..Full Backup successful" @@ -332,7 +369,7 @@ else rm -rf $HOME/lsn/inc1 mkdir -p $HOME/lsn/inc1 fi -$XTRABACKUP_DIR/bin/xtrabackup --backup --user=root -S $SOCKET --datadir=$DATADIR --extra-lsndir=$HOME/lsn/inc1 --incremental-basedir=$HOME/lsn/full --fifo-streams=$FIFO_STREAM --fifo-dir=$FIFO_DIR --core-file > $LOGDIR/inc1.log 2>&1 & +$XTRABACKUP_DIR/bin/xtrabackup --backup --user=root -S $SOCKET --datadir=$DATADIR --extra-lsndir=$HOME/lsn/inc1 --incremental-basedir=$HOME/lsn/full $keyring_backup_opts --fifo-streams=$FIFO_STREAM --fifo-dir=$FIFO_DIR --core-file > $LOGDIR/inc1.log 2>&1 & xbcloud_put inc1 echo "..Successful" @@ -345,7 +382,7 @@ else rm -rf $HOME/lsn/inc2 mkdir -p $HOME/lsn/inc2 fi -$XTRABACKUP_DIR/bin/xtrabackup --backup --user=root -S $SOCKET --datadir=$DATADIR --extra-lsndir=$HOME/lsn/inc2 --incremental-basedir=$HOME/lsn/inc1 --fifo-streams=$FIFO_STREAM --fifo-dir=$FIFO_DIR --core-file > $LOGDIR/inc2.log 2>&1 & +$XTRABACKUP_DIR/bin/xtrabackup --backup --user=root -S $SOCKET --datadir=$DATADIR --extra-lsndir=$HOME/lsn/inc2 --incremental-basedir=$HOME/lsn/inc1 $keyring_backup_opts --fifo-streams=$FIFO_STREAM --fifo-dir=$FIFO_DIR --core-file > $LOGDIR/inc2.log 2>&1 & xbcloud_put inc2 echo "..Successful" @@ -358,7 +395,7 @@ else rm -rf $HOME/lsn/inc3 mkdir -p $HOME/lsn/inc3 fi -$XTRABACKUP_DIR/bin/xtrabackup --backup --user=root -S $SOCKET --datadir=$DATADIR --extra-lsndir=$HOME/lsn/inc3 --incremental-basedir=$HOME/lsn/inc2 --fifo-streams=$FIFO_STREAM --fifo-dir=$FIFO_DIR --core-file > $LOGDIR/inc3.log 2>&1 & +$XTRABACKUP_DIR/bin/xtrabackup --backup --user=root -S $SOCKET --datadir=$DATADIR --extra-lsndir=$HOME/lsn/inc3 --incremental-basedir=$HOME/lsn/inc2 $keyring_backup_opts --fifo-streams=$FIFO_STREAM --fifo-dir=$FIFO_DIR --core-file > $LOGDIR/inc3.log 2>&1 & xbcloud_put inc3 echo "..Successful" @@ -423,7 +460,7 @@ echo "..Successful" } #Actual test begins here.. -start_minio +start_seaweedfs echo "###################################################" echo "# 1. Test FIFO xbstream: Full Backup and Restore #" echo "###################################################"