diff --git a/bin/db_maint.sh b/bin/db_maint.sh index e26b9f6..4e0467d 100755 --- a/bin/db_maint.sh +++ b/bin/db_maint.sh @@ -29,8 +29,7 @@ case "${1:-maint}" in verify_docker_service mysql_get_mywiki_tables mysql_redo_log 'Disable' - mysql_maintenance 'Check' - mysql_maintenance 'Optimize' + mysql_maintenance mysql_redo_log 'Enable' ;; *) echo "${NOTICE_HELP}";; diff --git a/bin/library_common.sh b/bin/library_common.sh index 8ec6d4e..f30788e 100644 --- a/bin/library_common.sh +++ b/bin/library_common.sh @@ -20,6 +20,7 @@ trap '_es=${?}; E0="$(printf "\e[0m")" # reset E30="$(printf "\e[30m")" # foreground: black E31="$(printf "\e[31m")" # foreground: red +E36="$(printf "\e[36m")" # foreground: cyan E90="$(printf "\e[90m")" # foreground: bright black (gray) E92="$(printf "\e[92m")" # foreground: bright green E93="$(printf "\e[93m")" # foreground: bright yellow @@ -106,6 +107,10 @@ repo_rel_path() { sed -e"s|${DIR_REPO}|DIR_REPO|" } +set_time_format() { + TIMEFORMAT="${E90}Duration: %3lR${E0}" +} + #### MAIN ##################################################################### cd "${DIR_REPO}" || exit diff --git a/bin/library_mysql.sh b/bin/library_mysql.sh index aaa63f0..34bcde5 100644 --- a/bin/library_mysql.sh +++ b/bin/library_mysql.sh @@ -6,65 +6,110 @@ # DIR_REPO="$(cd -P -- "${0%/*}/.." && pwd -P)" # source "${DIR_REPO}/bin/library_common.sh" # source "${DIR_REPO}/bin/library_mysql.sh" -# -# Analze maintenance is not included because when an InnoDB table is optimized, -# it does a recreate + analyze instead. For details, see: -# https://dev.mysql.com/doc/refman/9.7/en/optimize-table.html#optimize-table-innodb-details #### FUNCTIONS ################################################################ mysql_get_mywiki_tables() { - local _count + local _count_innodb _count_myisam print_header 'Get MediaWiki tables' - TABLES="$(docker compose exec 'db' sh -c \ - 'echo " SELECT TABLE_NAME - FROM information_schema.tables \ - WHERE TABLE_SCHEMA=\"my_wiki\";" \ - | /usr/bin/mysql --skip-column-names')" - _count="$(echo "${TABLES}" | wc -l)" - _count="${_count##* }" - print_key_val 'Table count' "${_count}" + TABLES_INNODB="$(echo ' + SELECT TABLE_NAME + FROM information_schema.TABLES + WHERE + ENGINE = "InnoDB" + AND TABLE_SCHEMA = "my_wiki";' \ + | docker compose exec --no-tty 'db' mysql --skip-column-names)" + _count_innodb="$(echo "${TABLES_INNODB}" | wc -l)" + _count_innodb="${_count_innodb##* }" + TABLES_MYISAM="$(echo ' + SELECT TABLE_NAME + FROM information_schema.TABLES + WHERE + ENGINE = "MyISAM" + AND TABLE_SCHEMA = "my_wiki";' \ + | docker compose exec --no-tty 'db' mysql --skip-column-names)" + _count_myisam="$(echo "${TABLES_MYISAM}" | wc -l)" + _count_myisam="${_count_myisam##* }" + TABLES_ALL="$(printf '%s\n%s' "${TABLES_INNODB}" "${TABLES_MYISAM}")" + print_key_val 'InnoDB tables' "${_count_innodb}" + print_key_val 'MyISAM tables' "${_count_myisam}" echo - } mysql_maintenance() { - # https://dev.mysql.com/doc/refman/9.7/en/optimize-table.html - local _i _op_lower _op_title _op_upper + mysql_maintenance_operation 'Check' + mysql_maintenance_operation 'Optimize' + mysql_maintenance_operation 'Analyze' +} + +mysql_maintenance_operation() { + local _header _i _op_lower _op_title _op_upper _option _query \ + _query_innodb _query_myisam _table _tables _op_title="${1}" _op_lower="$(echo "${_op_title}" | tr '[:upper:]' '[:lower:]')" _op_upper="$(echo "${_op_title}" | tr '[:lower:]' '[:upper:]')" - local _table + _option= print_header "${_op_title} MediaWiki tables" - print_bright 'Key' case "${_op_lower}" in + # https://dev.mysql.com/doc/refman/9.7/en/check-table.html check) - echo "${E92}.${E0} status OK" - TIMEFORMAT="${E97}${E100} check duration:${E0} %3lR" + _query_innodb="${_op_upper} TABLE tbl_name" + _query_myisam="${_query_innodb} EXTENDED" ;; + # https://dev.mysql.com/doc/refman/9.7/en/optimize-table.html optimize) - echo -n "${E92}.${E0} status OK (table does not support optimize," - echo ' doing recreate + analyze instead)' - TIMEFORMAT="${E97}${E100} optimize duration:${E0} %3lR" + _query="${_op_upper} TABLE tbl_name" + _header="InnoDB and MyISAM tables: running" + _header="${_header} ${E36}${_query}${E0}" + _tables="${TABLES_ALL}" + echo -n 'InnoDB tables do not support optimize, recreate +' + echo ' analyze will be done instead' + echo + mysql_maintenance_run "${_header}" "${_query}" "${_tables}" + ;; + # https://dev.mysql.com/doc/refman/9.7/en/analyze-table.html + analyze) + _query_innodb= + _query_myisam="${_op_upper} TABLE tbl_name" ;; esac - echo -n "${E93}_${E0} The storage engine for the table doesn't support" - echo " ${_op_lower}" - echo - print_bright "Running ${_op_upper} TABLE on all MediaWiki tables" + if [[ "${_op_lower}" != 'optimize' ]] + then + if [[ -n "${_query_innodb}" ]] + then + _header="InnoDB$ tables: running ${E36}${_query_innodb}${E0}" + _tables="${TABLES_INNODB}" + mysql_maintenance_run "${_header}" "${_query_innodb}" "${_tables}" + fi + if [[ -n "${_query_myisam}" ]] + then + _header="MyISAM tables: running ${E36}${_query_myisam}${E0}" + _tables="${TABLES_MYISAM}" + mysql_maintenance_run "${_header}" "${_query_myisam}" "${_tables}" + fi + fi +} + +mysql_maintenance_run() { + local _header _query _table _tables + _header="${1}" + _query="${2}" + _tables="${3}" + echo "${_header//tbl_name/${E90}tbl_name${E36}}" + set_time_format "${_op_lower}" time { declare -i _i=1 - for _table in ${TABLES} + for _table in ${_tables} do - docker compose exec 'db' sh -c \ - "echo '${_op_upper} TABLE ${_table};' \ - | /usr/bin/mysql --skip-column-names \ + echo "${_query//tbl_name/${_table}};" \ + | docker compose exec --no-tty 'db' sh -c \ + "mysql --skip-column-names \ | sed --null-data \ -e's/my[^\n]*analyze instead\n[^\n]*OK\n/${E92}.${E0}/g' \ -e's/my[^\n]*OK\n/${E92}.${E0}/g' \ - -e's/my[^\n]*support ${_op_lower}\n/${E93}_${E0}/g'" + -e's/my[^\n]*is already up to date\n/${E92}.${E0}/g'" if (( _i < 80 )) then (( _i = _i + 1 )) @@ -87,15 +132,15 @@ mysql_redo_log() { print_header "${_op_title} MySQL redo log" docker compose exec --env OPERATION="${_op_upper}" 'db' \ sh -c 'echo "ALTER INSTANCE ${OPERATION} INNODB REDO_LOG;" | mysql' - echo -n "InnoDB redo logging is ${_op_lower}d." + echo "InnoDB redo logging is ${_op_lower}d." case "${_op_lower}" in disable) - echo ' All data could be lost in case of a server' - echo 'crash.' + echo -n "⚠️ ${E93}All data could be lost in case of a server" + echo " crash.${E0}" ;; enable) - echo ' Data is now safe and can be recovered in case of' - echo 'a server crash.' + echo -n "🟢 ${E92}Data is now safe and can be recovered in case of" + echo " a server crash.${E0}" ;; esac echo diff --git a/bin/move_data.sh b/bin/move_data.sh index b26c990..723a112 100755 --- a/bin/move_data.sh +++ b/bin/move_data.sh @@ -129,7 +129,7 @@ files_export() { print_bright \ 'Rsync copy assets (remove MediaWiki files not in export cache)' \ | tr -d '\n' - TIMEFORMAT="${E97}${E100} rsync duration:${E0} %3lR" + set_time_format # The rsync options below are ordered to match `man rsync` time docker compose exec 'web' \ rsync \ @@ -153,7 +153,7 @@ files_export() { print_bright \ 'Rsync copy images (remove MediaWiki files not in export cache)' \ | tr -d '\n' - TIMEFORMAT="${E97}${E100} rsync duration:${E0} %3lR" + set_time_format # The rsync options below are ordered to match `man rsync` time docker compose exec 'web' \ rsync \ @@ -197,7 +197,7 @@ files_import() { print_bright \ 'Rsync copy assets (remove MediaWiki files not in pull cache)' \ | tr -d '\n' - TIMEFORMAT="${E97}${E100} rsync duration:${E0} %3lR" + set_time_format # The rsync options below are ordered to match `man rsync` time docker compose exec 'web' \ rsync \ @@ -218,7 +218,7 @@ files_import() { print_bright \ 'Rsync copy images (remove MediaWiki files not in pull cache)' \ | tr -d '\n' - TIMEFORMAT="${E97}${E100} rsync duration:${E0} %3lR" + set_time_format # The rsync options below are ordered to match `man rsync` time docker compose exec 'web' \ rsync \ @@ -263,7 +263,7 @@ files_pull() { #### Asssets print_bright 'Rsync download assets (remove local files not on remote)' \ | tr -d '\n' - TIMEFORMAT="${E97}${E100} rsync duration:${E0} %3lR" + set_time_format # The rsync options below are ordered to match `man rsync` time rsync \ --recursive \ @@ -286,7 +286,7 @@ files_pull() { ##### Images print_bright 'Rsync download assets (remove local files not on remote)' \ | tr -d '\n' - TIMEFORMAT="${E97}${E100} rsync duration:${E0} %3lR" + set_time_format # The rsync options below are ordered to match `man rsync` time rsync \ --recursive \ @@ -331,7 +331,7 @@ files_push() { echo #### Asssets print_bright 'Rsync upload assets (remove remote files not in local cache)' - TIMEFORMAT="${E97}${E100} rsync duration:${E0} %3lR" + set_time_format # The rsync options below are ordered to match `man rsync` time rsync \ --recursive \ @@ -350,7 +350,7 @@ files_push() { echo ##### Images print_bright 'Rsync upload images (remove remote files not in local cache)' - TIMEFORMAT="${E97}${E100} rsync duration:${E0} %3lR" + set_time_format # The rsync options below are ordered to match `man rsync` time rsync \ --recursive \ @@ -509,7 +509,7 @@ sql_export() { echo print_bright 'Dump database to local export cache sql file' echo '(overwrite existing file)' - TIMEFORMAT="${E97}${E100} mysqldump duration:${E0} %3lR" + set_time_format # https://dev.mysql.com/doc/refman/9.7/en/mysqldump.html # Options set in /root/.my.cnf in db container docker compose exec --env DOCKER_EXPORT_SQL="${DOCKER_EXPORT_SQL}" db \ @@ -521,7 +521,7 @@ sql_export() { "$(du -sh "${CACHE_EXPORT_SQL}" | awk '{print $1}')" echo echo 'Create compressed copy of database dump sql file' - TIMEFORMAT="${E97}${E100} compression duration:${E0} %3lR" + set_time_format time gzip --force --keep "${CACHE_EXPORT_SQL}" unset TIMEFORMAT print_key_val 'compressed file size' \ @@ -544,7 +544,7 @@ sql_import() { echo 'Disable autocommit' docker compose exec 'db' sh -c 'echo "SET autocommit=0;" | mysql' echo 'Import database dump SQL file from local pull cache' - TIMEFORMAT="${E97}${E100} import duration:${E0} %3lR" + set_time_format time docker compose exec --env DOCKER_PULL_SQL="${DOCKER_PULL_SQL}" 'db' \ sh -c 'mysql < "${DOCKER_PULL_SQL}"' unset TIMEFORMAT @@ -571,7 +571,7 @@ sql_pull() { echo print_bright 'Dump database to local pull cache sql file' echo '(overwrite existing file)' - TIMEFORMAT="${E97}${E100} mysqldump duration:${E0} %3lR" + set_time_format # https://dev.mysql.com/doc/refman/9.7/en/mysqldump.html # shellcheck disable=SC2016 time railway ssh --service "${RAILWAY_SERVICE_MYSQL}" \ @@ -588,7 +588,7 @@ sql_pull() { "$(du -sh "${CACHE_PULL_SQL}" | awk '{print $1}')" echo echo 'Create compressed copy of database dump sql file' - TIMEFORMAT="${E97}${E100} compression duration:${E0} %3lR" + set_time_format time gzip --force --keep "${CACHE_PULL_SQL}" unset TIMEFORMAT print_key_val 'compressed file size' \ @@ -614,7 +614,7 @@ sql_push() { --service "${RAILWAY_SERVICE_MYSQL}" echo echo 'Import database dump SQL file from local pull cache' - TIMEFORMAT="${E97}${E100} import duration:${E0} %3lR" + set_time_format time { railway ssh --service "${RAILWAY_SERVICE_MYSQL}" \ 'echo "ALTER INSTANCE DISABLE INNODB REDO_LOG;" | mysql' @@ -665,8 +665,7 @@ case "${COMMAND}" in mw_maintenance_titles mw_maintenance_rebuild mysql_get_mywiki_tables - mysql_maintenance 'Check' - mysql_maintenance 'Optimize' + mysql_maintenance mysql_redo_log 'Enable' ;; 'export') @@ -674,8 +673,7 @@ case "${COMMAND}" in verify_docker_services files_export mysql_get_mywiki_tables - mysql_maintenance 'Check' - mysql_maintenance 'Optimize' + mysql_maintenance sql_export ;; 'push')