diff --git a/lang/de_de.lang b/lang/de_de.lang index 7d6fdc6..e1fc184 100644 --- a/lang/de_de.lang +++ b/lang/de_de.lang @@ -71,7 +71,9 @@ statistics_delete_bots = Alle Bots löschen statistics_delete_referer = Alle Verweise löschen statistics_delete_media = Alle Media-Statistik löschen statistics_delete_noise = Störanfragen bereinigen +statistics_delete_non200_urls = Nicht-200-URLs bereinigen statistics_delete_old = Alte Statistikdaten bereinigen +statistics_delete_since_date = Daten ab Stichtag löschen statistics_delete_raw_old = Alte Rohdaten bereinigen statistics_maintenance_delete_heading = Löschaktionen statistics_maintenance_delete_note = Dieser Bereich entfernt Daten dauerhaft. Bitte vor größeren Löschungen Sicherungen prüfen. @@ -84,7 +86,9 @@ statistics_maintenance_scope_referer = Betroffene Daten: Referer-Statistik statistics_maintenance_scope_media = Betroffene Daten: Media-Statistik statistics_maintenance_scope_api = Betroffene Daten: API-/Kampagnen-Statistik statistics_maintenance_scope_noise = Betroffene Daten: Typische Probe-/Störanfragen +statistics_maintenance_scope_non200_urls = Betroffene Daten: URL-basierte Statistik für derzeitige Nicht-200-URLs (Näherungswert) statistics_maintenance_scope_keep_days = Betroffene Daten: Einträge älter als die angegebene Tageszahl +statistics_maintenance_scope_since_date = Betroffene Daten: Einträge ab dem gewählten Stichtag statistics_maintenance_scope_keep_days_raw = Betroffene Daten: Rohdaten älter als die angegebene Tageszahl statistics_maintenance_scope_optimize = Betroffene Daten: Tabellenstruktur und Speicherbelegung statistics_geo_status = Geo-Datenbank Status @@ -94,7 +98,9 @@ statistics_geo_status_not_available = nicht verfügbar statistics_geo_last_update = Letzte Aktualisierung statistics_geo_file_size = Dateigröße statistics_cleanup_keep_days = Tage behalten: +statistics_cleanup_since_date = Stichtag: statistics_cleanup_keep_days_raw = Rohdaten behalten (Tage): +statistics_cleanup_since_date_invalid = Ungültiges Stichtagsdatum. Bitte das Format JJJJ-MM-TT verwenden. statistics_monday = Montag statistics_tuesday = Dienstag statistics_wednesday = Mittwoch @@ -108,7 +114,9 @@ statistics_confirm_delete_bots = Wirklich alle Besuche von Bots löschen? statistics_confirm_delete_referer = Wirklich alle Referer löschen? statistics_confirm_delete_media = Wirklich die gesamte Media-Statistik löschen? statistics_confirm_delete_noise = Wirklich typische Probe- und WordPress-Anfragen aus den Statistiken löschen? +statistics_confirm_delete_non200_urls = Wirklich URL-basierte Statistikdaten für aktuell als Nicht-200 erkannte URLs bereinigen? statistics_confirm_delete_old = Wirklich alle Statistikdaten löschen, die älter als die angegebene Anzahl Tage sind? +statistics_confirm_delete_since_date = Wirklich alle Statistikdaten ab dem gewählten Stichtag löschen? statistics_confirm_delete_raw_old = Wirklich alte Rohdaten löschen? Tagesstatistiken bleiben erhalten. statistics_confirm_optimize_tables = Statistik-Tabellen jetzt optimieren? Das kann bei großen Tabellen etwas dauern. statistics_deleted_hashes = Einträge aus der Tabelle hashes gelöscht. @@ -121,7 +129,9 @@ statistics_geo_update_success = Geo-Datenbank aktualisiert. statistics_geo_update_error = Geo-Datenbank konnte nicht aktualisiert werden. statistics_deleted_noise = Es wurden %s Störanfragen gelöscht. statistics_deleted_noise_partial = Es wurden %s Störanfragen in diesem Lauf bereinigt. Für den Rest bitte den Vorgang erneut starten. +statistics_deleted_non200_urls = Es wurden %s URL-basierte Einträge für Nicht-200-URLs bereinigt. statistics_deleted_old = Es wurden %s alte Einträge gelöscht (älter als %s Tage). +statistics_deleted_since_date = Es wurden %s Einträge ab dem Stichtag %s gelöscht. statistics_deleted_raw_old = Es wurden %s alte Rohdaten-Einträge gelöscht (älter als %s Tage). statistics_batch_continue = Nächsten Batch ausführen statistics_batch_more_data = Dieser Vorgang läuft batchweise. Bitte den nächsten Batch starten, um weitere Datensätze zu verarbeiten. diff --git a/lang/en_gb.lang b/lang/en_gb.lang index 3fe7f3f..3e451a3 100644 --- a/lang/en_gb.lang +++ b/lang/en_gb.lang @@ -73,7 +73,9 @@ statistics_delete_bots = Delete all bots statistics_delete_referer = Delete all referers statistics_delete_media = Delete all media statistics statistics_delete_noise = Clean noise requests +statistics_delete_non200_urls = Clean non-200 URLs statistics_delete_old = Cleanup old statistics data +statistics_delete_since_date = Delete data from date statistics_delete_raw_old = Cleanup old raw data statistics_maintenance_delete_heading = Deletion actions statistics_maintenance_delete_note = This section permanently removes data. Please verify backups before larger deletions. @@ -86,7 +88,9 @@ statistics_maintenance_scope_referer = Affected data: referer statistics statistics_maintenance_scope_media = Affected data: media statistics statistics_maintenance_scope_api = Affected data: API/campaign statistics statistics_maintenance_scope_noise = Affected data: common probe/noise requests +statistics_maintenance_scope_non200_urls = Affected data: URL-based statistics for currently non-200 URLs (approximation) statistics_maintenance_scope_keep_days = Affected data: entries older than the configured number of days +statistics_maintenance_scope_since_date = Affected data: entries from the selected cutoff date statistics_maintenance_scope_keep_days_raw = Affected data: raw data older than the configured number of days statistics_maintenance_scope_optimize = Affected data: table structure and storage allocation statistics_geo_status = Geo database status @@ -96,7 +100,9 @@ statistics_geo_status_not_available = not available statistics_geo_last_update = Last update statistics_geo_file_size = File size statistics_cleanup_keep_days = Keep days: +statistics_cleanup_since_date = Cutoff date: statistics_cleanup_keep_days_raw = Keep raw data (days): +statistics_cleanup_since_date_invalid = Invalid cutoff date. Please use format YYYY-MM-DD. statistics_monday = Monday statistics_tuesday = Tuesday statistics_wednesday = Wednesday @@ -110,7 +116,9 @@ statistics_confirm_delete_bots = Really delete all visits from bots? statistics_confirm_delete_referer = Really delete all referers? statistics_confirm_delete_media = Really delete all media statistics? statistics_confirm_delete_noise = Really delete typical probe and WordPress requests from statistics? +statistics_confirm_delete_non200_urls = Really clean URL-based statistics for URLs currently recognized as non-200? statistics_confirm_delete_old = Really delete all statistics data older than the entered number of days? +statistics_confirm_delete_since_date = Really delete all statistics data from the selected cutoff date onward? statistics_confirm_delete_raw_old = Really delete old raw data? Daily statistics will be kept. statistics_confirm_optimize_tables = Optimize statistics tables now? This can take some time on large tables. statistics_deleted_hashes = entries deleted from the hashes table. @@ -123,7 +131,9 @@ statistics_geo_update_success = Geo database updated. statistics_geo_update_error = Geo database could not be updated. statistics_deleted_noise = %s noise requests were deleted. statistics_deleted_noise_partial = %s noise requests were cleaned in this run. Please run it again to continue. +statistics_deleted_non200_urls = %s URL-based entries for non-200 URLs were cleaned. statistics_deleted_old = %s old entries were deleted (older than %s days). +statistics_deleted_since_date = %s entries were deleted from cutoff date %s onward. statistics_deleted_raw_old = %s old raw-data entries were deleted (older than %s days). statistics_batch_continue = Run next batch statistics_batch_more_data = This operation runs in batches. Please start the next batch to process remaining records. diff --git a/lib/StatisticsMaintenanceCronjob.php b/lib/StatisticsMaintenanceCronjob.php index dc65b84..2a379a1 100644 --- a/lib/StatisticsMaintenanceCronjob.php +++ b/lib/StatisticsMaintenanceCronjob.php @@ -18,71 +18,92 @@ public function execute(): bool $cutoffDatetime = $cutoffDate . ' 00:00:00'; $deleted = 0; - $previousRuntimePause = (bool) rex_config::get('statistics', 'statistics_pause_tracking_runtime', false); + $previousRuntimePause = false; + $runtimePauseActive = false; + if ($pauseTrackingDuringMaintenance) { - rex_config::set('statistics', 'statistics_pause_tracking_runtime', true); + try { + $previousRuntimePause = (bool) rex_config::get('statistics', 'statistics_pause_tracking_runtime', false); + rex_config::set('statistics', 'statistics_pause_tracking_runtime', true); + $runtimePauseActive = true; + } catch (\Throwable $exception) { + rex_logger::logException($exception); + $this->setMessage('Statistics-Wartung fehlgeschlagen: Tracking-Pause konnte nicht gesetzt werden: ' . $exception->getMessage()); + + return false; + } } + $result = false; + $message = ''; + try { - try { - $deleted += $this->deleteChunked(rex::getTable('pagestats_visits_per_url'), 'date < :cutoff_date', [':cutoff_date' => $cutoffDate]); - $deleted += $this->deleteChunked(rex::getTable('pagestats_visitors_per_url'), 'date < :cutoff_date', [':cutoff_date' => $cutoffDate]); - $deleted += $this->deleteChunked(rex::getTable('pagestats_referer'), 'date < :cutoff_date', [':cutoff_date' => $cutoffDate]); - $deleted += $this->deleteChunked(rex::getTable('pagestats_media'), 'date < :cutoff_date', [':cutoff_date' => $cutoffDate]); - $deleted += $this->deleteChunked(rex::getTable('pagestats_api'), 'date < :cutoff_date', [':cutoff_date' => $cutoffDate]); - $deleted += $this->deleteChunked(rex::getTable('pagestats_sessionstats'), 'lastvisit < :cutoff_datetime', [':cutoff_datetime' => $cutoffDatetime]); - $deleted += $this->deleteChunked(rex::getTable('pagestats_hash'), 'datetime < :cutoff_datetime', [':cutoff_datetime' => $cutoffDatetime]); - - $deleted += $this->deleteOrphanUrlStatusChunked(); - - $noiseDeleted = 0; - $noiseHasMore = false; - if ($cleanupNoise) { - $noiseResult = $this->cleanupNoiseBatch(); - $noiseDeleted = $noiseResult['deleted']; - $noiseHasMore = $noiseResult['has_more']; - $deleted += $noiseDeleted; - } + $deleted += $this->deleteChunked(rex::getTable('pagestats_visits_per_url'), 'date < :cutoff_date', [':cutoff_date' => $cutoffDate]); + $deleted += $this->deleteChunked(rex::getTable('pagestats_visitors_per_url'), 'date < :cutoff_date', [':cutoff_date' => $cutoffDate]); + $deleted += $this->deleteChunked(rex::getTable('pagestats_referer'), 'date < :cutoff_date', [':cutoff_date' => $cutoffDate]); + $deleted += $this->deleteChunked(rex::getTable('pagestats_media'), 'date < :cutoff_date', [':cutoff_date' => $cutoffDate]); + $deleted += $this->deleteChunked(rex::getTable('pagestats_api'), 'date < :cutoff_date', [':cutoff_date' => $cutoffDate]); + $deleted += $this->deleteChunked(rex::getTable('pagestats_sessionstats'), 'lastvisit < :cutoff_datetime', [':cutoff_datetime' => $cutoffDatetime]); + $deleted += $this->deleteChunked(rex::getTable('pagestats_hash'), 'datetime < :cutoff_datetime', [':cutoff_datetime' => $cutoffDatetime]); + + $deleted += $this->deleteOrphanUrlStatusChunked(); + + $noiseDeleted = 0; + $noiseHasMore = false; + if ($cleanupNoise) { + $noiseResult = $this->cleanupNoiseBatch(); + $noiseDeleted = $noiseResult['deleted']; + $noiseHasMore = $noiseResult['has_more']; + $deleted += $noiseDeleted; + } - $optimized = 0; - $optimizedTotal = 0; - $optimizedRemaining = 0; - if ($optimizeTables) { - $optimizeResult = $this->optimizeTablesBatch($this->getTablesToOptimize(), $optimizeBatchSize); - $optimized = $optimizeResult['optimized']; - $optimizedTotal = $optimizeResult['total']; - $optimizedRemaining = $optimizeResult['remaining']; - } + $optimized = 0; + $optimizedTotal = 0; + $optimizedRemaining = 0; + if ($optimizeTables) { + $optimizeResult = $this->optimizeTablesBatch($this->getTablesToOptimize(), $optimizeBatchSize); + $optimized = $optimizeResult['optimized']; + $optimizedTotal = $optimizeResult['total']; + $optimizedRemaining = $optimizeResult['remaining']; + } - $message = 'Statistics-Wartung: ' . $deleted . ' Rohdaten-Einträge bereinigt (älter als ' . $daysToKeepRaw . ' Tage)'; - if ($cleanupNoise) { - $message .= ', ' . $noiseDeleted . ' Störanfragen bereinigt'; - if ($noiseHasMore) { - $message .= ' (weitere vorhanden)'; - } - } - if ($optimizeTables) { - $message .= ', ' . $optimized . ' Tabellen optimiert (Batchgröße: ' . $optimizeBatchSize . ')'; - if ($optimizedTotal > 0) { - $message .= ', verbleibend bis Vollzyklus: ' . $optimizedRemaining . ' von ' . $optimizedTotal; - } + $message = 'Statistics-Wartung: ' . $deleted . ' Rohdaten-Einträge bereinigt (älter als ' . $daysToKeepRaw . ' Tage)'; + if ($cleanupNoise) { + $message .= ', ' . $noiseDeleted . ' Störanfragen bereinigt'; + if ($noiseHasMore) { + $message .= ' (weitere vorhanden)'; } - if ($this->deferredDeleteBatches > 0) { - $message .= ', ' . $this->deferredDeleteBatches . ' Lösch-Batches wegen DB-Locks vertagt'; + } + if ($optimizeTables) { + $message .= ', ' . $optimized . ' Tabellen optimiert (Batchgröße: ' . $optimizeBatchSize . ')'; + if ($optimizedTotal > 0) { + $message .= ', verbleibend bis Vollzyklus: ' . $optimizedRemaining . ' von ' . $optimizedTotal; } - $this->setMessage($message); + } + if ($this->deferredDeleteBatches > 0) { + $message .= ', ' . $this->deferredDeleteBatches . ' Lösch-Batches wegen DB-Locks vertagt'; + } - return true; - } catch (rex_sql_exception $exception) { - $this->setMessage('Statistics-Wartung fehlgeschlagen: ' . $exception->getMessage()); + $result = true; + } catch (rex_sql_exception $exception) { + $message = 'Statistics-Wartung fehlgeschlagen: ' . $exception->getMessage(); + $result = false; + } - return false; - } - } finally { - if ($pauseTrackingDuringMaintenance) { + if ($runtimePauseActive) { + try { rex_config::set('statistics', 'statistics_pause_tracking_runtime', $previousRuntimePause); + } catch (\Throwable $exception) { + rex_logger::logException($exception); + $this->setMessage('Statistics-Wartung fehlgeschlagen: Tracking-Pause konnte nicht zurückgesetzt werden: ' . $exception->getMessage()); + + return false; } } + + $this->setMessage($message); + + return $result; } public function getTypeName(): string diff --git a/pages/settings.php b/pages/settings.php index 093a998..d18f800 100644 --- a/pages/settings.php +++ b/pages/settings.php @@ -355,6 +355,82 @@ $message = strtolower($exception->getMessage()); $isLockTimeout = false !== strpos($message, '1205') || false !== strpos($message, 'lock wait timeout'); + if ($isLockTimeout) { + echo rex_view::error($addon->i18n('statistics_cleanup_lock_timeout')); + } else { + rex_logger::logException($exception); + echo rex_view::error($exception->getMessage()); + } + } + } elseif ($function == 'delete_non200_urls') { + try { + $count = 0; + $hasMore = false; + $timedOut = false; + $chunkSize = $maintenanceChunkSize; + $maxRoundsPerRun = $maintenanceMaxRounds; + $round = 0; + + $statusTable = rex::getTable('pagestats_urlstatus'); + $visitsTable = rex::getTable('pagestats_visits_per_url'); + $visitorsTable = rex::getTable('pagestats_visitors_per_url'); + $continueLoop = false; + + do { + if (!$hasMaintenanceBudgetLeft()) { + $timedOut = true; + break; + } + + ++$round; + + $affectedVisits = $runDeleteWithRetry( + 'DELETE FROM ' . $visitsTable + . ' WHERE url IN (' + . 'SELECT stale.url FROM (' + . 'SELECT us.url FROM ' . $statusTable . ' us ' + . 'WHERE us.status <> :status_ok ' + . 'LIMIT ' . (int) $chunkSize + . ') stale' + . ')', + [':status_ok' => '200'] + ); + + $affectedVisitors = $runDeleteWithRetry( + 'DELETE FROM ' . $visitorsTable + . ' WHERE url IN (' + . 'SELECT stale.url FROM (' + . 'SELECT us.url FROM ' . $statusTable . ' us ' + . 'WHERE us.status <> :status_ok ' + . 'LIMIT ' . (int) $chunkSize + . ') stale' + . ')', + [':status_ok' => '200'] + ); + + $affectedStatus = $runDeleteWithRetry( + 'DELETE FROM ' . $statusTable . ' WHERE status <> :status_ok LIMIT ' . (int) $chunkSize, + [':status_ok' => '200'] + ); + + $count += $affectedVisits + $affectedVisitors + $affectedStatus; + $continueLoop = $affectedStatus >= $chunkSize || $affectedVisits >= $chunkSize || $affectedVisitors >= $chunkSize; + $hasMore = $continueLoop; + } while ($continueLoop && $round < $maxRoundsPerRun); + + if (!$timedOut && $hasMore && !$hasMaintenanceBudgetLeft()) { + $timedOut = true; + } + $hasMore = $hasMore || $timedOut; + + echo rex_view::success(sprintf($addon->i18n('statistics_deleted_non200_urls'), (string) $count)); + if ($hasMore) { + echo rex_view::warning($addon->i18n('statistics_batch_more_data') . $renderBatchContinueForm('delete_non200_urls')); + } + } catch (rex_sql_exception $exception) { + $message = strtolower($exception->getMessage()); + $isLockTimeout = false !== strpos($message, '1205') || false !== strpos($message, 'lock wait timeout'); + if ($isLockTimeout) { echo rex_view::error($addon->i18n('statistics_cleanup_lock_timeout')); } else { @@ -426,6 +502,78 @@ $message = strtolower($exception->getMessage()); $isLockTimeout = false !== strpos($message, '1205') || false !== strpos($message, 'lock wait timeout'); + if ($isLockTimeout) { + echo rex_view::error($addon->i18n('statistics_cleanup_lock_timeout')); + } else { + rex_logger::logException($exception); + echo rex_view::error($exception->getMessage()); + } + } + } elseif ($function == 'delete_since_date') { + try { + $sinceDateRaw = trim((string) rex_post('since_date', 'string', '')); + $sinceDateObj = DateTimeImmutable::createFromFormat('Y-m-d', $sinceDateRaw); + if (false === $sinceDateObj || $sinceDateObj->format('Y-m-d') !== $sinceDateRaw) { + echo rex_view::error($addon->i18n('statistics_cleanup_since_date_invalid')); + } else { + $sinceDate = $sinceDateObj->format('Y-m-d'); + $sinceDatetime = $sinceDate . ' 00:00:00'; + + $count = 0; + $hasMore = false; + $chunkSize = $maintenanceChunkSize; + $maxRoundsPerRun = $maintenanceMaxRounds; + + $result = $deleteChunkedLimited(rex::getTable('pagestats_visits_per_day'), 'date >= :since_date', [':since_date' => $sinceDate], $chunkSize, $maxRoundsPerRun); + $count += (int) $result['deleted']; + $hasMore = (bool) $result['has_more']; + + $result = $deleteChunkedLimited(rex::getTable('pagestats_visitors_per_day'), 'date >= :since_date', [':since_date' => $sinceDate], $chunkSize, $maxRoundsPerRun); + $count += (int) $result['deleted']; + $hasMore = $hasMore || (bool) $result['has_more']; + + $result = $deleteChunkedLimited(rex::getTable('pagestats_visits_per_url'), 'date >= :since_date', [':since_date' => $sinceDate], $chunkSize, $maxRoundsPerRun); + $count += (int) $result['deleted']; + $hasMore = $hasMore || (bool) $result['has_more']; + + $result = $deleteChunkedLimited(rex::getTable('pagestats_visitors_per_url'), 'date >= :since_date', [':since_date' => $sinceDate], $chunkSize, $maxRoundsPerRun); + $count += (int) $result['deleted']; + $hasMore = $hasMore || (bool) $result['has_more']; + + $result = $deleteChunkedLimited(rex::getTable('pagestats_referer'), 'date >= :since_date', [':since_date' => $sinceDate], $chunkSize, $maxRoundsPerRun); + $count += (int) $result['deleted']; + $hasMore = $hasMore || (bool) $result['has_more']; + + $result = $deleteChunkedLimited(rex::getTable('pagestats_media'), 'date >= :since_date', [':since_date' => $sinceDate], $chunkSize, $maxRoundsPerRun); + $count += (int) $result['deleted']; + $hasMore = $hasMore || (bool) $result['has_more']; + + $result = $deleteChunkedLimited(rex::getTable('pagestats_api'), 'date >= :since_date', [':since_date' => $sinceDate], $chunkSize, $maxRoundsPerRun); + $count += (int) $result['deleted']; + $hasMore = $hasMore || (bool) $result['has_more']; + + $result = $deleteChunkedLimited(rex::getTable('pagestats_hash'), 'datetime >= :since_datetime', [':since_datetime' => $sinceDatetime], $chunkSize, $maxRoundsPerRun); + $count += (int) $result['deleted']; + $hasMore = $hasMore || (bool) $result['has_more']; + + $result = $deleteChunkedLimited(rex::getTable('pagestats_sessionstats'), 'lastvisit >= :since_datetime', [':since_datetime' => $sinceDatetime], $chunkSize, $maxRoundsPerRun); + $count += (int) $result['deleted']; + $hasMore = $hasMore || (bool) $result['has_more']; + + // Remove stale URL status records when no matching URL stats remain. + $result = $deleteOrphanUrlStatusChunkedLimited($chunkSize, $maxRoundsPerRun); + $count += (int) $result['deleted']; + $hasMore = $hasMore || (bool) $result['has_more']; + + echo rex_view::success(sprintf($addon->i18n('statistics_deleted_since_date'), (string) $count, $sinceDate)); + if ($hasMore) { + echo rex_view::warning($addon->i18n('statistics_batch_more_data') . $renderBatchContinueForm('delete_since_date', ['since_date' => $sinceDate])); + } + } + } catch (rex_sql_exception $exception) { + $message = strtolower($exception->getMessage()); + $isLockTimeout = false !== strpos($message, '1205') || false !== strpos($message, 'lock wait timeout'); + if ($isLockTimeout) { echo rex_view::error($addon->i18n('statistics_cleanup_lock_timeout')); } else { @@ -933,6 +1081,14 @@ . '' . '' ); +$maintenanceTasksHtml .= $renderActionCard( + $addon->i18n('statistics_delete_non200_urls'), + $addon->i18n('statistics_maintenance_scope_non200_urls'), + '
' +); $maintenanceTasksHtml .= $renderActionCard( $addon->i18n('statistics_delete_old'), $addon->i18n('statistics_maintenance_scope_keep_days'), @@ -943,6 +1099,16 @@ . '' . '' ); +$maintenanceTasksHtml .= $renderActionCard( + $addon->i18n('statistics_delete_since_date'), + $addon->i18n('statistics_maintenance_scope_since_date'), + '' +); $maintenanceTasksHtml .= $renderActionCard( $addon->i18n('statistics_delete_raw_old'), $addon->i18n('statistics_maintenance_scope_keep_days_raw'),