From 1129c5295346a70f657465cec1ed2b3399e31c2c Mon Sep 17 00:00:00 2001 From: nagisml Date: Tue, 17 Nov 2026 21:29:55 +0100 Subject: [PATCH 1/3] =?UTF-8?q?Add=20a=20Repair=E2=80=A6=20action=20to=20t?= =?UTF-8?q?he=20database=20Maintenance=20dialog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Maintenance dialog (#1048) only checks the database file. Repair checks the data in it and fixes what can be fixed safely, like GSAK's Repair/Defrag. This is phase 1 of #1051: the check framework, rows whose cache is missing, broken links, and checks that only report. Repair… opens a dialog that checks the database in the background without changing it. It lists every check that found something, with the number of problems, the fix and a few example GC codes. The user ticks what to fix. Checks marked "Report only" can't be ticked. "Make a backup first" is ticked and uses the backup core (#952). The ticked fixes run in one transaction. Then the database is optimized, and compacted when more than a quarter of the file is free, as GSAK does. A summary follows and the check runs again. If quick_check fails, the file itself is damaged and SQLite can't repair it, so the dialog points to File → Restore from backup… (#954) instead. Fixable checks: - rows that point to a missing cache, found with foreign_key_check (logs, waypoints, attributes, trackables, user notes): deleted - custom waypoints whose parent_gc_code points to a cache that isn't in the database: link cleared - waypoints whose parent_gc_code isn't their own cache's code: set from the cache, as the importer and macros do - the corrected flag set without corrected coordinates, so filters count the cache as corrected while the map and distances don't: flag cleared. The cache keeps its last_user_update (the #1140 trigger would set it to now), because a repair is not a user edit. - companion database rows whose cache is gone: companion.cleanup() Report only: coordinates missing or out of range, corrected coordinates out of range, difficulty or terrain outside 1–5 or not in 0.5 steps, an empty GC code or name. There is no duplicate check. The unique constraints on attributes, log_id and waypoints exist in every database, and the importers don't fill their NULL gaps twice. The checks and fixes are in db/repair.py, which has no GUI code. The dialog works on the active database (its connections are released first, as for Compact) and on any other database in the list. The new texts are in all language files; the translations are machine-made. Part of #1051 --- src/opensak/db/repair.py | 336 +++++++++++++++++ .../dialogs/database_maintenance_dialog.py | 19 +- .../gui/dialogs/database_repair_dialog.py | 350 +++++++++++++++++ src/opensak/lang/cs.py | 32 ++ src/opensak/lang/da.py | 32 ++ src/opensak/lang/de.py | 32 ++ src/opensak/lang/de_CH.py | 32 ++ src/opensak/lang/en.py | 32 ++ src/opensak/lang/es.py | 32 ++ src/opensak/lang/fr.py | 32 ++ src/opensak/lang/nl.py | 32 ++ src/opensak/lang/pl.py | 32 ++ src/opensak/lang/pt.py | 32 ++ src/opensak/lang/se.py | 32 ++ tests/unit-tests/test_db_repair_1051.py | 351 ++++++++++++++++++ 15 files changed, 1406 insertions(+), 2 deletions(-) create mode 100644 src/opensak/db/repair.py create mode 100644 src/opensak/gui/dialogs/database_repair_dialog.py create mode 100644 tests/unit-tests/test_db_repair_1051.py diff --git a/src/opensak/db/repair.py b/src/opensak/db/repair.py new file mode 100644 index 00000000..ac8c042b --- /dev/null +++ b/src/opensak/db/repair.py @@ -0,0 +1,336 @@ +""" +src/opensak/db/repair.py — check the data of a database and fix what can be +fixed safely, like GSAK's Repair/Defrag (#1051, phase 1). + +#1048 checks the *file* (quick_check, integrity_check, foreign_key_check); +this module checks the *data*. ``scan()`` only reads and reports every check +with its number of problems and a few example GC codes. ``repair()`` fixes +the checks the user picked, in one transaction, and leaves everything else +as it is. Some checks are report only: there is no safe automatic fix, so +the user has to look at the caches. + +Checks in this phase: + +* rows that point to a missing parent row (``foreign_key_check``): logs, + waypoints, attributes, trackables and user notes of a cache that is gone. + Fix: delete them. Foreign keys are enforced and cache_delete.py removes + the child rows first, so these come from older databases, migrations that + run with foreign keys off, or a crash. +* a custom waypoint's link to its parent cache (``caches.parent_gc_code``) + that points to a cache that is not in the database. Fix: clear the link. +* a waypoint's ``parent_gc_code`` that isn't the code of its own cache. + Fix: set it, as the importer and macros do. +* the corrected-coordinates flag without corrected coordinates. Filters + count such a cache as corrected, the map and distances do not. Fix: clear + the flag. Coordinates "corrected" to the listed ones are fine: they are a + real user choice (the final is at the listed coordinates). +* rows of the companion database (#1034) whose cache is gone. Fix: + ``companion.cleanup()``. +* report only: coordinates out of range, difficulty or terrain outside + 1–5 or not in 0.5 steps, an empty GC code or name. + +No duplicate check: the unique constraints on attributes, ``logs.log_id`` +and waypoints exist in every database (models since v0.2.0, migrations 2 and +21), and the importers replace a cache's waypoints and match logs by id, so +the NULL gaps of those constraints are not filled twice either. + +Fixing a correction flag would fire the last_user_update trigger (#1140); +a repair is not a user edit, so the dates are put back. + +The functions open their own sqlite3 connection, like maintenance.py. The +scan also works while the database is open; for repair() the caller +releases the database's connections first. +""" + +from __future__ import annotations + +import logging +import sqlite3 +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Optional + +logger = logging.getLogger(__name__) + +# GC codes listed per check. +MAX_EXAMPLES = 5 +# Rows per DELETE … WHERE rowid IN (…), below SQLite's bound-parameter limit. +_CHUNK = 500 + +# Check ids. Their names are in database_repair_dialog.py. +ORPHANS = "orphans" +PARENT_LINK = "parent_link" +WAYPOINT_PARENT = "waypoint_parent" +CORRECTED_FLAG = "corrected_flag" +COMPANION_ORPHANS = "companion_orphans" +COORDS = "coords" +CORRECTED_COORDS = "corrected_coords" +RATINGS = "ratings" +IDENTITY = "identity" + + +@dataclass(frozen=True) +class Finding: + """One check's result. *table* is set for the orphan checks (one finding + per table). *fixable* False: report only.""" + + check: str + count: int + fixable: bool + examples: tuple[str, ...] = () + table: Optional[str] = None + + @property + def key(self) -> str: + """Identifies the finding to repair(): the check, or check:table.""" + return f"{self.check}:{self.table}" if self.table else self.check + + +@dataclass(frozen=True) +class ScanResult: + path: Path + findings: tuple[Finding, ...] = () + # quick_check messages. When set, the file itself is damaged: the data + # checks did not run, and only a backup can help. + damaged: tuple[str, ...] = () + + @property + def problems(self) -> tuple[Finding, ...]: + return tuple(f for f in self.findings if f.count) + + @property + def fixable(self) -> tuple[Finding, ...]: + return tuple(f for f in self.problems if f.fixable) + + +@dataclass +class RepairResult: + """What repair() changed: finding key → rows fixed.""" + + path: Path + fixed: dict[str, int] = field(default_factory=dict) + + @property + def total(self) -> int: + return sum(self.fixed.values()) + + +# ── Report-only checks: (id, WHERE on caches) ───────────────────────────────── + +_COORDS_WHERE = ( + "latitude IS NULL OR longitude IS NULL " + "OR latitude NOT BETWEEN -90 AND 90 OR longitude NOT BETWEEN -180 AND 180" +) + + +def _rating_bad(col: str) -> str: + # NULL is fine: custom waypoints have no rating. + return (f"({col} IS NOT NULL AND ({col} NOT BETWEEN 1 AND 5 " + f"OR {col} * 2 <> CAST({col} * 2 AS INTEGER)))") + + +_REPORT_CHECKS: tuple[tuple[str, str], ...] = ( + (COORDS, _COORDS_WHERE), + (RATINGS, f"{_rating_bad('difficulty')} OR {_rating_bad('terrain')}"), + (IDENTITY, "gc_code IS NULL OR trim(gc_code) = '' OR name IS NULL OR trim(name) = ''"), +) + +_CORRECTED_COORDS_WHERE = ( + "n.is_corrected AND n.corrected_lat IS NOT NULL AND n.corrected_lon IS NOT NULL " + "AND (n.corrected_lat NOT BETWEEN -90 AND 90 OR n.corrected_lon NOT BETWEEN -180 AND 180)" +) + +# ── Fixable checks on the main database ─────────────────────────────────────── + +_PARENT_LINK_WHERE = ( + "parent_gc_code IS NOT NULL AND trim(parent_gc_code) <> '' " + "AND upper(trim(parent_gc_code)) NOT IN (SELECT gc_code FROM caches)" +) +_WAYPOINT_PARENT_FROM = ( + "FROM waypoints w JOIN caches c ON c.id = w.cache_id " + "WHERE w.parent_gc_code IS NOT c.gc_code" +) +def _corrected_flag_where(alias: str = "") -> str: + a = f"{alias}." if alias else "" + return f"{a}is_corrected AND ({a}corrected_lat IS NULL OR {a}corrected_lon IS NULL)" + + +# ── Scan ────────────────────────────────────────────────────────────────────── + +def scan(db_path: Path, with_companion: bool = False, + step: Optional[Callable[[str], None]] = None) -> ScanResult: + """ + Run every check on the database *db_path* (and on its companion when + *with_companion*). Changes nothing. *step* is called with each check id + before it runs. Raises sqlite3.Error if the file can't be read. + """ + db_path = Path(db_path) + notify = step or (lambda _id: None) + conn = _connect(db_path) + try: + notify("quick_check") + rows = [str(r[0]) for r in conn.execute("PRAGMA quick_check").fetchall()] + if rows != ["ok"]: + return ScanResult(db_path, damaged=tuple(rows)) + findings: list[Finding] = [] + + notify(ORPHANS) + for table, count in sorted(_orphan_rowids_by_table(conn, count_only=True).items()): + findings.append(Finding(ORPHANS, count, True, table=table)) + + notify(PARENT_LINK) + findings.append(_cache_finding(conn, PARENT_LINK, _PARENT_LINK_WHERE, True)) + + notify(WAYPOINT_PARENT) + count = conn.execute(f"SELECT count(*) {_WAYPOINT_PARENT_FROM}").fetchone()[0] + examples = _codes(conn, f"SELECT DISTINCT c.gc_code {_WAYPOINT_PARENT_FROM} " + "ORDER BY c.gc_code") if count else () + findings.append(Finding(WAYPOINT_PARENT, count, True, examples)) + + notify(CORRECTED_FLAG) + findings.append(_note_finding(conn, CORRECTED_FLAG, _corrected_flag_where("n"), True)) + + if with_companion: + notify(COMPANION_ORPHANS) + findings.append(Finding(COMPANION_ORPHANS, + sum(_companion_orphans(db_path).values()), True)) + + for check, where in _REPORT_CHECKS: + notify(check) + findings.append(_cache_finding(conn, check, where, False)) + notify(CORRECTED_COORDS) + findings.append(_note_finding(conn, CORRECTED_COORDS, _CORRECTED_COORDS_WHERE, False)) + finally: + conn.close() + return ScanResult(db_path, tuple(findings)) + + +def _cache_finding(conn: sqlite3.Connection, check: str, where: str, fixable: bool) -> Finding: + count = conn.execute(f"SELECT count(*) FROM caches WHERE {where}").fetchone()[0] + examples = _codes(conn, f"SELECT gc_code FROM caches WHERE {where} ORDER BY gc_code") \ + if count else () + return Finding(check, count, fixable, examples) + + +def _note_finding(conn: sqlite3.Connection, check: str, where: str, fixable: bool) -> Finding: + base = f"FROM user_notes n JOIN caches c ON c.id = n.cache_id WHERE {where}" + count = conn.execute(f"SELECT count(*) {base}").fetchone()[0] + examples = _codes(conn, f"SELECT c.gc_code {base} ORDER BY c.gc_code") if count else () + return Finding(check, count, fixable, examples) + + +def _codes(conn: sqlite3.Connection, sql: str) -> tuple[str, ...]: + rows = conn.execute(f"{sql} LIMIT {MAX_EXAMPLES}").fetchall() + # An empty GC code is one of the problems; show it as "''". + return tuple(str(r[0]) if r[0] not in (None, "") else "''" for r in rows) + + +def _orphan_rowids_by_table(conn: sqlite3.Connection, count_only: bool = False) -> dict: + """foreign_key_check grouped by table: table → row count (*count_only*) + or table → list of rowids.""" + found: dict = {} + for table, rowid, _parent, _fkid in conn.execute("PRAGMA foreign_key_check").fetchall(): + if count_only: + found[table] = found.get(table, 0) + 1 + elif rowid is not None: + found.setdefault(table, []).append(rowid) + if not count_only: + # A row with two broken keys is listed twice. + found = {t: sorted(set(ids)) for t, ids in found.items()} + return found + + +def _companion_orphans(db_path: Path) -> dict[str, int]: + from opensak.db import companion + + if not companion.exists(db_path): + return {} + result = companion.check(db_path) + return dict(result.orphans) if result is not None else {} + + +# ── Repair ──────────────────────────────────────────────────────────────────── + +def repair(db_path: Path, keys: set[str], + step: Optional[Callable[[str], None]] = None) -> RepairResult: + """ + Fix the findings *keys* (Finding.key of fixable findings) in *db_path*. + The main database's fixes run in one transaction; the companion is + cleaned up after it is committed. Keys of report-only checks are + ignored. The caller has released the database's connections. Raises + sqlite3.Error; then nothing in the main database has changed. + """ + db_path = Path(db_path) + notify = step or (lambda _id: None) + result = RepairResult(db_path) + conn = _connect(db_path) + try: + conn.execute("PRAGMA foreign_keys=ON") + with conn: + orphan_tables = {k.split(":", 1)[1] for k in keys if k.startswith(f"{ORPHANS}:")} + if orphan_tables: + notify(ORPHANS) + for table, rowids in _orphan_rowids_by_table(conn).items(): + if table in orphan_tables: + result.fixed[f"{ORPHANS}:{table}"] = _delete_rowids(conn, table, rowids) + if PARENT_LINK in keys: + notify(PARENT_LINK) + result.fixed[PARENT_LINK] = conn.execute( + f"UPDATE caches SET parent_gc_code = NULL WHERE {_PARENT_LINK_WHERE}" + ).rowcount + if WAYPOINT_PARENT in keys: + notify(WAYPOINT_PARENT) + result.fixed[WAYPOINT_PARENT] = conn.execute( + "UPDATE waypoints SET parent_gc_code = (SELECT gc_code FROM caches " + "WHERE caches.id = waypoints.cache_id) " + "WHERE cache_id IN (SELECT id FROM caches) AND parent_gc_code IS NOT " + "(SELECT gc_code FROM caches WHERE caches.id = waypoints.cache_id)" + ).rowcount + if CORRECTED_FLAG in keys: + notify(CORRECTED_FLAG) + result.fixed[CORRECTED_FLAG] = _clear_corrected_flags(conn) + finally: + conn.close() + if COMPANION_ORPHANS in keys: + from opensak.db import companion + + notify(COMPANION_ORPHANS) + if companion.exists(db_path): + result.fixed[COMPANION_ORPHANS] = sum(companion.cleanup(db_path).orphans.values()) + result.fixed = {k: v for k, v in result.fixed.items() if v > 0} + if result.total: + logger.info("repair of %s: %s", db_path, result.fixed) + return result + + +def _delete_rowids(conn: sqlite3.Connection, table: str, rowids: list[int]) -> int: + removed = 0 + quoted = '"' + table.replace('"', '""') + '"' + for start in range(0, len(rowids), _CHUNK): + chunk = rowids[start:start + _CHUNK] + marks = ", ".join("?" * len(chunk)) + removed += conn.execute(f"DELETE FROM {quoted} WHERE rowid IN ({marks})", chunk).rowcount + return removed + + +def _clear_corrected_flags(conn: sqlite3.Connection) -> int: + """Clear the flag, keeping the caches' last_user_update: the trigger + would set it to now, but a repair is not a user edit.""" + kept = conn.execute( + "SELECT c.id, c.last_user_update FROM caches c JOIN user_notes n ON n.cache_id = c.id " + f"WHERE {_corrected_flag_where('n')}" + ).fetchall() + count = conn.execute( + f"UPDATE user_notes SET is_corrected = 0, corrected_source = NULL " + f"WHERE {_corrected_flag_where()}" + ).rowcount + conn.executemany("UPDATE caches SET last_user_update = ? WHERE id = ?", + [(stamp, cache_id) for cache_id, stamp in kept]) + return count + + +def _connect(db_path: Path) -> sqlite3.Connection: + """A connection to the existing file *db_path* — never creates one.""" + if not db_path.is_file(): + raise sqlite3.OperationalError(f"unable to open database file: {db_path}") + return sqlite3.connect(str(db_path), timeout=30) diff --git a/src/opensak/gui/dialogs/database_maintenance_dialog.py b/src/opensak/gui/dialogs/database_maintenance_dialog.py index 7246dd3f..3c30d517 100644 --- a/src/opensak/gui/dialogs/database_maintenance_dialog.py +++ b/src/opensak/gui/dialogs/database_maintenance_dialog.py @@ -13,6 +13,8 @@ * **Checkpoint** — write the WAL back and empty the -wal file. * **Compact…** — VACUUM, only when it pays off, after a disk space check and optionally a backup set. +* **Repair…** — check the data and fix what can be fixed safely, in its own + dialog (database_repair_dialog.py, #1051). While the ``companion-db`` feature flag is on, a "Companion data" section lists the companion's tables per owner (kind, rows, estimated size) and @@ -392,8 +394,11 @@ def _setup_ui(self) -> None: self._btn_compact = QPushButton(tr("maint_btn_compact")) self._btn_compact.setToolTip(tr("maint_tip_compact")) self._btn_compact.clicked.connect(self._compact) + self._btn_repair = QPushButton(tr("maint_btn_repair")) + self._btn_repair.setToolTip(tr("maint_tip_repair")) + self._btn_repair.clicked.connect(self._open_repair) for w in (self._btn_check, self._thorough, self._btn_optimize, - self._btn_checkpoint, self._btn_compact): + self._btn_checkpoint, self._btn_compact, self._btn_repair): actions.addWidget(w) actions.addStretch() layout.addLayout(actions) @@ -420,7 +425,7 @@ def _setup_ui(self) -> None: self._action_buttons: list[QWidget] = [ self._btn_check, self._thorough, self._btn_optimize, - self._btn_checkpoint, self._btn_compact, + self._btn_checkpoint, self._btn_compact, self._btn_repair, ] def _build_companion_group(self) -> QGroupBox: @@ -657,6 +662,16 @@ def _on_compact_failed(self, error: str) -> None: self._append([tr("maint_backup_failed", error=error)]) self.refresh() + def _open_repair(self) -> None: + from opensak.gui.dialogs.database_repair_dialog import DatabaseRepairDialog + + dlg = DatabaseRepairDialog(self._db, self) + dlg.exec() + dlg.wait_idle() + dlg.deleteLater() + # A repair can compact the file and remove rows. + self.refresh() + # ── Companion data ──────────────────────────────────────────────────────── def _clear_table(self) -> None: diff --git a/src/opensak/gui/dialogs/database_repair_dialog.py b/src/opensak/gui/dialogs/database_repair_dialog.py new file mode 100644 index 00000000..401f9dbd --- /dev/null +++ b/src/opensak/gui/dialogs/database_repair_dialog.py @@ -0,0 +1,350 @@ +""" +src/opensak/gui/dialogs/database_repair_dialog.py — Repair… in the +Maintenance dialog (#1051, phase 1): check the data of a database and fix +what can be fixed safely, like GSAK's Repair/Defrag. + +1. Scan (read only, in the background): every check of db/repair.py with + its number of problems and a few example GC codes. +2. The user ticks what to fix. Report-only checks can't be ticked. "Make a + backup first" is ticked. +3. Repair: backup set, the fixes in one transaction, Optimize, and Compact + when more than a quarter of the file is free (as GSAK does). Then a + summary, and the scan runs again. + +A damaged file (quick_check fails) can't be repaired by SQLite; the dialog +points to File → Restore from backup… instead. + +Works on the active database (its connections are released first, as for +Compact) and on any other database in the list. The work is in run_scan() +and run_repair(), which return plain results, so they can be tested +without threads. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Optional + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QCheckBox, QDialog, QDialogButtonBox, QHBoxLayout, QHeaderView, QLabel, + QPlainTextEdit, QProgressBar, QPushButton, QTreeWidget, QTreeWidgetItem, + QVBoxLayout, QWidget, +) + +from opensak.db import companion, maintenance, repair +from opensak.gui.dialogs.database_maintenance_dialog import ( + CompactOutcome, MaintenanceWorker, Step, compact_lines, +) +from opensak.gui.dialogs.widgets import clamp_dialog_height_to_screen +from opensak.lang import tr +from opensak.utils import flags + +logger = logging.getLogger(__name__) + +# Compact after a repair when at least this share of the file is free (GSAK). +COMPACT_FRACTION = 0.25 +# The finding key of a row in the tree. +KEY_ROLE = Qt.ItemDataRole.UserRole +# Damage messages listed; the rest is in the log. +_MAX_DAMAGE_LINES = 20 + +# Each check's name, as a translation key. +_CHECK_TEXT = { + repair.ORPHANS: "repair_chk_orphans", + repair.PARENT_LINK: "repair_chk_parent_link", + repair.WAYPOINT_PARENT: "repair_chk_waypoint_parent", + repair.CORRECTED_FLAG: "repair_chk_corrected_flag", + repair.COMPANION_ORPHANS: "repair_chk_companion_orphans", + repair.COORDS: "repair_chk_coords", + repair.CORRECTED_COORDS: "repair_chk_corrected_coords", + repair.RATINGS: "repair_chk_ratings", + repair.IDENTITY: "repair_chk_identity", +} +# The fix each fixable check makes, as a translation key. +_FIX_TEXT = { + repair.ORPHANS: "repair_fix_delete", + repair.PARENT_LINK: "repair_fix_clear_link", + repair.WAYPOINT_PARENT: "repair_fix_set_parent", + repair.CORRECTED_FLAG: "repair_fix_clear_flag", + repair.COMPANION_ORPHANS: "repair_fix_delete", +} + + +@dataclass +class RepairOutcome: + result: repair.RepairResult + backup_path: Optional[Path] = None + compact: Optional[maintenance.VacuumResult] = None + + +# ── The work (worker thread) ────────────────────────────────────────────────── + +def _with_companion(db_path: Path) -> bool: + return bool(flags.companion_db) and companion.exists(db_path) + + +def run_scan(db_path: Path, step: Step) -> repair.ScanResult: + step(tr("repair_step_scan")) + return repair.scan(db_path, with_companion=_with_companion(db_path)) + + +def run_repair(db: Any, keys: set[str], backup: bool, step: Step) -> RepairOutcome: + """Back up *db* (when *backup*), fix *keys*, then Optimize and Compact + when it pays off. The caller has released the database's connections. + A failed backup raises BackupError, and then nothing is changed.""" + backup_path = None + if backup: + from opensak.backup.backupset import KIND_MANUAL, write_backup_set + + step(tr("maint_step_backup")) + backup_path = write_backup_set([db], KIND_MANUAL).backup_set.path + step(tr("repair_step_fix")) + outcome = RepairOutcome(repair.repair(db.path, keys), backup_path) + step(tr("maint_step_optimize", file=tr("maint_db_group"))) + maintenance.optimize_file(db.path) + stats = maintenance.file_info(db.path).stats + if maintenance.is_worthwhile(stats, maintenance.MANUAL_MIN_RECLAIM_BYTES, COMPACT_FRACTION): + step(tr("maint_step_compact", file=tr("maint_db_group"))) + # vacuum() checks the free disk space itself. + outcome.compact = maintenance.vacuum_file( + db.path, min_bytes=maintenance.MANUAL_MIN_RECLAIM_BYTES, + min_fraction=COMPACT_FRACTION) + if outcome.compact.ran: + maintenance.optimize_file(db.path) + return outcome + + +# ── Text ────────────────────────────────────────────────────────────────────── + +def check_label(finding: repair.Finding) -> str: + key = _CHECK_TEXT[finding.check] + return tr(key, table=finding.table) if finding.table else tr(key) + + +def _label_for_key(key: str) -> str: + check, _, table = key.partition(":") + return check_label(repair.Finding(check, 0, True, table=table or None)) + + +def repair_lines(outcome: RepairOutcome) -> list[str]: + lines = [] + if outcome.backup_path is not None: + lines.append(tr("maint_backup_done", path=str(outcome.backup_path))) + if not outcome.result.fixed: + lines.append(tr("repair_nothing_fixed")) + for key, count in outcome.result.fixed.items(): + lines.append(tr("repair_fixed", count=f"{count:,}", check=_label_for_key(key))) + if outcome.compact is not None: + lines += compact_lines(CompactOutcome(results=[outcome.compact])) + return lines + + +# ── Dialog ──────────────────────────────────────────────────────────────────── + +class DatabaseRepairDialog(QDialog): + """Repair of the database *db* (an entry of the database list).""" + + def __init__(self, db: Any, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._db = db + self.scan_result: Optional[repair.ScanResult] = None + self._worker: Optional[MaintenanceWorker] = None + self.setWindowTitle(tr("repair_title", name=db.name)) + self.setMinimumSize(720, 460) + self._setup_ui() + clamp_dialog_height_to_screen(self, parent) + self._scan() + + def _setup_ui(self) -> None: + layout = QVBoxLayout(self) + intro = QLabel(tr("repair_intro")) + intro.setWordWrap(True) + layout.addWidget(intro) + + self._summary = QLabel("") + self._summary.setWordWrap(True) + layout.addWidget(self._summary) + + self._tree = QTreeWidget() + self._tree.setRootIsDecorated(False) + self._tree.setHeaderLabels([ + tr("repair_col_check"), tr("repair_col_count"), tr("repair_col_fix"), + tr("repair_col_examples"), + ]) + header = self._tree.header() + if header is not None: + header.setSectionResizeMode(0, QHeaderView.ResizeMode.Stretch) + for col in (1, 2, 3): + header.setSectionResizeMode(col, QHeaderView.ResizeMode.ResizeToContents) + self._tree.itemChanged.connect(lambda *_: self._update_buttons()) + layout.addWidget(self._tree, 2) + + row = QHBoxLayout() + self._backup = QCheckBox(tr("maint_compact_backup")) + self._backup.setChecked(True) + self._btn_rescan = QPushButton(tr("repair_btn_rescan")) + self._btn_rescan.clicked.connect(self._scan) + self._btn_repair = QPushButton(tr("repair_btn_repair")) + self._btn_repair.clicked.connect(self._repair) + row.addWidget(self._backup) + row.addStretch() + row.addWidget(self._btn_rescan) + row.addWidget(self._btn_repair) + layout.addLayout(row) + + busy_row = QHBoxLayout() + self._busy = QProgressBar() + self._busy.setRange(0, 0) + self._busy.setMaximumWidth(160) + self._busy.setVisible(False) + self._step = QLabel("") + busy_row.addWidget(self._busy) + busy_row.addWidget(self._step, 1) + layout.addLayout(busy_row) + + layout.addWidget(QLabel(f"{tr('dist_result_group')}")) + self._results = QPlainTextEdit() + self._results.setReadOnly(True) + self._results.setMinimumHeight(80) + layout.addWidget(self._results, 1) + + buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Close) + buttons.rejected.connect(self.reject) + layout.addWidget(buttons) + + # ── Scan ────────────────────────────────────────────────────────────────── + + def _scan(self) -> None: + self.scan_result = None + self._run(lambda step: run_scan(self._db.path, step), self._show_scan) + + def _show_scan(self, result: repair.ScanResult) -> None: + self._tree.blockSignals(True) + self._tree.clear() + for finding in result.problems: + item = QTreeWidgetItem([ + check_label(finding), f"{finding.count:,}", + tr(_FIX_TEXT[finding.check]) if finding.fixable else tr("repair_report_only"), + ", ".join(finding.examples) + (" …" if finding.count > len(finding.examples) + and finding.examples else ""), + ]) + item.setData(0, KEY_ROLE, finding.key) + item.setTextAlignment(1, Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter) + if finding.fixable: + item.setFlags(item.flags() | Qt.ItemFlag.ItemIsUserCheckable) + item.setCheckState(0, Qt.CheckState.Checked) + else: + item.setFlags(item.flags() & ~Qt.ItemFlag.ItemIsUserCheckable) + self._tree.addTopLevelItem(item) + self._tree.blockSignals(False) + + if result.damaged: + self._summary.setText(f"{tr('repair_damaged')} {tr('maint_check_main_failed')}") + self._append(list(result.damaged[:_MAX_DAMAGE_LINES])) + elif not result.problems: + self._summary.setText(tr("repair_scan_clean")) + else: + self._summary.setText(tr("repair_scan_summary", problems=len(result.problems), + fixable=len(result.fixable))) + self.scan_result = result + self._set_busy(False) + + def checked_keys(self) -> set[str]: + keys = set() + for i in range(self._tree.topLevelItemCount()): + item = self._tree.topLevelItem(i) + if (item is not None and item.flags() & Qt.ItemFlag.ItemIsUserCheckable + and item.checkState(0) == Qt.CheckState.Checked): + keys.add(str(item.data(0, KEY_ROLE))) + return keys + + # ── Repair ──────────────────────────────────────────────────────────────── + + def _repair(self) -> None: + keys = self.checked_keys() + if not keys or self.is_running(): + return + backup = self._backup.isChecked() + from opensak.db.database import release_connections + + release_connections(self._db.path) + self._run(lambda step: run_repair(self._db, keys, backup, step), self._on_repaired, + failed=self._on_repair_failed) + + def _on_repaired(self, outcome: RepairOutcome) -> None: + if outcome.backup_path is not None: + try: + from opensak.backup.exit_state import record_backup_state + record_backup_state([self._db]) + except Exception: + logger.warning("repair: could not record the backup state", exc_info=True) + self._set_busy(False) + self._append(repair_lines(outcome)) + self._rescan_when_done() + + def _on_repair_failed(self, error: str) -> None: + self._set_busy(False) + self._append([tr("repair_failed", error=error)]) + self._rescan_when_done() + + def _rescan_when_done(self) -> None: + # Called from the worker's last signal: its thread is about to end, + # and _run() starts nothing while it still runs. + self.wait_idle() + self._scan() + + # ── Running jobs ────────────────────────────────────────────────────────── + + def is_running(self) -> bool: + return self._worker is not None and self._worker.isRunning() + + def _run(self, job: Callable[[Step], object], done: Callable[[Any], None], + failed: Optional[Callable[[str], None]] = None) -> None: + if self.is_running(): + return + self._set_busy(True) + worker = MaintenanceWorker(job, self) + worker.step.connect(self._step.setText) + worker.succeeded.connect(done) + worker.failed.connect(failed or self._on_failed) + self._worker = worker + worker.start() + + def _on_failed(self, error: str) -> None: + self._set_busy(False) + self._append([tr("maint_failed", file=self._db.name, error=error)]) + self._summary.setText("") + self.scan_result = repair.ScanResult(Path(self._db.path)) + + def _set_busy(self, busy: bool) -> None: + self._busy.setVisible(busy) + if not busy: + self._step.setText("") + self._tree.setEnabled(not busy) + self._backup.setEnabled(not busy) + self._btn_rescan.setEnabled(not busy) + self._btn_repair.setEnabled(not busy and bool(self.checked_keys())) + + def _update_buttons(self) -> None: + self._btn_repair.setEnabled(not self.is_running() and bool(self.checked_keys())) + + def _append(self, lines: list[str]) -> None: + if not lines: + return + stamp = time.strftime("%H:%M:%S") + self._results.appendPlainText("\n".join(f"{stamp} {line}" for line in lines)) + + def reject(self) -> None: + # A running repair or VACUUM can't be interrupted safely. + if self.is_running(): + return + super().reject() + + def wait_idle(self, msecs: int = 60000) -> None: + """Block until the running step is done (for tests and closing).""" + if self._worker is not None: + self._worker.wait(msecs) diff --git a/src/opensak/lang/cs.py b/src/opensak/lang/cs.py index d2ca9622..ab54ce84 100644 --- a/src/opensak/lang/cs.py +++ b/src/opensak/lang/cs.py @@ -1107,6 +1107,38 @@ "maint_compact_total": "Celkem: {before} → {after}.", "maint_backup_failed": "Nic nebylo zkomprimováno:\n{error}", "maint_backup_done": "Záloha vytvořena: {path}", + "maint_btn_repair": "Opravit…", + "maint_tip_repair": "Zkontroluje data (řádky bez keše, poškozené odkazy, neplatné hodnoty) a opraví, co lze bezpečně opravit.", + "repair_title": "Oprava — {name}", + "repair_intro": "Zkontroluje data databáze. Zaškrtnuté problémy budou opraveny. Problémy označené \"Pouze hlásit\" nemají bezpečnou automatickou opravu: zkontrolujte tyto keše sami.", + "repair_col_check": "Kontrola", + "repair_col_count": "Problémy", + "repair_col_fix": "Oprava", + "repair_col_examples": "Příklady", + "repair_chk_orphans": "Řádky v {table}, jejichž keš již neexistuje", + "repair_chk_parent_link": "Vlastní waypointy propojené s keší, která není v databázi", + "repair_chk_waypoint_parent": "Waypointy s nesprávným nadřazeným GC kódem", + "repair_chk_corrected_flag": "Keše označené jako opravené bez opravených souřadnic", + "repair_chk_companion_orphans": "Řádky v doprovodné databázi, jejichž keš již neexistuje", + "repair_chk_coords": "Souřadnice chybí nebo jsou mimo rozsah", + "repair_chk_corrected_coords": "Opravené souřadnice mimo rozsah", + "repair_chk_ratings": "Obtížnost nebo terén není mezi 1 a 5 v krocích po 0,5", + "repair_chk_identity": "Prázdný GC kód nebo název", + "repair_fix_delete": "Smazat", + "repair_fix_clear_link": "Odstranit odkaz", + "repair_fix_set_parent": "Nastavit kód keše", + "repair_fix_clear_flag": "Zrušit označení", + "repair_report_only": "Pouze hlásit", + "repair_btn_repair": "Opravit", + "repair_btn_rescan": "Zkontrolovat znovu", + "repair_step_scan": "Kontrola dat…", + "repair_step_fix": "Oprava zaškrtnutých problémů…", + "repair_scan_clean": "Nebyly nalezeny žádné problémy.", + "repair_scan_summary": "Problémy našlo kontrol: {problems}, z toho lze zde opravit: {fixable}.", + "repair_damaged": "Samotný soubor databáze je poškozen a SQLite jej nedokáže opravit.", + "repair_fixed": "Opraveno {count}: {check}", + "repair_nothing_fixed": "Nic nebylo změněno.", + "repair_failed": "Oprava selhala, nic nebylo změněno:\n{error}", "maint_cleanup_done": "{file}: odstraněny zbylé řádky ({orphans}) a záznamy registru ({registry}).", "maint_cleanup_nothing": "{file}: není co uklízet.", "maint_clear_confirm": "Smazat všechny řádky ({rows}) tabulky {table}?\nVlastník: {owner}\n\nTabulka zůstane. Řádky lze vrátit jen ze zálohy.", diff --git a/src/opensak/lang/da.py b/src/opensak/lang/da.py index 2d02da6c..c6362ff5 100644 --- a/src/opensak/lang/da.py +++ b/src/opensak/lang/da.py @@ -1107,6 +1107,38 @@ "maint_compact_total": "I alt: {before} → {after}.", "maint_backup_failed": "Intet blev komprimeret:\n{error}", "maint_backup_done": "Sikkerhedskopi lavet: {path}", + "maint_btn_repair": "Reparér…", + "maint_tip_repair": "Kontrollerer data (rækker uden cache, brudte links, ugyldige værdier) og retter det, der sikkert kan rettes.", + "repair_title": "Reparér — {name}", + "repair_intro": "Kontrollerer databasens data. De afkrydsede problemer bliver rettet. Problemer markeret \"Kun rapport\" har ingen sikker automatisk rettelse: kontrollér selv de caches.", + "repair_col_check": "Kontrol", + "repair_col_count": "Problemer", + "repair_col_fix": "Rettelse", + "repair_col_examples": "Eksempler", + "repair_chk_orphans": "Rækker i {table}, hvis cache ikke længere findes", + "repair_chk_parent_link": "Brugerdefinerede waypoints knyttet til en cache, der ikke er i databasen", + "repair_chk_waypoint_parent": "Waypoints med forkert overordnet GC-kode", + "repair_chk_corrected_flag": "Caches markeret som korrigeret uden korrigerede koordinater", + "repair_chk_companion_orphans": "Rækker i ledsagerdatabasen, hvis cache ikke længere findes", + "repair_chk_coords": "Koordinater mangler eller er uden for gyldigt område", + "repair_chk_corrected_coords": "Korrigerede koordinater uden for gyldigt område", + "repair_chk_ratings": "Sværhedsgrad eller terræn ikke mellem 1 og 5 i trin på 0,5", + "repair_chk_identity": "Tom GC-kode eller navn", + "repair_fix_delete": "Slet", + "repair_fix_clear_link": "Fjern linket", + "repair_fix_set_parent": "Sæt cachens kode", + "repair_fix_clear_flag": "Fjern markeringen", + "repair_report_only": "Kun rapport", + "repair_btn_repair": "Reparér", + "repair_btn_rescan": "Kontrollér igen", + "repair_step_scan": "Kontrollerer data…", + "repair_step_fix": "Retter de afkrydsede problemer…", + "repair_scan_clean": "Ingen problemer fundet.", + "repair_scan_summary": "{problems} kontrol(ler) fandt problemer, {fixable} af dem kan rettes her.", + "repair_damaged": "Selve databasefilen er beskadiget, og SQLite kan ikke reparere den.", + "repair_fixed": "Rettet {count}: {check}", + "repair_nothing_fixed": "Intet blev ændret.", + "repair_failed": "Reparationen mislykkedes, intet blev ændret:\n{error}", "maint_cleanup_done": "{file}: fjernede {orphans} overskydende række(r) og {registry} registreringer.", "maint_cleanup_nothing": "{file}: intet at rydde op.", "maint_clear_confirm": "Slet alle {rows} rækker i tabellen {table}?\nEjer: {owner}\n\nTabellen bliver. Rækkerne kan kun hentes tilbage fra en sikkerhedskopi.", diff --git a/src/opensak/lang/de.py b/src/opensak/lang/de.py index b48419a6..3b747f0c 100644 --- a/src/opensak/lang/de.py +++ b/src/opensak/lang/de.py @@ -1108,6 +1108,38 @@ "maint_compact_total": "Gesamt: {before} → {after}.", "maint_backup_failed": "Es wurde nichts komprimiert:\n{error}", "maint_backup_done": "Sicherung erstellt: {path}", + "maint_btn_repair": "Reparieren…", + "maint_tip_repair": "Prüft die Daten (Zeilen ohne Cache, defekte Verknüpfungen, ungültige Werte) und behebt, was sicher behoben werden kann.", + "repair_title": "Reparieren — {name}", + "repair_intro": "Prüft die Daten der Datenbank. Die angekreuzten Probleme werden behoben. Für Probleme mit \"Nur melden\" gibt es keine sichere automatische Lösung: Prüfen Sie diese Caches selbst.", + "repair_col_check": "Prüfung", + "repair_col_count": "Probleme", + "repair_col_fix": "Behebung", + "repair_col_examples": "Beispiele", + "repair_chk_orphans": "Zeilen in {table}, deren Cache nicht mehr existiert", + "repair_chk_parent_link": "Eigene Wegpunkte, die mit einem Cache verknüpft sind, der nicht in der Datenbank ist", + "repair_chk_waypoint_parent": "Wegpunkte mit falschem übergeordnetem GC-Code", + "repair_chk_corrected_flag": "Caches, die als korrigiert markiert sind, aber keine korrigierten Koordinaten haben", + "repair_chk_companion_orphans": "Zeilen in der Begleitdatenbank, deren Cache nicht mehr existiert", + "repair_chk_coords": "Koordinaten fehlen oder liegen außerhalb des gültigen Bereichs", + "repair_chk_corrected_coords": "Korrigierte Koordinaten außerhalb des gültigen Bereichs", + "repair_chk_ratings": "Schwierigkeit oder Gelände nicht zwischen 1 und 5 in Schritten von 0,5", + "repair_chk_identity": "Leerer GC-Code oder Name", + "repair_fix_delete": "Löschen", + "repair_fix_clear_link": "Verknüpfung entfernen", + "repair_fix_set_parent": "Code des Caches setzen", + "repair_fix_clear_flag": "Markierung entfernen", + "repair_report_only": "Nur melden", + "repair_btn_repair": "Reparieren", + "repair_btn_rescan": "Erneut prüfen", + "repair_step_scan": "Daten werden geprüft…", + "repair_step_fix": "Angekreuzte Probleme werden behoben…", + "repair_scan_clean": "Keine Probleme gefunden.", + "repair_scan_summary": "{problems} Prüfung(en) haben Probleme gefunden, {fixable} davon können hier behoben werden.", + "repair_damaged": "Die Datenbankdatei selbst ist beschädigt, und SQLite kann sie nicht reparieren.", + "repair_fixed": "Behoben {count}: {check}", + "repair_nothing_fixed": "Es wurde nichts geändert.", + "repair_failed": "Reparatur fehlgeschlagen, es wurde nichts geändert:\n{error}", "maint_cleanup_done": "{file}: {orphans} übrig gebliebene Zeile(n) und {registry} Registereinträge entfernt.", "maint_cleanup_nothing": "{file}: nichts aufzuräumen.", "maint_clear_confirm": "Alle {rows} Zeilen der Tabelle {table} löschen?\nBesitzer: {owner}\n\nDie Tabelle bleibt. Die Zeilen lassen sich nur aus einer Sicherung zurückholen.", diff --git a/src/opensak/lang/de_CH.py b/src/opensak/lang/de_CH.py index 414a0805..7e82d18c 100644 --- a/src/opensak/lang/de_CH.py +++ b/src/opensak/lang/de_CH.py @@ -1109,6 +1109,38 @@ "maint_compact_total": "Gesamt: {before} → {after}.", "maint_backup_failed": "Es wurde nichts komprimiert:\n{error}", "maint_backup_done": "Sicherung erstellt: {path}", + "maint_btn_repair": "Reparieren…", + "maint_tip_repair": "Prüft die Daten (Zeilen ohne Cache, defekte Verknüpfungen, ungültige Werte) und behebt, was sicher behoben werden kann.", + "repair_title": "Reparieren — {name}", + "repair_intro": "Prüft die Daten der Datenbank. Die angekreuzten Probleme werden behoben. Für Probleme mit \"Nur melden\" gibt es keine sichere automatische Lösung: Prüfen Sie diese Caches selbst.", + "repair_col_check": "Prüfung", + "repair_col_count": "Probleme", + "repair_col_fix": "Behebung", + "repair_col_examples": "Beispiele", + "repair_chk_orphans": "Zeilen in {table}, deren Cache nicht mehr existiert", + "repair_chk_parent_link": "Eigene Wegpunkte, die mit einem Cache verknüpft sind, der nicht in der Datenbank ist", + "repair_chk_waypoint_parent": "Wegpunkte mit falschem übergeordnetem GC-Code", + "repair_chk_corrected_flag": "Caches, die als korrigiert markiert sind, aber keine korrigierten Koordinaten haben", + "repair_chk_companion_orphans": "Zeilen in der Begleitdatenbank, deren Cache nicht mehr existiert", + "repair_chk_coords": "Koordinaten fehlen oder liegen ausserhalb des gültigen Bereichs", + "repair_chk_corrected_coords": "Korrigierte Koordinaten ausserhalb des gültigen Bereichs", + "repair_chk_ratings": "Schwierigkeit oder Gelände nicht zwischen 1 und 5 in Schritten von 0,5", + "repair_chk_identity": "Leerer GC-Code oder Name", + "repair_fix_delete": "Löschen", + "repair_fix_clear_link": "Verknüpfung entfernen", + "repair_fix_set_parent": "Code des Caches setzen", + "repair_fix_clear_flag": "Markierung entfernen", + "repair_report_only": "Nur melden", + "repair_btn_repair": "Reparieren", + "repair_btn_rescan": "Erneut prüfen", + "repair_step_scan": "Daten werden geprüft…", + "repair_step_fix": "Angekreuzte Probleme werden behoben…", + "repair_scan_clean": "Keine Probleme gefunden.", + "repair_scan_summary": "{problems} Prüfung(en) haben Probleme gefunden, {fixable} davon können hier behoben werden.", + "repair_damaged": "Die Datenbankdatei selbst ist beschädigt, und SQLite kann sie nicht reparieren.", + "repair_fixed": "Behoben {count}: {check}", + "repair_nothing_fixed": "Es wurde nichts geändert.", + "repair_failed": "Reparatur fehlgeschlagen, es wurde nichts geändert:\n{error}", "maint_cleanup_done": "{file}: {orphans} übrig gebliebene Zeile(n) und {registry} Registereinträge entfernt.", "maint_cleanup_nothing": "{file}: nichts aufzuräumen.", "maint_clear_confirm": "Alle {rows} Zeilen der Tabelle {table} löschen?\nBesitzer: {owner}\n\nDie Tabelle bleibt. Die Zeilen lassen sich nur aus einer Sicherung zurückholen.", diff --git a/src/opensak/lang/en.py b/src/opensak/lang/en.py index 815eee48..27553250 100644 --- a/src/opensak/lang/en.py +++ b/src/opensak/lang/en.py @@ -1106,6 +1106,38 @@ "maint_compact_total": "Total: {before} → {after}.", "maint_backup_failed": "Nothing was compacted:\n{error}", "maint_backup_done": "Backup made: {path}", + "maint_btn_repair": "Repair…", + "maint_tip_repair": "Checks the data (rows without a cache, broken links, invalid values) and fixes what can be fixed safely.", + "repair_title": "Repair — {name}", + "repair_intro": "Checks the data of the database. The ticked problems are fixed. Problems marked \"Report only\" have no safe automatic fix: check those caches yourself.", + "repair_col_check": "Check", + "repair_col_count": "Problems", + "repair_col_fix": "Fix", + "repair_col_examples": "Examples", + "repair_chk_orphans": "Rows in {table} whose cache no longer exists", + "repair_chk_parent_link": "Custom waypoints linked to a cache that isn't in the database", + "repair_chk_waypoint_parent": "Waypoints with the wrong parent GC code", + "repair_chk_corrected_flag": "Caches marked as corrected without corrected coordinates", + "repair_chk_companion_orphans": "Rows in the companion database whose cache no longer exists", + "repair_chk_coords": "Coordinates missing or out of range", + "repair_chk_corrected_coords": "Corrected coordinates out of range", + "repair_chk_ratings": "Difficulty or terrain not between 1 and 5 in steps of 0.5", + "repair_chk_identity": "Empty GC code or name", + "repair_fix_delete": "Delete", + "repair_fix_clear_link": "Clear the link", + "repair_fix_set_parent": "Set the cache's code", + "repair_fix_clear_flag": "Clear the mark", + "repair_report_only": "Report only", + "repair_btn_repair": "Repair", + "repair_btn_rescan": "Scan again", + "repair_step_scan": "Checking the data…", + "repair_step_fix": "Fixing the ticked problems…", + "repair_scan_clean": "No problems found.", + "repair_scan_summary": "{problems} check(s) found problems, {fixable} of them can be fixed here.", + "repair_damaged": "The database file itself is damaged, and SQLite can't repair it.", + "repair_fixed": "Fixed {count}: {check}", + "repair_nothing_fixed": "Nothing was changed.", + "repair_failed": "Repair failed, nothing was changed:\n{error}", "maint_cleanup_done": "{file}: removed {orphans} leftover row(s) and {registry} registry entries.", "maint_cleanup_nothing": "{file}: nothing to clean up.", "maint_clear_confirm": "Delete all {rows} rows of the table {table}?\nOwner: {owner}\n\nThe table stays. The rows can only be brought back from a backup.", diff --git a/src/opensak/lang/es.py b/src/opensak/lang/es.py index 29d4a26c..743e6140 100644 --- a/src/opensak/lang/es.py +++ b/src/opensak/lang/es.py @@ -1108,6 +1108,38 @@ "maint_compact_total": "Total: {before} → {after}.", "maint_backup_failed": "No se compactó nada:\n{error}", "maint_backup_done": "Copia de seguridad hecha: {path}", + "maint_btn_repair": "Reparar…", + "maint_tip_repair": "Comprueba los datos (filas sin caché, enlaces rotos, valores no válidos) y corrige lo que se puede corregir de forma segura.", + "repair_title": "Reparar — {name}", + "repair_intro": "Comprueba los datos de la base de datos. Se corrigen los problemas marcados. Los problemas con \"Solo informar\" no tienen una corrección automática segura: revise esos cachés usted mismo.", + "repair_col_check": "Comprobación", + "repair_col_count": "Problemas", + "repair_col_fix": "Corrección", + "repair_col_examples": "Ejemplos", + "repair_chk_orphans": "Filas en {table} cuyo caché ya no existe", + "repair_chk_parent_link": "Waypoints personalizados vinculados a un caché que no está en la base de datos", + "repair_chk_waypoint_parent": "Waypoints con el código GC principal incorrecto", + "repair_chk_corrected_flag": "Cachés marcados como corregidos sin coordenadas corregidas", + "repair_chk_companion_orphans": "Filas en la base de datos complementaria cuyo caché ya no existe", + "repair_chk_coords": "Coordenadas que faltan o fuera de rango", + "repair_chk_corrected_coords": "Coordenadas corregidas fuera de rango", + "repair_chk_ratings": "Dificultad o terreno no entre 1 y 5 en pasos de 0,5", + "repair_chk_identity": "Código GC o nombre vacío", + "repair_fix_delete": "Eliminar", + "repair_fix_clear_link": "Quitar el vínculo", + "repair_fix_set_parent": "Poner el código del caché", + "repair_fix_clear_flag": "Quitar la marca", + "repair_report_only": "Solo informar", + "repair_btn_repair": "Reparar", + "repair_btn_rescan": "Comprobar de nuevo", + "repair_step_scan": "Comprobando los datos…", + "repair_step_fix": "Corrigiendo los problemas marcados…", + "repair_scan_clean": "No se encontraron problemas.", + "repair_scan_summary": "{problems} comprobación(es) encontraron problemas, {fixable} se pueden corregir aquí.", + "repair_damaged": "El propio archivo de la base de datos está dañado y SQLite no puede repararlo.", + "repair_fixed": "Corregido {count}: {check}", + "repair_nothing_fixed": "No se cambió nada.", + "repair_failed": "La reparación falló, no se cambió nada:\n{error}", "maint_cleanup_done": "{file}: eliminadas {orphans} fila(s) sobrantes y {registry} entradas del registro.", "maint_cleanup_nothing": "{file}: nada que limpiar.", "maint_clear_confirm": "¿Borrar las {rows} filas de la tabla {table}?\nPropietario: {owner}\n\nLa tabla se queda. Las filas solo se pueden recuperar desde una copia de seguridad.", diff --git a/src/opensak/lang/fr.py b/src/opensak/lang/fr.py index 94dd43ad..59fe588b 100644 --- a/src/opensak/lang/fr.py +++ b/src/opensak/lang/fr.py @@ -1108,6 +1108,38 @@ "maint_compact_total": "Total : {before} → {after}.", "maint_backup_failed": "Rien n'a été compacté :\n{error}", "maint_backup_done": "Sauvegarde faite : {path}", + "maint_btn_repair": "Réparer…", + "maint_tip_repair": "Vérifie les données (lignes sans cache, liens rompus, valeurs non valides) et corrige ce qui peut l'être sans risque.", + "repair_title": "Réparer — {name}", + "repair_intro": "Vérifie les données de la base. Les problèmes cochés sont corrigés. Les problèmes marqués « Signaler seulement » n'ont pas de correction automatique sûre : vérifiez ces caches vous-même.", + "repair_col_check": "Vérification", + "repair_col_count": "Problèmes", + "repair_col_fix": "Correction", + "repair_col_examples": "Exemples", + "repair_chk_orphans": "Lignes de {table} dont la cache n'existe plus", + "repair_chk_parent_link": "Waypoints personnalisés liés à une cache absente de la base", + "repair_chk_waypoint_parent": "Waypoints avec un mauvais code GC parent", + "repair_chk_corrected_flag": "Caches marquées comme corrigées sans coordonnées corrigées", + "repair_chk_companion_orphans": "Lignes de la base compagnon dont la cache n'existe plus", + "repair_chk_coords": "Coordonnées manquantes ou hors limites", + "repair_chk_corrected_coords": "Coordonnées corrigées hors limites", + "repair_chk_ratings": "Difficulté ou terrain hors de 1 à 5 ou pas par pas de 0,5", + "repair_chk_identity": "Code GC ou nom vide", + "repair_fix_delete": "Supprimer", + "repair_fix_clear_link": "Retirer le lien", + "repair_fix_set_parent": "Mettre le code de la cache", + "repair_fix_clear_flag": "Retirer la marque", + "repair_report_only": "Signaler seulement", + "repair_btn_repair": "Réparer", + "repair_btn_rescan": "Vérifier à nouveau", + "repair_step_scan": "Vérification des données…", + "repair_step_fix": "Correction des problèmes cochés…", + "repair_scan_clean": "Aucun problème trouvé.", + "repair_scan_summary": "{problems} vérification(s) ont trouvé des problèmes, {fixable} peuvent être corrigées ici.", + "repair_damaged": "Le fichier de la base lui-même est endommagé et SQLite ne peut pas le réparer.", + "repair_fixed": "Corrigé {count} : {check}", + "repair_nothing_fixed": "Rien n'a été modifié.", + "repair_failed": "La réparation a échoué, rien n'a été modifié :\n{error}", "maint_cleanup_done": "{file} : {orphans} ligne(s) restante(s) et {registry} entrées du registre supprimées.", "maint_cleanup_nothing": "{file} : rien à nettoyer.", "maint_clear_confirm": "Supprimer les {rows} lignes de la table {table} ?\nPropriétaire : {owner}\n\nLa table reste. Les lignes ne peuvent être récupérées que depuis une sauvegarde.", diff --git a/src/opensak/lang/nl.py b/src/opensak/lang/nl.py index 6f629247..7c0dc0c2 100644 --- a/src/opensak/lang/nl.py +++ b/src/opensak/lang/nl.py @@ -1110,6 +1110,38 @@ "maint_compact_total": "Totaal: {before} → {after}.", "maint_backup_failed": "Er is niets gecomprimeerd:\n{error}", "maint_backup_done": "Back-up gemaakt: {path}", + "maint_btn_repair": "Repareren…", + "maint_tip_repair": "Controleert de gegevens (rijen zonder cache, verbroken koppelingen, ongeldige waarden) en herstelt wat veilig hersteld kan worden.", + "repair_title": "Repareren — {name}", + "repair_intro": "Controleert de gegevens van de database. De aangevinkte problemen worden hersteld. Problemen met \"Alleen melden\" hebben geen veilige automatische oplossing: controleer die caches zelf.", + "repair_col_check": "Controle", + "repair_col_count": "Problemen", + "repair_col_fix": "Oplossing", + "repair_col_examples": "Voorbeelden", + "repair_chk_orphans": "Rijen in {table} waarvan de cache niet meer bestaat", + "repair_chk_parent_link": "Eigen waypoints gekoppeld aan een cache die niet in de database staat", + "repair_chk_waypoint_parent": "Waypoints met de verkeerde bovenliggende GC-code", + "repair_chk_corrected_flag": "Caches gemarkeerd als gecorrigeerd zonder gecorrigeerde coördinaten", + "repair_chk_companion_orphans": "Rijen in de begeleidende database waarvan de cache niet meer bestaat", + "repair_chk_coords": "Coördinaten ontbreken of buiten bereik", + "repair_chk_corrected_coords": "Gecorrigeerde coördinaten buiten bereik", + "repair_chk_ratings": "Moeilijkheid of terrein niet tussen 1 en 5 in stappen van 0,5", + "repair_chk_identity": "Lege GC-code of naam", + "repair_fix_delete": "Verwijderen", + "repair_fix_clear_link": "Koppeling verwijderen", + "repair_fix_set_parent": "Code van de cache instellen", + "repair_fix_clear_flag": "Markering verwijderen", + "repair_report_only": "Alleen melden", + "repair_btn_repair": "Repareren", + "repair_btn_rescan": "Opnieuw controleren", + "repair_step_scan": "Gegevens controleren…", + "repair_step_fix": "Aangevinkte problemen herstellen…", + "repair_scan_clean": "Geen problemen gevonden.", + "repair_scan_summary": "{problems} controle(s) vonden problemen, {fixable} daarvan kunnen hier hersteld worden.", + "repair_damaged": "Het databasebestand zelf is beschadigd en SQLite kan het niet repareren.", + "repair_fixed": "Hersteld {count}: {check}", + "repair_nothing_fixed": "Er is niets gewijzigd.", + "repair_failed": "Repareren mislukt, er is niets gewijzigd:\n{error}", "maint_cleanup_done": "{file}: {orphans} overgebleven rij(en) en {registry} registervermeldingen verwijderd.", "maint_cleanup_nothing": "{file}: niets op te ruimen.", "maint_clear_confirm": "Alle {rows} rijen van de tabel {table} verwijderen?\nEigenaar: {owner}\n\nDe tabel blijft. De rijen zijn alleen terug te halen uit een back-up.", diff --git a/src/opensak/lang/pl.py b/src/opensak/lang/pl.py index d797e5a5..dba80515 100644 --- a/src/opensak/lang/pl.py +++ b/src/opensak/lang/pl.py @@ -1108,6 +1108,38 @@ "maint_compact_total": "Razem: {before} → {after}.", "maint_backup_failed": "Nic nie zostało skompaktowane:\n{error}", "maint_backup_done": "Utworzono kopię zapasową: {path}", + "maint_btn_repair": "Napraw…", + "maint_tip_repair": "Sprawdza dane (wiersze bez skrzynki, zerwane powiązania, nieprawidłowe wartości) i naprawia to, co można bezpiecznie naprawić.", + "repair_title": "Naprawa — {name}", + "repair_intro": "Sprawdza dane bazy. Zaznaczone problemy zostaną naprawione. Problemy oznaczone \"Tylko zgłoś\" nie mają bezpiecznej automatycznej naprawy: sprawdź te skrzynki samodzielnie.", + "repair_col_check": "Sprawdzenie", + "repair_col_count": "Problemy", + "repair_col_fix": "Naprawa", + "repair_col_examples": "Przykłady", + "repair_chk_orphans": "Wiersze w {table}, których skrzynka już nie istnieje", + "repair_chk_parent_link": "Własne waypointy powiązane ze skrzynką, której nie ma w bazie", + "repair_chk_waypoint_parent": "Waypointy z błędnym nadrzędnym kodem GC", + "repair_chk_corrected_flag": "Skrzynki oznaczone jako skorygowane bez skorygowanych współrzędnych", + "repair_chk_companion_orphans": "Wiersze w bazie towarzyszącej, których skrzynka już nie istnieje", + "repair_chk_coords": "Brak współrzędnych lub poza zakresem", + "repair_chk_corrected_coords": "Skorygowane współrzędne poza zakresem", + "repair_chk_ratings": "Trudność lub teren nie od 1 do 5 w krokach co 0,5", + "repair_chk_identity": "Pusty kod GC lub nazwa", + "repair_fix_delete": "Usuń", + "repair_fix_clear_link": "Usuń powiązanie", + "repair_fix_set_parent": "Ustaw kod skrzynki", + "repair_fix_clear_flag": "Usuń oznaczenie", + "repair_report_only": "Tylko zgłoś", + "repair_btn_repair": "Napraw", + "repair_btn_rescan": "Sprawdź ponownie", + "repair_step_scan": "Sprawdzanie danych…", + "repair_step_fix": "Naprawianie zaznaczonych problemów…", + "repair_scan_clean": "Nie znaleziono problemów.", + "repair_scan_summary": "Sprawdzenia z problemami: {problems}, w tym do naprawy tutaj: {fixable}.", + "repair_damaged": "Sam plik bazy jest uszkodzony i SQLite nie może go naprawić.", + "repair_fixed": "Naprawiono {count}: {check}", + "repair_nothing_fixed": "Nic nie zostało zmienione.", + "repair_failed": "Naprawa nie powiodła się, nic nie zostało zmienione:\n{error}", "maint_cleanup_done": "{file}: usunięto pozostałe wiersze ({orphans}) i wpisy rejestru ({registry}).", "maint_cleanup_nothing": "{file}: nie ma czego czyścić.", "maint_clear_confirm": "Usunąć wszystkie wiersze ({rows}) tabeli {table}?\nWłaściciel: {owner}\n\nTabela zostaje. Wiersze można odzyskać tylko z kopii zapasowej.", diff --git a/src/opensak/lang/pt.py b/src/opensak/lang/pt.py index 83337881..b1ef5ff5 100644 --- a/src/opensak/lang/pt.py +++ b/src/opensak/lang/pt.py @@ -1107,6 +1107,38 @@ "maint_compact_total": "Total: {before} → {after}.", "maint_backup_failed": "Nada foi compactado:\n{error}", "maint_backup_done": "Cópia de segurança feita: {path}", + "maint_btn_repair": "Reparar…", + "maint_tip_repair": "Verifica os dados (linhas sem cache, ligações quebradas, valores inválidos) e corrige o que pode ser corrigido com segurança.", + "repair_title": "Reparar — {name}", + "repair_intro": "Verifica os dados da base de dados. Os problemas marcados são corrigidos. Os problemas com \"Só reportar\" não têm uma correção automática segura: verifique essas caches.", + "repair_col_check": "Verificação", + "repair_col_count": "Problemas", + "repair_col_fix": "Correção", + "repair_col_examples": "Exemplos", + "repair_chk_orphans": "Linhas em {table} cuja cache já não existe", + "repair_chk_parent_link": "Waypoints personalizados ligados a uma cache que não está na base de dados", + "repair_chk_waypoint_parent": "Waypoints com o código GC principal errado", + "repair_chk_corrected_flag": "Caches marcadas como corrigidas sem coordenadas corrigidas", + "repair_chk_companion_orphans": "Linhas na base de dados complementar cuja cache já não existe", + "repair_chk_coords": "Coordenadas em falta ou fora do intervalo", + "repair_chk_corrected_coords": "Coordenadas corrigidas fora do intervalo", + "repair_chk_ratings": "Dificuldade ou terreno fora de 1 a 5 em passos de 0,5", + "repair_chk_identity": "Código GC ou nome vazio", + "repair_fix_delete": "Eliminar", + "repair_fix_clear_link": "Remover a ligação", + "repair_fix_set_parent": "Definir o código da cache", + "repair_fix_clear_flag": "Remover a marca", + "repair_report_only": "Só reportar", + "repair_btn_repair": "Reparar", + "repair_btn_rescan": "Verificar de novo", + "repair_step_scan": "A verificar os dados…", + "repair_step_fix": "A corrigir os problemas marcados…", + "repair_scan_clean": "Nenhum problema encontrado.", + "repair_scan_summary": "{problems} verificação(ões) encontraram problemas, {fixable} podem ser corrigidas aqui.", + "repair_damaged": "O próprio ficheiro da base de dados está danificado e o SQLite não o consegue reparar.", + "repair_fixed": "Corrigido {count}: {check}", + "repair_nothing_fixed": "Nada foi alterado.", + "repair_failed": "A reparação falhou, nada foi alterado:\n{error}", "maint_cleanup_done": "{file}: removidas {orphans} linha(s) que sobravam e {registry} entradas do registo.", "maint_cleanup_nothing": "{file}: nada para limpar.", "maint_clear_confirm": "Apagar as {rows} linhas da tabela {table}?\nDono: {owner}\n\nA tabela fica. As linhas só podem ser recuperadas a partir de uma cópia de segurança.", diff --git a/src/opensak/lang/se.py b/src/opensak/lang/se.py index 63247692..9a76f2d8 100644 --- a/src/opensak/lang/se.py +++ b/src/opensak/lang/se.py @@ -1107,6 +1107,38 @@ "maint_compact_total": "Totalt: {before} → {after}.", "maint_backup_failed": "Inget komprimerades:\n{error}", "maint_backup_done": "Säkerhetskopia gjord: {path}", + "maint_btn_repair": "Reparera…", + "maint_tip_repair": "Kontrollerar data (rader utan cache, trasiga länkar, ogiltiga värden) och rättar det som kan rättas säkert.", + "repair_title": "Reparera — {name}", + "repair_intro": "Kontrollerar databasens data. De markerade problemen rättas. Problem märkta \"Endast rapport\" har ingen säker automatisk rättning: kontrollera de cacherna själv.", + "repair_col_check": "Kontroll", + "repair_col_count": "Problem", + "repair_col_fix": "Rättning", + "repair_col_examples": "Exempel", + "repair_chk_orphans": "Rader i {table} vars cache inte längre finns", + "repair_chk_parent_link": "Egna waypoints länkade till en cache som inte finns i databasen", + "repair_chk_waypoint_parent": "Waypoints med fel överordnad GC-kod", + "repair_chk_corrected_flag": "Cacher markerade som korrigerade utan korrigerade koordinater", + "repair_chk_companion_orphans": "Rader i följedatabasen vars cache inte längre finns", + "repair_chk_coords": "Koordinater saknas eller utanför giltigt område", + "repair_chk_corrected_coords": "Korrigerade koordinater utanför giltigt område", + "repair_chk_ratings": "Svårighet eller terräng inte mellan 1 och 5 i steg om 0,5", + "repair_chk_identity": "Tom GC-kod eller namn", + "repair_fix_delete": "Ta bort", + "repair_fix_clear_link": "Ta bort länken", + "repair_fix_set_parent": "Sätt cachens kod", + "repair_fix_clear_flag": "Ta bort markeringen", + "repair_report_only": "Endast rapport", + "repair_btn_repair": "Reparera", + "repair_btn_rescan": "Kontrollera igen", + "repair_step_scan": "Kontrollerar data…", + "repair_step_fix": "Rättar de markerade problemen…", + "repair_scan_clean": "Inga problem hittades.", + "repair_scan_summary": "{problems} kontroll(er) hittade problem, {fixable} av dem kan rättas här.", + "repair_damaged": "Själva databasfilen är skadad och SQLite kan inte reparera den.", + "repair_fixed": "Rättat {count}: {check}", + "repair_nothing_fixed": "Inget ändrades.", + "repair_failed": "Reparationen misslyckades, inget ändrades:\n{error}", "maint_cleanup_done": "{file}: tog bort {orphans} överbliven(a) rad(er) och {registry} registerposter.", "maint_cleanup_nothing": "{file}: inget att städa upp.", "maint_clear_confirm": "Radera alla {rows} rader i tabellen {table}?\nÄgare: {owner}\n\nTabellen finns kvar. Raderna kan bara hämtas tillbaka från en säkerhetskopia.", diff --git a/tests/unit-tests/test_db_repair_1051.py b/tests/unit-tests/test_db_repair_1051.py new file mode 100644 index 00000000..639b1f91 --- /dev/null +++ b/tests/unit-tests/test_db_repair_1051.py @@ -0,0 +1,351 @@ +""" +tests/unit-tests/test_db_repair_1051.py — database repair, phase 1 (#1051): +the data checks of db/repair.py on a database with deliberately broken +rows, the fixes, a second run that finds nothing, and the Repair dialog +opened from the Maintenance dialog. +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +pytest.importorskip("pytestqt") + +from opensak.db import companion, maintenance, repair +from opensak.db.database import dispose_engine, get_session, init_db +from opensak.db.models import Cache, Log, UserNote, Waypoint +from opensak.gui.dialogs import database_maintenance_dialog as dmd +from opensak.gui.dialogs import database_repair_dialog as drd +from opensak.lang import load_language, tr +from opensak.utils import flags + +USER_DATE = "2025-01-02 03:04:05.000000" + + +@pytest.fixture(autouse=True) +def _fresh(monkeypatch): + load_language("en") + monkeypatch.setattr(flags, "companion_db", True) + companion.reset() + yield + dispose_engine() + companion.reset() + + +def _healthy(path: Path) -> Path: + """A database without problems: two caches, a custom waypoint linked to + GC1, a waypoint, a log and a correction to the listed coordinates.""" + path.parent.mkdir(parents=True, exist_ok=True) + init_db(db_path=path) + with get_session() as s: + gc1 = Cache(gc_code="GC1", name="One", cache_type="Traditional Cache", + latitude=47.0, longitude=8.0, difficulty=1.5, terrain=5.0) + gc2 = Cache(gc_code="GC2", name="Two", cache_type="Mystery Cache", + latitude=47.1, longitude=8.1, difficulty=2.0, terrain=2.0) + cw = Cache(gc_code="CW1", name="Parking", cache_type="Custom", + latitude=47.2, longitude=8.2, parent_gc_code="GC1") + s.add_all([gc1, gc2, cw]) + s.flush() + s.add(Waypoint(cache_id=gc1.id, parent_gc_code="GC1", prefix="PK", name="Parking")) + s.add(Log(cache_id=gc1.id, log_id="1", log_type="Found it", finder="me")) + # Corrected to the listed coordinates: a real choice, not a problem. + s.add(UserNote(cache_id=gc2.id, corrected_lat=47.1, corrected_lon=8.1, + is_corrected=True)) + dispose_engine(path) + return path + + +def _broken(path: Path) -> Path: + """_healthy() plus one problem of every kind.""" + _healthy(path) + conn = sqlite3.connect(str(path)) + try: + conn.execute("PRAGMA foreign_keys=OFF") + gc1 = conn.execute("SELECT id FROM caches WHERE gc_code = 'GC1'").fetchone()[0] + gc2 = conn.execute("SELECT id FROM caches WHERE gc_code = 'GC2'").fetchone()[0] + conn.executescript(f""" + INSERT INTO logs (cache_id, log_id, log_type) VALUES (9999, 'x1', 'Note'); + INSERT INTO logs (cache_id, log_id, log_type) VALUES (9999, 'x2', 'Note'); + INSERT INTO waypoints (cache_id, prefix, name) VALUES (9999, 'S1', 'Stage'); + INSERT INTO attributes (cache_id, attribute_id) VALUES (9998, 1); + INSERT INTO trackables (cache_id, name) VALUES (9998, 'TB'); + INSERT INTO user_notes (cache_id, note) VALUES (9997, 'gone'); + INSERT INTO caches (gc_code, name, cache_type, latitude, longitude, parent_gc_code) + VALUES ('CW2', 'Lost link', 'Custom', 47.3, 8.3, 'gc404'); + UPDATE waypoints SET parent_gc_code = 'GC2' WHERE cache_id = {gc1}; + UPDATE user_notes SET corrected_lat = NULL WHERE cache_id = {gc2}; + INSERT INTO caches (gc_code, name, cache_type, latitude, longitude, + difficulty, terrain) + VALUES ('GC3', 'Bad', 'Traditional Cache', 95.0, 8.0, 1.3, 6.0); + INSERT INTO caches (gc_code, name, cache_type, latitude, longitude) + VALUES ('GC4', ' ', 'Traditional Cache', 47.0, 8.0); + """) + conn.execute("UPDATE caches SET last_user_update = ? WHERE id = ?", (USER_DATE, gc2)) + conn.commit() + finally: + conn.close() + return path + + +def _counts(result: repair.ScanResult) -> dict[str, int]: + return {f.key: f.count for f in result.problems} + + +class TestScan: + def test_healthy_database_has_no_problems(self, tmp_path): + result = repair.scan(_healthy(tmp_path / "Ok.db"), with_companion=True) + assert not result.damaged + assert result.problems == () + + def test_finds_every_problem(self, tmp_path): + result = repair.scan(_broken(tmp_path / "Broken.db")) + assert _counts(result) == { + "orphans:logs": 2, "orphans:waypoints": 1, "orphans:attributes": 1, + "orphans:trackables": 1, "orphans:user_notes": 1, + repair.PARENT_LINK: 1, repair.WAYPOINT_PARENT: 1, repair.CORRECTED_FLAG: 1, + repair.COORDS: 1, repair.RATINGS: 1, repair.IDENTITY: 1, + } + by_key = {f.key: f for f in result.findings} + assert by_key[repair.PARENT_LINK].examples == ("CW2",) + assert by_key[repair.WAYPOINT_PARENT].examples == ("GC1",) + assert by_key[repair.CORRECTED_FLAG].examples == ("GC2",) + assert by_key[repair.RATINGS].examples == ("GC3",) + assert by_key[repair.IDENTITY].examples == ("GC4",) + assert not by_key[repair.COORDS].fixable + assert {f.key for f in result.fixable} == { + "orphans:logs", "orphans:waypoints", "orphans:attributes", "orphans:trackables", + "orphans:user_notes", repair.PARENT_LINK, repair.WAYPOINT_PARENT, + repair.CORRECTED_FLAG, + } + + def test_ratings_in_half_steps_are_fine(self, tmp_path): + db = _healthy(tmp_path / "Ok.db") + conn = sqlite3.connect(str(db)) + with conn: + conn.execute("UPDATE caches SET difficulty = 4.5, terrain = 1.0 WHERE gc_code = 'GC1'") + conn.close() + assert repair.RATINGS not in _counts(repair.scan(db)) + + def test_corrected_coordinates_out_of_range(self, tmp_path): + db = _healthy(tmp_path / "Ok.db") + conn = sqlite3.connect(str(db)) + with conn: + conn.execute("UPDATE user_notes SET corrected_lon = 200") + conn.close() + assert _counts(repair.scan(db)) == {repair.CORRECTED_COORDS: 1} + + def test_examples_are_limited(self, tmp_path): + db = _healthy(tmp_path / "Ok.db") + conn = sqlite3.connect(str(db)) + with conn: + for n in range(repair.MAX_EXAMPLES + 3): + conn.execute("INSERT INTO caches (gc_code, name, cache_type, latitude, longitude, " + f"difficulty) VALUES ('GCX{n}', 'x', 'T', 1, 1, 7)") + conn.close() + finding = next(f for f in repair.scan(db).findings if f.check == repair.RATINGS) + assert finding.count == repair.MAX_EXAMPLES + 3 + assert len(finding.examples) == repair.MAX_EXAMPLES + + def test_damaged_file_skips_the_data_checks(self, tmp_path): + db = _healthy(tmp_path / "Ok.db") + with patch.object(repair, "_connect") as connect: + connect.return_value.execute.return_value.fetchall.return_value = [ + ("*** in database main ***",)] + result = repair.scan(db) + assert result.damaged == ("*** in database main ***",) + assert result.findings == () + + def test_companion_orphans(self, tmp_path): + db = _healthy(tmp_path / "Ok.db") + companion.create(db) + conn = sqlite3.connect(str(companion.companion_path(db))) + with conn: + conn.execute("CREATE TABLE notes (gc_code TEXT PRIMARY KEY, note TEXT)") + conn.execute("INSERT INTO notes VALUES ('GC1', 'kept'), ('GC404', 'orphan')") + conn.close() + result = repair.scan(db, with_companion=True) + assert _counts(result) == {repair.COMPANION_ORPHANS: 1} + assert repair.repair(db, {repair.COMPANION_ORPHANS}).fixed == {repair.COMPANION_ORPHANS: 1} + assert repair.scan(db, with_companion=True).problems == () + + +class TestRepair: + def test_fixes_everything_and_a_second_run_finds_nothing_fixable(self, tmp_path): + db = _broken(tmp_path / "Broken.db") + keys = {f.key for f in repair.scan(db).fixable} + result = repair.repair(db, keys) + assert result.fixed == { + "orphans:logs": 2, "orphans:waypoints": 1, "orphans:attributes": 1, + "orphans:trackables": 1, "orphans:user_notes": 1, + repair.PARENT_LINK: 1, repair.WAYPOINT_PARENT: 1, repair.CORRECTED_FLAG: 1, + } + again = repair.scan(db) + assert again.fixable == () + # Report-only problems are left for the user. + assert set(_counts(again)) == {repair.COORDS, repair.RATINGS, repair.IDENTITY} + assert maintenance.check_file(db).ok + assert repair.repair(db, keys).fixed == {} + + def test_keeps_the_good_rows(self, tmp_path): + db = _broken(tmp_path / "Broken.db") + repair.repair(db, {f.key for f in repair.scan(db).fixable}) + init_db(db_path=db) + with get_session() as s: + gc1 = s.query(Cache).filter_by(gc_code="GC1").one() + assert [lg.log_id for lg in gc1.logs] == ["1"] + assert [w.parent_gc_code for w in gc1.waypoints] == ["GC1"] + assert s.query(Cache).filter_by(gc_code="CW1").one().parent_gc_code == "GC1" + assert s.query(Cache).filter_by(gc_code="CW2").one().parent_gc_code is None + gc2 = s.query(Cache).filter_by(gc_code="GC2").one() + assert gc2.user_note is not None and not gc2.user_note.is_corrected + assert gc2.user_note.corrected_source is None + + def test_clearing_the_flag_keeps_last_user_update(self, tmp_path): + db = _broken(tmp_path / "Broken.db") + repair.repair(db, {repair.CORRECTED_FLAG}) + conn = sqlite3.connect(str(db)) + stamp = conn.execute( + "SELECT last_user_update FROM caches WHERE gc_code = 'GC2'").fetchone()[0] + conn.close() + assert stamp == USER_DATE + + def test_only_the_chosen_fixes(self, tmp_path): + db = _broken(tmp_path / "Broken.db") + result = repair.repair(db, {"orphans:logs", repair.COORDS}) + assert result.fixed == {"orphans:logs": 2} + counts = _counts(repair.scan(db)) + assert "orphans:logs" not in counts + assert counts["orphans:waypoints"] == 1 + assert counts[repair.COORDS] == 1 + + def test_a_failure_changes_nothing(self, tmp_path): + db = _broken(tmp_path / "Broken.db") + with patch.object(repair, "_clear_corrected_flags", side_effect=sqlite3.OperationalError): + with pytest.raises(sqlite3.OperationalError): + repair.repair(db, {"orphans:logs", repair.CORRECTED_FLAG}) + assert _counts(repair.scan(db))["orphans:logs"] == 2 + + +# ── The dialog's work and text ──────────────────────────────────────────────── + +class TestRunRepair: + def test_backup_fix_and_summary(self, tmp_path): + db = _broken(tmp_path / "Broken.db") + keys = {f.key for f in repair.scan(db).fixable} + made = SimpleNamespace(backup_set=SimpleNamespace(path=tmp_path / "set")) + with patch("opensak.backup.backupset.write_backup_set", return_value=made) as write: + outcome = drd.run_repair(SimpleNamespace(name="Broken", path=db), keys, True, + lambda _s: None) + write.assert_called_once() + lines = drd.repair_lines(outcome) + assert lines[0] == tr("maint_backup_done", path=str(tmp_path / "set")) + assert tr("repair_fixed", count=2, check=tr("repair_chk_orphans", table="logs")) in lines + assert outcome.compact is None or outcome.compact.status in ( + maintenance.DONE, maintenance.NOT_WORTHWHILE) + + def test_nothing_to_fix(self, tmp_path): + db = _healthy(tmp_path / "Ok.db") + outcome = drd.run_repair(SimpleNamespace(name="Ok", path=db), set(), False, + lambda _s: None) + assert drd.repair_lines(outcome)[0] == tr("repair_nothing_fixed") + + def test_compacts_when_a_quarter_is_free(self, tmp_path): + db = _healthy(tmp_path / "Ok.db") + conn = sqlite3.connect(str(db)) + with conn: + conn.execute("CREATE TABLE filler (x BLOB)") + for _ in range(60): + conn.execute("INSERT INTO filler VALUES (zeroblob(1024 * 1024))") + with conn: + conn.execute("DROP TABLE filler") + conn.close() + with patch.object(maintenance, "MANUAL_MIN_RECLAIM_BYTES", 0): + outcome = drd.run_repair(SimpleNamespace(name="Ok", path=db), set(), False, + lambda _s: None) + assert outcome.compact is not None and outcome.compact.ran + assert maintenance.file_info(db).stats.free_bytes == 0 + + +# ── The dialog ──────────────────────────────────────────────────────────────── + +def _dialog(qtbot, db: Path) -> drd.DatabaseRepairDialog: + dlg = drd.DatabaseRepairDialog(SimpleNamespace(name=db.stem, path=db)) + qtbot.addWidget(dlg) + qtbot.waitUntil(lambda: not dlg.is_running() and dlg.scan_result is not None, timeout=10000) + return dlg + + +def _rows(dlg) -> dict[str, object]: + out = {} + for i in range(dlg._tree.topLevelItemCount()): + item = dlg._tree.topLevelItem(i) + assert item is not None + out[item.data(0, drd.KEY_ROLE)] = item + return out + + +class TestDialog: + def test_lists_the_problems_and_ticks_the_fixable_ones(self, qtbot, tmp_path): + dlg = _dialog(qtbot, _broken(tmp_path / "Broken.db")) + rows = _rows(dlg) + assert rows["orphans:logs"].checkState(0) == drd.Qt.CheckState.Checked + report = rows[repair.COORDS] + assert not report.flags() & drd.Qt.ItemFlag.ItemIsUserCheckable + assert report.text(2) == tr("repair_report_only") + assert "GC3" in rows[repair.COORDS].text(3) + assert dlg._backup.isChecked() + assert dlg._btn_repair.isEnabled() + assert dlg.checked_keys() == {f.key for f in dlg.scan_result.fixable} + + def test_unticking_everything_disables_repair(self, qtbot, tmp_path): + dlg = _dialog(qtbot, _broken(tmp_path / "Broken.db")) + for item in _rows(dlg).values(): + if item.flags() & drd.Qt.ItemFlag.ItemIsUserCheckable: + item.setCheckState(0, drd.Qt.CheckState.Unchecked) + assert not dlg._btn_repair.isEnabled() + + def test_healthy_database(self, qtbot, tmp_path): + dlg = _dialog(qtbot, _healthy(tmp_path / "Ok.db")) + assert dlg._tree.topLevelItemCount() == 0 + assert dlg._summary.text() == tr("repair_scan_clean") + assert not dlg._btn_repair.isEnabled() + + def test_repair_then_scans_again(self, qtbot, tmp_path): + db = _broken(tmp_path / "Broken.db") + init_db(db_path=db) # the active database + dlg = _dialog(qtbot, db) + dlg._backup.setChecked(False) + dlg._repair() + qtbot.waitUntil(lambda: not dlg.is_running() and dlg.scan_result is not None + and not dlg.scan_result.fixable, timeout=20000) + assert tr("repair_fixed", count=2, check=tr("repair_chk_orphans", table="logs")) \ + in dlg._results.toPlainText() + assert set(_rows(dlg)) == {repair.COORDS, repair.RATINGS, repair.IDENTITY} + with get_session() as s: # the active database is still usable + assert s.query(Log).count() == 1 + + def test_damaged_file_points_to_restore(self, qtbot, tmp_path): + db = _healthy(tmp_path / "Ok.db") + damaged = repair.ScanResult(db, damaged=("page 3 is never used",)) + with patch.object(repair, "scan", return_value=damaged): + dlg = _dialog(qtbot, db) + assert dlg._summary.text().startswith(tr("repair_damaged")) + assert tr("maint_check_main_failed") in dlg._summary.text() + assert "page 3 is never used" in dlg._results.toPlainText() + assert not dlg._btn_repair.isEnabled() + + def test_maintenance_dialog_opens_repair(self, qtbot, tmp_path): + db = _healthy(tmp_path / "Ok.db") + dlg = dmd.DatabaseMaintenanceDialog(SimpleNamespace(name="Ok", path=db)) + qtbot.addWidget(dlg) + qtbot.waitUntil(lambda: not dlg.is_running() and dlg._info.main is not None, + timeout=10000) + with patch.object(drd.DatabaseRepairDialog, "exec", return_value=0) as run: + dlg._btn_repair.click() + run.assert_called_once() + qtbot.waitUntil(lambda: not dlg.is_running(), timeout=10000) From b63d77bcdf4ad9ef01e5962af584d1c8ca214c4e Mon Sep 17 00:00:00 2001 From: nagisml Date: Wed, 18 Nov 2026 18:42:25 +0100 Subject: [PATCH 2/3] Check and repair dates in the database Repair dialog Phase 2 of #1051. SQLite has no date type. SQLAlchemy stores a DateTime as text ("YYYY-MM-DD HH:MM:SS.ffffff") and reads it back with datetime.fromisoformat(), in both 2.0 and 2.1. If a value can't be read, loading the cache, its logs or its waypoints fails. A value in another format may still load, but it compares wrongly as text, and OpenSAK's filters and sorting compare dates as text. A DATETIME column has NUMERIC affinity, so "20240305" is stored as the integer 20240305. OpenSAK itself always writes the stored format: the ORM, the GSAK fast path through Core, the triggers and the macros' cache writes. Other formats only come from raw SQL in macros or from other tools. Every DATETIME column of the database is checked, so new columns are covered too. Values in the stored format are recognized in SQL, and only the other values are read into Python. New checks: - Dates in a format OpenSAK doesn't use (fixable): converted to the stored format. This covers ISO dates with or without a time, a "T" or an offset; one-digit month or day; "/" or "." as separator; DD.MM.YYYY; and an 8-digit YYYYMMDD number. An offset is converted to UTC, as the GPX importer does (#617), so a repair and a re-import store the same value. - Dates that can't be read (fixable): cleared. This covers impossible dates such as 2024-02-30 or 0000-00-00, empty text, other numbers and blobs. The two NOT NULL bookkeeping columns, caches.imported_at and user_notes.updated_at, are set to now, as their database default does. - Placeholder date 1970-01-01 (fixable): cleared. GSAK stores the Unix epoch as "no date" (Waypoints.cDate), and the GSAK import copies it. A real 150k-cache database had 792 such waypoints on 318 caches, exactly as many as in GSAK. - Dates before May 2000 or more than a day in the future (report only). The placed date of an event or a GPS Adventures Maze exhibit may be in the future, because it is the day the event or exhibit takes place (e.g. GCBGCCC). Each affected row is now listed with its GC code, the field (for a waypoint also its prefix, e.g. "waypoints.wp_date S1") and the value as stored. This applies to every check, so report-only problems can be found and fixed by hand. Each check keeps up to 1000 rows; the count is always complete. The examples cell shows the first five, and its tooltip shows twenty. "Copy list", the context menu and Ctrl+C copy them for the selected rows, or all rows, as tab-separated text (check, GC code, field, value), ready for a spreadsheet. The new texts are in all language files; the translations are machine-made. Part of #1051 --- src/opensak/db/repair.py | 158 +++++++-- src/opensak/db/repair_dates.py | 304 ++++++++++++++++++ .../gui/dialogs/database_repair_dialog.py | 93 +++++- src/opensak/lang/cs.py | 13 + src/opensak/lang/da.py | 13 + src/opensak/lang/de.py | 13 + src/opensak/lang/de_CH.py | 13 + src/opensak/lang/en.py | 13 + src/opensak/lang/es.py | 13 + src/opensak/lang/fr.py | 13 + src/opensak/lang/nl.py | 13 + src/opensak/lang/pl.py | 13 + src/opensak/lang/pt.py | 13 + src/opensak/lang/se.py | 13 + tests/unit-tests/test_db_repair_1051.py | 65 +++- tests/unit-tests/test_db_repair_dates_1051.py | 284 ++++++++++++++++ 16 files changed, 1003 insertions(+), 44 deletions(-) create mode 100644 src/opensak/db/repair_dates.py create mode 100644 tests/unit-tests/test_db_repair_dates_1051.py diff --git a/src/opensak/db/repair.py b/src/opensak/db/repair.py index ac8c042b..6bb821bf 100644 --- a/src/opensak/db/repair.py +++ b/src/opensak/db/repair.py @@ -1,6 +1,6 @@ """ src/opensak/db/repair.py — check the data of a database and fix what can be -fixed safely, like GSAK's Repair/Defrag (#1051, phase 1). +fixed safely, like GSAK's Repair/Defrag (#1051). #1048 checks the *file* (quick_check, integrity_check, foreign_key_check); this module checks the *data*. ``scan()`` only reads and reports every check @@ -26,8 +26,14 @@ real user choice (the final is at the listed coordinates). * rows of the companion database (#1034) whose cache is gone. Fix: ``companion.cleanup()``. +* dates (phase 2, details in db/repair_dates.py): values SQLAlchemy can't + read, which break loading a cache, or that are in another format, which + sorts and filters wrongly. Fix: convert them to the stored format, or + clear the ones that can't be read as a date at all. GSAK's "no date", + 1970-01-01, is a check of its own. Fix: clear it. * report only: coordinates out of range, difficulty or terrain outside - 1–5 or not in 0.5 steps, an empty GC code or name. + 1–5 or not in 0.5 steps, an empty GC code or name, dates before May 2000 + or in the future. No duplicate check: the unique constraints on attributes, ``logs.log_id`` and waypoints exist in every database (models since v0.2.0, migrations 2 and @@ -48,12 +54,18 @@ import sqlite3 from dataclasses import dataclass, field from pathlib import Path -from typing import Callable, Optional +from typing import Callable, Iterator, Optional + +from opensak.db import repair_dates +from opensak.db.repair_dates import Detail logger = logging.getLogger(__name__) -# GC codes listed per check. +# Affected rows shown in the dialog per check … MAX_EXAMPLES = 5 +# … and kept for "Copy list", so the user can fix report-only problems by +# hand. The count is always complete. +MAX_DETAILS = 1000 # Rows per DELETE … WHERE rowid IN (…), below SQLite's bound-parameter limit. _CHUNK = 500 @@ -67,19 +79,29 @@ CORRECTED_COORDS = "corrected_coords" RATINGS = "ratings" IDENTITY = "identity" +DATE_FORMAT = "date_format" +DATE_UNREADABLE = "date_unreadable" +DATE_PLACEHOLDER = "date_placeholder" +DATE_RANGE = "date_range" @dataclass(frozen=True) class Finding: """One check's result. *table* is set for the orphan checks (one finding - per table). *fixable* False: report only.""" + per table). *fixable* False: report only. *details*: the first + MAX_DETAILS affected rows, ordered by GC code (none for orphans, whose + cache is gone).""" check: str count: int fixable: bool - examples: tuple[str, ...] = () + details: tuple[Detail, ...] = () table: Optional[str] = None + @property + def examples(self) -> tuple[str, ...]: + return tuple(d.text for d in self.details[:MAX_EXAMPLES]) + @property def key(self) -> str: """Identifies the finding to repair(): the check, or check:table.""" @@ -129,11 +151,16 @@ def _rating_bad(col: str) -> str: f"OR {col} * 2 <> CAST({col} * 2 AS INTEGER)))") -_REPORT_CHECKS: tuple[tuple[str, str], ...] = ( - (COORDS, _COORDS_WHERE), - (RATINGS, f"{_rating_bad('difficulty')} OR {_rating_bad('terrain')}"), - (IDENTITY, "gc_code IS NULL OR trim(gc_code) = '' OR name IS NULL OR trim(name) = ''"), +# (id, WHERE on caches, field shown, SQL for the value shown) +_REPORT_CHECKS: tuple[tuple[str, str, str, str], ...] = ( + (COORDS, _COORDS_WHERE, "latitude, longitude", + "quote(latitude) || ', ' || quote(longitude)"), + (RATINGS, f"{_rating_bad('difficulty')} OR {_rating_bad('terrain')}", "difficulty / terrain", + "quote(difficulty) || ' / ' || quote(terrain)"), + (IDENTITY, "gc_code IS NULL OR trim(gc_code) = '' OR name IS NULL OR trim(name) = ''", + "gc_code / name", "quote(gc_code) || ' / ' || quote(name)"), ) +_CORRECTED_VALUE = "quote(n.corrected_lat) || ', ' || quote(n.corrected_lon)" _CORRECTED_COORDS_WHERE = ( "n.is_corrected AND n.corrected_lat IS NOT NULL AND n.corrected_lon IS NOT NULL " @@ -179,50 +206,89 @@ def scan(db_path: Path, with_companion: bool = False, findings.append(Finding(ORPHANS, count, True, table=table)) notify(PARENT_LINK) - findings.append(_cache_finding(conn, PARENT_LINK, _PARENT_LINK_WHERE, True)) + findings.append(_cache_finding(conn, PARENT_LINK, _PARENT_LINK_WHERE, True, + "parent_gc_code", "parent_gc_code")) notify(WAYPOINT_PARENT) - count = conn.execute(f"SELECT count(*) {_WAYPOINT_PARENT_FROM}").fetchone()[0] - examples = _codes(conn, f"SELECT DISTINCT c.gc_code {_WAYPOINT_PARENT_FROM} " - "ORDER BY c.gc_code") if count else () - findings.append(Finding(WAYPOINT_PARENT, count, True, examples)) + findings.append(_finding(conn, WAYPOINT_PARENT, True, _WAYPOINT_PARENT_FROM, "c.gc_code", + "waypoints.parent_gc_code", "w.parent_gc_code")) notify(CORRECTED_FLAG) findings.append(_note_finding(conn, CORRECTED_FLAG, _corrected_flag_where("n"), True)) + notify(DATE_FORMAT) + findings += _date_findings(conn) + if with_companion: notify(COMPANION_ORPHANS) findings.append(Finding(COMPANION_ORPHANS, sum(_companion_orphans(db_path).values()), True)) - for check, where in _REPORT_CHECKS: + for check, where, label, value in _REPORT_CHECKS: notify(check) - findings.append(_cache_finding(conn, check, where, False)) + findings.append(_cache_finding(conn, check, where, False, label, value)) notify(CORRECTED_COORDS) findings.append(_note_finding(conn, CORRECTED_COORDS, _CORRECTED_COORDS_WHERE, False)) + notify(DATE_RANGE) + count, dates = repair_dates.out_of_range(conn, MAX_DETAILS) + findings.append(Finding(DATE_RANGE, count, False, tuple(dates))) finally: conn.close() return ScanResult(db_path, tuple(findings)) -def _cache_finding(conn: sqlite3.Connection, check: str, where: str, fixable: bool) -> Finding: - count = conn.execute(f"SELECT count(*) FROM caches WHERE {where}").fetchone()[0] - examples = _codes(conn, f"SELECT gc_code FROM caches WHERE {where} ORDER BY gc_code") \ - if count else () - return Finding(check, count, fixable, examples) +_DATE_CHECKS = (DATE_FORMAT, DATE_UNREADABLE, DATE_PLACEHOLDER) -def _note_finding(conn: sqlite3.Connection, check: str, where: str, fixable: bool) -> Finding: - base = f"FROM user_notes n JOIN caches c ON c.id = n.cache_id WHERE {where}" - count = conn.execute(f"SELECT count(*) {base}").fetchone()[0] - examples = _codes(conn, f"SELECT c.gc_code {base} ORDER BY c.gc_code") if count else () - return Finding(check, count, fixable, examples) +def _date_check(bad: repair_dates.BadDate) -> str: + if bad.fixed == repair_dates.EPOCH: + return DATE_PLACEHOLDER + return DATE_FORMAT if bad.fixed is not None else DATE_UNREADABLE + + +def _all_bad_dates(conn: sqlite3.Connection) -> Iterator[repair_dates.BadDate]: + yield from repair_dates.bad_dates(conn) + yield from repair_dates.placeholder_dates(conn) + + +def _date_findings(conn: sqlite3.Connection) -> list[Finding]: + counts = {check: 0 for check in _DATE_CHECKS} + details: dict[str, list[Detail]] = {check: [] for check in _DATE_CHECKS} + for bad in _all_bad_dates(conn): + check = _date_check(bad) + counts[check] += 1 + kept = details[check] + kept.append(bad.detail) + if len(kept) > 2 * MAX_DETAILS: # keep memory flat on a huge log table + kept.sort() + del kept[MAX_DETAILS:] + return [Finding(check, counts[check], True, tuple(sorted(details[check])[:MAX_DETAILS])) + for check in counts] -def _codes(conn: sqlite3.Connection, sql: str) -> tuple[str, ...]: - rows = conn.execute(f"{sql} LIMIT {MAX_EXAMPLES}").fetchall() - # An empty GC code is one of the problems; show it as "''". - return tuple(str(r[0]) if r[0] not in (None, "") else "''" for r in rows) +def _finding(conn: sqlite3.Connection, check: str, fixable: bool, from_where: str, + code_sql: str, label: str, value_sql: str) -> Finding: + """A finding from "FROM … WHERE …" *from_where*, with the GC code and + value of each affected row from *code_sql* and *value_sql*.""" + count = conn.execute(f"SELECT count(*) {from_where}").fetchone()[0] + rows = conn.execute( + f"SELECT {code_sql}, {value_sql} {from_where} ORDER BY 1 LIMIT {MAX_DETAILS}" + ).fetchall() if count else [] + return Finding(check, count, fixable, + tuple(Detail(repair_dates.shown(code), label, repair_dates.shown(value)) + for code, value in rows)) + + +def _cache_finding(conn: sqlite3.Connection, check: str, where: str, fixable: bool, + label: str, value_sql: str) -> Finding: + return _finding(conn, check, fixable, f"FROM caches WHERE {where}", "gc_code", + label, value_sql) + + +def _note_finding(conn: sqlite3.Connection, check: str, where: str, fixable: bool) -> Finding: + return _finding(conn, check, fixable, + f"FROM user_notes n JOIN caches c ON c.id = n.cache_id WHERE {where}", + "c.gc_code", "corrected", _CORRECTED_VALUE) def _orphan_rowids_by_table(conn: sqlite3.Connection, count_only: bool = False) -> dict: @@ -289,6 +355,9 @@ def repair(db_path: Path, keys: set[str], if CORRECTED_FLAG in keys: notify(CORRECTED_FLAG) result.fixed[CORRECTED_FLAG] = _clear_corrected_flags(conn) + if keys & set(_DATE_CHECKS): + notify(DATE_FORMAT) + result.fixed.update(_fix_dates(conn, keys)) finally: conn.close() if COMPANION_ORPHANS in keys: @@ -303,9 +372,34 @@ def repair(db_path: Path, keys: set[str], return result +def _fix_dates(conn: sqlite3.Connection, keys: set[str]) -> dict[str, int]: + """Convert (DATE_FORMAT) the dates not in the stored format, or clear + the unreadable ones (DATE_UNREADABLE) and GSAK's 1970-01-01 "no date" + (DATE_PLACEHOLDER). A NOT NULL column gets now instead of NULL.""" + fixed = {check: 0 for check in _DATE_CHECKS} + now = repair_dates.now_stored() + for bad in list(_all_bad_dates(conn)): + check = _date_check(bad) + if check not in keys: + continue + if check == DATE_FORMAT: + value: Optional[str] = bad.fixed + else: + value = now if bad.column.not_null else None + conn.execute( + f"UPDATE {_quote(bad.column.table)} SET {_quote(bad.column.column)} = ? " + "WHERE rowid = ?", (value, bad.rowid)) + fixed[check] += 1 + return fixed + + +def _quote(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + def _delete_rowids(conn: sqlite3.Connection, table: str, rowids: list[int]) -> int: removed = 0 - quoted = '"' + table.replace('"', '""') + '"' + quoted = _quote(table) for start in range(0, len(rowids), _CHUNK): chunk = rowids[start:start + _CHUNK] marks = ", ".join("?" * len(chunk)) diff --git a/src/opensak/db/repair_dates.py b/src/opensak/db/repair_dates.py new file mode 100644 index 00000000..caa43e56 --- /dev/null +++ b/src/opensak/db/repair_dates.py @@ -0,0 +1,304 @@ +""" +src/opensak/db/repair_dates.py — the date checks of the database repair +(#1051, phase 2), used by db/repair.py. + +SQLite has no date type: SQLAlchemy stores a DateTime as text in the form +``YYYY-MM-DD HH:MM:SS.ffffff`` and reads it back with +``datetime.fromisoformat()`` (SQLAlchemy 2.0 and 2.1). A value it can't read +(not text, or text fromisoformat rejects) raises while a cache, its logs or +its waypoints are loaded. A value in another format that does load (a bare +date, a "T", an offset) still compares wrongly as text, and OpenSAK's +filters and sorting compare dates as text. A DATETIME column has NUMERIC +affinity, so "20240305" is stored as the integer 20240305. + +Every DATETIME column of the main database is checked, found through +``PRAGMA table_info``, so new columns are covered without changes here. +Values in the stored format are recognised in SQL; only the others are read +into Python. Each of those is either + +* readable: an ISO date or timestamp with or without time, "T" or offset, + one-digit month or day, "/" or "." between the parts, a day-first + "DD.MM.YYYY", or an 8-digit YYYYMMDD number. Fix: convert it to the + stored format. A value with an offset is converted to UTC, as the GPX + importer does (importer._parse_datetime(), #617), so a repair and a + re-import store the same value; or +* unreadable: anything else (impossible dates such as 2024-02-30 or + 0000-00-00, empty text, other numbers, blobs). Fix: clear it, or for the + two NOT NULL bookkeeping columns (``caches.imported_at``, + ``user_notes.updated_at``) set it to now, like their database default. + +1970-01-01 00:00:00 (the Unix epoch) is GSAK's "no date": GSAK's own +Waypoints.cDate holds it, and the GSAK import copies it (792 waypoints in a +real 150k-cache database). It is its own fixable check: clear it, or for a +NOT NULL column set it to now. + +Every affected row is listed with its GC code, the field (for a waypoint +also its prefix, e.g. "waypoints.wp_date S1") and the value as stored. + +Report only: dates before May 2000 (geocaching began on 3 May 2000) or +more than a day in the future, except the placed date of an event or a GPS +Adventures Maze exhibit (DATED_CACHE_TYPES), which is the day it takes +place. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Iterator, Optional + +from opensak.utils.constants import EVENT_LOG_CACHE_TYPES + +# Cache types whose placed date is the day they take place, so it may lie +# in the future: the events, and the GPS Adventures Maze exhibits (logged +# "Found it", so they aren't in EVENT_LOG_CACHE_TYPES). +DATED_CACHE_TYPES: frozenset[str] = EVENT_LOG_CACHE_TYPES | {"GPS Adventures Maze"} + +# Earlier dates are reported: the first geocache was placed on 3 May 2000. +EARLIEST = "2000-05-01" +# The Unix epoch: GSAK stores it as "no date" (e.g. Waypoints.cDate), and +# the GSAK import copies it. Its own fixable check, not "too early". +EPOCH = "1970-01-01 00:00:00.000000" +# Later than now plus this is reported (time zones, clocks a bit ahead). +FUTURE_MARGIN = timedelta(days=1) + +_DATE_TYPES = ("DATETIME", "DATE", "TIMESTAMP") + +_ISO_LOOSE = re.compile( + r"(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})" + r"(?:[ T](\d{1,2}):(\d{1,2})(?::(\d{1,2})(?:[.,](\d{1,6}))?)?)?" +) +_DAY_FIRST = re.compile( + r"(\d{1,2})\.(\d{1,2})\.(\d{4})(?: (\d{1,2}):(\d{1,2})(?::(\d{1,2}))?)?" +) + + +@dataclass(frozen=True, order=True) +class Detail: + """One affected row of a repair check: its cache, the field and the + value as stored, so the user can find and fix it by hand.""" + + gc_code: str + field: str + value: str + + @property + def text(self) -> str: + return f"{self.gc_code} {self.field} {self.value}" + + +def shown(value: object) -> str: + """A stored value as the dialog shows it, with NULL and empty text + visible.""" + if value is None: + return "NULL" + if isinstance(value, bytes): + return "x'" + value.hex() + "'" + text = str(value) + return text if text.strip() else repr(text) + + +@dataclass(frozen=True) +class DateColumn: + table: str + column: str + not_null: bool + + @property + def label(self) -> str: + return self.column if self.table == "caches" else f"{self.table}.{self.column}" + + +@dataclass(frozen=True) +class BadDate: + """A stored value that isn't in the stored format. *fixed* is the value + in the stored format, or None when it can't be read.""" + + column: DateColumn + rowid: int + value: object + fixed: Optional[str] + gc_code: Optional[str] + place: Optional[str] = None # a waypoint's prefix + + @property + def detail(self) -> Detail: + return Detail(self.gc_code or "?", _field(self.column, self.place), shown(self.value)) + + +def to_stored(dt: datetime) -> str: + """*dt* in the format SQLAlchemy stores a DateTime in. Formatted by hand: + strftime("%Y") doesn't pad years before 1000 on every platform.""" + return (f"{dt.year:04d}-{dt.month:02d}-{dt.day:02d} " + f"{dt.hour:02d}:{dt.minute:02d}:{dt.second:02d}.{dt.microsecond:06d}") + + +def normalize(value: object) -> Optional[str]: + """*value* in the stored format, or None if it can't be read as a date.""" + if isinstance(value, bytes): + try: + value = value.decode("utf-8") + except UnicodeDecodeError: + return None + if isinstance(value, float) and value.is_integer(): + value = int(value) + if isinstance(value, int) and not isinstance(value, bool): + value = str(value) if 19000101 <= value <= 99991231 else "" + if not isinstance(value, str): + return None + text = value.strip() + if not text: + return None + dt = _parse(text) + if dt is None: + return None + if dt.tzinfo is not None: + try: + dt = dt.astimezone(timezone.utc) + except OverflowError: # e.g. 0001-01-01T00:00:00+01:00 + return None + return to_stored(dt.replace(tzinfo=None)) + + +def _parse(text: str) -> Optional[datetime]: + try: + return datetime.fromisoformat(text) + except ValueError: + pass + # Unmatched optional groups are None. Year, month and day always match; + # "or 0" only satisfies the type (year 0 raises ValueError below). + parts: Optional[tuple[Optional[str], ...]] = None + m = _ISO_LOOSE.fullmatch(text) + if m: + parts = m.groups() + else: + m = _DAY_FIRST.fullmatch(text) + if m: + day, month, year, *clock = m.groups() + parts = (year, month, day, *clock, None) + if parts is None: + return None + year, month, day, hour, minute, second, fraction = parts + try: + return datetime(int(year or 0), int(month or 0), int(day or 0), int(hour or 0), + int(minute or 0), + int(second or 0), int((fraction or "0").ljust(6, "0"))) + except ValueError: + return None + + +def date_columns(conn) -> list[DateColumn]: + """Every date column of the main database open on *conn*.""" + columns = [] + tables = [name for (name,) in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' " + "ORDER BY name")] + for table in tables: + for _cid, name, type_, notnull, _default, _pk in conn.execute( + f"PRAGMA table_info({_q(table)})"): + if (type_ or "").upper() in _DATE_TYPES: + columns.append(DateColumn(table, name, bool(notnull))) + return columns + + +def _stored_format(col: str) -> str: + """SQL: *col* is text in the stored format and a real date and time.""" + return ( + f"(typeof({col}) = 'text' AND length({col}) = 26 AND {col} GLOB " + "'[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]" + ".[0-9][0-9][0-9][0-9][0-9][0-9]' " + f"AND date({col}) = substr({col}, 1, 10) " + f"AND time(substr({col}, 12, 8)) = substr({col}, 12, 8) " + f"AND substr({col}, 1, 4) <> '0000')" + ) + + +def _code_sql(conn, column: DateColumn) -> str: + """SQL for the GC code of the row being checked (alias t), or NULL.""" + if column.table == "caches": + return "t.gc_code" + if "cache_id" in _names(conn, column.table): + return "(SELECT c.gc_code FROM caches c WHERE c.id = t.cache_id)" + return "NULL" + + +def _place_sql(conn, column: DateColumn) -> str: + """SQL for where in the cache the row is: a waypoint's prefix, so the + user finds the waypoint (in GSAK too). NULL for other tables.""" + if column.table == "waypoints" and "prefix" in _names(conn, column.table): + return "t.prefix" + return "NULL" + + +def _field(column: DateColumn, place: Optional[str]) -> str: + return f"{column.label} {place}" if place else column.label + + +def _names(conn, table: str) -> set[str]: + return {r[1] for r in conn.execute(f"PRAGMA table_info({_q(table)})")} + + +def _rows(conn, column: DateColumn, where: str, params: Optional[dict] = None, + extra: str = "") -> list: + """(rowid, value, GC code, place) of the rows of *column* matching + *where* (on alias t, column as {col}).""" + col = f"t.{_q(column.column)}" + return conn.execute( + f"SELECT t.rowid, {col}, {_code_sql(conn, column)}, {_place_sql(conn, column)} " + f"FROM {_q(column.table)} t WHERE {where.format(col=col)} {extra}", params or {} + ).fetchall() + + +def bad_dates(conn, columns: Optional[list[DateColumn]] = None) -> Iterator[BadDate]: + """Every value of the date *columns* (all by default) that isn't NULL + and isn't in the stored format.""" + for column in columns if columns is not None else date_columns(conn): + where = "{col} IS NOT NULL AND NOT " + _stored_format("{col}") + for rowid, value, code, place in _rows(conn, column, where): + yield BadDate(column, rowid, value, normalize(value), code, place) + + +def placeholder_dates(conn) -> Iterator[BadDate]: + """Every value that is EPOCH in the stored format. (One in another + format comes from bad_dates(), with *fixed* == EPOCH.)""" + for column in date_columns(conn): + for rowid, value, code, place in _rows(conn, column, "{col} = :epoch", + {"epoch": EPOCH}): + yield BadDate(column, rowid, value, EPOCH, code, place) + + +def out_of_range(conn, limit: int) -> tuple[int, list[Detail]]: + """(number of, the first *limit* of) the dates in the stored format + before EARLIEST or after now + FUTURE_MARGIN, ordered by GC code. The + placed date of a DATED_CACHE_TYPES cache may be in the future.""" + future = to_stored(datetime.now(timezone.utc).replace(tzinfo=None) + FUTURE_MARGIN) + dated = ", ".join("'" + t.replace("'", "''") + "'" for t in sorted(DATED_CACHE_TYPES)) + total = 0 + details: list[Detail] = [] + params = {"future": future, "earliest": EARLIEST, "epoch": EPOCH} + for column in date_columns(conn): + too_late = "{col} > :future" + if column.table == "caches" and column.column == "hidden_date": + too_late += f" AND t.cache_type NOT IN ({dated})" + where = (_stored_format("{col}") + " AND {col} <> :epoch " + "AND ({col} < :earliest OR (" + too_late + "))") + count = conn.execute( + f"SELECT count(*) FROM {_q(column.table)} t " + f"WHERE {where.format(col='t.' + _q(column.column))}", params).fetchone()[0] + if not count: + continue + total += count + for _rowid, value, code, place in _rows(conn, column, where, params, + f"ORDER BY 3 LIMIT {limit}"): + details.append(Detail(code or "?", _field(column, place), shown(value))) + return total, sorted(details)[:limit] + + +def now_stored() -> str: + """Now as naive UTC in the stored format, like the columns' defaults.""" + return to_stored(datetime.now(timezone.utc).replace(tzinfo=None)) + + +def _q(name: str) -> str: + return '"' + name.replace('"', '""') + '"' diff --git a/src/opensak/gui/dialogs/database_repair_dialog.py b/src/opensak/gui/dialogs/database_repair_dialog.py index 401f9dbd..4189f484 100644 --- a/src/opensak/gui/dialogs/database_repair_dialog.py +++ b/src/opensak/gui/dialogs/database_repair_dialog.py @@ -1,6 +1,6 @@ """ src/opensak/gui/dialogs/database_repair_dialog.py — Repair… in the -Maintenance dialog (#1051, phase 1): check the data of a database and fix +Maintenance dialog (#1051): check the data of a database and fix what can be fixed safely, like GSAK's Repair/Defrag. 1. Scan (read only, in the background): every check of db/repair.py with @@ -11,6 +11,11 @@ when more than a quarter of the file is free (as GSAK does). Then a summary, and the scan runs again. +Every problem row shows a few affected caches with the field and its stored +value. "Copy list" (also Ctrl+C and the context menu) copies them for the +selected rows, or all rows, as tab-separated text, so report-only problems +can be found and fixed by hand. + A damaged file (quick_check fails) can't be repaired by SQLite; the dialog points to File → Restore from backup… instead. @@ -28,11 +33,12 @@ from pathlib import Path from typing import Any, Callable, Optional -from PySide6.QtCore import Qt +from PySide6.QtCore import QPoint, Qt +from PySide6.QtGui import QGuiApplication, QKeySequence, QShortcut from PySide6.QtWidgets import ( - QCheckBox, QDialog, QDialogButtonBox, QHBoxLayout, QHeaderView, QLabel, - QPlainTextEdit, QProgressBar, QPushButton, QTreeWidget, QTreeWidgetItem, - QVBoxLayout, QWidget, + QAbstractItemView, QCheckBox, QDialog, QDialogButtonBox, QHBoxLayout, QHeaderView, + QLabel, QMenu, QPlainTextEdit, QProgressBar, QPushButton, QTreeWidget, + QTreeWidgetItem, QVBoxLayout, QWidget, ) from opensak.db import companion, maintenance, repair @@ -51,6 +57,8 @@ KEY_ROLE = Qt.ItemDataRole.UserRole # Damage messages listed; the rest is in the log. _MAX_DAMAGE_LINES = 20 +# Affected rows in an examples cell's tooltip. +_TOOLTIP_DETAILS = 20 # Each check's name, as a translation key. _CHECK_TEXT = { @@ -63,6 +71,10 @@ repair.CORRECTED_COORDS: "repair_chk_corrected_coords", repair.RATINGS: "repair_chk_ratings", repair.IDENTITY: "repair_chk_identity", + repair.DATE_FORMAT: "repair_chk_date_format", + repair.DATE_UNREADABLE: "repair_chk_date_unreadable", + repair.DATE_RANGE: "repair_chk_date_range", + repair.DATE_PLACEHOLDER: "repair_chk_date_placeholder", } # The fix each fixable check makes, as a translation key. _FIX_TEXT = { @@ -71,6 +83,9 @@ repair.WAYPOINT_PARENT: "repair_fix_set_parent", repair.CORRECTED_FLAG: "repair_fix_clear_flag", repair.COMPANION_ORPHANS: "repair_fix_delete", + repair.DATE_FORMAT: "repair_fix_convert", + repair.DATE_UNREADABLE: "repair_fix_clear", + repair.DATE_PLACEHOLDER: "repair_fix_clear", } @@ -143,6 +158,27 @@ def repair_lines(outcome: RepairOutcome) -> list[str]: return lines +def copy_text(findings: list[repair.Finding]) -> str: + """The affected rows of *findings* as tab-separated text with a header: + check, GC code, field, value. A finding without details (orphans, whose + cache is gone) gets one line with its count; one with more rows than + kept says how many are not listed.""" + lines = ["\t".join([tr("repair_col_check"), tr("col_gc_code"), tr("repair_col_field"), + tr("repair_col_value")])] + for finding in findings: + label = check_label(finding) + if not finding.details: + lines.append("\t".join([label, "", "", tr("repair_copy_no_code", + count=f"{finding.count:,}")])) + continue + for d in finding.details: + lines.append("\t".join([label, d.gc_code, d.field, d.value])) + if finding.count > len(finding.details): + lines.append("\t".join([label, "", "", tr( + "repair_copy_more", count=f"{finding.count - len(finding.details):,}")])) + return "\n".join(lines) + "\n" + + # ── Dialog ──────────────────────────────────────────────────────────────────── class DatabaseRepairDialog(QDialog): @@ -181,17 +217,27 @@ def _setup_ui(self) -> None: for col in (1, 2, 3): header.setSectionResizeMode(col, QHeaderView.ResizeMode.ResizeToContents) self._tree.itemChanged.connect(lambda *_: self._update_buttons()) + self._tree.setSelectionMode(QAbstractItemView.SelectionMode.ExtendedSelection) + self._tree.setContextMenuPolicy(Qt.ContextMenuPolicy.CustomContextMenu) + self._tree.customContextMenuRequested.connect(self._tree_menu) + copy = QShortcut(QKeySequence(QKeySequence.StandardKey.Copy), self._tree) + copy.setContext(Qt.ShortcutContext.WidgetShortcut) + copy.activated.connect(self.copy_list) layout.addWidget(self._tree, 2) row = QHBoxLayout() self._backup = QCheckBox(tr("maint_compact_backup")) self._backup.setChecked(True) + self._btn_copy = QPushButton(tr("repair_btn_copy")) + self._btn_copy.setToolTip(tr("repair_tip_copy")) + self._btn_copy.clicked.connect(self.copy_list) self._btn_rescan = QPushButton(tr("repair_btn_rescan")) self._btn_rescan.clicked.connect(self._scan) self._btn_repair = QPushButton(tr("repair_btn_repair")) self._btn_repair.clicked.connect(self._repair) row.addWidget(self._backup) row.addStretch() + row.addWidget(self._btn_copy) row.addWidget(self._btn_rescan) row.addWidget(self._btn_repair) layout.addLayout(row) @@ -229,10 +275,15 @@ def _show_scan(self, result: repair.ScanResult) -> None: item = QTreeWidgetItem([ check_label(finding), f"{finding.count:,}", tr(_FIX_TEXT[finding.check]) if finding.fixable else tr("repair_report_only"), - ", ".join(finding.examples) + (" …" if finding.count > len(finding.examples) + "; ".join(finding.examples) + (" …" if finding.count > len(finding.examples) and finding.examples else ""), ]) item.setData(0, KEY_ROLE, finding.key) + if finding.details: + tip = [d.text for d in finding.details[:_TOOLTIP_DETAILS]] + if finding.count > len(tip): + tip.append(tr("repair_copy_more", count=f"{finding.count - len(tip):,}")) + item.setToolTip(3, "\n".join(tip + ["", tr("repair_tip_copy")])) item.setTextAlignment(1, Qt.AlignmentFlag.AlignRight | Qt.AlignmentFlag.AlignVCenter) if finding.fixable: item.setFlags(item.flags() | Qt.ItemFlag.ItemIsUserCheckable) @@ -262,6 +313,34 @@ def checked_keys(self) -> set[str]: keys.add(str(item.data(0, KEY_ROLE))) return keys + # ── Copy ────────────────────────────────────────────────────────────────── + + def _findings_to_copy(self) -> list[repair.Finding]: + """The findings of the selected rows, or all problems.""" + if self.scan_result is None: + return [] + problems = list(self.scan_result.problems) + selected = {str(item.data(0, KEY_ROLE)) for item in self._tree.selectedItems()} + return [f for f in problems if f.key in selected] if selected else problems + + def copy_list(self) -> None: + findings = self._findings_to_copy() + if not findings: + return + text = copy_text(findings) + clipboard = QGuiApplication.clipboard() + if clipboard is not None: + clipboard.setText(text) + self._append([tr("repair_copied", count=f"{text.count(chr(10)) - 1:,}")]) + + def _tree_menu(self, pos: QPoint) -> None: + if self.scan_result is None or not self.scan_result.problems: + return + menu = QMenu(self) + menu.addAction(tr("repair_btn_copy"), self.copy_list) + viewport = self._tree.viewport() + menu.exec(viewport.mapToGlobal(pos) if viewport is not None else pos) + # ── Repair ──────────────────────────────────────────────────────────────── def _repair(self) -> None: @@ -327,6 +406,8 @@ def _set_busy(self, busy: bool) -> None: self._tree.setEnabled(not busy) self._backup.setEnabled(not busy) self._btn_rescan.setEnabled(not busy) + self._btn_copy.setEnabled(not busy and self.scan_result is not None + and bool(self.scan_result.problems)) self._btn_repair.setEnabled(not busy and bool(self.checked_keys())) def _update_buttons(self) -> None: diff --git a/src/opensak/lang/cs.py b/src/opensak/lang/cs.py index ab54ce84..790b6c0c 100644 --- a/src/opensak/lang/cs.py +++ b/src/opensak/lang/cs.py @@ -1124,6 +1124,19 @@ "repair_chk_corrected_coords": "Opravené souřadnice mimo rozsah", "repair_chk_ratings": "Obtížnost nebo terén není mezi 1 a 5 v krocích po 0,5", "repair_chk_identity": "Prázdný GC kód nebo název", + "repair_chk_date_format": "Data ve formátu, který OpenSAK nepoužívá", + "repair_chk_date_unreadable": "Data, která nelze přečíst", + "repair_chk_date_range": "Data před květnem 2000 nebo v budoucnosti", + "repair_chk_date_placeholder": "Zástupné datum 1970-01-01 (\"bez data\" z GSAK)", + "repair_fix_convert": "Převést", + "repair_fix_clear": "Vymazat", + "repair_col_field": "Pole", + "repair_col_value": "Hodnota", + "repair_btn_copy": "Kopírovat seznam", + "repair_tip_copy": "Zkopíruje dotčené keše s polem a jeho hodnotou, pro vybrané řádky nebo všechny, pro vložení do tabulky.", + "repair_copy_more": "… a dalších {count}, neuvedeno", + "repair_copy_no_code": "Řádky, jejichž keš již neexistuje: {count}", + "repair_copied": "Zkopírováno řádků do schránky: {count}.", "repair_fix_delete": "Smazat", "repair_fix_clear_link": "Odstranit odkaz", "repair_fix_set_parent": "Nastavit kód keše", diff --git a/src/opensak/lang/da.py b/src/opensak/lang/da.py index c6362ff5..44be958d 100644 --- a/src/opensak/lang/da.py +++ b/src/opensak/lang/da.py @@ -1124,6 +1124,19 @@ "repair_chk_corrected_coords": "Korrigerede koordinater uden for gyldigt område", "repair_chk_ratings": "Sværhedsgrad eller terræn ikke mellem 1 og 5 i trin på 0,5", "repair_chk_identity": "Tom GC-kode eller navn", + "repair_chk_date_format": "Datoer i et format, OpenSAK ikke bruger", + "repair_chk_date_unreadable": "Datoer, der ikke kan læses", + "repair_chk_date_range": "Datoer før maj 2000 eller i fremtiden", + "repair_chk_date_placeholder": "Pladsholderdato 1970-01-01 (\"ingen dato\" fra GSAK)", + "repair_fix_convert": "Konvertér", + "repair_fix_clear": "Ryd", + "repair_col_field": "Felt", + "repair_col_value": "Værdi", + "repair_btn_copy": "Kopiér liste", + "repair_tip_copy": "Kopierer de berørte caches med feltet og dets værdi, for de valgte rækker eller alle, til indsættelse i et regneark.", + "repair_copy_more": "… og {count} mere, ikke vist", + "repair_copy_no_code": "{count} række(r), hvis cache ikke længere findes", + "repair_copied": "Kopierede {count} linje(r) til udklipsholderen.", "repair_fix_delete": "Slet", "repair_fix_clear_link": "Fjern linket", "repair_fix_set_parent": "Sæt cachens kode", diff --git a/src/opensak/lang/de.py b/src/opensak/lang/de.py index 3b747f0c..7870b7f6 100644 --- a/src/opensak/lang/de.py +++ b/src/opensak/lang/de.py @@ -1125,6 +1125,19 @@ "repair_chk_corrected_coords": "Korrigierte Koordinaten außerhalb des gültigen Bereichs", "repair_chk_ratings": "Schwierigkeit oder Gelände nicht zwischen 1 und 5 in Schritten von 0,5", "repair_chk_identity": "Leerer GC-Code oder Name", + "repair_chk_date_format": "Daten in einem Format, das OpenSAK nicht verwendet", + "repair_chk_date_unreadable": "Daten, die nicht gelesen werden können", + "repair_chk_date_range": "Daten vor Mai 2000 oder in der Zukunft", + "repair_chk_date_placeholder": "Platzhalterdatum 1970-01-01 (\"kein Datum\" aus GSAK)", + "repair_fix_convert": "Umwandeln", + "repair_fix_clear": "Leeren", + "repair_col_field": "Feld", + "repair_col_value": "Wert", + "repair_btn_copy": "Liste kopieren", + "repair_tip_copy": "Kopiert die betroffenen Caches mit dem Feld und seinem Wert, für die ausgewählten Zeilen oder alle, zum Einfügen in eine Tabelle.", + "repair_copy_more": "… und {count} weitere, nicht aufgeführt", + "repair_copy_no_code": "{count} Zeile(n), deren Cache nicht mehr existiert", + "repair_copied": "{count} Zeile(n) in die Zwischenablage kopiert.", "repair_fix_delete": "Löschen", "repair_fix_clear_link": "Verknüpfung entfernen", "repair_fix_set_parent": "Code des Caches setzen", diff --git a/src/opensak/lang/de_CH.py b/src/opensak/lang/de_CH.py index 7e82d18c..688bab64 100644 --- a/src/opensak/lang/de_CH.py +++ b/src/opensak/lang/de_CH.py @@ -1126,6 +1126,19 @@ "repair_chk_corrected_coords": "Korrigierte Koordinaten ausserhalb des gültigen Bereichs", "repair_chk_ratings": "Schwierigkeit oder Gelände nicht zwischen 1 und 5 in Schritten von 0,5", "repair_chk_identity": "Leerer GC-Code oder Name", + "repair_chk_date_format": "Daten in einem Format, das OpenSAK nicht verwendet", + "repair_chk_date_unreadable": "Daten, die nicht gelesen werden können", + "repair_chk_date_range": "Daten vor Mai 2000 oder in der Zukunft", + "repair_chk_date_placeholder": "Platzhalterdatum 1970-01-01 (\"kein Datum\" aus GSAK)", + "repair_fix_convert": "Umwandeln", + "repair_fix_clear": "Leeren", + "repair_col_field": "Feld", + "repair_col_value": "Wert", + "repair_btn_copy": "Liste kopieren", + "repair_tip_copy": "Kopiert die betroffenen Caches mit dem Feld und seinem Wert, für die ausgewählten Zeilen oder alle, zum Einfügen in eine Tabelle.", + "repair_copy_more": "… und {count} weitere, nicht aufgeführt", + "repair_copy_no_code": "{count} Zeile(n), deren Cache nicht mehr existiert", + "repair_copied": "{count} Zeile(n) in die Zwischenablage kopiert.", "repair_fix_delete": "Löschen", "repair_fix_clear_link": "Verknüpfung entfernen", "repair_fix_set_parent": "Code des Caches setzen", diff --git a/src/opensak/lang/en.py b/src/opensak/lang/en.py index 27553250..e3b61392 100644 --- a/src/opensak/lang/en.py +++ b/src/opensak/lang/en.py @@ -1123,6 +1123,19 @@ "repair_chk_corrected_coords": "Corrected coordinates out of range", "repair_chk_ratings": "Difficulty or terrain not between 1 and 5 in steps of 0.5", "repair_chk_identity": "Empty GC code or name", + "repair_chk_date_format": "Dates in a format OpenSAK doesn't use", + "repair_chk_date_unreadable": "Dates that can't be read", + "repair_chk_date_range": "Dates before May 2000 or in the future", + "repair_chk_date_placeholder": "Placeholder date 1970-01-01 (\"no date\" from GSAK)", + "repair_fix_convert": "Convert", + "repair_fix_clear": "Clear", + "repair_col_field": "Field", + "repair_col_value": "Value", + "repair_btn_copy": "Copy list", + "repair_tip_copy": "Copies the affected caches with the field and its value, for the selected rows or all, to paste into a spreadsheet.", + "repair_copy_more": "… and {count} more, not listed", + "repair_copy_no_code": "{count} row(s) whose cache no longer exists", + "repair_copied": "Copied {count} line(s) to the clipboard.", "repair_fix_delete": "Delete", "repair_fix_clear_link": "Clear the link", "repair_fix_set_parent": "Set the cache's code", diff --git a/src/opensak/lang/es.py b/src/opensak/lang/es.py index 743e6140..26237bcd 100644 --- a/src/opensak/lang/es.py +++ b/src/opensak/lang/es.py @@ -1125,6 +1125,19 @@ "repair_chk_corrected_coords": "Coordenadas corregidas fuera de rango", "repair_chk_ratings": "Dificultad o terreno no entre 1 y 5 en pasos de 0,5", "repair_chk_identity": "Código GC o nombre vacío", + "repair_chk_date_format": "Fechas en un formato que OpenSAK no usa", + "repair_chk_date_unreadable": "Fechas que no se pueden leer", + "repair_chk_date_range": "Fechas anteriores a mayo de 2000 o en el futuro", + "repair_chk_date_placeholder": "Fecha de relleno 1970-01-01 (\"sin fecha\" de GSAK)", + "repair_fix_convert": "Convertir", + "repair_fix_clear": "Borrar", + "repair_col_field": "Campo", + "repair_col_value": "Valor", + "repair_btn_copy": "Copiar lista", + "repair_tip_copy": "Copia los cachés afectados con el campo y su valor, de las filas seleccionadas o de todas, para pegarlos en una hoja de cálculo.", + "repair_copy_more": "… y {count} más, no listados", + "repair_copy_no_code": "{count} fila(s) cuyo caché ya no existe", + "repair_copied": "{count} línea(s) copiadas al portapapeles.", "repair_fix_delete": "Eliminar", "repair_fix_clear_link": "Quitar el vínculo", "repair_fix_set_parent": "Poner el código del caché", diff --git a/src/opensak/lang/fr.py b/src/opensak/lang/fr.py index 59fe588b..87d55aef 100644 --- a/src/opensak/lang/fr.py +++ b/src/opensak/lang/fr.py @@ -1125,6 +1125,19 @@ "repair_chk_corrected_coords": "Coordonnées corrigées hors limites", "repair_chk_ratings": "Difficulté ou terrain hors de 1 à 5 ou pas par pas de 0,5", "repair_chk_identity": "Code GC ou nom vide", + "repair_chk_date_format": "Dates dans un format qu'OpenSAK n'utilise pas", + "repair_chk_date_unreadable": "Dates illisibles", + "repair_chk_date_range": "Dates antérieures à mai 2000 ou dans le futur", + "repair_chk_date_placeholder": "Date fictive 1970-01-01 (« pas de date » de GSAK)", + "repair_fix_convert": "Convertir", + "repair_fix_clear": "Effacer", + "repair_col_field": "Champ", + "repair_col_value": "Valeur", + "repair_btn_copy": "Copier la liste", + "repair_tip_copy": "Copie les caches concernées avec le champ et sa valeur, pour les lignes sélectionnées ou toutes, à coller dans un tableur.", + "repair_copy_more": "… et {count} de plus, non listées", + "repair_copy_no_code": "{count} ligne(s) dont la cache n'existe plus", + "repair_copied": "{count} ligne(s) copiée(s) dans le presse-papiers.", "repair_fix_delete": "Supprimer", "repair_fix_clear_link": "Retirer le lien", "repair_fix_set_parent": "Mettre le code de la cache", diff --git a/src/opensak/lang/nl.py b/src/opensak/lang/nl.py index 7c0dc0c2..e2892dde 100644 --- a/src/opensak/lang/nl.py +++ b/src/opensak/lang/nl.py @@ -1127,6 +1127,19 @@ "repair_chk_corrected_coords": "Gecorrigeerde coördinaten buiten bereik", "repair_chk_ratings": "Moeilijkheid of terrein niet tussen 1 en 5 in stappen van 0,5", "repair_chk_identity": "Lege GC-code of naam", + "repair_chk_date_format": "Datums in een formaat dat OpenSAK niet gebruikt", + "repair_chk_date_unreadable": "Datums die niet gelezen kunnen worden", + "repair_chk_date_range": "Datums vóór mei 2000 of in de toekomst", + "repair_chk_date_placeholder": "Plaatsvervangende datum 1970-01-01 (\"geen datum\" uit GSAK)", + "repair_fix_convert": "Omzetten", + "repair_fix_clear": "Wissen", + "repair_col_field": "Veld", + "repair_col_value": "Waarde", + "repair_btn_copy": "Lijst kopiëren", + "repair_tip_copy": "Kopieert de betrokken caches met het veld en de waarde, voor de geselecteerde rijen of alle, om in een spreadsheet te plakken.", + "repair_copy_more": "… en nog {count}, niet vermeld", + "repair_copy_no_code": "{count} rij(en) waarvan de cache niet meer bestaat", + "repair_copied": "{count} regel(s) naar het klembord gekopieerd.", "repair_fix_delete": "Verwijderen", "repair_fix_clear_link": "Koppeling verwijderen", "repair_fix_set_parent": "Code van de cache instellen", diff --git a/src/opensak/lang/pl.py b/src/opensak/lang/pl.py index dba80515..a3e4b901 100644 --- a/src/opensak/lang/pl.py +++ b/src/opensak/lang/pl.py @@ -1125,6 +1125,19 @@ "repair_chk_corrected_coords": "Skorygowane współrzędne poza zakresem", "repair_chk_ratings": "Trudność lub teren nie od 1 do 5 w krokach co 0,5", "repair_chk_identity": "Pusty kod GC lub nazwa", + "repair_chk_date_format": "Daty w formacie, którego OpenSAK nie używa", + "repair_chk_date_unreadable": "Daty, których nie można odczytać", + "repair_chk_date_range": "Daty sprzed maja 2000 lub z przyszłości", + "repair_chk_date_placeholder": "Data zastępcza 1970-01-01 (\"brak daty\" z GSAK)", + "repair_fix_convert": "Konwertuj", + "repair_fix_clear": "Wyczyść", + "repair_col_field": "Pole", + "repair_col_value": "Wartość", + "repair_btn_copy": "Kopiuj listę", + "repair_tip_copy": "Kopiuje dotknięte skrzynki z polem i jego wartością, dla zaznaczonych wierszy lub wszystkich, do wklejenia w arkuszu.", + "repair_copy_more": "… i {count} więcej, niewymienione", + "repair_copy_no_code": "Wiersze, których skrzynka już nie istnieje: {count}", + "repair_copied": "Skopiowano wierszy do schowka: {count}.", "repair_fix_delete": "Usuń", "repair_fix_clear_link": "Usuń powiązanie", "repair_fix_set_parent": "Ustaw kod skrzynki", diff --git a/src/opensak/lang/pt.py b/src/opensak/lang/pt.py index b1ef5ff5..b01c79ae 100644 --- a/src/opensak/lang/pt.py +++ b/src/opensak/lang/pt.py @@ -1124,6 +1124,19 @@ "repair_chk_corrected_coords": "Coordenadas corrigidas fora do intervalo", "repair_chk_ratings": "Dificuldade ou terreno fora de 1 a 5 em passos de 0,5", "repair_chk_identity": "Código GC ou nome vazio", + "repair_chk_date_format": "Datas num formato que o OpenSAK não usa", + "repair_chk_date_unreadable": "Datas que não podem ser lidas", + "repair_chk_date_range": "Datas anteriores a maio de 2000 ou no futuro", + "repair_chk_date_placeholder": "Data de substituição 1970-01-01 (\"sem data\" do GSAK)", + "repair_fix_convert": "Converter", + "repair_fix_clear": "Limpar", + "repair_col_field": "Campo", + "repair_col_value": "Valor", + "repair_btn_copy": "Copiar lista", + "repair_tip_copy": "Copia as caches afetadas com o campo e o seu valor, das linhas selecionadas ou de todas, para colar numa folha de cálculo.", + "repair_copy_more": "… e mais {count}, não listadas", + "repair_copy_no_code": "{count} linha(s) cuja cache já não existe", + "repair_copied": "{count} linha(s) copiadas para a área de transferência.", "repair_fix_delete": "Eliminar", "repair_fix_clear_link": "Remover a ligação", "repair_fix_set_parent": "Definir o código da cache", diff --git a/src/opensak/lang/se.py b/src/opensak/lang/se.py index 9a76f2d8..be2bc60d 100644 --- a/src/opensak/lang/se.py +++ b/src/opensak/lang/se.py @@ -1124,6 +1124,19 @@ "repair_chk_corrected_coords": "Korrigerade koordinater utanför giltigt område", "repair_chk_ratings": "Svårighet eller terräng inte mellan 1 och 5 i steg om 0,5", "repair_chk_identity": "Tom GC-kod eller namn", + "repair_chk_date_format": "Datum i ett format som OpenSAK inte använder", + "repair_chk_date_unreadable": "Datum som inte kan läsas", + "repair_chk_date_range": "Datum före maj 2000 eller i framtiden", + "repair_chk_date_placeholder": "Platshållardatum 1970-01-01 (\"inget datum\" från GSAK)", + "repair_fix_convert": "Konvertera", + "repair_fix_clear": "Rensa", + "repair_col_field": "Fält", + "repair_col_value": "Värde", + "repair_btn_copy": "Kopiera lista", + "repair_tip_copy": "Kopierar de berörda cacherna med fältet och dess värde, för de markerade raderna eller alla, för att klistra in i ett kalkylark.", + "repair_copy_more": "… och {count} till, inte listade", + "repair_copy_no_code": "{count} rad(er) vars cache inte längre finns", + "repair_copied": "Kopierade {count} rad(er) till urklipp.", "repair_fix_delete": "Ta bort", "repair_fix_clear_link": "Ta bort länken", "repair_fix_set_parent": "Sätt cachens kod", diff --git a/tests/unit-tests/test_db_repair_1051.py b/tests/unit-tests/test_db_repair_1051.py index 639b1f91..ba6990b7 100644 --- a/tests/unit-tests/test_db_repair_1051.py +++ b/tests/unit-tests/test_db_repair_1051.py @@ -19,6 +19,7 @@ from opensak.db import companion, maintenance, repair from opensak.db.database import dispose_engine, get_session, init_db from opensak.db.models import Cache, Log, UserNote, Waypoint +from opensak.db.repair_dates import Detail from opensak.gui.dialogs import database_maintenance_dialog as dmd from opensak.gui.dialogs import database_repair_dialog as drd from opensak.lang import load_language, tr @@ -111,11 +112,18 @@ def test_finds_every_problem(self, tmp_path): repair.COORDS: 1, repair.RATINGS: 1, repair.IDENTITY: 1, } by_key = {f.key: f for f in result.findings} - assert by_key[repair.PARENT_LINK].examples == ("CW2",) - assert by_key[repair.WAYPOINT_PARENT].examples == ("GC1",) - assert by_key[repair.CORRECTED_FLAG].examples == ("GC2",) - assert by_key[repair.RATINGS].examples == ("GC3",) - assert by_key[repair.IDENTITY].examples == ("GC4",) + # Each affected row with the field and its stored value (#1051 feedback). + assert by_key[repair.PARENT_LINK].details == (Detail("CW2", "parent_gc_code", "gc404"),) + assert by_key[repair.WAYPOINT_PARENT].details == ( + Detail("GC1", "waypoints.parent_gc_code", "GC2"),) + assert by_key[repair.CORRECTED_FLAG].details == (Detail("GC2", "corrected", "NULL, 8.1"),) + assert by_key[repair.RATINGS].details == ( + Detail("GC3", "difficulty / terrain", "1.3 / 6.0"),) + assert by_key[repair.IDENTITY].details == ( + Detail("GC4", "gc_code / name", "'GC4' / ' '"),) + assert by_key[repair.COORDS].details == ( + Detail("GC3", "latitude, longitude", "95.0, 8.0"),) + assert by_key[repair.RATINGS].examples == ("GC3 difficulty / terrain 1.3 / 6.0",) assert not by_key[repair.COORDS].fixable assert {f.key for f in result.fixable} == { "orphans:logs", "orphans:waypoints", "orphans:attributes", "orphans:trackables", @@ -147,9 +155,12 @@ def test_examples_are_limited(self, tmp_path): conn.execute("INSERT INTO caches (gc_code, name, cache_type, latitude, longitude, " f"difficulty) VALUES ('GCX{n}', 'x', 'T', 1, 1, 7)") conn.close() - finding = next(f for f in repair.scan(db).findings if f.check == repair.RATINGS) + with patch.object(repair, "MAX_DETAILS", repair.MAX_EXAMPLES + 1): + finding = next(f for f in repair.scan(db).findings if f.check == repair.RATINGS) assert finding.count == repair.MAX_EXAMPLES + 3 assert len(finding.examples) == repair.MAX_EXAMPLES + assert len(finding.details) == repair.MAX_EXAMPLES + 1 + assert [d.gc_code for d in finding.details] == sorted(d.gc_code for d in finding.details) def test_damaged_file_skips_the_data_checks(self, tmp_path): db = _healthy(tmp_path / "Ok.db") @@ -349,3 +360,45 @@ def test_maintenance_dialog_opens_repair(self, qtbot, tmp_path): dlg._btn_repair.click() run.assert_called_once() qtbot.waitUntil(lambda: not dlg.is_running(), timeout=10000) + + +# ── Copy list (#1051 feedback: fix report-only problems by hand) ────────────── + +class TestCopy: + def test_copy_text(self): + findings = [ + repair.Finding(repair.RATINGS, 3, False, ( + Detail("GC3", "difficulty / terrain", "1.3 / 6.0"), + Detail("GC5", "difficulty / terrain", "7.0 / 1.0"))), + repair.Finding(repair.ORPHANS, 4, True, table="logs"), + ] + lines = drd.copy_text(findings).splitlines() + assert lines[0].split("\t") == [tr("repair_col_check"), tr("col_gc_code"), + tr("repair_col_field"), tr("repair_col_value")] + ratings = tr("repair_chk_ratings") + assert lines[1].split("\t") == [ratings, "GC3", "difficulty / terrain", "1.3 / 6.0"] + assert lines[2].split("\t") == [ratings, "GC5", "difficulty / terrain", "7.0 / 1.0"] + assert lines[3].split("\t") == [ratings, "", "", tr("repair_copy_more", count="1")] + assert lines[4].split("\t") == [tr("repair_chk_orphans", table="logs"), "", "", + tr("repair_copy_no_code", count="4")] + + def test_copies_all_rows_or_the_selected_ones(self, qtbot, tmp_path): + dlg = _dialog(qtbot, _broken(tmp_path / "Broken.db")) + clipboard = drd.QGuiApplication.clipboard() + assert clipboard is not None and dlg._btn_copy.isEnabled() + dlg._btn_copy.click() + text = clipboard.text() + assert "GC3\tlatitude, longitude\t95.0, 8.0" in text + assert "CW2\tparent_gc_code\tgc404" in text + assert tr("repair_copied", count=f"{len(text.splitlines()) - 1:,}") \ + in dlg._results.toPlainText() + + _rows(dlg)[repair.RATINGS].setSelected(True) + dlg.copy_list() + lines = clipboard.text().splitlines() + assert len(lines) == 2 + assert lines[1].startswith(tr("repair_chk_ratings") + "\tGC3\t") + + def test_nothing_to_copy_on_a_healthy_database(self, qtbot, tmp_path): + dlg = _dialog(qtbot, _healthy(tmp_path / "Ok.db")) + assert not dlg._btn_copy.isEnabled() diff --git a/tests/unit-tests/test_db_repair_dates_1051.py b/tests/unit-tests/test_db_repair_dates_1051.py new file mode 100644 index 00000000..7de3d541 --- /dev/null +++ b/tests/unit-tests/test_db_repair_dates_1051.py @@ -0,0 +1,284 @@ +""" +tests/unit-tests/test_db_repair_dates_1051.py — database repair, phase 2 +(#1051): dates SQLAlchemy can't read (a cache with one fails to load), dates +in another format, and dates before May 2000 or in the future. +""" + +from __future__ import annotations + +import sqlite3 +from datetime import datetime, timedelta +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytest.importorskip("pytestqt") + +from opensak.db import companion, repair, repair_dates +from opensak.db.database import dispose_engine, get_session, init_db +from opensak.db.models import Cache, Log +from opensak.db.repair_dates import Detail +from opensak.gui.dialogs import database_repair_dialog as drd +from opensak.lang import load_language, tr +from opensak.utils import flags + + +@pytest.fixture(autouse=True) +def _fresh(monkeypatch): + load_language("en") + monkeypatch.setattr(flags, "companion_db", False) + companion.reset() + yield + dispose_engine() + companion.reset() + + +def _database(path: Path) -> Path: + """Caches GC1 (with a log), GC2 and the event GC3, all dates valid.""" + path.parent.mkdir(parents=True, exist_ok=True) + init_db(db_path=path) + with get_session() as s: + gc1 = Cache(gc_code="GC1", name="One", cache_type="Traditional Cache", + latitude=47.0, longitude=8.0, hidden_date=datetime(2010, 5, 1)) + s.add_all([gc1, + Cache(gc_code="GC2", name="Two", cache_type="Traditional Cache", + latitude=47.1, longitude=8.1), + Cache(gc_code="GC3", name="Event", cache_type="Event Cache", + latitude=47.2, longitude=8.2)]) + s.flush() + s.add(Log(cache_id=gc1.id, log_id="1", log_type="Found it", + log_date=datetime(2020, 1, 2, 3, 4, 5))) + dispose_engine(path) + return path + + +def _sql(path: Path, *statements: str, params: tuple = ()) -> None: + conn = sqlite3.connect(str(path)) + try: + with conn: + for statement in statements: + conn.execute(statement, params) if params else conn.execute(statement) + finally: + conn.close() + + +def _value(path: Path, sql: str): + conn = sqlite3.connect(str(path)) + try: + return conn.execute(sql).fetchone()[0] + finally: + conn.close() + + +def _counts(result: repair.ScanResult) -> dict[str, int]: + return {f.key: f.count for f in result.problems} + + +def _load_all(path: Path) -> None: + """Load every cache and log like the GUI does; raises on a bad date.""" + init_db(db_path=path) + with get_session() as s: + for cache in s.query(Cache).all(): + _ = [lg.log_date for lg in cache.logs] + dispose_engine(path) + + +class TestNormalize: + @pytest.mark.parametrize("value, expected", [ + ("2024-03-05", "2024-03-05 00:00:00.000000"), + ("2024-03-05 10:11:12", "2024-03-05 10:11:12.000000"), + ("2024-03-05T10:11:12", "2024-03-05 10:11:12.000000"), + ("2024-03-05T10:11:12Z", "2024-03-05 10:11:12.000000"), + ("2024-03-05T10:11:12.5", "2024-03-05 10:11:12.500000"), + # Converted to UTC, as the GPX importer does. + ("2024-03-05T01:00:00+02:00", "2024-03-04 23:00:00.000000"), + (" 2024-03-05 ", "2024-03-05 00:00:00.000000"), + ("2024-3-5", "2024-03-05 00:00:00.000000"), + ("2024/03/05 7:08", "2024-03-05 07:08:00.000000"), + ("05.03.2024", "2024-03-05 00:00:00.000000"), + ("5.3.2024 10:11", "2024-03-05 10:11:00.000000"), + (20240305, "2024-03-05 00:00:00.000000"), + (20240305.0, "2024-03-05 00:00:00.000000"), + (b"2024-03-05", "2024-03-05 00:00:00.000000"), + ]) + def test_readable(self, value, expected): + assert repair_dates.normalize(value) == expected + + @pytest.mark.parametrize("value", [ + "", " ", "abc", "2024-02-30", "0000-00-00 00:00:00", "2024-13-01", + "2024-03-05 25:00:00", 45000, 45000.5, 1709632800, b"\xff\xfe", True, + ]) + def test_unreadable(self, value): + assert repair_dates.normalize(value) is None + + def test_years_before_1000_are_padded(self): + assert repair_dates.to_stored(datetime(999, 1, 2)) == "0999-01-02 00:00:00.000000" + + +class TestScan: + def test_valid_dates_are_not_reported(self, tmp_path): + assert repair.scan(_database(tmp_path / "Ok.db")).problems == () + + def test_finds_wrong_formats_and_unreadable_dates(self, tmp_path): + db = _database(tmp_path / "Dates.db") + _sql(db, + "UPDATE caches SET hidden_date = '2011-04-03T10:00:00Z' WHERE gc_code = 'GC1'", + "UPDATE caches SET found_date = '20240305' WHERE gc_code = 'GC2'", + "UPDATE logs SET log_date = 'yesterday'", + "UPDATE caches SET last_updated = '2024-02-30 00:00:00.000000' WHERE gc_code = 'GC2'") + result = repair.scan(db) + assert _counts(result) == {repair.DATE_FORMAT: 2, repair.DATE_UNREADABLE: 2} + by_key = {f.key: f for f in result.findings} + assert by_key[repair.DATE_FORMAT].details == ( + Detail("GC1", "hidden_date", "2011-04-03T10:00:00Z"), + Detail("GC2", "found_date", "20240305")) + assert by_key[repair.DATE_UNREADABLE].details == ( + Detail("GC1", "logs.log_date", "yesterday"), + Detail("GC2", "last_updated", "2024-02-30 00:00:00.000000")) + assert by_key[repair.DATE_FORMAT].fixable and by_key[repair.DATE_UNREADABLE].fixable + + def test_too_early_and_future_dates_are_report_only(self, tmp_path): + db = _database(tmp_path / "Dates.db") + future = repair_dates.to_stored(datetime.now() + timedelta(days=30)) + _sql(db, "UPDATE logs SET log_date = '1999-12-31 00:00:00.000000'") + _sql(db, "UPDATE caches SET hidden_date = ? WHERE gc_code IN ('GC2', 'GC3')", + params=(future,)) + result = repair.scan(db) + # GC3 is an event: its placed date is the event date. + assert _counts(result) == {repair.DATE_RANGE: 2} + finding = next(f for f in result.findings if f.check == repair.DATE_RANGE) + assert not finding.fixable + assert finding.details == (Detail("GC1", "logs.log_date", "1999-12-31 00:00:00.000000"), + Detail("GC2", "hidden_date", future)) + + def test_a_maze_exhibit_may_be_in_the_future(self, tmp_path): + # #1051 feedback: e.g. GCBGCCC, a GPS Adventures Maze, placed on the + # day of the exhibit. + db = _database(tmp_path / "Dates.db") + future = repair_dates.to_stored(datetime.now() + timedelta(days=300)) + _sql(db, "UPDATE caches SET cache_type = 'GPS Adventures Maze', hidden_date = ? " + "WHERE gc_code = 'GC2'", params=(future,)) + assert repair.scan(db).problems == () + assert "GPS Adventures Maze" in repair_dates.DATED_CACHE_TYPES + + def test_first_geocaching_day_is_fine(self, tmp_path): + db = _database(tmp_path / "Dates.db") + _sql(db, "UPDATE caches SET hidden_date = '2000-05-03 00:00:00.000000'") + assert repair.scan(db).problems == () + + +class TestRepair: + def test_a_cache_with_a_bad_date_loads_again(self, tmp_path): + db = _database(tmp_path / "Dates.db") + _sql(db, "UPDATE caches SET found_date = '20240305' WHERE gc_code = 'GC2'", + "UPDATE logs SET log_date = 'yesterday'") + with pytest.raises((ValueError, TypeError)): + _load_all(db) + dispose_engine(db) + keys = {f.key for f in repair.scan(db).fixable} + assert repair.repair(db, keys).fixed == {repair.DATE_FORMAT: 1, + repair.DATE_UNREADABLE: 1} + _load_all(db) + assert _value(db, "SELECT found_date FROM caches WHERE gc_code = 'GC2'") \ + == "2024-03-05 00:00:00.000000" + assert _value(db, "SELECT log_date FROM logs") is None + assert repair.scan(db).problems == () + + def test_not_null_columns_get_now(self, tmp_path): + db = _database(tmp_path / "Dates.db") + _sql(db, "UPDATE caches SET imported_at = 'broken' WHERE gc_code = 'GC1'") + before = repair_dates.now_stored() + repair.repair(db, {repair.DATE_UNREADABLE}) + stamp = _value(db, "SELECT imported_at FROM caches WHERE gc_code = 'GC1'") + assert stamp >= before + assert repair.scan(db).problems == () + + def test_only_the_ticked_date_fix(self, tmp_path): + db = _database(tmp_path / "Dates.db") + _sql(db, "UPDATE caches SET hidden_date = '2011-04-03' WHERE gc_code = 'GC1'", + "UPDATE logs SET log_date = 'yesterday'") + assert repair.repair(db, {repair.DATE_FORMAT}).fixed == {repair.DATE_FORMAT: 1} + assert _counts(repair.scan(db)) == {repair.DATE_UNREADABLE: 1} + assert _value(db, "SELECT log_date FROM logs") == "yesterday" + + def test_dates_of_other_tables_are_checked_too(self, tmp_path): + db = _database(tmp_path / "Dates.db") + _sql(db, "CREATE TABLE extra (id INTEGER PRIMARY KEY, seen DATETIME)", + "INSERT INTO extra (seen) VALUES ('2024-03-05')") + result = repair.scan(db) + finding = next(f for f in result.findings if f.check == repair.DATE_FORMAT) + assert finding.count == 1 and finding.details == (Detail("?", "extra.seen", "2024-03-05"),) + repair.repair(db, {repair.DATE_FORMAT}) + assert _value(db, "SELECT seen FROM extra") == "2024-03-05 00:00:00.000000" + + def test_report_only_dates_are_left_alone(self, tmp_path): + db = _database(tmp_path / "Dates.db") + _sql(db, "UPDATE logs SET log_date = '1999-12-31 00:00:00.000000'") + assert repair.repair(db, {repair.DATE_RANGE}).fixed == {} + assert _value(db, "SELECT log_date FROM logs") == "1999-12-31 00:00:00.000000" + + +class TestDialog: + def test_lists_the_date_checks(self, qtbot, tmp_path): + db = _database(tmp_path / "Dates.db") + _sql(db, "UPDATE caches SET hidden_date = '2011-04-03' WHERE gc_code = 'GC1'", + "UPDATE logs SET log_date = '1999-12-31 00:00:00.000000'") + dlg = drd.DatabaseRepairDialog(SimpleNamespace(name="Dates", path=db)) + qtbot.addWidget(dlg) + qtbot.waitUntil(lambda: not dlg.is_running() and dlg.scan_result is not None, + timeout=10000) + rows = {} + for i in range(dlg._tree.topLevelItemCount()): + item = dlg._tree.topLevelItem(i) + assert item is not None + rows[item.data(0, drd.KEY_ROLE)] = item + assert rows[repair.DATE_FORMAT].text(0) == tr("repair_chk_date_format") + assert rows[repair.DATE_FORMAT].text(2) == tr("repair_fix_convert") + assert rows[repair.DATE_FORMAT].text(3) == "GC1 hidden_date 2011-04-03" + assert "GC1 hidden_date 2011-04-03" in rows[repair.DATE_FORMAT].toolTip(3) + assert rows[repair.DATE_RANGE].text(2) == tr("repair_report_only") + assert dlg.checked_keys() == {repair.DATE_FORMAT} + + +class TestPlaceholder: + """#1051 feedback: GSAK's "no date" 1970-01-01 (e.g. Waypoints.cDate, + copied by the GSAK import) is fixable, with the waypoint in the example.""" + + def _with_waypoints(self, path: Path) -> Path: + db = _database(path) + _sql(db, + "INSERT INTO waypoints (cache_id, parent_gc_code, prefix, name, wp_date) " + "SELECT id, gc_code, 'S1', 'Stage', " + "'1970-01-01 00:00:00.000000' FROM caches WHERE gc_code = 'GC1'", + # In another format: converted, it is the placeholder too. + "INSERT INTO waypoints (cache_id, parent_gc_code, prefix, name, wp_date) " + "SELECT id, gc_code, 'S2', 'Stage', " + "'1970-01-01' FROM caches WHERE gc_code = 'GC1'") + return db + + def test_reported_as_placeholder_not_as_too_early(self, tmp_path): + result = repair.scan(self._with_waypoints(tmp_path / "Dates.db")) + assert _counts(result) == {repair.DATE_PLACEHOLDER: 2} + finding = next(f for f in result.findings if f.check == repair.DATE_PLACEHOLDER) + assert finding.fixable + assert finding.details == ( + Detail("GC1", "waypoints.wp_date S1", "1970-01-01 00:00:00.000000"), + Detail("GC1", "waypoints.wp_date S2", "1970-01-01")) + + def test_fix_clears_it(self, tmp_path): + db = self._with_waypoints(tmp_path / "Dates.db") + _sql(db, "UPDATE caches SET imported_at = '1970-01-01 00:00:00.000000' " + "WHERE gc_code = 'GC2'") + assert repair.repair(db, {repair.DATE_PLACEHOLDER}).fixed == {repair.DATE_PLACEHOLDER: 3} + assert _value(db, "SELECT count(*) FROM waypoints WHERE wp_date IS NULL") == 2 + # imported_at is NOT NULL: now instead. + assert _value(db, "SELECT imported_at FROM caches WHERE gc_code = 'GC2'") > "2000" + assert repair.scan(db).problems == () + + def test_other_old_dates_are_still_too_early(self, tmp_path): + db = self._with_waypoints(tmp_path / "Dates.db") + _sql(db, "UPDATE waypoints SET wp_date = '1970-01-02 00:00:00.000000' " + "WHERE prefix = 'S1'") + counts = _counts(repair.scan(db)) + assert counts == {repair.DATE_PLACEHOLDER: 1, repair.DATE_RANGE: 1} From beeaac0c7e54b7a83483969334f9ea3cf685eb85 Mon Sep 17 00:00:00 2001 From: nagisml Date: Wed, 18 Nov 2026 19:02:32 +0100 Subject: [PATCH 3/3] Recalculate log totals and the found mark in the database Repair dialog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 of #1051. A cache keeps values derived from its logs, waypoints and trackables, so the grid, the filters and sorting don't have to count them each time. The importers and macros keep these values up to date. They can still drift: in older databases, after raw SQL, or after a repair has removed rows or cleared log dates. The totals mean "what is in the database", which is how every import and macro already computes them: - log_count, last_log_date, last_found_date (latest found-type log by anyone) and last_four_logs (logs with the same date may be in any order, so the same lines in another order are not a difference) - waypoint_count and trackable_count - found_log_count, the user's own found-type logs. It is only raised, never lowered: a missing log doesn't prove fewer finds, because a Pocket Query brings only some logs and #1310 will delete logs. find_count (the site's total, or GSAK's) is not derived from the logs and is never touched. New checks: - Counts or dates that don't match the cache's rows (fixable): recalculated. The dialog lists each change as "old → new". - Caches with one of the user's own found logs that aren't marked found (fixable): marked found. If the found date is empty, it is set from the latest own found log with log_calendar_midnight() (#875). found is never cleared, because Adventure Labs and manual marks have no log behind them. dnf is not checked, because the importers only take it from GSAK's flag. The checks of the user's own logs need the account from Settings and are skipped without one, as the importer does. The totals are calculated from the log dates as the date checks leave them (converted, or no date when unreadable or 1970-01-01). The repair recalculates them after deleting rows without a cache and fixing dates, in the same transaction. Caches and logs are read in one pass ordered by cache id, so memory stays flat. On a real database of 152k caches and 732k logs, every stored value matched these rules, and the scan took 4 s longer with an 11 MB peak. The new texts are in all language files; the translations are machine-made. Fixes #1051 --- src/opensak/db/repair.py | 68 ++++- src/opensak/db/repair_totals.py | 187 ++++++++++++++ .../gui/dialogs/database_repair_dialog.py | 16 +- src/opensak/lang/cs.py | 4 + src/opensak/lang/da.py | 4 + src/opensak/lang/de.py | 4 + src/opensak/lang/de_CH.py | 4 + src/opensak/lang/en.py | 4 + src/opensak/lang/es.py | 4 + src/opensak/lang/fr.py | 4 + src/opensak/lang/nl.py | 4 + src/opensak/lang/pl.py | 4 + src/opensak/lang/pt.py | 4 + src/opensak/lang/se.py | 4 + tests/unit-tests/test_db_repair_1051.py | 3 +- tests/unit-tests/test_db_repair_dates_1051.py | 29 ++- .../unit-tests/test_db_repair_totals_1051.py | 240 ++++++++++++++++++ 17 files changed, 571 insertions(+), 16 deletions(-) create mode 100644 src/opensak/db/repair_totals.py create mode 100644 tests/unit-tests/test_db_repair_totals_1051.py diff --git a/src/opensak/db/repair.py b/src/opensak/db/repair.py index 6bb821bf..9351c04f 100644 --- a/src/opensak/db/repair.py +++ b/src/opensak/db/repair.py @@ -31,6 +31,11 @@ sorts and filters wrongly. Fix: convert them to the stored format, or clear the ones that can't be read as a date at all. GSAK's "no date", 1970-01-01, is a check of its own. Fix: clear it. +* totals (phase 3, details in db/repair_totals.py): log and found-log + counts, last log and found dates, the last four logs, waypoint and + trackable counts that don't match the cache's rows. Fix: recalculate them + as the importer does. And a cache with one of the user's own found logs + that isn't marked found. Fix: mark it (found is never cleared). * report only: coordinates out of range, difficulty or terrain outside 1–5 or not in 0.5 steps, an empty GC code or name, dates before May 2000 or in the future. @@ -56,7 +61,8 @@ from pathlib import Path from typing import Callable, Iterator, Optional -from opensak.db import repair_dates +from opensak.db import repair_dates, repair_totals +from opensak.db.repair_totals import Account from opensak.db.repair_dates import Detail logger = logging.getLogger(__name__) @@ -82,6 +88,8 @@ DATE_FORMAT = "date_format" DATE_UNREADABLE = "date_unreadable" DATE_PLACEHOLDER = "date_placeholder" +TOTALS = "totals" +FOUND_MARK = "found_mark" DATE_RANGE = "date_range" @@ -185,11 +193,14 @@ def _corrected_flag_where(alias: str = "") -> str: # ── Scan ────────────────────────────────────────────────────────────────────── def scan(db_path: Path, with_companion: bool = False, - step: Optional[Callable[[str], None]] = None) -> ScanResult: + step: Optional[Callable[[str], None]] = None, + account: Optional[Account] = None) -> ScanResult: """ Run every check on the database *db_path* (and on its companion when *with_companion*). Changes nothing. *step* is called with each check id - before it runs. Raises sqlite3.Error if the file can't be read. + before it runs. *account* (finder id, username) is needed for the + checks of the user's own logs. Raises sqlite3.Error if the file can't + be read. """ db_path = Path(db_path) notify = step or (lambda _id: None) @@ -219,6 +230,9 @@ def scan(db_path: Path, with_companion: bool = False, notify(DATE_FORMAT) findings += _date_findings(conn) + notify(TOTALS) + findings += _totals_findings(conn, account) + if with_companion: notify(COMPANION_ORPHANS) findings.append(Finding(COMPANION_ORPHANS, @@ -318,7 +332,8 @@ def _companion_orphans(db_path: Path) -> dict[str, int]: # ── Repair ──────────────────────────────────────────────────────────────────── def repair(db_path: Path, keys: set[str], - step: Optional[Callable[[str], None]] = None) -> RepairResult: + step: Optional[Callable[[str], None]] = None, + account: Optional[Account] = None) -> RepairResult: """ Fix the findings *keys* (Finding.key of fixable findings) in *db_path*. The main database's fixes run in one transaction; the companion is @@ -358,6 +373,10 @@ def repair(db_path: Path, keys: set[str], if keys & set(_DATE_CHECKS): notify(DATE_FORMAT) result.fixed.update(_fix_dates(conn, keys)) + # After the rows and dates they are derived from were fixed. + if keys & {TOTALS, FOUND_MARK}: + notify(TOTALS) + result.fixed.update(_fix_totals(conn, keys, account)) finally: conn.close() if COMPANION_ORPHANS in keys: @@ -393,6 +412,47 @@ def _fix_dates(conn: sqlite3.Connection, keys: set[str]) -> dict[str, int]: return fixed +def _totals_findings(conn: sqlite3.Connection, account: Optional[Account]) -> list[Finding]: + counts = {TOTALS: 0, FOUND_MARK: 0} + details: dict[str, list[Detail]] = {TOTALS: [], FOUND_MARK: []} + for cache in repair_totals.differences(conn, account): + if cache.changes: + counts[TOTALS] += 1 + for column, (old, new) in cache.changes.items(): + if len(details[TOTALS]) < MAX_DETAILS: + details[TOTALS].append(Detail( + cache.gc_code, column, + f"{repair_totals.shown(old)} → {repair_totals.shown(new)}")) + if cache.mark_found: + counts[FOUND_MARK] += 1 + if len(details[FOUND_MARK]) < MAX_DETAILS: + value = "0 → 1" + if cache.found_date: + value += f", found_date {cache.found_date[:10]}" + details[FOUND_MARK].append(Detail(cache.gc_code, "found", value)) + return [Finding(check, counts[check], True, tuple(sorted(details[check]))) + for check in counts] + + +def _fix_totals(conn: sqlite3.Connection, keys: set[str], + account: Optional[Account]) -> dict[str, int]: + """Recalculate the totals (TOTALS) and mark found (FOUND_MARK).""" + fixed = {TOTALS: 0, FOUND_MARK: 0} + for cache in list(repair_totals.differences(conn, account)): + if TOTALS in keys and cache.changes: + columns = list(cache.changes) + conn.execute( + f"UPDATE caches SET {', '.join(f'{_quote(c)} = ?' for c in columns)} " + "WHERE id = ?", [cache.changes[c][1] for c in columns] + [cache.cache_id]) + fixed[TOTALS] += 1 + if FOUND_MARK in keys and cache.mark_found: + conn.execute("UPDATE caches SET found = 1, " + "found_date = COALESCE(found_date, ?) WHERE id = ?", + (cache.found_date, cache.cache_id)) + fixed[FOUND_MARK] += 1 + return fixed + + def _quote(name: str) -> str: return '"' + name.replace('"', '""') + '"' diff --git a/src/opensak/db/repair_totals.py b/src/opensak/db/repair_totals.py new file mode 100644 index 00000000..7e064ffa --- /dev/null +++ b/src/opensak/db/repair_totals.py @@ -0,0 +1,187 @@ +""" +src/opensak/db/repair_totals.py — the totals checks of the database repair +(#1051, phase 3), used by db/repair.py. + +A cache keeps values derived from its logs, waypoints and trackables, so +the grid, the filters and sorting don't count them each time. The +importers and macros keep them up to date; they drift in older databases, +after raw SQL, or after a repair removed rows or cleared log dates. + +The totals mean "what is in the database", as every import and macro +already computes them (importer/__init__.py ``_upsert_cache()``, +gsak_importer.py, macro/cache_write.py ``refresh_derived()``): + +* ``log_count``: the cache's logs. ``last_log_date``: the latest log date. + ``last_found_date``: the latest found-type log (FOUND_LOG_TYPES) by + anyone. ``last_four_logs``: the four latest dated logs, one + "datetypefinder" line each. Logs with the same date may be in + any order, so the same lines in another order are not a difference. +* ``waypoint_count``, ``trackable_count``: the cache's rows. +* ``found_log_count``: the user's own found-type logs (is_own_log()). Only + raised, never lowered: a missing log doesn't prove fewer finds (a Pocket + Query brings only some logs, and #1310 will delete logs). + +``find_count`` (the site's total, or GSAK's) is not derived from the logs +and is never touched. + +Found: a cache with one of the user's own found logs that isn't marked +found is marked, and gets the found date of the latest such log (as +log_calendar_midnight() makes it, #875) when it has none. ``found`` is never +cleared: Adventure Labs and manual marks have no log behind them. ``dnf`` is +not checked: the importers take it only from GSAK's flag. + +The checks of the user's own logs need the account (Settings → GC finder +id / username); without one they are skipped, as the importer does. + +Caches and logs are read in one pass ordered by cache id, so memory stays +flat on a large database. On a real database of 152k caches and 732k logs +the stored values matched these rules exactly. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import datetime +from typing import Iterator, Optional + +from opensak.utils.constants import FOUND_LOG_TYPES +from opensak.utils.log_dates import is_own_log, log_calendar_midnight + +# (finder id, username) from Settings. +Account = tuple[Optional[str], Optional[str]] + +# The derived columns this module checks, in the order shown. +COLUMNS = ("log_count", "found_log_count", "last_log_date", "last_found_date", + "last_four_logs", "waypoint_count", "trackable_count") + + +@dataclass +class CacheTotals: + """One cache whose stored values differ from its rows. *changes*: + column → (stored, recalculated). *mark_found*: it has an own found log + but isn't marked found; *found_date* then is the date to set (None: keep + the stored one).""" + + cache_id: int + gc_code: str + changes: dict[str, tuple[object, object]] = field(default_factory=dict) + mark_found: bool = False + found_date: Optional[str] = None + + +def has_account(account: Optional[Account]) -> bool: + return account is not None and bool((account[0] or "").strip() or (account[1] or "").strip()) + + +def differences(conn, account: Optional[Account]) -> Iterator[CacheTotals]: + """Every cache whose derived values or found mark don't match its rows.""" + own = has_account(account) + finder_id, username = account if account is not None else (None, None) + waypoints = dict(conn.execute("SELECT cache_id, count(*) FROM waypoints GROUP BY cache_id")) + trackables = dict(conn.execute( + "SELECT cache_id, count(*) FROM trackables GROUP BY cache_id")) + logs = conn.cursor().execute( + "SELECT cache_id, log_date, log_type, finder, finder_id FROM logs ORDER BY cache_id") + pending = logs.fetchone() + caches = conn.cursor().execute( + f"SELECT id, gc_code, found, found_date, {', '.join(COLUMNS)} FROM caches ORDER BY id") + for row in caches: + cache_id, code, found, found_date = row[:4] + stored = dict(zip(COLUMNS, row[4:])) + cache_logs = [] + while pending is not None and pending[0] < cache_id: + pending = logs.fetchone() # a log without a cache + while pending is not None and pending[0] == cache_id: + cache_logs.append(pending[1:]) + pending = logs.fetchone() + result = _check(cache_id, code, stored, cache_logs, waypoints.get(cache_id, 0), + trackables.get(cache_id, 0), bool(found), found_date, + own, finder_id, username) + if result is not None: + yield result + + +def _check(cache_id: int, code: str, stored: dict, logs: list, waypoint_count: int, + trackable_count: int, found: bool, found_date: Optional[str], own: bool, + finder_id: Optional[str], username: Optional[str]) -> Optional[CacheTotals]: + # Log dates as the date checks leave them: converted to the stored + # format, or None when unreadable or GSAK's 1970 "no date". + logs = [(log_date(lg[0]), *lg[1:]) for lg in logs] + dated = [lg for lg in logs if lg[0] is not None] + calc: dict[str, object] = { + "log_count": len(logs), + "last_log_date": max((lg[0] for lg in dated), default=None), + "last_found_date": max((lg[0] for lg in dated if lg[1] in FOUND_LOG_TYPES), + default=None), + "last_four_logs": four_logs(dated), + "waypoint_count": waypoint_count, + "trackable_count": trackable_count, + } + own_found = [lg for lg in logs if lg[1] in FOUND_LOG_TYPES + and is_own_log(lg[3], lg[2], finder_id, username)] if own else [] + if own and len(own_found) > (stored["found_log_count"] or 0): + calc["found_log_count"] = len(own_found) + result = CacheTotals(cache_id, code) + for column, value in calc.items(): + if column == "last_four_logs" and _same_lines(stored[column], value): + continue + if stored[column] != value: + result.changes[column] = (stored[column], value) + if own_found and not found: + result.mark_found = True + if not found_date: + latest = max((lg[0] for lg in own_found if lg[0] is not None), default=None) + result.found_date = _found_day(latest) + return result if result.changes or result.mark_found else None + + +_STORED = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}") + + +def log_date(value: object) -> Optional[str]: + """A stored log date in the stored format, as the date checks would + leave it; None for no date.""" + from opensak.db.repair_dates import EPOCH, normalize + + if isinstance(value, str) and _STORED.fullmatch(value) and value != EPOCH: + return value # the common case, kept fast + if value is None: + return None + fixed = normalize(value) + return fixed if fixed != EPOCH else None + + +def four_logs(dated: list) -> Optional[str]: + """last_four_logs as the importer writes it, from (date, type, finder, …) + rows with a date in the stored format.""" + latest = sorted(dated, key=lambda lg: lg[0], reverse=True)[:4] + return "\n".join(f"{lg[0][:10]}T{lg[0][11:19]}\t{lg[1]}\t{lg[2] or ''}" + for lg in latest) or None + + +def _same_lines(stored: object, calc: object) -> bool: + """The same lines, maybe in another order (logs with the same date).""" + if not isinstance(stored, str) or not isinstance(calc, str): + return False + return sorted(stored.split("\n")) == sorted(calc.split("\n")) + + +def _found_day(stored_date: Optional[str]) -> Optional[str]: + from opensak.db.repair_dates import to_stored + + if not stored_date: + return None + try: + day = log_calendar_midnight(datetime.fromisoformat(stored_date), own=True) + except ValueError: + return None + return to_stored(day) if day is not None else None + + +def shown(value: object) -> str: + """A stored or recalculated value on one line without tabs, so it fits + a tree cell and the tab-separated "Copy list" (last_four_logs has both).""" + if value is None: + return "NULL" + return str(value).replace("\t", " ").replace("\n", " | ") diff --git a/src/opensak/gui/dialogs/database_repair_dialog.py b/src/opensak/gui/dialogs/database_repair_dialog.py index 4189f484..d845d157 100644 --- a/src/opensak/gui/dialogs/database_repair_dialog.py +++ b/src/opensak/gui/dialogs/database_repair_dialog.py @@ -75,6 +75,8 @@ repair.DATE_UNREADABLE: "repair_chk_date_unreadable", repair.DATE_RANGE: "repair_chk_date_range", repair.DATE_PLACEHOLDER: "repair_chk_date_placeholder", + repair.TOTALS: "repair_chk_totals", + repair.FOUND_MARK: "repair_chk_found_mark", } # The fix each fixable check makes, as a translation key. _FIX_TEXT = { @@ -86,6 +88,8 @@ repair.DATE_FORMAT: "repair_fix_convert", repair.DATE_UNREADABLE: "repair_fix_clear", repair.DATE_PLACEHOLDER: "repair_fix_clear", + repair.TOTALS: "repair_fix_recalculate", + repair.FOUND_MARK: "repair_fix_mark_found", } @@ -102,9 +106,17 @@ def _with_companion(db_path: Path) -> bool: return bool(flags.companion_db) and companion.exists(db_path) +def _account() -> repair.Account: + """The account in Settings, for the checks of the user's own logs.""" + from opensak.gui.settings import get_settings + + settings = get_settings() + return settings.gc_finder_id, settings.gc_username + + def run_scan(db_path: Path, step: Step) -> repair.ScanResult: step(tr("repair_step_scan")) - return repair.scan(db_path, with_companion=_with_companion(db_path)) + return repair.scan(db_path, with_companion=_with_companion(db_path), account=_account()) def run_repair(db: Any, keys: set[str], backup: bool, step: Step) -> RepairOutcome: @@ -118,7 +130,7 @@ def run_repair(db: Any, keys: set[str], backup: bool, step: Step) -> RepairOutco step(tr("maint_step_backup")) backup_path = write_backup_set([db], KIND_MANUAL).backup_set.path step(tr("repair_step_fix")) - outcome = RepairOutcome(repair.repair(db.path, keys), backup_path) + outcome = RepairOutcome(repair.repair(db.path, keys, account=_account()), backup_path) step(tr("maint_step_optimize", file=tr("maint_db_group"))) maintenance.optimize_file(db.path) stats = maintenance.file_info(db.path).stats diff --git a/src/opensak/lang/cs.py b/src/opensak/lang/cs.py index 790b6c0c..3ea7a154 100644 --- a/src/opensak/lang/cs.py +++ b/src/opensak/lang/cs.py @@ -1128,6 +1128,10 @@ "repair_chk_date_unreadable": "Data, která nelze přečíst", "repair_chk_date_range": "Data před květnem 2000 nebo v budoucnosti", "repair_chk_date_placeholder": "Zástupné datum 1970-01-01 (\"bez data\" z GSAK)", + "repair_chk_totals": "Počty logů, waypointů a trackovatelných předmětů nebo data, která neodpovídají řádkům keše", + "repair_chk_found_mark": "Keše s vaším vlastním logem nálezu, které nejsou označené jako nalezené", + "repair_fix_recalculate": "Přepočítat", + "repair_fix_mark_found": "Označit jako nalezenou", "repair_fix_convert": "Převést", "repair_fix_clear": "Vymazat", "repair_col_field": "Pole", diff --git a/src/opensak/lang/da.py b/src/opensak/lang/da.py index 44be958d..61bbb384 100644 --- a/src/opensak/lang/da.py +++ b/src/opensak/lang/da.py @@ -1128,6 +1128,10 @@ "repair_chk_date_unreadable": "Datoer, der ikke kan læses", "repair_chk_date_range": "Datoer før maj 2000 eller i fremtiden", "repair_chk_date_placeholder": "Pladsholderdato 1970-01-01 (\"ingen dato\" fra GSAK)", + "repair_chk_totals": "Antal logs, waypoints og trackables eller datoer, der ikke passer med cachens rækker", + "repair_chk_found_mark": "Caches med din egen fundlog, som ikke er markeret som fundet", + "repair_fix_recalculate": "Genberegn", + "repair_fix_mark_found": "Markér som fundet", "repair_fix_convert": "Konvertér", "repair_fix_clear": "Ryd", "repair_col_field": "Felt", diff --git a/src/opensak/lang/de.py b/src/opensak/lang/de.py index 7870b7f6..cb2d5849 100644 --- a/src/opensak/lang/de.py +++ b/src/opensak/lang/de.py @@ -1129,6 +1129,10 @@ "repair_chk_date_unreadable": "Daten, die nicht gelesen werden können", "repair_chk_date_range": "Daten vor Mai 2000 oder in der Zukunft", "repair_chk_date_placeholder": "Platzhalterdatum 1970-01-01 (\"kein Datum\" aus GSAK)", + "repair_chk_totals": "Anzahl Logs, Wegpunkte und Trackables oder Daten, die nicht zu den Zeilen des Caches passen", + "repair_chk_found_mark": "Caches mit Ihrem eigenen Fund-Log, die nicht als gefunden markiert sind", + "repair_fix_recalculate": "Neu berechnen", + "repair_fix_mark_found": "Als gefunden markieren", "repair_fix_convert": "Umwandeln", "repair_fix_clear": "Leeren", "repair_col_field": "Feld", diff --git a/src/opensak/lang/de_CH.py b/src/opensak/lang/de_CH.py index 688bab64..cba174e3 100644 --- a/src/opensak/lang/de_CH.py +++ b/src/opensak/lang/de_CH.py @@ -1130,6 +1130,10 @@ "repair_chk_date_unreadable": "Daten, die nicht gelesen werden können", "repair_chk_date_range": "Daten vor Mai 2000 oder in der Zukunft", "repair_chk_date_placeholder": "Platzhalterdatum 1970-01-01 (\"kein Datum\" aus GSAK)", + "repair_chk_totals": "Anzahl Logs, Wegpunkte und Trackables oder Daten, die nicht zu den Zeilen des Caches passen", + "repair_chk_found_mark": "Caches mit Ihrem eigenen Fund-Log, die nicht als gefunden markiert sind", + "repair_fix_recalculate": "Neu berechnen", + "repair_fix_mark_found": "Als gefunden markieren", "repair_fix_convert": "Umwandeln", "repair_fix_clear": "Leeren", "repair_col_field": "Feld", diff --git a/src/opensak/lang/en.py b/src/opensak/lang/en.py index e3b61392..80f0d57a 100644 --- a/src/opensak/lang/en.py +++ b/src/opensak/lang/en.py @@ -1127,6 +1127,10 @@ "repair_chk_date_unreadable": "Dates that can't be read", "repair_chk_date_range": "Dates before May 2000 or in the future", "repair_chk_date_placeholder": "Placeholder date 1970-01-01 (\"no date\" from GSAK)", + "repair_chk_totals": "Log, waypoint and trackable counts or dates that don't match the cache's rows", + "repair_chk_found_mark": "Caches with your own found log that aren't marked as found", + "repair_fix_recalculate": "Recalculate", + "repair_fix_mark_found": "Mark as found", "repair_fix_convert": "Convert", "repair_fix_clear": "Clear", "repair_col_field": "Field", diff --git a/src/opensak/lang/es.py b/src/opensak/lang/es.py index 26237bcd..57249f83 100644 --- a/src/opensak/lang/es.py +++ b/src/opensak/lang/es.py @@ -1129,6 +1129,10 @@ "repair_chk_date_unreadable": "Fechas que no se pueden leer", "repair_chk_date_range": "Fechas anteriores a mayo de 2000 o en el futuro", "repair_chk_date_placeholder": "Fecha de relleno 1970-01-01 (\"sin fecha\" de GSAK)", + "repair_chk_totals": "Recuentos o fechas de registros, waypoints y rastreables que no coinciden con las filas del caché", + "repair_chk_found_mark": "Cachés con tu propio registro de encontrado que no están marcados como encontrados", + "repair_fix_recalculate": "Recalcular", + "repair_fix_mark_found": "Marcar como encontrado", "repair_fix_convert": "Convertir", "repair_fix_clear": "Borrar", "repair_col_field": "Campo", diff --git a/src/opensak/lang/fr.py b/src/opensak/lang/fr.py index 87d55aef..543a2615 100644 --- a/src/opensak/lang/fr.py +++ b/src/opensak/lang/fr.py @@ -1129,6 +1129,10 @@ "repair_chk_date_unreadable": "Dates illisibles", "repair_chk_date_range": "Dates antérieures à mai 2000 ou dans le futur", "repair_chk_date_placeholder": "Date fictive 1970-01-01 (« pas de date » de GSAK)", + "repair_chk_totals": "Nombres ou dates de logs, waypoints et objets voyageurs qui ne correspondent pas aux lignes de la cache", + "repair_chk_found_mark": "Caches avec votre propre log de trouvaille qui ne sont pas marquées comme trouvées", + "repair_fix_recalculate": "Recalculer", + "repair_fix_mark_found": "Marquer comme trouvée", "repair_fix_convert": "Convertir", "repair_fix_clear": "Effacer", "repair_col_field": "Champ", diff --git a/src/opensak/lang/nl.py b/src/opensak/lang/nl.py index e2892dde..c4a68dde 100644 --- a/src/opensak/lang/nl.py +++ b/src/opensak/lang/nl.py @@ -1131,6 +1131,10 @@ "repair_chk_date_unreadable": "Datums die niet gelezen kunnen worden", "repair_chk_date_range": "Datums vóór mei 2000 of in de toekomst", "repair_chk_date_placeholder": "Plaatsvervangende datum 1970-01-01 (\"geen datum\" uit GSAK)", + "repair_chk_totals": "Aantallen of datums van logs, waypoints en trackables die niet kloppen met de rijen van de cache", + "repair_chk_found_mark": "Caches met je eigen vondstlog die niet als gevonden gemarkeerd zijn", + "repair_fix_recalculate": "Opnieuw berekenen", + "repair_fix_mark_found": "Markeren als gevonden", "repair_fix_convert": "Omzetten", "repair_fix_clear": "Wissen", "repair_col_field": "Veld", diff --git a/src/opensak/lang/pl.py b/src/opensak/lang/pl.py index a3e4b901..4244d4bb 100644 --- a/src/opensak/lang/pl.py +++ b/src/opensak/lang/pl.py @@ -1129,6 +1129,10 @@ "repair_chk_date_unreadable": "Daty, których nie można odczytać", "repair_chk_date_range": "Daty sprzed maja 2000 lub z przyszłości", "repair_chk_date_placeholder": "Data zastępcza 1970-01-01 (\"brak daty\" z GSAK)", + "repair_chk_totals": "Liczby lub daty logów, waypointów i trackabli niezgodne z wierszami skrzynki", + "repair_chk_found_mark": "Skrzynki z Twoim logiem znalezienia, które nie są oznaczone jako znalezione", + "repair_fix_recalculate": "Przelicz", + "repair_fix_mark_found": "Oznacz jako znalezioną", "repair_fix_convert": "Konwertuj", "repair_fix_clear": "Wyczyść", "repair_col_field": "Pole", diff --git a/src/opensak/lang/pt.py b/src/opensak/lang/pt.py index b01c79ae..96ccfcae 100644 --- a/src/opensak/lang/pt.py +++ b/src/opensak/lang/pt.py @@ -1128,6 +1128,10 @@ "repair_chk_date_unreadable": "Datas que não podem ser lidas", "repair_chk_date_range": "Datas anteriores a maio de 2000 ou no futuro", "repair_chk_date_placeholder": "Data de substituição 1970-01-01 (\"sem data\" do GSAK)", + "repair_chk_totals": "Contagens ou datas de registos, waypoints e trackables que não batem com as linhas da cache", + "repair_chk_found_mark": "Caches com o seu próprio registo de encontrada que não estão marcadas como encontradas", + "repair_fix_recalculate": "Recalcular", + "repair_fix_mark_found": "Marcar como encontrada", "repair_fix_convert": "Converter", "repair_fix_clear": "Limpar", "repair_col_field": "Campo", diff --git a/src/opensak/lang/se.py b/src/opensak/lang/se.py index be2bc60d..3f8c938d 100644 --- a/src/opensak/lang/se.py +++ b/src/opensak/lang/se.py @@ -1128,6 +1128,10 @@ "repair_chk_date_unreadable": "Datum som inte kan läsas", "repair_chk_date_range": "Datum före maj 2000 eller i framtiden", "repair_chk_date_placeholder": "Platshållardatum 1970-01-01 (\"inget datum\" från GSAK)", + "repair_chk_totals": "Antal eller datum för loggar, waypoints och trackables som inte stämmer med cachens rader", + "repair_chk_found_mark": "Cacher med din egen hittad-logg som inte är markerade som hittade", + "repair_fix_recalculate": "Räkna om", + "repair_fix_mark_found": "Markera som hittad", "repair_fix_convert": "Konvertera", "repair_fix_clear": "Rensa", "repair_col_field": "Fält", diff --git a/tests/unit-tests/test_db_repair_1051.py b/tests/unit-tests/test_db_repair_1051.py index ba6990b7..d6b673ce 100644 --- a/tests/unit-tests/test_db_repair_1051.py +++ b/tests/unit-tests/test_db_repair_1051.py @@ -45,7 +45,8 @@ def _healthy(path: Path) -> Path: init_db(db_path=path) with get_session() as s: gc1 = Cache(gc_code="GC1", name="One", cache_type="Traditional Cache", - latitude=47.0, longitude=8.0, difficulty=1.5, terrain=5.0) + latitude=47.0, longitude=8.0, difficulty=1.5, terrain=5.0, + log_count=1, waypoint_count=1) gc2 = Cache(gc_code="GC2", name="Two", cache_type="Mystery Cache", latitude=47.1, longitude=8.1, difficulty=2.0, terrain=2.0) cw = Cache(gc_code="CW1", name="Parking", cache_type="Custom", diff --git a/tests/unit-tests/test_db_repair_dates_1051.py b/tests/unit-tests/test_db_repair_dates_1051.py index 7de3d541..2309ff6d 100644 --- a/tests/unit-tests/test_db_repair_dates_1051.py +++ b/tests/unit-tests/test_db_repair_dates_1051.py @@ -40,7 +40,11 @@ def _database(path: Path) -> Path: init_db(db_path=path) with get_session() as s: gc1 = Cache(gc_code="GC1", name="One", cache_type="Traditional Cache", - latitude=47.0, longitude=8.0, hidden_date=datetime(2010, 5, 1)) + latitude=47.0, longitude=8.0, hidden_date=datetime(2010, 5, 1), + # Its totals, as the importer sets them (#1051 phase 3). + log_count=1, last_log_date=datetime(2020, 1, 2, 3, 4, 5), + last_found_date=datetime(2020, 1, 2, 3, 4, 5), + last_four_logs="2020-01-02T03:04:05 Found it ") s.add_all([gc1, Cache(gc_code="GC2", name="Two", cache_type="Traditional Cache", latitude=47.1, longitude=8.1), @@ -72,7 +76,10 @@ def _value(path: Path, sql: str): def _counts(result: repair.ScanResult) -> dict[str, int]: - return {f.key: f.count for f in result.problems} + """The problems found, without the totals: changing log dates or adding + waypoints by SQL makes those differ too (tested in TestTotals).""" + return {f.key: f.count for f in result.problems + if f.check not in (repair.TOTALS, repair.FOUND_MARK)} def _load_all(path: Path) -> None: @@ -159,13 +166,13 @@ def test_a_maze_exhibit_may_be_in_the_future(self, tmp_path): future = repair_dates.to_stored(datetime.now() + timedelta(days=300)) _sql(db, "UPDATE caches SET cache_type = 'GPS Adventures Maze', hidden_date = ? " "WHERE gc_code = 'GC2'", params=(future,)) - assert repair.scan(db).problems == () + assert _counts(repair.scan(db)) == {} assert "GPS Adventures Maze" in repair_dates.DATED_CACHE_TYPES def test_first_geocaching_day_is_fine(self, tmp_path): db = _database(tmp_path / "Dates.db") _sql(db, "UPDATE caches SET hidden_date = '2000-05-03 00:00:00.000000'") - assert repair.scan(db).problems == () + assert _counts(repair.scan(db)) == {} class TestRepair: @@ -177,13 +184,16 @@ def test_a_cache_with_a_bad_date_loads_again(self, tmp_path): _load_all(db) dispose_engine(db) keys = {f.key for f in repair.scan(db).fixable} + # The cleared log date also changes GC1's totals (phase 3), which the + # same repair recalculates after the dates. assert repair.repair(db, keys).fixed == {repair.DATE_FORMAT: 1, - repair.DATE_UNREADABLE: 1} + repair.DATE_UNREADABLE: 1, repair.TOTALS: 1} + assert _value(db, "SELECT last_log_date FROM caches WHERE gc_code = 'GC1'") is None _load_all(db) assert _value(db, "SELECT found_date FROM caches WHERE gc_code = 'GC2'") \ == "2024-03-05 00:00:00.000000" assert _value(db, "SELECT log_date FROM logs") is None - assert repair.scan(db).problems == () + assert _counts(repair.scan(db)) == {} def test_not_null_columns_get_now(self, tmp_path): db = _database(tmp_path / "Dates.db") @@ -192,7 +202,7 @@ def test_not_null_columns_get_now(self, tmp_path): repair.repair(db, {repair.DATE_UNREADABLE}) stamp = _value(db, "SELECT imported_at FROM caches WHERE gc_code = 'GC1'") assert stamp >= before - assert repair.scan(db).problems == () + assert _counts(repair.scan(db)) == {} def test_only_the_ticked_date_fix(self, tmp_path): db = _database(tmp_path / "Dates.db") @@ -238,7 +248,8 @@ def test_lists_the_date_checks(self, qtbot, tmp_path): assert rows[repair.DATE_FORMAT].text(3) == "GC1 hidden_date 2011-04-03" assert "GC1 hidden_date 2011-04-03" in rows[repair.DATE_FORMAT].toolTip(3) assert rows[repair.DATE_RANGE].text(2) == tr("repair_report_only") - assert dlg.checked_keys() == {repair.DATE_FORMAT} + assert repair.DATE_FORMAT in dlg.checked_keys() + assert repair.DATE_RANGE not in dlg.checked_keys() class TestPlaceholder: @@ -274,7 +285,7 @@ def test_fix_clears_it(self, tmp_path): assert _value(db, "SELECT count(*) FROM waypoints WHERE wp_date IS NULL") == 2 # imported_at is NOT NULL: now instead. assert _value(db, "SELECT imported_at FROM caches WHERE gc_code = 'GC2'") > "2000" - assert repair.scan(db).problems == () + assert _counts(repair.scan(db)) == {} def test_other_old_dates_are_still_too_early(self, tmp_path): db = self._with_waypoints(tmp_path / "Dates.db") diff --git a/tests/unit-tests/test_db_repair_totals_1051.py b/tests/unit-tests/test_db_repair_totals_1051.py new file mode 100644 index 00000000..8e55b503 --- /dev/null +++ b/tests/unit-tests/test_db_repair_totals_1051.py @@ -0,0 +1,240 @@ +""" +tests/unit-tests/test_db_repair_totals_1051.py — database repair, phase 3 +(#1051): the counts and dates derived from a cache's logs, waypoints and +trackables, and the found mark from the user's own found logs. +""" + +from __future__ import annotations + +import sqlite3 +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +pytest.importorskip("pytestqt") + +from opensak.db import companion, repair, repair_totals +from opensak.db.database import dispose_engine, get_session, init_db +from opensak.db.models import Cache, Log, Trackable, Waypoint +from opensak.db.repair_dates import Detail +from opensak.gui.dialogs import database_repair_dialog as drd +from opensak.lang import load_language, tr +from opensak.utils import flags + +ME = ("4711", "Me") +D1 = datetime(2021, 3, 4, 12, 0, 0) # whole hours: a fixed time, calendar date as is +D2 = datetime(2023, 6, 10, 12, 0, 0) + + +@pytest.fixture(autouse=True) +def _fresh(monkeypatch): + load_language("en") + monkeypatch.setattr(flags, "companion_db", False) + companion.reset() + yield + dispose_engine() + companion.reset() + + +def _database(path: Path) -> Path: + """GC1: two logs (one my own find), a waypoint and a trackable, with the + totals as the importer sets them; GC2: nothing, found by hand.""" + path.parent.mkdir(parents=True, exist_ok=True) + init_db(db_path=path) + with get_session() as s: + gc1 = Cache(gc_code="GC1", name="One", cache_type="Traditional Cache", + latitude=47.0, longitude=8.0, found=True, found_date=datetime(2021, 3, 4), + log_count=2, found_log_count=1, last_log_date=D2, last_found_date=D2, + last_four_logs="2023-06-10T12:00:00\tFound it\tOther\n" + "2021-03-04T12:00:00\tFound it\tMe", + waypoint_count=1, trackable_count=1, find_count=99) + gc2 = Cache(gc_code="GC2", name="Lab", cache_type="Lab Cache", + latitude=47.1, longitude=8.1, found=True) + s.add_all([gc1, gc2]) + s.flush() + s.add_all([ + Log(cache_id=gc1.id, log_id="1", log_type="Found it", finder="Me", + finder_id="4711", log_date=D1), + Log(cache_id=gc1.id, log_id="2", log_type="Found it", finder="Other", + finder_id="9", log_date=D2), + Waypoint(cache_id=gc1.id, parent_gc_code="GC1", prefix="PK", name="Parking"), + Trackable(cache_id=gc1.id, name="TB"), + ]) + dispose_engine(path) + return path + + +def _sql(path: Path, *statements: str) -> None: + conn = sqlite3.connect(str(path)) + try: + with conn: + for statement in statements: + conn.execute(statement) + finally: + conn.close() + + +def _row(path: Path, sql: str): + conn = sqlite3.connect(str(path)) + try: + return conn.execute(sql).fetchone() + finally: + conn.close() + + +def _finding(result: repair.ScanResult, check: str) -> repair.Finding: + return next(f for f in result.findings if f.check == check) + + +class TestScan: + def test_matching_totals_are_not_reported(self, tmp_path): + assert repair.scan(_database(tmp_path / "Ok.db"), account=ME).problems == () + + def test_every_derived_column(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET log_count = 12, last_log_date = NULL, " + "last_found_date = '2020-01-01 00:00:00.000000', last_four_logs = NULL, " + "waypoint_count = 0, trackable_count = 5 WHERE gc_code = 'GC1'") + finding = _finding(repair.scan(db, account=ME), repair.TOTALS) + assert finding.count == 1 and finding.fixable + assert finding.details == ( + Detail("GC1", "last_found_date", + "2020-01-01 00:00:00.000000 → 2023-06-10 12:00:00.000000"), + Detail("GC1", "last_four_logs", "NULL → 2023-06-10T12:00:00 Found it Other | " + "2021-03-04T12:00:00 Found it Me"), + Detail("GC1", "last_log_date", "NULL → 2023-06-10 12:00:00.000000"), + Detail("GC1", "log_count", "12 → 2"), + Detail("GC1", "trackable_count", "5 → 1"), + Detail("GC1", "waypoint_count", "0 → 1"), + ) + + def test_logs_with_the_same_date_in_another_order_are_fine(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE logs SET log_date = '2023-06-10 12:00:00.000000'", + "UPDATE caches SET last_found_date = '2023-06-10 12:00:00.000000', " + "last_four_logs = '2023-06-10T12:00:00' || char(9) || 'Found it' || char(9) || 'Me'" + " || char(10) || '2023-06-10T12:00:00' || char(9) || 'Found it' || char(9) || " + "'Other' WHERE gc_code = 'GC1'") + assert repair.scan(db, account=ME).problems == () + + def test_own_found_log_count_is_raised_never_lowered(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET found_log_count = 0 WHERE gc_code = 'GC1'") + finding = _finding(repair.scan(db, account=ME), repair.TOTALS) + assert finding.details == (Detail("GC1", "found_log_count", "0 → 1"),) + # More finds than logs (a Pocket Query brings only some): kept. + _sql(db, "UPDATE caches SET found_log_count = 3 WHERE gc_code = 'GC1'") + assert repair.scan(db, account=ME).problems == () + + def test_own_log_matched_by_name_too(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET found_log_count = 0 WHERE gc_code = 'GC1'") + finding = _finding(repair.scan(db, account=(None, "me")), repair.TOTALS) + assert finding.details == (Detail("GC1", "found_log_count", "0 → 1"),) + + def test_found_mark(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET found = 0, found_date = NULL WHERE gc_code = 'GC1'") + finding = _finding(repair.scan(db, account=ME), repair.FOUND_MARK) + assert finding.fixable + assert finding.details == (Detail("GC1", "found", "0 → 1, found_date 2021-03-04"),) + + def test_without_an_account_own_logs_are_not_checked(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET found = 0, found_log_count = 0 WHERE gc_code = 'GC1'") + assert repair.scan(db, account=None).problems == () + assert repair.scan(db, account=("", " ")).problems == () + + def test_found_without_an_own_log_is_kept(self, tmp_path): + # GC2 is a Lab marked found by hand: no log behind it, never cleared. + result = repair.scan(_database(tmp_path / "T.db"), account=ME) + assert result.problems == () + + def test_logs_without_a_cache_are_skipped(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "PRAGMA foreign_keys=OFF", + "INSERT INTO logs (cache_id, log_id, log_type) VALUES (0, 'x', 'Note')", + "INSERT INTO logs (cache_id, log_id, log_type) VALUES (99999, 'y', 'Note')") + totals = _finding(repair.scan(db, account=ME), repair.TOTALS) + assert totals.count == 0 + + +class TestRepair: + def test_recalculates_and_a_second_run_finds_nothing(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET log_count = 12, last_log_date = NULL, last_four_logs = NULL, " + "waypoint_count = 0, trackable_count = 5, found_log_count = 0 " + "WHERE gc_code = 'GC1'") + assert repair.repair(db, {repair.TOTALS}, account=ME).fixed == {repair.TOTALS: 1} + assert _row(db, "SELECT log_count, last_log_date, waypoint_count, trackable_count, " + "found_log_count, last_four_logs, find_count FROM caches " + "WHERE gc_code = 'GC1'") == ( + 2, "2023-06-10 12:00:00.000000", 1, 1, 1, + "2023-06-10T12:00:00\tFound it\tOther\n2021-03-04T12:00:00\tFound it\tMe", 99) + assert repair.scan(db, account=ME).problems == () + + def test_recalculated_values_load(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET last_log_date = NULL WHERE gc_code = 'GC1'") + repair.repair(db, {repair.TOTALS}, account=ME) + init_db(db_path=db) + with get_session() as s: + assert s.query(Cache).filter_by(gc_code="GC1").one().last_log_date == D2 + + def test_marks_found_with_the_date_of_the_own_log(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET found = 0, found_date = NULL WHERE gc_code = 'GC1'") + assert repair.repair(db, {repair.FOUND_MARK}, account=ME).fixed == {repair.FOUND_MARK: 1} + assert _row(db, "SELECT found, found_date FROM caches WHERE gc_code = 'GC1'") == ( + 1, "2021-03-04 00:00:00.000000") + + def test_marking_found_keeps_a_found_date(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET found = 0, found_date = '2021-03-05 00:00:00.000000' " + "WHERE gc_code = 'GC1'") + repair.repair(db, {repair.FOUND_MARK}, account=ME) + assert _row(db, "SELECT found, found_date FROM caches WHERE gc_code = 'GC1'") == ( + 1, "2021-03-05 00:00:00.000000") + + def test_only_the_ticked_fix(self, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET found = 0, log_count = 7 WHERE gc_code = 'GC1'") + assert repair.repair(db, {repair.FOUND_MARK}, account=ME).fixed == {repair.FOUND_MARK: 1} + assert _row(db, "SELECT log_count FROM caches WHERE gc_code = 'GC1'") == (7,) + + def test_after_deleting_orphans_and_clearing_dates(self, tmp_path): + # The totals are recalculated from what the other fixes leave. + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE logs SET log_date = 'garbage' WHERE log_id = '2'") + keys = {f.key for f in repair.scan(db, account=ME).fixable} + assert keys == {repair.DATE_UNREADABLE, repair.TOTALS} + repair.repair(db, keys, account=ME) + assert _row(db, "SELECT last_log_date, last_found_date, log_count FROM caches " + "WHERE gc_code = 'GC1'") == ( + "2021-03-04 12:00:00.000000", "2021-03-04 12:00:00.000000", 2) + assert repair.scan(db, account=ME).problems == () + + +class TestDialog: + def test_uses_the_account_from_settings(self, qtbot, tmp_path): + db = _database(tmp_path / "T.db") + _sql(db, "UPDATE caches SET found = 0 WHERE gc_code = 'GC1'") + with patch.object(drd, "_account", return_value=ME): + dlg = drd.DatabaseRepairDialog(SimpleNamespace(name="T", path=db)) + qtbot.addWidget(dlg) + qtbot.waitUntil(lambda: not dlg.is_running() and dlg.scan_result is not None, + timeout=10000) + item = dlg._tree.topLevelItem(0) + assert item is not None + assert item.text(0) == tr("repair_chk_found_mark") + assert item.text(2) == tr("repair_fix_mark_found") + assert item.text(3) == "GC1 found 0 → 1" + assert dlg.checked_keys() == {repair.FOUND_MARK} + + +def test_shown_keeps_tab_separated_copies_intact(): + assert repair_totals.shown("a\tb\nc\td") == "a b | c d" + assert repair_totals.shown(None) == "NULL"