From 8abe56e15a9df746b55cc61b11a000971b14c951 Mon Sep 17 00:00:00 2001 From: sahil-singh1107 Date: Mon, 28 Sep 2026 12:45:47 +0530 Subject: [PATCH 1/8] fix(wal): improve replay and recovery --- src/seriousdb/wal.py | 53 ++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/seriousdb/wal.py b/src/seriousdb/wal.py index e6b04f5..baa099c 100644 --- a/src/seriousdb/wal.py +++ b/src/seriousdb/wal.py @@ -92,6 +92,10 @@ class WriteAheadLog: def __init__(self, filename: str): self.filename = filename self._offset = os.path.getsize(filename) if os.path.isfile(filename) else 0 + self._file = open(filename, "a+b") + + def close(self) -> None: + self._file.close() def append(self, entry: WalEntry) -> None: """Append `entry` to the log and fsync it. @@ -107,11 +111,13 @@ def append(self, entry: WalEntry) -> None: If the log file cannot be written. """ self._repair_torn_tail() - with open(self.filename, "ab") as f: - data = (json.dumps(entry.to_dict()) + "\n").encode() - f.write(data) - f.flush() - os.fsync(f.fileno()) + + data = (json.dumps(entry.to_dict()) + "\n").encode() + + self._file.write(data) + self._file.flush() + os.fsync(self._file.fileno()) + self._offset += len(data) def replay(self) -> list[WalEntry]: @@ -138,23 +144,25 @@ def replay(self) -> list[WalEntry]: entries: list[WalEntry] = [] good_offset = 0 found_bad_entry = False - with open(self.filename, "rb") as f: - for raw_line in f: - if not raw_line.endswith(b"\n"): + + self._file.seek(0) + + for raw_line in self._file: + if not raw_line.endswith(b"\n"): + found_bad_entry = True + break + line = raw_line.strip() + if line: + try: + entries.append(WalEntry.from_dict(json.loads(line.decode()))) + except (json.JSONDecodeError, UnicodeDecodeError, ValueError) as e: + logger.warning( + "Corrupt entry in write-ahead log %s (%s)", + self.filename, + e, + ) found_bad_entry = True break - line = raw_line.strip() - if line: - try: - entries.append(WalEntry.from_dict(json.loads(line.decode()))) - except (json.JSONDecodeError, UnicodeDecodeError, ValueError) as e: - logger.warning( - "Corrupt entry in write-ahead log %s (%s)", - self.filename, - e, - ) - found_bad_entry = True - break good_offset += len(raw_line) if found_bad_entry: @@ -162,8 +170,7 @@ def replay(self) -> list[WalEntry]: "Truncating write-ahead log %s to its last known-good entry", self.filename, ) - with open(self.filename, "r+b") as f: - f.truncate(good_offset) + self._file.truncate(good_offset) self._offset = good_offset return entries @@ -180,7 +187,9 @@ def clear(self) -> None: with tempfile.NamedTemporaryFile("wb", dir=dir_name, delete=False) as tmp_file: pass try: + self._file.close() os.replace(tmp_file.name, self.filename) + self._file = open(self.filename, "a+b") except OSError: os.unlink(tmp_file.name) raise From 61d43280d3bf46614afa17b83c10f6391cf69c2d Mon Sep 17 00:00:00 2001 From: sahil-singh1107 Date: Wed, 30 Sep 2026 21:40:58 +0530 Subject: [PATCH 2/8] fix: manage WAL file handles during load, clear and unsuccessful writes --- src/seriousdb/cache.py | 2 ++ src/seriousdb/wal.py | 42 +++++++++++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/seriousdb/cache.py b/src/seriousdb/cache.py index 93d7057..ba8a348 100644 --- a/src/seriousdb/cache.py +++ b/src/seriousdb/cache.py @@ -321,6 +321,8 @@ def load(self, filename: str) -> None: ) self.db = _write_default(filename) self.filename = filename + if self.wal is not None: + self.wal.close() self.wal = WriteAheadLog(f"{filename}.wal") replayed = self.wal.replay() self._writes_since_compact = len(replayed) diff --git a/src/seriousdb/wal.py b/src/seriousdb/wal.py index baa099c..81e0a26 100644 --- a/src/seriousdb/wal.py +++ b/src/seriousdb/wal.py @@ -95,7 +95,13 @@ def __init__(self, filename: str): self._file = open(filename, "a+b") def close(self) -> None: - self._file.close() + if self._file is not None: + self._file.close() + self._file = None + + def _ensure_open(self): + if self._file is None: + self._file = open(self.filename, "a+b") def append(self, entry: WalEntry) -> None: """Append `entry` to the log and fsync it. @@ -111,14 +117,24 @@ def append(self, entry: WalEntry) -> None: If the log file cannot be written. """ self._repair_torn_tail() + self._ensure_open() data = (json.dumps(entry.to_dict()) + "\n").encode() - self._file.write(data) - self._file.flush() - os.fsync(self._file.fileno()) - - self._offset += len(data) + try: + self._file.write(data) + self._file.flush() + os.fsync(self._file.fileno()) + + self._offset += len(data) + except Exception: + try: + self._file.close() + except OSError: + pass + finally: + self._file = None + raise def replay(self) -> list[WalEntry]: """Return every entry durably written to the log, oldest first. @@ -141,6 +157,8 @@ def replay(self) -> list[WalEntry]: self._offset = 0 return [] + self._ensure_open() + entries: list[WalEntry] = [] good_offset = 0 found_bad_entry = False @@ -186,14 +204,20 @@ def clear(self) -> None: dir_name = os.path.dirname(self.filename) or "." with tempfile.NamedTemporaryFile("wb", dir=dir_name, delete=False) as tmp_file: pass + + self.close() + try: - self._file.close() os.replace(tmp_file.name, self.filename) - self._file = open(self.filename, "a+b") except OSError: - os.unlink(tmp_file.name) + try: + os.unlink(tmp_file.name) + except FileNotFoundError: + pass raise + self._offset = 0 + self._file = None def _repair_torn_tail(self) -> None: """Truncate any bytes left by a previous failed write. From c9b704b750aeef45bd6b913fcfc2d640142212f9 Mon Sep 17 00:00:00 2001 From: sahil-singh1107 Date: Wed, 30 Sep 2026 21:56:45 +0530 Subject: [PATCH 3/8] add: simple benchmark test for wal append --- benchmarks/test_wal.py | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 benchmarks/test_wal.py diff --git a/benchmarks/test_wal.py b/benchmarks/test_wal.py new file mode 100644 index 0000000..a6bb8ff --- /dev/null +++ b/benchmarks/test_wal.py @@ -0,0 +1,46 @@ +"""Benchmarks for write-ahead log persistence.""" + +from pathlib import Path + +import pytest + +from seriousdb.wal import SetEntry, WriteAheadLog + +from ._support import WARMUP_ROUNDS + + +@pytest.mark.benchmark(group="wal") +def test_wal_append( + benchmark, + database_file: Path, + measured_rounds: int, +) -> None: + """Measure appending one entry to the WAL, including fsync.""" + + entry = SetEntry(key="key", value="value") + + def prepare_wal(): + database_file.unlink(missing_ok=True) + wal = WriteAheadLog(str(database_file)) + return (wal,), {} + + def append_entry(wal: WriteAheadLog) -> None: + wal.append(entry) + + def close_wal(wal: WriteAheadLog) -> None: + wal.close() + + benchmark.extra_info.update( + operation="WAL append", + persistence="write + flush + fsync", + key_bytes=len(entry.key.encode()), + value_bytes=len(entry.value.encode()), + ) + + benchmark.pedantic( + append_entry, + setup=prepare_wal, + teardown=close_wal, + rounds=50, + warmup_rounds=WARMUP_ROUNDS, + ) From d4953c119cede3ef845a8970e6004951f7930617 Mon Sep 17 00:00:00 2001 From: sahil-singh1107 Date: Thu, 8 Oct 2026 15:16:26 +0530 Subject: [PATCH 4/8] resolve merge conflicts --- src/seriousdb/wal.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/seriousdb/wal.py b/src/seriousdb/wal.py index 81e0a26..e350b0d 100644 --- a/src/seriousdb/wal.py +++ b/src/seriousdb/wal.py @@ -94,7 +94,7 @@ def __init__(self, filename: str): self._offset = os.path.getsize(filename) if os.path.isfile(filename) else 0 self._file = open(filename, "a+b") - def close(self) -> None: + def _close(self) -> None: if self._file is not None: self._file.close() self._file = None @@ -173,7 +173,7 @@ def replay(self) -> list[WalEntry]: if line: try: entries.append(WalEntry.from_dict(json.loads(line.decode()))) - except (json.JSONDecodeError, UnicodeDecodeError, ValueError) as e: + except (json.JSONDecodeError, UnicodeDecodeError, TypeError, ValueError) as e: logger.warning( "Corrupt entry in write-ahead log %s (%s)", self.filename, @@ -181,7 +181,7 @@ def replay(self) -> list[WalEntry]: ) found_bad_entry = True break - good_offset += len(raw_line) + good_offset += len(raw_line) if found_bad_entry: logger.warning( @@ -205,7 +205,7 @@ def clear(self) -> None: with tempfile.NamedTemporaryFile("wb", dir=dir_name, delete=False) as tmp_file: pass - self.close() + self._close() try: os.replace(tmp_file.name, self.filename) From 99f9bae7ac870d5bce8089ed6191c90b521399d2 Mon Sep 17 00:00:00 2001 From: sahil-singh1107 Date: Mon, 12 Oct 2026 18:58:35 +0530 Subject: [PATCH 5/8] fix Cache.load --- src/seriousdb/cache.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/seriousdb/cache.py b/src/seriousdb/cache.py index 2e8491b..18966e1 100644 --- a/src/seriousdb/cache.py +++ b/src/seriousdb/cache.py @@ -317,19 +317,16 @@ def load(self, filename: str) -> None: e, backup, ) - self.db = _write_default(filename) - self.filename = filename + db = _write_default(filename) if self.wal is not None: self.wal.close() self.wal = WriteAheadLog(f"{filename}.wal") replayed = self.wal.replay() - self._writes_since_compact = len(replayed) for entry in replayed: entry.apply(db) self.db = db self.filename = filename - self.wal = wal self._writes_since_compact = len(replayed) def flush(self) -> None: From fedd60d5456e414d47a4987f00e2a1be5c5f0d39 Mon Sep 17 00:00:00 2001 From: chrollo-lucifer-12 Date: Mon, 12 Oct 2026 13:38:45 +0000 Subject: [PATCH 6/8] fix: keep cache uninitialized when WAL replay fails --- src/seriousdb/cache.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/seriousdb/cache.py b/src/seriousdb/cache.py index 18966e1..4a93a90 100644 --- a/src/seriousdb/cache.py +++ b/src/seriousdb/cache.py @@ -318,16 +318,25 @@ def load(self, filename: str) -> None: backup, ) db = _write_default(filename) - if self.wal is not None: - self.wal.close() - self.wal = WriteAheadLog(f"{filename}.wal") - replayed = self.wal.replay() - for entry in replayed: - entry.apply(db) + wal = WriteAheadLog(f"{filename}.wal") + try: + replayed = wal.replay() + for entry in replayed: + entry.apply(db) + except Exception: + try: + wal.close() + except OSError: + pass + raise + old_wal = self.wal self.db = db self.filename = filename + self.wal = wal self._writes_since_compact = len(replayed) + if old_wal is not None: + old_wal.close() def flush(self) -> None: """No-op, kept for backward compatibility. From de18445ee6631530a73b8a35c0180b85a3338117 Mon Sep 17 00:00:00 2001 From: chrollo-lucifer-12 Date: Mon, 12 Oct 2026 13:43:41 +0000 Subject: [PATCH 7/8] fix: add docstring for close and _ensure_open --- src/seriousdb/wal.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/seriousdb/wal.py b/src/seriousdb/wal.py index a4fcb9f..87567b0 100644 --- a/src/seriousdb/wal.py +++ b/src/seriousdb/wal.py @@ -105,11 +105,13 @@ def __init__(self, filename: str): self._file = open(filename, "a+b") def close(self) -> None: - f, self._file = self._file, None - if f is not None: + """Close the WAL file if it is open.""" + f, self._file = self._file, None + if f is not None: f.close() def _ensure_open(self): + """Return the open WAL file, reopening it if necessary.""" if self._file is None: self._file = open(self.filename, "a+b") return self._file From e583d10c7c1a2e0fe54248e24259ae5ce9ae25ee Mon Sep 17 00:00:00 2001 From: chrollo-lucifer-12 Date: Thu, 15 Oct 2026 16:21:37 +0000 Subject: [PATCH 8/8] fix the format checks and benchmark test --- benchmarks/test_wal.py | 38 ++++++++++++++++++++++++++++++++++++++ src/seriousdb/wal.py | 20 ++++++++++++-------- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/benchmarks/test_wal.py b/benchmarks/test_wal.py index a6bb8ff..9742738 100644 --- a/benchmarks/test_wal.py +++ b/benchmarks/test_wal.py @@ -44,3 +44,41 @@ def close_wal(wal: WriteAheadLog) -> None: rounds=50, warmup_rounds=WARMUP_ROUNDS, ) + + +@pytest.mark.benchmark(group="wal") +def test_wal_append_many( + benchmark, + database_file: Path, +) -> None: + """Measure repeated appends using one open WAL.""" + + entry = SetEntry(key="key", value="value") + + def prepare_wal(): + database_file.unlink(missing_ok=True) + wal = WriteAheadLog(str(database_file)) + return (wal,), {} + + def append_many(wal: WriteAheadLog) -> None: + for _ in range(1000): + wal.append(entry) + + def close_wal(wal: WriteAheadLog) -> None: + wal.close() + + benchmark.extra_info.update( + operation="1000 WAL appends", + persistence="write + flush + fsync", + entries=1000, + key_bytes=len(entry.key.encode()), + value_bytes=len(entry.value.encode()), + ) + + benchmark.pedantic( + append_many, + setup=prepare_wal, + teardown=close_wal, + rounds=10, + warmup_rounds=WARMUP_ROUNDS, + ) diff --git a/src/seriousdb/wal.py b/src/seriousdb/wal.py index 87567b0..e832013 100644 --- a/src/seriousdb/wal.py +++ b/src/seriousdb/wal.py @@ -102,18 +102,18 @@ class WriteAheadLog: def __init__(self, filename: str): self.filename = filename self._offset = os.path.getsize(filename) if os.path.isfile(filename) else 0 - self._file = open(filename, "a+b") + self._file = open(filename, "a+b") # noqa: SIM115 - kept open, closed in close() def close(self) -> None: - """Close the WAL file if it is open.""" - f, self._file = self._file, None - if f is not None: + """Close the WAL file if it is open.""" + f, self._file = self._file, None + if f is not None: f.close() def _ensure_open(self): """Return the open WAL file, reopening it if necessary.""" if self._file is None: - self._file = open(self.filename, "a+b") + self._file = open(self.filename, "a+b") # noqa: SIM115 - reopened lazily, closed in close() return self._file def append(self, entry: WalEntry) -> None: @@ -186,7 +186,12 @@ def replay(self) -> list[WalEntry]: if line: try: entries.append(WalEntry.from_dict(json.loads(line.decode()))) - except (json.JSONDecodeError, UnicodeDecodeError, TypeError, ValueError) as e: + except ( + json.JSONDecodeError, + UnicodeDecodeError, + TypeError, + ValueError, + ) as e: logger.warning( "Corrupt entry in write-ahead log %s (%s)", self.filename, @@ -218,9 +223,8 @@ def clear(self) -> None: with tempfile.NamedTemporaryFile("wb", dir=dir_name, delete=False) as tmp_file: pass - self.close() - try: + self.close() os.replace(tmp_file.name, self.filename) except OSError: try: