From ce84950a53ef7c586b25f6f1a088ce4ee16a8f74 Mon Sep 17 00:00:00 2001 From: nammovuivui Date: Wed, 21 Oct 2026 17:05:55 +0700 Subject: [PATCH] perf(translate): remember BabelDOC font hash passes BabelDOC re-hashes all 34 target-language fonts (266 MB) in every pipeline stage, 7-8 times per page. Memoize assets.verify_file on passes only, tied to size+mtime, and pre-verify cached fonts on the engine-warm thread. 3-page Argos run: 28.1 s -> 16.7 s. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/translation-pipeline.md | 60 ++- .../api/routes/translation.py | 4 + src/desktop_pdf_translator/api/server.py | 21 +- .../processors/font_verify_cache.py | 123 ++++++ tests/test_engine_warm_gate.py | 42 +- tests/test_font_verify_cache.py | 386 ++++++++++++++++++ 6 files changed, 623 insertions(+), 13 deletions(-) create mode 100644 src/desktop_pdf_translator/processors/font_verify_cache.py create mode 100644 tests/test_font_verify_cache.py diff --git a/docs/architecture/translation-pipeline.md b/docs/architecture/translation-pipeline.md index 9c821d7..64140c8 100644 --- a/docs/architecture/translation-pipeline.md +++ b/docs/architecture/translation-pipeline.md @@ -99,17 +99,55 @@ it into `babeldoc_chunk_error`. **What this does not fix, and why chunk size is a live question.** The layout model was never the largest per-chunk fixed cost. `FontMapper.__init__` sha3_256-verifies and loads **all 34 embedding fonts -(~250 MB)** for `lang_out`, and nothing memoizes it — not `fontmap.py`, not -`assets.py`. BabelDOC constructs one per stage: `ILCreater`, `ParagraphFinder`, -`StylesAndFormulas`, `ILTranslator`, `Typesetting`, `PDFCreater`. Measured -against the installed babeldoc on a warm cache: **1.02 s per `FontMapper`, ≈6 s -per chunk**, none of which this cache touches. So "the reload is fixed" is not -on its own a reason to shrink a chunk — shrinking one multiplies ~6 s by the -extra chunks it creates, and on the Argos path (`_MAX_PARALLEL_CHUNKS_ARGOS = 2`) -that can outweigh what the cache saves. `_PAGES_PER_CHUNK_ARGOS` is 1 today, -like the LLM path; measure an end-to-end Argos run before trusting that, or -before changing either constant. What a larger chunk buys is amortized -per-chunk fixed overhead in general, not the ONNX reload specifically. +(266 MB)** for `lang_out`, and BabelDOC memoizes neither — not `fontmap.py`, not +`assets.py`. It constructs one per stage: `ILCreater`, `ParagraphFinder`, +`StylesAndFormulas`, `ILTranslator` (a second inside `ILTranslatorLLMOnly`), +`Typesetting`, `PDFCreater` (a second for scanned-page detection) — 7 per page +on the Argos path, 8 on the LLM path. The hash is now remembered (next section); +what is left is ~0.26 s per `FontMapper` of `pymupdf.Font` parsing, ≈2 s per +chunk, which this cache does not touch either. So "the reload is fixed" is not +on its own a reason to shrink a chunk — shrinking one multiplies that fixed cost +by the extra chunks it creates, and on the Argos path +(`_MAX_PARALLEL_CHUNKS_ARGOS = 2`) that can outweigh what the cache saves. +`_PAGES_PER_CHUNK_ARGOS` is 1 today, like the LLM path; measure an end-to-end +Argos run before trusting that, or before changing either constant. What a +larger chunk buys is amortized per-chunk fixed overhead in general, not the ONNX +reload specifically. + +## A font that passed its hash check is not hashed again + +A `FontMapper` took ~1.3 s, of which ~1.0 s was `assets.verify_file` hashing +the 34 fonts (measured alone: 0.99 s; parsing them alone: 0.13 s; the ~0.26 s a +mapper still costs with the hash remembered is that parse plus the rest of the +constructor, under the contention of a real run). Traced over real one-page +LLM-path runs on two different documents: **272 checks per page — 34 fonts × 8 +mappers (238 on Argos, with 7) — against the same 34 (file, expected hash) +pairs**, none derived from the document; every pair passed, +and every file was byte-identical before and after both runs. Only a process's +first check can find anything (a missing or corrupt font, which BabelDOC then +re-downloads); the rest re-hash unchanged bytes. + +`processors/font_verify_cache.py` replaces the module global +`babeldoc.assets.assets.verify_file` with a memo — BabelDOC's asset functions +call it by that global name, so that one attribute reaches every stage; re-check +it when bumping babeldoc. Measured on a one-page run: **15.5 s → 7.9 s**, the +`FontMapper` share 60% → 23%. + +- **Only a pass is remembered**, keyed on (path, expected hash) and tied to the + file's size and mtime as they stood both before and after the hash. A failure, + a font rewritten later (a re-download, the setup flow's offline restore) and a + font that changed mid-hash are all hashed again. Nothing persists across + processes. There is no single-flight: chunks that start together before the + warm thread has finished each hash the fonts once — extra work, never a wrong + answer. +- **Installed in two places**, both idempotent: the `engine-warm` thread, and + `routes/translation.py:_load_engine` for a Translate click that beats it. +- **The warm thread pre-checks the fonts on disk** (`preverify_cached_fonts`, + ~1 s), so the first page doesn't pay that round either. It calls `verify_file` + on existing files only, never `get_font_and_metadata`, which downloads — the + same rule that gates the layout-model load. It needs no `babeldoc_core_ready()` + gate for that reason, and has its own `try`, after the model, so a failure + never costs the larger warm-up. **Rolling-PDF pruning keeps two files, not one.** `cleanup_partial_artifacts` runs after every chunk now, not only on cancel, so a long run no longer piles diff --git a/src/desktop_pdf_translator/api/routes/translation.py b/src/desktop_pdf_translator/api/routes/translation.py index 203f834..2e356d0 100644 --- a/src/desktop_pdf_translator/api/routes/translation.py +++ b/src/desktop_pdf_translator/api/routes/translation.py @@ -81,9 +81,13 @@ def _load_engine() -> tuple[type, type]: beats that thread — or a warm-up that failed, which it does silently — lands here instead. Callers must run it off the event loop. """ + from ...processors import font_verify_cache from ...processors.events import EventType from ...processors.processor import PDFProcessor + # Normally done by the warm thread too; a click that beats it would + # otherwise re-hash every font in every stage of the first job. + font_verify_cache.install() return EventType, PDFProcessor diff --git a/src/desktop_pdf_translator/api/server.py b/src/desktop_pdf_translator/api/server.py index bf1f7db..f65c7b9 100644 --- a/src/desktop_pdf_translator/api/server.py +++ b/src/desktop_pdf_translator/api/server.py @@ -126,7 +126,8 @@ def _warm_translation_engine() -> None: only ever warms what is already on disk. Importing BabelDOC is still unconditional — it touches no assets, and it is - the larger half of the cold-start cost. + the larger half of the cold-start cost. So is the font pre-check that + follows (`_preverify_babeldoc_fonts`), which only reads. A Translate click can still beat this thread, so the job path imports BabelDOC in a thread of its own rather than assuming this one won the race @@ -149,9 +150,27 @@ def _warm_translation_engine() -> None: except Exception as exc: # noqa: BLE001 logger.warning("Translation engine warm-up failed (non-fatal): %s", exc) return + _preverify_babeldoc_fonts() logger.info("Translation engine warm in %.1fs", time.perf_counter() - started) +def _preverify_babeldoc_fonts() -> None: + """Hash BabelDOC's cached fonts once now, so the first page doesn't. + + Every stage of every page re-checks them; `font_verify_cache` remembers the + passes, and this pays for the first round (~1 s) before the user clicks. + Ungated by `babeldoc_core_ready()`, unlike the layout model: it only reads + fonts already on disk and never downloads. Its own try, after the model, + so a failure here never costs the larger warm-up. + """ + try: + from ..processors import font_verify_cache + + font_verify_cache.preverify_cached_fonts() + except Exception as exc: # noqa: BLE001 + logger.warning("BabelDOC font pre-check failed (non-fatal): %s", exc) + + def _sweep_orphan_translate_dirs(max_age_seconds: int = 3600) -> int: """Remove `pdfusion-translate-*` dirs left behind by a prior sidecar that crashed or was killed before its next-job cleanup could fire. diff --git a/src/desktop_pdf_translator/processors/font_verify_cache.py b/src/desktop_pdf_translator/processors/font_verify_cache.py new file mode 100644 index 0000000..6de5dbf --- /dev/null +++ b/src/desktop_pdf_translator/processors/font_verify_cache.py @@ -0,0 +1,123 @@ +"""Remembers which of BabelDOC's cached assets have passed their hash check. + +BabelDOC builds a `FontMapper` in every pipeline stage — `ILCreater`, +`ParagraphFinder`, `StylesAndFormulas`, `ILTranslator` (twice on the LLM path), +`Typesetting`, `PDFCreater` (twice when scanned-page detection runs) — and each +one runs `assets.verify_file` over every font of the target language: 34 files, +266 MB of SHA3-256 for Vietnamese, ~1 s. Traced over two different documents on +the LLM path, that is 272 checks per page (238 on Argos, which skips one +mapper) against the same 34 (file, expected hash) pairs, none of them derived +from the document, every file byte-identical before and after. Measured on a +one-page run: 15.5 s → 7.9 s with the passes remembered. + +Only the first check in a process can find something new — a missing or corrupt +font, which BabelDOC then re-downloads — so a pass is remembered and a failure +never is. A pass is tied to the file's size and mtime as they were on both +sides of the hash, so a font rewritten afterwards (a re-download, the setup +flow restoring the offline zip) or while it was being hashed is checked again. + +Kept free of babeldoc imports at module scope, like `doc_layout_cache.py`: the +boot warm thread imports this, and so does the suite. +""" + +from __future__ import annotations + +import logging +import os +import threading +import time +from pathlib import Path +from typing import Callable, Optional + +logger = logging.getLogger(__name__) + +Verify = Callable[[Path, str], bool] + +# Marks a function this module produced, so `install()` can tell a memo it +# already put in place from BabelDOC's own function — or from a test's fake, +# which `monkeypatch` restores, taking the mark with it. +_MEMO_MARK = "_pdfusion_memoized" + +_install_lock = threading.Lock() + + +def _signature(path: Path) -> Optional[tuple[int, int]]: + try: + st = os.stat(path) + except OSError: + return None + return st.st_size, st.st_mtime_ns + + +def memoize_verify(verify: Verify) -> Verify: + """Wrap `verify` so a (file, expected hash) that already passed, and has not + changed since, is answered without hashing it again.""" + passed: dict[tuple[str, str], tuple[int, int]] = {} + lock = threading.Lock() + + def memoized(path: Path, sha3_256: str) -> bool: + key = (os.fspath(path), sha3_256) + before = _signature(path) + if before is not None: + with lock: + if passed.get(key) == before: + return True + ok = verify(path, sha3_256) + # Stat again after hashing: a pass on bytes that changed underneath it + # vouches for a file nobody verified. + with lock: + if ok and before is not None and _signature(path) == before: + passed[key] = before + else: + passed.pop(key, None) + return ok + + setattr(memoized, _MEMO_MARK, True) + memoized.__wrapped__ = verify # type: ignore[attr-defined] + return memoized + + +def install() -> None: + """Put the memo in front of `babeldoc.assets.assets.verify_file`. + + BabelDOC's asset functions call `verify_file` as a module global, so + replacing the attribute is enough to reach every stage's font check. + Re-check that against `assets.py` when bumping babeldoc. Blocking (imports + babeldoc's asset module); call off the event loop. + """ + from babeldoc.assets import assets + + with _install_lock: + if getattr(assets.verify_file, _MEMO_MARK, False): + return + assets.verify_file = memoize_verify(assets.verify_file) + + +def preverify_cached_fonts() -> int: + """Check every embedding font already on disk, so the first page doesn't. + + Only fonts that exist are checked, and through `verify_file` rather than + `get_font_and_metadata`, which downloads what is missing: this runs on the + boot warm thread, with no UI to report a download, and BabelDOC writes + downloads in place — racing the setup flow's copy can corrupt it. Returns + how many fonts passed. Blocking; call off the event loop. + """ + install() + from babeldoc.assets import assets + from babeldoc.assets.embedding_assets_metadata import EMBEDDING_FONT_METADATA + from babeldoc.const import get_cache_file_path + + started = time.perf_counter() + checked = passed = 0 + for name, meta in EMBEDDING_FONT_METADATA.items(): + path = get_cache_file_path(name, "fonts") + if not path.exists(): + continue + checked += 1 + if assets.verify_file(path, meta["sha3_256"]): + passed += 1 + logger.info( + "BabelDOC fonts pre-verified: %d of %d on disk passed in %.1fs", + passed, checked, time.perf_counter() - started, + ) + return passed diff --git a/tests/test_engine_warm_gate.py b/tests/test_engine_warm_gate.py index a3aad8c..da8f919 100644 --- a/tests/test_engine_warm_gate.py +++ b/tests/test_engine_warm_gate.py @@ -21,7 +21,7 @@ from desktop_pdf_translator import engine_assets from desktop_pdf_translator.api import server -from desktop_pdf_translator.processors import doc_layout_cache +from desktop_pdf_translator.processors import doc_layout_cache, font_verify_cache import desktop_pdf_translator.processors as processors_pkg @@ -32,10 +32,16 @@ def loads(monkeypatch: pytest.MonkeyPatch) -> list[int]: `from ..processors import processor` resolves through `getattr` on the package before it reaches the import machinery, so setting the attribute is enough to keep babeldoc + torch out of this test. + + The font pre-check is stubbed too: run for real it imports babeldoc, hashes + the developer's whole font cache and leaves `verify_file` patched for the + rest of the session. The tests that are about it re-stub it with recorders. """ monkeypatch.setattr( processors_pkg, "processor", types.ModuleType("processor"), raising=False ) + monkeypatch.setattr(font_verify_cache, "install", lambda: None) + monkeypatch.setattr(font_verify_cache, "preverify_cached_fonts", lambda: 0) calls: list[int] = [] monkeypatch.setattr( doc_layout_cache, @@ -64,6 +70,40 @@ def test_skips_the_model_when_the_asset_is_missing(monkeypatch, loads): assert loads == [] +# --------------------------------------------------------------------------- +# Font hash memo (BabelDOC re-hashes every cached font per pipeline stage) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("core_ready", [True, False]) +def test_warm_up_preverifies_the_fonts(monkeypatch, loads, core_ready): + """The pre-check only reads fonts already on disk, so it needs no + `babeldoc_core_ready()` gate. (It installs the memo itself — pinned by + `test_preverified_fonts_are_not_hashed_again_by_babeldoc`.)""" + monkeypatch.setattr(engine_assets, "babeldoc_core_ready", lambda: core_ready) + calls: list[str] = [] + monkeypatch.setattr( + font_verify_cache, "preverify_cached_fonts", lambda: calls.append("preverify") or 0 + ) + + server._warm_translation_engine() + + assert calls == ["preverify"] + + +def test_a_failing_font_precheck_does_not_cost_the_layout_model_warm_up(monkeypatch, loads): + monkeypatch.setattr(engine_assets, "babeldoc_core_ready", lambda: True) + + def boom(): + raise RuntimeError("font pre-check failed") + + monkeypatch.setattr(font_verify_cache, "preverify_cached_fonts", boom) + + server._warm_translation_engine() + + assert loads == [1] + + # --------------------------------------------------------------------------- # Argos is warmed when it is what would run # --------------------------------------------------------------------------- diff --git a/tests/test_font_verify_cache.py b/tests/test_font_verify_cache.py new file mode 100644 index 0000000..af964e8 --- /dev/null +++ b/tests/test_font_verify_cache.py @@ -0,0 +1,386 @@ +"""BabelDOC re-hashes the same cached fonts on every pipeline stage. + +Every stage builds a font mapper, and every mapper runs `verify_file` over all +34 fonts (266 MB, ~1 s) — 7-8 times per translated page. The fonts don't change +between those checks, so `font_verify_cache` remembers a pass. The risky half is +remembering too much: a failure, a font rewritten by a re-download, or a file +changing mid-hash must each be re-checked, or BabelDOC would trust bytes it +never verified. These tests pin both halves. + +The pre-check tests inject fake `babeldoc` modules via `sys.modules` (the +module under test imports babeldoc only inside functions) so they don't pay for +the real import; one integration test uses the real package to prove the hook +point BabelDOC actually calls is the one we replace. +""" + +from __future__ import annotations + +import hashlib +import os +import sys +import types +from pathlib import Path + +import pytest + +from desktop_pdf_translator.processors import font_verify_cache + + +def _sha(data: bytes) -> str: + return hashlib.sha3_256(data).hexdigest() + + +def _real_verify(path: Path, expected: str) -> bool: + if not path.exists(): + return False + return _sha(path.read_bytes()) == expected + + +class Counting: + """A verify function that records every call and really hashes.""" + + def __init__(self) -> None: + self.calls: list[tuple[Path, str]] = [] + + def __call__(self, path: Path, expected: str) -> bool: + self.calls.append((path, expected)) + return _real_verify(path, expected) + + +def _bump_mtime(path: Path) -> None: + st = path.stat() + os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns + 5_000_000_000)) + + +# --------------------------------------------------------------------------- +# memoize_verify +# --------------------------------------------------------------------------- + + +def test_a_passing_file_is_hashed_once(tmp_path): + f = tmp_path / "a.ttf" + f.write_bytes(b"font") + inner = Counting() + verify = font_verify_cache.memoize_verify(inner) + + assert verify(f, _sha(b"font")) is True + assert verify(f, _sha(b"font")) is True + + assert len(inner.calls) == 1 + + +def test_a_failure_is_never_remembered(tmp_path): + """A font BabelDOC repairs by re-downloading must be re-checked and pass.""" + f = tmp_path / "a.ttf" + f.write_bytes(b"corrupt") + inner = Counting() + verify = font_verify_cache.memoize_verify(inner) + good = _sha(b"font") + + assert verify(f, good) is False + assert verify(f, good) is False + assert len(inner.calls) == 2 + + f.write_bytes(b"font") + assert verify(f, good) is True + + +def test_a_missing_file_fails_and_is_not_remembered(tmp_path): + f = tmp_path / "missing.ttf" + inner = Counting() + verify = font_verify_cache.memoize_verify(inner) + + assert verify(f, _sha(b"font")) is False + + f.write_bytes(b"font") + assert verify(f, _sha(b"font")) is True + assert len(inner.calls) == 2 + + +def test_a_resized_file_is_verified_again(tmp_path): + f = tmp_path / "a.ttf" + f.write_bytes(b"font") + inner = Counting() + verify = font_verify_cache.memoize_verify(inner) + expected = _sha(b"font") + assert verify(f, expected) is True + + f.write_bytes(b"corrupt-and-longer") + + assert verify(f, expected) is False + assert len(inner.calls) == 2 + + +def test_a_file_rewritten_with_the_same_size_is_verified_again(tmp_path): + """Same length, new mtime: size alone would wave a swapped font through.""" + f = tmp_path / "a.ttf" + f.write_bytes(b"font") + inner = Counting() + verify = font_verify_cache.memoize_verify(inner) + expected = _sha(b"font") + assert verify(f, expected) is True + + f.write_bytes(b"FONT") + _bump_mtime(f) + + assert verify(f, expected) is False + assert len(inner.calls) == 2 + + +def test_the_same_path_with_another_expected_hash_is_not_answered_from_memory(tmp_path): + f = tmp_path / "a.ttf" + f.write_bytes(b"font") + inner = Counting() + verify = font_verify_cache.memoize_verify(inner) + assert verify(f, _sha(b"font")) is True + + assert verify(f, _sha(b"something else")) is False + assert len(inner.calls) == 2 + + +def test_a_file_changing_while_it_is_hashed_is_not_remembered(tmp_path): + """The pass describes bytes that no longer exist; trusting it would + vouch for a file nobody verified.""" + f = tmp_path / "a.ttf" + f.write_bytes(b"font") + calls = [] + + def racing(path, expected): + calls.append(1) + path.write_bytes(b"rewritten-with-a-different-length") + return True + + verify = font_verify_cache.memoize_verify(racing) + + assert verify(f, "x") is True + assert verify(f, "x") is True + + assert len(calls) == 2 + + +def test_a_file_changed_mid_hash_then_restored_to_its_old_stat_is_not_remembered(tmp_path): + """Same length, and the old mtime put back afterwards (a `copy2`-style + restore): only a stat taken after the hash sees that the bytes moved.""" + f = tmp_path / "a.ttf" + f.write_bytes(b"font") + old_mtime_ns = f.stat().st_mtime_ns + calls = [] + + def racing(path, expected): + calls.append(1) + path.write_bytes(b"FONT") + os.utime(path, ns=(old_mtime_ns + 5_000_000_000, old_mtime_ns + 5_000_000_000)) + return True + + verify = font_verify_cache.memoize_verify(racing) + + assert verify(f, "x") is True + os.utime(f, ns=(old_mtime_ns, old_mtime_ns)) + assert verify(f, "x") is True + + assert len(calls) == 2 + + +def test_results_for_different_paths_are_independent(tmp_path): + a, b = tmp_path / "a.ttf", tmp_path / "b.ttf" + a.write_bytes(b"aaa") + b.write_bytes(b"bbb") + inner = Counting() + verify = font_verify_cache.memoize_verify(inner) + + assert verify(a, _sha(b"aaa")) is True + assert verify(b, _sha(b"bbb")) is True + assert verify(a, _sha(b"aaa")) is True + assert verify(b, _sha(b"bbb")) is True + + assert len(inner.calls) == 2 + + +# --------------------------------------------------------------------------- +# install, against the real BabelDOC +# --------------------------------------------------------------------------- + + +@pytest.fixture +def real_assets(monkeypatch): + return pytest.importorskip("babeldoc.assets.assets") + + +def test_install_memoizes_the_function_babeldoc_calls(monkeypatch, tmp_path, real_assets): + f = tmp_path / "a.ttf" + f.write_bytes(b"font") + original = Counting() + # setattr first so teardown restores the real global after install(). + monkeypatch.setattr(real_assets, "verify_file", original) + + font_verify_cache.install() + assert real_assets.verify_file(f, _sha(b"font")) is True + assert real_assets.verify_file(f, _sha(b"font")) is True + + assert len(original.calls) == 1 + + +def test_install_twice_does_not_wrap_a_second_time(monkeypatch, tmp_path, real_assets): + f = tmp_path / "a.ttf" + f.write_bytes(b"font") + original = Counting() + monkeypatch.setattr(real_assets, "verify_file", original) + + font_verify_cache.install() + first = real_assets.verify_file + font_verify_cache.install() + # Call counts alone can't see a double wrap: the inner memo absorbs it. + assert real_assets.verify_file is first + assert first.__wrapped__ is original + assert real_assets.verify_file(f, _sha(b"font")) is True + assert real_assets.verify_file(f, _sha(b"font")) is True + + assert len(original.calls) == 1 + + +def test_babeldocs_own_font_lookup_hashes_a_font_once(monkeypatch, tmp_path, real_assets): + """The end-to-end claim: `get_font_and_metadata` is what every stage's + font mapper calls, and it must reach the memo without downloading.""" + from babeldoc.assets import embedding_assets_metadata as meta + + font = tmp_path / "fake-font.ttf" + font.write_bytes(b"pretend this is a font") + monkeypatch.setitem( + meta.EMBEDDING_FONT_METADATA, "fake-font.ttf", {"sha3_256": _sha(font.read_bytes())} + ) + monkeypatch.setattr(real_assets, "get_cache_file_path", lambda name, sub: font) + + def no_download(*a, **k): + raise AssertionError("a verified font must not be downloaded") + + # A wrong False goes to the upstream probe before `download_file`; stub + # both so that fails here instead of reaching the network. + monkeypatch.setattr(real_assets, "get_fastest_upstream_for_font", no_download) + monkeypatch.setattr(real_assets, "download_file", no_download) + original = Counting() + monkeypatch.setattr(real_assets, "verify_file", original) + + font_verify_cache.install() + first = real_assets.get_font_and_metadata("fake-font.ttf") + second = real_assets.get_font_and_metadata("fake-font.ttf") + + assert first[0] == second[0] == font + assert len(original.calls) == 1 + + +# --------------------------------------------------------------------------- +# preverify_cached_fonts, against fake BabelDOC modules +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fake_babeldoc(monkeypatch, tmp_path): + """A fake BabelDOC with three fonts in metadata, two on disk (one corrupt). + + Downloading entry points raise, so any test that calls into them fails + loudly instead of reaching the network. + """ + good, corrupt = tmp_path / "good.ttf", tmp_path / "corrupt.ttf" + good.write_bytes(b"good font") + corrupt.write_bytes(b"bad bytes") + paths = { + "good.ttf": good, + "corrupt.ttf": corrupt, + "absent.ttf": tmp_path / "absent.ttf", + } + metadata = { + "good.ttf": {"sha3_256": _sha(b"good font")}, + "corrupt.ttf": {"sha3_256": _sha(b"the real bytes")}, + "absent.ttf": {"sha3_256": _sha(b"never downloaded")}, + } + counter = Counting() + downloads: list[tuple] = [] + + def forbidden(*a, **k): + downloads.append(a) + raise AssertionError("pre-verification must never download") + + pkg = types.ModuleType("babeldoc") + pkg.__path__ = [] + assets_pkg = types.ModuleType("babeldoc.assets") + assets_pkg.__path__ = [] + assets = types.ModuleType("babeldoc.assets.assets") + assets.verify_file = counter + assets.get_font_and_metadata = forbidden + assets.get_font_and_metadata_async = forbidden + assets.download_file = forbidden + assets.get_cache_file_path = lambda name, sub: paths[name] + meta = types.ModuleType("babeldoc.assets.embedding_assets_metadata") + meta.EMBEDDING_FONT_METADATA = metadata + const = types.ModuleType("babeldoc.const") + const.get_cache_file_path = lambda name, sub: paths[name] + pkg.assets, pkg.const = assets_pkg, const + assets_pkg.assets, assets_pkg.embedding_assets_metadata = assets, meta + assets.EMBEDDING_FONT_METADATA = metadata + for name, mod in { + "babeldoc": pkg, + "babeldoc.assets": assets_pkg, + "babeldoc.assets.assets": assets, + "babeldoc.assets.embedding_assets_metadata": meta, + "babeldoc.const": const, + }.items(): + monkeypatch.setitem(sys.modules, name, mod) + return types.SimpleNamespace( + assets=assets, counter=counter, paths=paths, metadata=metadata, downloads=downloads + ) + + +def test_preverify_checks_only_fonts_on_disk_and_counts_the_passes(fake_babeldoc): + passed = font_verify_cache.preverify_cached_fonts() + + checked = {p.name for p, _ in fake_babeldoc.counter.calls} + assert checked == {"good.ttf", "corrupt.ttf"} + assert passed == 1 + + +def test_preverify_never_downloads_a_missing_font(fake_babeldoc): + # Recorded as well as raised: a pre-check that swallowed the AssertionError + # would otherwise hide the breach. + font_verify_cache.preverify_cached_fonts() + + assert fake_babeldoc.downloads == [] + assert not fake_babeldoc.paths["absent.ttf"].exists() + assert all(p.name != "absent.ttf" for p, _ in fake_babeldoc.counter.calls) + + +def test_preverified_fonts_are_not_hashed_again_by_babeldoc(fake_babeldoc): + font_verify_cache.preverify_cached_fonts() + hashed = len(fake_babeldoc.counter.calls) + + good = fake_babeldoc.paths["good.ttf"] + expected = fake_babeldoc.metadata["good.ttf"]["sha3_256"] + assert fake_babeldoc.assets.verify_file(good, expected) is True + + assert len(fake_babeldoc.counter.calls) == hashed + + +# --------------------------------------------------------------------------- +# The job path installs it too +# --------------------------------------------------------------------------- + + +def test_the_job_path_installs_the_memo(monkeypatch): + """A Translate click can beat the boot warm thread; without this the first + job would run every stage's font check unmemoized.""" + import desktop_pdf_translator.processors as processors_pkg + from desktop_pdf_translator.api.routes import translation + + stub = types.ModuleType("processor") + stub.PDFProcessor = type("PDFProcessor", (), {}) + events = types.ModuleType("events") + events.EventType = type("EventType", (), {}) + monkeypatch.setattr(processors_pkg, "processor", stub, raising=False) + monkeypatch.setattr(processors_pkg, "events", events, raising=False) + monkeypatch.setitem(sys.modules, "desktop_pdf_translator.processors.processor", stub) + monkeypatch.setitem(sys.modules, "desktop_pdf_translator.processors.events", events) + calls: list[int] = [] + monkeypatch.setattr(font_verify_cache, "install", lambda: calls.append(1)) + + translation._load_engine() + + assert calls == [1]