From 31d9ba393892fa9df097230cf74b4c4e07e6f27b Mon Sep 17 00:00:00 2001 From: nammovuivui Date: Fri, 23 Oct 2026 13:57:55 +0700 Subject: [PATCH 1/2] docs(translate): delete comments the code contradicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each claim below was checked against the code or measured, and is false: - "Each in-flight BabelDOC sub-job peaks at ~9 GB" and "four chunks = ~36 GB": BabelDOC's "Peak memory usage" line is the whole sidecar plus its children, not one sub-job. The 120 s save timeouts blamed on swap are, per CLAUDE.md, the freeze_support bug. - "~150-300 MB of intermediate state" per sub-job: each page loaded ~1.6 GB of fonts. - `_MAX_PARALLEL_CHUNKS` "may be in flight at once": it was referenced nowhere; `_effective_parallel_chunks` decides. Removed with its comment. - "batches of 4": `batch_size` is 16. - "CTranslate2 serializes the decode": `inter_threads` is 2 on four or more cores. - "process-lifetime translation cache": it is the on-disk paragraph cache. - "≈16 paragraphs in flight": Argos runs 2 sub-jobs of 4 workers, so 8. - `enhance_compatibility` "forces extra typesetting passes": in babeldoc 0.5.24 it only turns on three flags that are already on or moot here. - Glossary extraction "invokes the translator" on LLM paths: BabelDOC extracts terms only through `do_llm_translate`, which no PDFusion translator has. - "ILTranslator twice on the LLM path", "272 checks per page (238 on Argos)": the measured counts are 7 FontMappers a page on the LLM path and 6 on Argos. - Argos "gains nothing from extra parallelism" / "saturates at low parallelism": 4 chunks in flight ran 31 pages 13% faster than 2. Co-Authored-By: Claude Opus 5.5 --- .../processors/font_verify_cache.py | 16 ++-- .../processors/processor.py | 73 ++++++------------- .../translators/argos_translator.py | 17 ++--- 3 files changed, 38 insertions(+), 68 deletions(-) diff --git a/src/desktop_pdf_translator/processors/font_verify_cache.py b/src/desktop_pdf_translator/processors/font_verify_cache.py index 6de5dbf..db21793 100644 --- a/src/desktop_pdf_translator/processors/font_verify_cache.py +++ b/src/desktop_pdf_translator/processors/font_verify_cache.py @@ -1,14 +1,14 @@ """Remembers which of BabelDOC's cached assets have passed their hash check. BabelDOC builds a `FontMapper` in every pipeline stage — `ILCreater`, -`ParagraphFinder`, `StylesAndFormulas`, `ILTranslator` (twice on the LLM path), -`Typesetting`, `PDFCreater` (twice when scanned-page detection runs) — and each -one runs `assets.verify_file` over every font of the target language: 34 files, -266 MB of SHA3-256 for Vietnamese, ~1 s. Traced over two different documents on -the LLM path, that is 272 checks per page (238 on Argos, which skips one -mapper) against the same 34 (file, expected hash) pairs, none of them derived -from the document, every file byte-identical before and after. Measured on a -one-page run: 15.5 s → 7.9 s with the passes remembered. +`ParagraphFinder`, `StylesAndFormulas`, `ILTranslator`, `Typesetting`, +`PDFCreater` (twice when scanned-page detection runs) — and each one runs +`assets.verify_file` over every font of the target language: 34 files, 266 MB +of SHA3-256 for Vietnamese, ~1 s. That is 238 checks per page on the LLM path +(204 on Argos, which skips scanned-page detection) against the same 34 (file, +expected hash) pairs, none of them derived from the document, every file +byte-identical before and after. Measured on a one-page run: 15.5 s → 7.9 s +with the passes remembered. Only the first check in a process can find something new — a missing or corrupt font, which BabelDOC then re-downloads — so a pass is remembered and a failure diff --git a/src/desktop_pdf_translator/processors/processor.py b/src/desktop_pdf_translator/processors/processor.py index d96dbe0..d0110b7 100644 --- a/src/desktop_pdf_translator/processors/processor.py +++ b/src/desktop_pdf_translator/processors/processor.py @@ -130,8 +130,7 @@ def _schedule_temp_cleanup(path: Path) -> None: # Number of source pages per BabelDOC sub-job. 1 = page-by-page streaming: the # viewer hot-swaps every time a single page finishes its full BabelDOC pipeline -# (translate → typeset → render → save). Combined with `_MAX_PARALLEL_CHUNKS` -# below, this gives the user the maximum streaming feel. +# (translate → typeset → render → save). _PAGES_PER_CHUNK = 1 # Argos-only: pages per chunk when translating offline. Previously 3, to @@ -141,27 +140,9 @@ def _schedule_temp_cleanup(path: Path) -> None: # streaming granularity as the LLM path. _PAGES_PER_CHUNK_ARGOS = 1 -# Argos-only cap on parallel chunks. Each in-flight BabelDOC sub-job peaks at -# ~9 GB RAM on an academic paper (DocLayoutYOLO + IR + font maps). Four -# parallel chunks = ~36 GB → swap thrashing on most desktops, which is what -# produced the 120-second "PDF save timeout" warnings. Argos itself is the -# bottleneck-side rate limiter and gains nothing from extra parallelism, so -# we cap conservatively. +# Argos-only cap on parallel chunks. _MAX_PARALLEL_CHUNKS_ARGOS = 2 -# How many BabelDOC sub-jobs may be in flight at once. BabelDOC's pipeline is -# monolithic (you can't externally separate translate from render), but by -# running multiple jobs concurrently we get natural pipelining: while job N is -# in its typeset/render/save phase, jobs N+1..N+k can be in their translate -# phase calling Argos. The shared `ArgosTranslator` instance has a single -# batch queue, so the 4 BabelDOC worker threads from each job naturally feed -# one Argos batch at a time — no double-buffering needed, no contention beyond -# Argos's own CTranslate2 serialization. -# -# Higher values increase memory (each BabelDOC job holds its own IR/font maps) -# but improve pipeline depth. 4 is a reasonable default for desktop machines. -_MAX_PARALLEL_CHUNKS = 4 - # Tag for the "stop now, the credentials are being rejected" item that # `_handle_translation_failure` pushes onto the multiplexer queue. _FATAL_KIND = "fatal" @@ -171,15 +152,11 @@ def _effective_parallel_chunks(settings, translator=None) -> int: """Resolve the chunk-concurrency knob. `processing.max_parallel_chunks = 0` (default) means auto: roughly half the - CPU cores, clamped to [2, 8]. Each in-flight BabelDOC sub-job carries - ~150-300 MB of intermediate state on a typical academic paper, so we leave - half the cores (and roughly half the RAM headroom) to the rest of the app - plus the WebView2 renderer. + CPU cores, clamped to [2, 8], which leaves the other half to the rest of + the app plus the webview's renderer. When `translator` is an ArgosTranslator, the result is clamped down to - `_MAX_PARALLEL_CHUNKS_ARGOS` to control memory — Argos throughput - saturates at low parallelism anyway, and per-chunk RAM is ~9 GB on - formula-heavy papers. + `_MAX_PARALLEL_CHUNKS_ARGOS` to control memory. """ try: configured = int(settings.processing.max_parallel_chunks) @@ -747,22 +724,20 @@ async def _process_with_babeldoc( 1. Split the selected pages (`file_metadata.selected_pages`, or all of them) into 1-page chunks up front. - 2. Launch up to `_MAX_PARALLEL_CHUNKS` BabelDOC sub-jobs concurrently. - While chunk N is in its typeset/render/save phase, chunk N+1 can be - in its translate phase calling Argos. + 2. Launch up to `_effective_parallel_chunks()` BabelDOC sub-jobs + concurrently. While chunk N is in its typeset/render/save phase, + chunk N+1 can be in its translate phase calling Argos. 3. The single shared `ArgosTranslator` instance has one batch queue; - paragraphs from all concurrent chunks fill the same batches of 4, - and CTranslate2 serializes the actual NMT inference. Argos is the - natural rate limiter — extra parallelism doesn't make it faster, - but it does hide BabelDOC's per-chunk non-translate overhead. + paragraphs from all concurrent chunks fill the same batches, and + running chunks side by side hides BabelDOC's per-chunk + non-translate overhead. 4. Completed chunks are merged into a rolling versioned PDF (in strict page order, even when chunks finish out of order; every other page comes from the original) and a `chunk_ready` SSE event is emitted so the viewer hot-swaps. The translator instance is reused across chunks so the Argos batch - queue and process-lifetime translation cache (argos_translator.py) - persist — repeated paragraphs across chunks translate for free. + queue persists. """ chunk_tasks: list[asyncio.Task] = [] max_parallel = _effective_parallel_chunks(self.settings, translator) @@ -1231,13 +1206,11 @@ def _create_babeldoc_config(self, file_path: Path, translator, output_dir: Path, logger.debug(f" lang_out: {translator.lang_out}") logger.debug(f" output_dir: {output_dir}") - # Per-translator overrides for heavy non-translation stages. Argos - # has no LLM available, so any flag that secretly triggers an LLM - # call (glossary extraction) is pure overhead — and skipping the - # SSIM scanned-page detector + the compatibility-enhancement pass - # both shave significant wall time on academic papers. LLM paths - # keep the previous behavior so glossary terms and OCR-quality - # detection still work for paid backends. + # Per-translator overrides for heavy non-translation stages. Skipping + # the SSIM scanned-page detector shaves significant wall time on + # academic papers, so Argos skips it. LLM paths keep it, and with + # `auto_enable_ocr_workaround` off a page it finds scanned raises + # BabelDOC's `ScannedPDFError`. is_argos = isinstance(translator, ArgosTranslator) # Configure for single translated PDF output only (no dual, no decompressed, no bounding boxes) @@ -1266,9 +1239,9 @@ def _create_babeldoc_config(self, file_path: Path, translator, output_dir: Path, short_line_split_factor=0.8, disable_rich_text_translate=True, dual_translate_first=False, - # `enhance_compatibility=True` forces extra typesetting passes - # that double-render text. Argos doesn't need them; LLMs benefit - # for tricky fonts. + # A no-op on both paths in babeldoc 0.5.24: it only turns on + # `skip_clean` and `disable_rich_text_translate`, both already on + # here, and `dual_translate_first`, which `no_dual` makes moot. enhance_compatibility=not is_argos, use_alternating_pages_dual=False, # Set watermark mode to NoWatermark to avoid bounding boxes @@ -1287,9 +1260,9 @@ def _create_babeldoc_config(self, file_path: Path, translator, output_dir: Path, glossaries=None, auto_enable_ocr_workaround=False, pool_max_workers=processing_settings.max_workers, - # Heaviest non-translation stage (weight 30 in BabelDOC). It - # invokes the translator with extraction prompts — Argos is an - # NMT model, can't follow those prompts, so the work is wasted. + # BabelDOC extracts terms only through a translator's + # `do_llm_translate`, which none of ours has, so the stage never + # runs on either path. auto_extract_glossary=not is_argos, primary_font_family=None, only_include_translated_page=False, diff --git a/src/desktop_pdf_translator/translators/argos_translator.py b/src/desktop_pdf_translator/translators/argos_translator.py index ea0443e..65d80fa 100644 --- a/src/desktop_pdf_translator/translators/argos_translator.py +++ b/src/desktop_pdf_translator/translators/argos_translator.py @@ -304,13 +304,11 @@ def _setup_translator(self, **kwargs): self.model = kwargs.get("model", "argostranslate") # Streaming/batching: BabelDOC's worker pool calls translate() from up - # to `pool_max_workers` threads concurrently across up to - # `max_parallel_chunks` BabelDOC sub-jobs (≈16 paragraphs in flight at - # peak with the qps throttle disabled — see processor.py). We coalesce - # them and feed the whole batch into one CTranslate2 translate_batch - # call so the underlying engine packs sentences across paragraphs. - # `batch_size=16` matches that peak; `batch_timeout=0.4s` keeps the - # tail of each chunk snappy without losing batching benefit. + # to `pool_max_workers` threads concurrently in every in-flight + # BabelDOC sub-job. We coalesce them and feed the whole batch into one + # CTranslate2 translate_batch call so the underlying engine packs + # sentences across paragraphs. A batch goes once it holds `batch_size` + # paragraphs, or `batch_timeout` after the last one joined. self.batch_size = max(1, int(kwargs.get("batch_size", 16))) self.batch_timeout = float(kwargs.get("batch_timeout", 0.4)) self._batch_lock = threading.Lock() @@ -330,8 +328,7 @@ def _setup_translator(self, **kwargs): # Pipeline batches across 2 workers: while one thread is mid-inference # on batch N, the other can start batch N+1's pre-processing / - # tokenization. CTranslate2 serializes the actual decode internally, - # but this overlapping cuts the inter-batch gap. ~15-30% throughput + # tokenization, which cuts the inter-batch gap. ~15-30% throughput # win on typical academic PDFs. self._batch_executor = ThreadPoolExecutor( max_workers=2, thread_name_prefix="argos-batch" @@ -399,7 +396,7 @@ def translate(self, text: str, **kwargs) -> str: cached = _llm_cache_get(processed, "en", "vi", "argos", self.model) if cached is not None: - # Step 3 — served from the process-lifetime cache; no NMT call. + # Step 3 — served from the paragraph cache; no NMT call. _debug.record( "3.cache_hit", call=self.translate_call_count, From 23bfaf02cd2a4d146eb01556732c2478bb572862 Mon Sep 17 00:00:00 2001 From: nammovuivui Date: Fri, 23 Oct 2026 13:59:05 +0700 Subject: [PATCH 2/2] fix(translate): load BabelDOC's fonts once per process A 346-page book translated with Argos was killed by the kernel's OOM killer 238 pages in, with the sidecar at ~12 GB. The LLM path has the same problem, with three times the pages in flight. BabelDOC 0.5.24 builds a FontMapper in every pipeline stage, 6 per page on Argos and 7 on the LLM path. Each one reads all 34 embedding fonts (266 MB) into pymupdf.Font objects. Its constructor wraps bound methods in lru_cache, so every mapper and every font sits in a reference cycle that only a full GC frees, and full collections follow object counts, which font buffers barely move. Replayed through the real pipeline, 4 pages took the process from 0.26 GB to 6.5 GB with no full collection in between. font_cache.install() replaces FontMapper.__init__ with a copy of BabelDOC's whose font loop takes each font from a process-wide cache, decorated once. It only patches a babeldoc in VERIFIED_BABELDOC: on any other version it logs a warning and leaves the constructor alone, which costs memory instead of breaking every stage. routes/translation.py _load_engine installs it before BabelDOC builds anything. Sharing is safe because no stage writes to a mapper's fonts after construction, and PyMuPDF never releases the GIL: neither _mupdf.so nor _extra.so imports PyEval_SaveThread. Measured on the same book afterwards: its 301 pages with text ran in 23 minutes with the sidecar peaking at 2.38 GB. Page 25's output is identical before and after, word for word and font for font, on both paths. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 15 +- docs/architecture/translation-pipeline.md | 97 ++- .../api/routes/translation.py | 5 +- .../processors/font_cache.py | 182 +++++ .../processors/processor.py | 4 +- tests/test_font_cache.py | 746 ++++++++++++++++++ tests/test_font_verify_cache.py | 7 +- 7 files changed, 1029 insertions(+), 27 deletions(-) create mode 100644 src/desktop_pdf_translator/processors/font_cache.py create mode 100644 tests/test_font_cache.py diff --git a/CLAUDE.md b/CLAUDE.md index c75fd7b..46934a8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,7 +117,7 @@ singleton with no job id, so it is polled via `GET /setup/status`. | `engine_assets.py` | Single source of truth for "the offline engine is installed" | | `config/` | `ConfigManager` + Pydantic `AppSettings` (`providers` map, `ModelRef`s) | | `providers/` | `registry.py` — one `ProviderSpec` per provider, the source of every per-provider rule (stdlib-only); `listing.py` — one model lister per protocol; `catalog.py` — the model catalog cache and `list_models` | -| `processors/` | `PDFProcessor` (BabelDOC), `page_selection.py`, `pdf_pages.py`, `pdf_cache.py`, `doc_layout_cache.py` | +| `processors/` | `PDFProcessor` (BabelDOC), `page_selection.py`, `pdf_pages.py`, `pdf_cache.py`, `doc_layout_cache.py`, `font_verify_cache.py`, `font_cache.py` | | `translators/` | `BaseTranslator` + OpenAI/Gemini/Anthropic/Argos, `factory.py`, `capabilities.py`, `rate_limiter.py`, `translation_cache.py`, `param_compat.py`, `usage_estimate.py` | | `rag/` | `rag_chain.py` (`EnhancedRAGChain`), `conversation.py` (`ChatTurn`, stdlib-only), `ingest.py` (PDF → chunks, page kinds), `ocr.py`, `retrieval.py` (RRF, page evidence), `citations.py`, `page_images.py`, `vector_store.py`, `index_spec.py` (embedding model + `EmbeddingSpec`s), `onnx_embeddings.py`, `keyword_search.py` | | `benchmarks/rag/` | Corpus, questions with gold pages, harness (baseline from a `main` worktree), results — `README.md` there has the numbers | @@ -262,6 +262,13 @@ rule has, unless noted, a test. They are the things most easily undone by (and a hit is linked back), via `pdf_cache.link_or_copy`, so **never rewrite a translated PDF in place**. Write a new file and `os.replace` it, or the write reaches the cache entry too. +- **BabelDOC's fonts are loaded once per process** (`processors/font_cache.py`). + BabelDOC builds a `FontMapper` per stage, 6–7 a page, and each loaded all 34 + embedding fonts (266 MB) into reference cycles only a full GC frees — on a + book that ran the sidecar out of memory. The shared constructor is a copy of + BabelDOC's, installed only on a babeldoc in `VERIFIED_BABELDOC`: re-check it + against `fontmap.py` when bumping babeldoc. Sharing a font across pages relies + on PyMuPDF never releasing the GIL; re-check that when bumping PyMuPDF. **Data and chat** → `data-and-chat.md` - Three stores under one root: `pdfusion.db` (system of record), `vectors/` @@ -537,9 +544,9 @@ isolation (real ChromaDB under `tmp_path`), chat ingestion (real PDFs built with PyMuPDF), OCR respacing, rank fusion and page evidence, citation checking, page images, multimodal and streamed `generate()` per SDK, the answer chain's image, citation, fence and conversation rules, answer ratings, the benchmark's -interval statistics, chunking and rolling-PDF assembly, -the data root per platform, packaging config agreement, and the viewer's pure -half. +interval statistics, chunking and rolling-PDF assembly, BabelDOC's shared +fonts, the data root per platform, packaging config agreement, and the viewer's +pure half. Not covered: `processors/processor.py` proper, chat quality with the real embedding/OCR models and a real LLM (that is what `benchmarks/rag/` measures — diff --git a/docs/architecture/translation-pipeline.md b/docs/architecture/translation-pipeline.md index d8ff782..a447fe5 100644 --- a/docs/architecture/translation-pipeline.md +++ b/docs/architecture/translation-pipeline.md @@ -97,33 +97,31 @@ A failed load leaves the module global `None` rather than caching the failure, so the next chunk retries; `run_one_chunk`'s `except BaseException` still routes it into `babeldoc_chunk_error`. -**What this does not fix, and why chunk size is a live question.** The layout model was never the largest per-chunk fixed cost. +**What this does not fix, and why chunk size is a live question.** The layout +model was never the largest per-chunk fixed cost: the fonts were. `FontMapper.__init__` sha3_256-verifies and loads **all 34 embedding fonts (266 MB)** for `lang_out`, and BabelDOC memoizes neither — not `fontmap.py`, not `assets.py`. It constructs one per stage: `ILCreater`, `ParagraphFinder`, -`StylesAndFormulas`, `ILTranslator` (a second inside `ILTranslatorLLMOnly`), -`Typesetting`, `PDFCreater` (a second for scanned-page detection) — 7 per page -on the Argos path, 8 on the LLM path. The hash is now remembered (next section); -what is left is ~0.26 s per `FontMapper` of `pymupdf.Font` parsing, ≈2 s per -chunk, which this cache does not touch either. So "the reload is fixed" is not -on its own a reason to shrink a chunk — shrinking one multiplies that fixed cost -by the extra chunks it creates, and on the Argos path -(`_MAX_PARALLEL_CHUNKS_ARGOS = 2`) that can outweigh what the cache saves. -`_PAGES_PER_CHUNK_ARGOS` is 1 today, like the LLM path; measure an end-to-end -Argos run before trusting that, or before changing either constant. What a -larger chunk buys is amortized per-chunk fixed overhead in general, not the ONNX -reload specifically. +`StylesAndFormulas`, `ILTranslator`, `Typesetting`, `PDFCreater` (a second for +scanned-page detection) — 6 per page on the Argos path, 7 on the LLM path, +counted on real runs. The hash is now remembered and the fonts loaded once per +process (the next two sections), but every chunk is still a whole BabelDOC run, +with its own IR, rendering and PDF save. `_PAGES_PER_CHUNK_ARGOS` is 1 today, +like the LLM path; measure an end-to-end Argos run before changing either +constant. What a larger chunk buys is amortized per-chunk fixed overhead in +general, not the ONNX reload specifically. ## A font that passed its hash check is not hashed again A `FontMapper` took ~1.3 s, of which ~1.0 s was `assets.verify_file` hashing the 34 fonts (measured alone: 0.99 s; parsing them alone: 0.13 s; the ~0.26 s a -mapper still costs with the hash remembered is that parse plus the rest of the -constructor, under the contention of a real run). Traced over real one-page -LLM-path runs on two different documents: **272 checks per page — 34 fonts × 8 -mappers (238 on Argos, with 7) — against the same 34 (file, expected hash) -pairs**, none derived from the document; every pair passed, -and every file was byte-identical before and after both runs. Only a process's +mapper still cost with the hash remembered was that parse plus the rest of the +constructor, under the contention of a real run — a parse only a process's first +mapper pays now, as the next section explains). That is **238 checks per page +— 34 fonts × 7 mappers (204 on Argos, with 6) — against the same 34 (file, +expected hash) pairs**, none derived from the document; traced over real +one-page runs on two different documents, every pair passed, and every file was +byte-identical before and after both runs. Only a process's first check can find anything (a missing or corrupt font, which BabelDOC then re-downloads); the rest re-hash unchanged bytes. @@ -163,6 +161,67 @@ usually loses that race and just logs, which is what makes the symptom intermittent rather than absent — do not read "it works here" as "the ordering is safe".) +## Each font is loaded once per process + +A `FontMapper` also *loads* all 34 fonts, each into a `pymupdf.Font` that holds +the whole file: 266 MB for any target language, because BabelDOC's +`__add_fallback_to_font_family` appends every other family's fonts to each one. +At 6 mappers a page (7 on the LLM path) that is 1.6 GB of fonts a page, and +BabelDOC never gave it back on time. The constructor memoizes by wrapping bound +methods in `lru_cache` — its own `has_char`/`map_in_type`, and every font's +`has_glyph`/`char_lengths` — so each mapper and each font sits in a reference +cycle that only CPython's full collection frees. Full collections are scheduled +by counts of objects, which font buffers barely move, and nothing in BabelDOC, +Argos or PDFusion calls `gc.collect()`. + +On a long document that was an out-of-memory kill, whatever the translator. A +346-page book on Argos (2 pages in flight) took the sidecar to 12 GB, and the +kernel killed it 238 pages in. Replayed through the real pipeline, four pages +took the process from 0.26 GB to 6.5 GB with no full collection in between; the +LLM path's 6 pages in flight (`cpu // 2` on 12 cores; a stub translator that +returns its input, so no request was made) passed 6.5 GB before its first page +finished. Two things about the numbers in BabelDOC's log: "Peak memory usage" is +the whole sidecar and its children, not one sub-job, and on Linux it +over-counts. Between its two-second PSS samples it adds up the RSS of the +sidecar and of each forked PDF-save child, and a forked child's RSS includes +every page it shares with the sidecar. + +`processors/font_cache.py` replaces `FontMapper.__init__` with a copy of +BabelDOC's whose font loop takes each font from a process-wide cache, which gives +a font BabelDOC's per-font attributes and memos once, at its first load. On the +same book afterwards, its 301 pages with text went through in 23 minutes with +the sidecar never above 2.4 GB — 1,806 mappers sharing one set of fonts — and a +page translated before and after the change came out the same, every word in +the same place. Memory still drifts up by about 1.7 MB a page: most likely the +rolling PDF, rebuilt whole after every page (43.5 MB by the end), and per-page +garbage that still waits for a full collection, such as the source PDF's own +fonts, which `ILCreater` memoizes the same way. + +- **Sharing is safe because nothing writes to a font after construction.** The + stages only read a mapper (`has_char`, `map`, `fontid2font`, `base_font`, + `add_font`), and PyMuPDF never releases the GIL — neither `_mupdf.so` nor + `_extra.so` imports `PyEval_SaveThread` — so MuPDF calls from different pages + run one at a time. Python does run inside a call, in PyMuPDF's warning and + error callbacks, where threads can switch: the same exposure as the one MuPDF + context every page already shares. The shared memos are keyed by a character + (and, for `char_lengths`, a font size) and capped at 10,240 entries each; + `char_lengths` hands every caller the same list, which they only read. +- **The fonts stay.** ~266 MB is resident from the first translation until the + sidecar exits. The cache is never pruned: every mapper wants all 34. +- **It is a copy of BabelDOC 0.5.24's constructor**, so `install()` patches only + a BabelDOC whose version is in `VERIFIED_BABELDOC`; any other keeps its own + constructor and logs a warning. babeldoc is pinned to a range, and a copy that + no longer matches could break every stage, where an unpatched one only costs + the memory. `test_font_cache.py` compares the copy with the installed + constructor attribute by attribute, and fails by name on an unverified + version: re-check the copy against `fontmap.py` and extend the set when + bumping babeldoc. Re-check the GIL claim above when bumping PyMuPDF. +- **Installed by `routes/translation.py:_load_engine`**, which every job runs + before BabelDOC builds anything. The class is patched, not the module's name + for it: each stage imported `FontMapper` by name. +- **A failed load is not remembered**, so a font BabelDOC re-downloads is loaded + by the next mapper. + ## Page ranges A translation covers the pages the toolbar's Pages box names (#33) — `1-20, 35`, diff --git a/src/desktop_pdf_translator/api/routes/translation.py b/src/desktop_pdf_translator/api/routes/translation.py index 2e356d0..e06dc51 100644 --- a/src/desktop_pdf_translator/api/routes/translation.py +++ b/src/desktop_pdf_translator/api/routes/translation.py @@ -81,13 +81,16 @@ def _load_engine() -> tuple[type, type]: beats that thread — or a warm-up that failed, which it does silently — lands here instead. Callers must run it off the event loop. """ - from ...processors import font_verify_cache + from ...processors import font_cache, font_verify_cache from ...processors.events import EventType from ...processors.processor import PDFProcessor # Normally done by the warm thread too; a click that beats it would # otherwise re-hash every font in every stage of the first job. font_verify_cache.install() + # Before any stage builds a `FontMapper`: BabelDOC's own loads all 34 fonts + # for each, and on a book that runs the sidecar out of memory. + font_cache.install() return EventType, PDFProcessor diff --git a/src/desktop_pdf_translator/processors/font_cache.py b/src/desktop_pdf_translator/processors/font_cache.py new file mode 100644 index 0000000..a96d346 --- /dev/null +++ b/src/desktop_pdf_translator/processors/font_cache.py @@ -0,0 +1,182 @@ +"""BabelDOC's embedding fonts, loaded once per process instead of once per +`FontMapper`. + +BabelDOC builds a `FontMapper` in every pipeline stage — six per page on the +Argos path, seven on the LLM path, which also runs scanned-page detection — +and each one reads every embedding font of the target language into a +`pymupdf.Font`: all 34 of them, 266 MB, for any language, because BabelDOC +appends every other family's fonts to each family as fallbacks. None of it was +freed when a stage ended. The constructor memoizes by wrapping bound methods +in `lru_cache` — its own `has_char`/`map_in_type`, and each font's +`has_glyph`/`char_lengths` — so every mapper and every font sits in a +reference cycle, which only a full collection frees, and full collections +follow counts of objects, which 266 MB of font buffers barely move. On a book +that ran the sidecar out of memory: replayed through the real pipeline, four +pages took it from 0.26 GB to 6.5 GB without one full collection. + +`install()` replaces `FontMapper.__init__` with a copy of BabelDOC's whose font +loop asks `_shared_font` instead. Sharing is safe because nothing in BabelDOC +writes to a mapper's fonts after construction — the stages only read them — +and PyMuPDF never releases the GIL, so MuPDF calls from different pages run +one at a time (Python runs inside a call only in PyMuPDF's warning and error +callbacks; see "Each font is loaded once per process" in +docs/architecture/translation-pipeline.md). The cost is that the fonts stay: +~266 MB from the first translation until the sidecar exits. + +The copy is babeldoc 0.5.24's constructor, so it is only installed on a version +in `VERIFIED_BABELDOC`: babeldoc is pinned to a range, and a newer constructor +the copy doesn't match could break every stage, where leaving it alone only +costs the memory. `tests/test_font_cache.py` compares the copy with the +installed BabelDOC's own constructor; re-check it, then extend the set, when +bumping babeldoc. + +Kept free of babeldoc imports at module scope, like `doc_layout_cache.py`. +""" + +from __future__ import annotations + +import functools +import logging +import os +import threading +from pathlib import Path +from typing import Any, Mapping + +logger = logging.getLogger(__name__) + +# The babeldoc versions whose `FontMapper.__init__` `_sharing_init` was checked +# against. +VERIFIED_BABELDOC = frozenset({"0.5.24"}) + +# Marks the constructor this module installed, so `install()` can tell it from +# BabelDOC's own — or from a test's, which `monkeypatch` restores. +_MARK = "_pdfusion_shared_fonts" + +_install_lock = threading.Lock() +_fonts_lock = threading.Lock() +# Font file path → its one `pymupdf.Font`. Never pruned: every mapper of every +# page wants all of them. +_fonts: dict[str, Any] = {} + + +def _shared_font(font_file_name: str, font_path: Path, font_metadata: Mapping[str, Any]): + """The process's `pymupdf.Font` for `font_path`, loaded on first use and + given what BabelDOC's constructor gives each font it loads — once, where + BabelDOC did it again for every mapper. A load that raises is not + remembered, so the next mapper tries again.""" + key = os.fspath(font_path) + with _fonts_lock: + font = _fonts.get(key) + if font is None: + import pymupdf + + font = pymupdf.Font(fontfile=key) + font.has_glyph = functools.lru_cache(maxsize=10240, typed=True)(font.has_glyph) + font.char_lengths = functools.lru_cache(maxsize=10240, typed=True)( + font.char_lengths + ) + font.font_id = font_file_name + font.font_path = font_path + font.ascent_fontmap = font_metadata["ascent"] + font.descent_fontmap = font_metadata["descent"] + font.encoding_length = font_metadata["encoding_length"] + _fonts[key] = font + return font + + +def _sharing_init(fontmap, original): + """BabelDOC 0.5.24's `FontMapper.__init__`, with its font loop asking + `_shared_font`. Everything else is BabelDOC's, line for line, and reaches + BabelDOC's `assets` through the fontmap module, as the original does.""" + + def __init__(self, translation_config): + self.translation_config = translation_config + assert translation_config.primary_font_family in [ + None, + "serif", + "sans-serif", + "script", + ] + self.primary_font_family = fontmap.PrimaryFontFamily.from_str( + translation_config.primary_font_family, + ) + + font_family = fontmap.assets.get_font_family(translation_config.lang_out) + self.font_file_names = [] + for k in ( + "normal", + "script", + "fallback", + "base", + ): + self.font_file_names.extend(font_family[k]) + + self.fonts = {} + self.fontid2fontpath = {} + for font_file_name in self.font_file_names: + if font_file_name in self.fontid2fontpath: + continue + font_path, font_metadata = fontmap.assets.get_font_and_metadata(font_file_name) + self.fonts[font_file_name] = _shared_font(font_file_name, font_path, font_metadata) + self.fontid2fontpath[font_file_name] = font_path + + self.normal_font_ids = font_family["normal"] + self.script_font_ids = font_family["script"] + self.fallback_font_ids = font_family["fallback"] + self.base_font_ids = font_family["base"] + self.fontid2fontpath["base"] = self.fontid2fontpath[font_family["base"][0]] + + self.fontid2font = {f.font_id: f for f in self.fonts.values()} + + self.fontid2font["base"] = self.fontid2font[self.base_font_ids[0]] + + self.normal_fonts = [self.fontid2font[font_id] for font_id in self.normal_font_ids] + self.script_fonts = [self.fontid2font[font_id] for font_id in self.script_font_ids] + self.fallback_fonts = [ + self.fontid2font[font_id] for font_id in self.fallback_font_ids + ] + + self.base_font = self.fontid2font["base"] + + self.type2font = { + "normal": self.normal_fonts, + "script": self.script_fonts, + "fallback": self.fallback_fonts, + "base": [self.base_font], + } + + self.has_char = functools.lru_cache(maxsize=10240, typed=True)(self.has_char) + self.map_in_type = functools.lru_cache(maxsize=10240, typed=True)( + self.map_in_type + ) + + __init__.__wrapped__ = original # type: ignore[attr-defined] + setattr(__init__, _MARK, True) + return __init__ + + +def install() -> None: + """Have every BabelDOC `FontMapper` share one loaded copy of each font. + + Patches the class, not the module's name for it: every stage imported + `FontMapper` by name, so only the class object reaches them all. + Idempotent. Blocking (imports BabelDOC); call off the event loop. + """ + import babeldoc + from babeldoc.format.pdf.document_il.utils import fontmap + + with _install_lock: + current = fontmap.FontMapper.__init__ + if getattr(current, _MARK, False): + return + # A babeldoc that reports no version is as unchecked as a new one, and + # leaving its constructor alone only costs memory. + version = getattr(babeldoc, "__version__", None) + if version not in VERIFIED_BABELDOC: + logger.warning( + "babeldoc %s is not one font_cache was checked against; its " + "FontMapper keeps loading every font for every stage", + version, + ) + return + fontmap.FontMapper.__init__ = _sharing_init(fontmap, current) diff --git a/src/desktop_pdf_translator/processors/processor.py b/src/desktop_pdf_translator/processors/processor.py index d0110b7..e2d9a89 100644 --- a/src/desktop_pdf_translator/processors/processor.py +++ b/src/desktop_pdf_translator/processors/processor.py @@ -140,7 +140,9 @@ def _schedule_temp_cleanup(path: Path) -> None: # streaming granularity as the LLM path. _PAGES_PER_CHUNK_ARGOS = 1 -# Argos-only cap on parallel chunks. +# Argos-only cap on parallel chunks, set for memory when every page loaded all +# of BabelDOC's fonts again. With them shared (`font_cache.py`), 4 in flight +# translated 31 pages 13% faster than 2, at 0.25 GB more (one run each). _MAX_PARALLEL_CHUNKS_ARGOS = 2 # Tag for the "stop now, the credentials are being rejected" item that diff --git a/tests/test_font_cache.py b/tests/test_font_cache.py new file mode 100644 index 0000000..60533b3 --- /dev/null +++ b/tests/test_font_cache.py @@ -0,0 +1,746 @@ +"""BabelDOC loads every embedding font again for every `FontMapper` it builds. + +A mapper reads each font of the target language from disk into a +`pymupdf.Font` (34 files, ~266 MB for Vietnamese), and BabelDOC builds six or +seven mappers per translated page, one per pipeline stage. A book is translated +page by page, so a run loaded hundreds of GB of fonts and the sidecar ran out of +memory. `font_cache.install()` makes every mapper in the process share one +loaded font per file. The risky half is sharing more than the fonts: a mapper +that also shared its config or containers would couple stages that must stay +independent, and a memo wrapped around a shared font's methods once per mapper +makes every lookup pay for all the layers before it. + +These tests pin that the fonts really are shared (by every stage that builds a +mapper, so the hook has to be the one they all use; across languages; between +threads that start together; for as long as the process lives) and that nothing +else is: a mapper keeps its own config and containers, and a shared font is +still the font BabelDOC's own constructor would have built. A load that failed +leaves nothing behind. The hook goes only onto a BabelDOC whose constructor the +module's copy was checked against, since a constructor it doesn't match could +break every stage while an unpatched one only costs memory. And the translate +job path installs it. + +`fontmap.assets` is stubbed with real but tiny fonts (PyMuPDF's built-in ones, +written under `tmp_path`), so nothing downloads and the suite doesn't load 266 +MB. Expected values come from that stub's data, or from a mapper built by +BabelDOC's own constructor, never from the code under test. +""" + +from __future__ import annotations + +import gc +import importlib +import inspect +import logging +import sys +import threading +import time +import types +import uuid +import weakref + +import pymupdf +import pytest + +from desktop_pdf_translator.processors import font_cache, font_verify_cache + +babeldoc = pytest.importorskip("babeldoc") +fontmap = pytest.importorskip("babeldoc.format.pdf.document_il.utils.fontmap") + +_LOGGER = "desktop_pdf_translator.processors.font_cache" + +# Captured now: a test may swap `pymupdf.Font` for a slower subclass, and the +# helpers below must still recognise every font however it was built. +_FONT = pymupdf.Font + +# In none of the stub fonts (they are Latin), so lookups that must come up +# empty are exercised too. +_CJK = chr(0x4E2D) + +# Named like BabelDOC's own fonts: `FontMapper.map_in_type` tells serif from sans +# by looking for "serif" in the font id, because the fonts' own flags say serif +# for all of them. +_BUILTIN_FONT = { + "TestSerif-Regular": "tiro", + "TestSerif-Bold": "tibo", + "TestSans-Regular": "helv", + "TestSans-Bold": "hebo", + "TestSans-Italic": "heit", + "TestFallback-Regular": "cour", + # In the store, but in no family the stub hands out unless a test says so. + "TestExtra-Regular": "cobo", +} + +# Stages that take nothing but a config and build a mapper from it. +_STAGES = ( + ("styles_and_formulas", "StylesAndFormulas"), + ("paragraph_finder", "ParagraphFinder"), + ("typesetting", "Typesetting"), +) + +# Containers the constructor builds fresh. The `*_font_ids` lists are left out: +# BabelDOC aliases them to the lists in the font family it is handed. +_OWN_CONTAINERS = ( + "font_file_names", + "fonts", + "fontid2fontpath", + "fontid2font", + "normal_fonts", + "script_fonts", + "fallback_fonts", + "type2font", +) + +_MEMOIZED_METHODS = ("has_glyph", "char_lengths") + + +@pytest.fixture +def original_init(monkeypatch): + """BabelDOC's own constructor, with `fontmap` put back after the test + whatever `install()` did to it. + + Unwrapped first: if anything earlier in the run installed the hook for real, + the wrapper it left must not be mistaken for BabelDOC's own. The class + binding is restored too, so a hook that rebinds the name instead of patching + the class can't leak into the next test's stage imports and pass there. + """ + original = inspect.unwrap(fontmap.FontMapper.__init__) + monkeypatch.setattr(fontmap.FontMapper, "__init__", original) + monkeypatch.setattr(fontmap, "FontMapper", fontmap.FontMapper) + return original + + +@pytest.fixture +def stub_fonts(monkeypatch, tmp_path, original_init): + """Seven real font files behind `fontmap.assets`, as a mapper asks for them. + + Six make up the family the stub hands out for any language; `extra` is in + the store but in no family. Every test gets font names of its own: the + shared cache outlives a test, and nothing it remembers from another one + should be able to answer this one. + """ + token = uuid.uuid4().hex[:8] + + def named(stem): + return f"{stem}-{token}.ttf" + + paths = {} + for stem, builtin in _BUILTIN_FONT.items(): + path = tmp_path / named(stem) + path.write_bytes(pymupdf.Font(builtin).buffer) + paths[path.name] = path + # A different value per font, so a font handed another font's metadata shows. + metadata = { + name: {"ascent": 900 + 7 * i, "descent": -(250 + 3 * i), "encoding_length": 1 + i % 2} + for i, name in enumerate(paths) + } + family = { + "normal": [ + named("TestSerif-Regular"), + named("TestSerif-Bold"), + named("TestSans-Regular"), + named("TestSans-Bold"), + ], + "script": [named("TestSans-Italic")], + "fallback": [named("TestFallback-Regular")], + # Listed under "normal" as well, like BabelDOC's own base font: its + # constructor loads a name once however many lists carry it. + "base": [named("TestSans-Regular")], + } + names = list(dict.fromkeys(n for kind in ("normal", "script", "fallback", "base") for n in family[kind])) + stub = types.SimpleNamespace( + get_font_family=lambda lang_code: family, + get_font_and_metadata=lambda name: (paths[name], metadata[name]), + ) + + def not_through_the_global(*args, **kwargs): + raise AssertionError("fonts must come through fontmap.assets") + + # A hook that reached for the real asset store would otherwise start on the + # 266 MB of real fonts, or the network; fail here instead. + real_assets = fontmap.assets + monkeypatch.setattr(real_assets, "get_font_family", not_through_the_global) + monkeypatch.setattr(real_assets, "get_font_and_metadata", not_through_the_global) + monkeypatch.setattr(fontmap, "assets", stub) + return types.SimpleNamespace( + names=names, paths=paths, metadata=metadata, extra=named("TestExtra-Regular") + ) + + +@pytest.fixture +def eager_thread_switching(): + """Switch threads between almost any two bytecodes, so a missing lock shows. + + The interval is process-wide, hence put back whatever the test does. + """ + previous = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + try: + yield + finally: + sys.setswitchinterval(previous) + + +def _config(**fields): + """What `FontMapper.__init__` reads from a `TranslationConfig`, and keeps.""" + return types.SimpleNamespace(**{"lang_out": "vi", "primary_font_family": None, **fields}) + + +def _restub(monkeypatch, **replacements): + """Swap parts of the stubbed `fontmap.assets` for a test's own; the rest answers as before.""" + current = fontmap.assets + parts = { + "get_font_family": current.get_font_family, + "get_font_and_metadata": current.get_font_and_metadata, + **replacements, + } + monkeypatch.setattr(fontmap, "assets", types.SimpleNamespace(**parts)) + + +def _assert_same_font_attributes(font, reference, name): + """`font` carries what BabelDOC's constructor left on `reference`. + + Compared as instance attributes, so one BabelDOC starts setting on a font + later is held to as well. PyMuPDF's own handle (`this`) differs by + construction. A memo is compared by being one, with the same settings, + around the same method: what it wraps is a different font's bound method. + """ + own = {key: value for key, value in vars(font).items() if key != "this"} + expected = {key: value for key, value in vars(reference).items() if key != "this"} + assert set(own) == set(expected), f"{name}: attributes differ" + for key, value in expected.items(): + if hasattr(value, "cache_parameters"): + assert hasattr(own[key], "cache_parameters"), f"{name}.{key} is not memoized" + assert own[key].cache_parameters() == value.cache_parameters(), f"{name}.{key}" + assert own[key].__wrapped__.__name__ == value.__wrapped__.__name__, f"{name}.{key}" + else: + assert own[key] == value, f"{name}.{key}" + + +def _count_font_loads(monkeypatch): + """Record every font file read into a `pymupdf.Font`, as `{file: times}`. + + Identity of what mappers hold says nothing about how often it was built: a + cache that loads first and keeps whichever copy arrived first hands out one + object while every mapper pays for 266 MB and leaves its copy to the GC. + """ + loads: dict[str, int] = {} + + class CountingFont(_FONT): + def __init__(self, *args, **kwargs): + key = str(kwargs.get("fontfile", args[0] if args else None)) + loads[key] = loads.get(key, 0) + 1 + super().__init__(*args, **kwargs) + + monkeypatch.setattr(pymupdf, "Font", CountingFont) + return loads + + +def _slow_font_loads(monkeypatch, seconds): + """Make loading a font take a while, as a real font's megabytes do. + + The stub's fonts load in microseconds, which hides the window between a + cache looking for a font and storing it. Sleeping releases the GIL, so + every thread gets into that window however the platform schedules them. + Only bites where fonts are built through `pymupdf.Font` looked up at call + time; elsewhere the test falls back on its other aids. + """ + + class SlowFont(_FONT): + def __init__(self, *args, **kwargs): + time.sleep(seconds) + super().__init__(*args, **kwargs) + + monkeypatch.setattr(pymupdf, "Font", SlowFont) + + +def _fonts_in(value): + """Every `pymupdf.Font` reachable through dicts, lists and tuples.""" + if isinstance(value, _FONT): + yield value + elif isinstance(value, dict): + for item in value.values(): + yield from _fonts_in(item) + elif isinstance(value, (list, tuple)): + for item in value: + yield from _fonts_in(item) + + +def _shape(mapper): + """What a mapper holds, each font reduced to its id. + + Mappers that loaded their fonts separately can then be compared. The two + per-instance memos are callables; only their presence is compared. + """ + + def reduced(value): + if isinstance(value, _FONT): + return ("font", getattr(value, "font_id", None)) + if isinstance(value, dict): + return {key: reduced(item) for key, item in value.items()} + if isinstance(value, list): + return [reduced(item) for item in value] + if callable(value): + return "" + return value + + return {name: reduced(value) for name, value in vars(mapper).items()} + + +def _font_id(font): + return None if font is None else font.font_id + + +class _Rendezvous: + """Holds the threads that ask for one font until all of them have, or a + moment has passed. + + That lines them up at the point where a cache decides whether it has the + font. Not a `threading.Barrier`: an implementation that keeps its lock across + the lookup would deadlock on it, the thread holding the lock waiting for + peers that are waiting for the lock. A timeout that opens the gate for good + costs the race its edge, never a false failure. + """ + + def __init__(self, parties, timeout): + self._parties = parties + self._timeout = timeout + self._lock = threading.Lock() + self._gates = {} + + def __call__(self, key): + with self._lock: + state = self._gates.setdefault(key, [0, threading.Event()]) + state[0] += 1 + if state[0] >= self._parties: + state[1].set() + gate = state[1] + if not gate.wait(self._timeout): + gate.set() + + +# --------------------------------------------------------------------------- +# The fonts are shared +# --------------------------------------------------------------------------- + + +def test_two_mappers_for_one_language_share_every_font(stub_fonts): + font_cache.install() + + a, b, c = (fontmap.FontMapper(_config()) for _ in range(3)) + + assert set(a.fonts) == set(stub_fonts.names) + for name in stub_fonts.names: + assert a.fonts[name] is b.fonts[name], f"{name} was loaded again for the next mapper" + # However a mapper hands a font out (`fonts`, `fontid2font`, the per-type + # lists) it is the one object: a copy on any route still fills memory. + loaded = {id(font) for mapper in (a, b, c) for font in _fonts_in(vars(mapper))} + assert len(loaded) == len(stub_fonts.names) + + +def test_the_stages_that_build_mappers_get_the_shared_fonts(stub_fonts): + # Imported before install(): a stage binds `FontMapper` when its module is + # imported, so a hook that swapped the class inside `fontmap` would never + # reach it. + stages = [ + getattr(importlib.import_module(f"babeldoc.format.pdf.document_il.midend.{module}"), name) + for module, name in _STAGES + ] + font_cache.install() + config = _config() + + baseline = fontmap.FontMapper(config) + for stage in stages: + mapper = stage(config).font_mapper + assert isinstance(mapper, fontmap.FontMapper) + for name in stub_fonts.names: + assert mapper.fonts[name] is baseline.fonts[name], ( + f"{stage.__name__} loaded {name} for itself" + ) + + +# --------------------------------------------------------------------------- +# Nothing else is +# --------------------------------------------------------------------------- + + +def test_each_mapper_keeps_the_config_it_was_built_with(stub_fonts): + serif_config = _config(primary_font_family="serif") + sans_config = _config(primary_font_family="sans-serif") + font_cache.install() + + a = fontmap.FontMapper(serif_config) + b = fontmap.FontMapper(sans_config) + + assert a.translation_config is serif_config + assert b.translation_config is sans_config + # Derived from the config in the constructor, so a mapper rebuilt from a + # remembered one would carry the first config's choice. + assert a.primary_font_family == fontmap.PrimaryFontFamily.SERIF + assert b.primary_font_family == fontmap.PrimaryFontFamily.SANS_SERIF + + +def test_mappers_share_fonts_but_own_everything_else(stub_fonts): + font_cache.install() + a = fontmap.FontMapper(_config()) + b = fontmap.FontMapper(_config()) + untouched = _shape(a) + + for attr in _OWN_CONTAINERS: + assert getattr(a, attr) is not getattr(b, attr), f"{attr} is shared between mappers" + assert a.has_char is not b.has_char + assert a.map_in_type is not b.map_in_type + + # Emptying every container of one mapper reaches neither a mapper built + # before it nor one built after: a cache that handed out its own dicts and + # lists would empty the next mapper's too. + for attr in _OWN_CONTAINERS: + getattr(a, attr).clear() + c = fontmap.FontMapper(_config()) + assert _shape(b) == untouched + assert _shape(c) == untouched + + +def test_a_mapper_is_what_babeldocs_own_constructor_builds(stub_fonts, original_init): + font_cache.install() + config = _config(primary_font_family="serif") + + built = fontmap.FontMapper(config) + reference = object.__new__(fontmap.FontMapper) + original_init(reference, config) + + # A constructor that re-creates BabelDOC's would silently miss an attribute + # added by the next release, to the mapper or to a font; this is where that + # shows, and the check to lean on when bumping babeldoc. + assert _shape(built) == _shape(reference) + assert set(built.fonts) == set(reference.fonts) + for name, expected in reference.fonts.items(): + _assert_same_font_attributes(built.fonts[name], expected, name) + queries = [ + (bold, italic, False, serif, char, font_type) + for bold in (False, True) + for italic in (False, True) + for serif in (False, True) + for char in ("a", _CJK) + for font_type in ("normal", "script", "fallback", "base") + ] + for query in queries: + assert _font_id(built.map_in_type(*query)) == _font_id(reference.map_in_type(*query)), query + + +# --------------------------------------------------------------------------- +# A shared font is the font BabelDOC would have built +# --------------------------------------------------------------------------- + + +def test_a_shared_font_carries_what_babeldocs_constructor_sets(stub_fonts): + font_cache.install() + first = fontmap.FontMapper(_config()) + later = fontmap.FontMapper(_config()) + + for name in stub_fonts.names: + font = later.fonts[name] + assert font is first.fonts[name], f"{name} is not shared" + expected = stub_fonts.metadata[name] + assert font.font_id == name + assert font.font_path == stub_fonts.paths[name] + assert font.ascent_fontmap == expected["ascent"] + assert font.descent_fontmap == expected["descent"] + assert font.encoding_length == expected["encoding_length"] + + +def test_later_mappers_do_not_wrap_a_shared_fonts_methods_again(stub_fonts): + font_cache.install() + first = fontmap.FontMapper(_config()) + memoized = { + (name, method): getattr(font, method) + for name, font in first.fonts.items() + for method in _MEMOIZED_METHODS + } + + for _ in range(3): + later = fontmap.FontMapper(_config()) + + for name, font in later.fonts.items(): + assert font is first.fonts[name], f"{name} is not shared" + for method in _MEMOIZED_METHODS: + current = getattr(font, method) + assert current is memoized[name, method], f"{name}.{method} was replaced" + assert hasattr(current, "cache_info"), f"{name}.{method} is not memoized" + # `lru_cache` keeps what it wraps as `__wrapped__`: the font's own + # bound method, unless a second layer was put around the first. + assert getattr(current.__wrapped__, "__self__", None) is font, ( + f"{name}.{method} is wrapped more than once" + ) + # Expected values from a font nothing here has touched. Twice, so the + # second answer comes from the memo. + oracle = pymupdf.Font(fontfile=str(stub_fonts.paths[name])) + for _ in range(2): + for char in ("a", "Z", _CJK): + assert font.has_glyph(ord(char)) == oracle.has_glyph(ord(char)) + assert font.char_lengths("Hello", fontsize=10) == oracle.char_lengths( + "Hello", fontsize=10 + ) + for method in _MEMOIZED_METHODS: + assert getattr(font, method).cache_info().hits > 0, f"{name}.{method} never hit" + + +# --------------------------------------------------------------------------- +# install() is idempotent +# --------------------------------------------------------------------------- + + +def test_installing_twice_wraps_the_constructor_once(original_init): + font_cache.install() + first = fontmap.FontMapper.__init__ + font_cache.install() + + # Whether the second call stacked a wrapper shows only in what `__wrapped__` + # leads to: the fonts would be shared either way. + assert fontmap.FontMapper.__init__ is first + assert getattr(first, "__wrapped__", None) is original_init + + +# --------------------------------------------------------------------------- +# One font per file, whoever asks and whenever +# --------------------------------------------------------------------------- + + +def test_a_font_is_cached_per_file_not_per_language(stub_fonts, monkeypatch): + # BabelDOC's own families are the same 34 files, arranged differently per + # language, so a cache keyed by language would load all 266 MB again for each + # language a session translates into. + shared = [name for name in stub_fonts.names if "Sans" in name] + other_family = { + "normal": [*shared, stub_fonts.extra], + "script": [], + "fallback": [], + "base": [shared[0]], + } + default_family = fontmap.assets.get_font_family("vi") + _restub( + monkeypatch, + get_font_family=lambda lang_code: other_family if lang_code == "ja" else default_family, + ) + font_cache.install() + + vi = fontmap.FontMapper(_config(lang_out="vi")) + ja = fontmap.FontMapper(_config(lang_out="ja")) + + assert set(vi.fonts) == set(stub_fonts.names) + assert set(ja.fonts) == {*shared, stub_fonts.extra} + for name in shared: + assert vi.fonts[name] is ja.fonts[name], f"{name} was loaded again for the second language" + loaded = {id(font) for mapper in (vi, ja) for font in _fonts_in(vars(mapper))} + assert len(loaded) == len({*stub_fonts.names, stub_fonts.extra}) + + +def test_each_font_file_is_loaded_once_however_many_mappers_ask(stub_fonts, monkeypatch): + # BabelDOC's families overlap, so the second language must not reload the + # files it shares with the first. + shared = [name for name in stub_fonts.names if "Sans" in name] + other_family = { + "normal": [*shared, stub_fonts.extra], + "script": [], + "fallback": [], + "base": [shared[0]], + } + default_family = fontmap.assets.get_font_family("vi") + _restub( + monkeypatch, + get_font_family=lambda lang_code: other_family if lang_code == "ja" else default_family, + ) + loads = _count_font_loads(monkeypatch) + font_cache.install() + + for _ in range(3): + fontmap.FontMapper(_config(lang_out="vi")) + for _ in range(2): + fontmap.FontMapper(_config(lang_out="ja")) + + expected = {str(stub_fonts.paths[name]) for name in {*stub_fonts.names, stub_fonts.extra}} + assert set(loads) == expected + assert {file: times for file, times in loads.items() if times != 1} == {} + + +def test_threads_building_mappers_at_once_still_share_one_font_per_file( + stub_fonts, monkeypatch, eager_thread_switching +): + # Everything here is there to make a cache without a lock lose: threads that + # start together, are held at each font until all of them have asked for it, + # and switch constantly, while a font takes a while to load. + threads = 8 + rendezvous = _Rendezvous(parties=threads, timeout=0.1) + look_up = fontmap.assets.get_font_and_metadata + + def look_up_once_all_have_asked(name): + rendezvous(name) + return look_up(name) + + _restub(monkeypatch, get_font_and_metadata=look_up_once_all_have_asked) + _slow_font_loads(monkeypatch, 0.01) + font_cache.install() + start = threading.Barrier(threads) + built = [None] * threads + failures = [] + + def build(index): + try: + start.wait(timeout=30) + built[index] = fontmap.FontMapper(_config()) + except BaseException as error: # noqa: BLE001 - reported below, not lost in the thread + failures.append(error) + + workers = [threading.Thread(target=build, args=(i,), daemon=True) for i in range(threads)] + for worker in workers: + worker.start() + for worker in workers: + worker.join(timeout=60) + + assert not any(worker.is_alive() for worker in workers), "a thread never finished" + assert not failures, failures + for name in stub_fonts.names: + distinct = {id(mapper.fonts[name]) for mapper in built} + assert len(distinct) == 1, f"{name} was loaded by {len(distinct)} threads" + loaded = {id(font) for mapper in built for font in _fonts_in(vars(mapper))} + assert len(loaded) == len(stub_fonts.names) + + +def test_a_failed_load_is_not_remembered(stub_fonts, original_init): + # The third font a mapper loads: two are already cached when it fails. + broken = stub_fonts.names[2] + path = stub_fonts.paths[broken] + real_font = path.read_bytes() + path.write_bytes(b"this is not a font") + config = _config() + + with pytest.raises(Exception) as expected: + original_init(object.__new__(fontmap.FontMapper), config) + + font_cache.install() + # Twice: a failure left behind as a half-built entry would let the second + # attempt through. + for _ in range(2): + with pytest.raises(Exception) as raised: + fontmap.FontMapper(config) + assert type(raised.value) is type(expected.value), raised.value + + path.write_bytes(real_font) + try: + a = fontmap.FontMapper(config) + except Exception as error: # noqa: BLE001 - the point is what it says about the repair + pytest.fail(f"{broken} still fails to load after it was repaired: {error!r}") + b = fontmap.FontMapper(config) + + assert set(a.fonts) == set(stub_fonts.names) + for name in stub_fonts.names: + assert a.fonts[name] is b.fonts[name], f"{name} is not shared after the repair" + assert a.fonts[broken].has_glyph(ord("a")) + + +def test_shared_fonts_outlive_the_mappers_that_asked_for_them(stub_fonts): + font_cache.install() + mappers = [fontmap.FontMapper(_config()) for _ in range(3)] + watching = {name: weakref.ref(font) for name, font in mappers[0].fonts.items()} + assert set(watching) == set(stub_fonts.names) + + del mappers + # A mapper and its fonts sit in reference cycles (a bound method memoized on + # the object it belongs to), so only a collection frees them; run one, so a + # cache that kept its fonts weakly really loses them. + gc.collect() + again = fontmap.FontMapper(_config()) + + for name, ref in watching.items(): + assert ref() is not None, f"{name} was freed with the mappers that held it" + assert ref() is again.fonts[name], f"{name} was loaded again" + oracle = pymupdf.Font(fontfile=str(stub_fonts.paths[name])) + for char in ("a", "Z", _CJK): + assert again.fonts[name].has_glyph(ord(char)) == oracle.has_glyph(ord(char)) + + +# --------------------------------------------------------------------------- +# Only for a BabelDOC the copy was checked against +# --------------------------------------------------------------------------- + + +def test_an_unverified_babeldoc_is_left_unpatched_with_a_warning( + stub_fonts, original_init, monkeypatch, caplog +): + version = "0.0.0+unverified" + assert version not in font_cache.VERIFIED_BABELDOC + monkeypatch.setattr(babeldoc, "__version__", version) + + with caplog.at_level(logging.WARNING, logger=_LOGGER): + font_cache.install() + + assert fontmap.FontMapper.__init__ is original_init + assert not hasattr(fontmap.FontMapper.__init__, "__wrapped__") + a = fontmap.FontMapper(_config()) + b = fontmap.FontMapper(_config()) + for name in stub_fonts.names: + assert a.fonts[name] is not b.fonts[name], f"{name} is shared on an unverified BabelDOC" + warnings = [ + record.getMessage() + for record in caplog.records + if record.name == _LOGGER and record.levelno == logging.WARNING + ] + assert any(version in message for message in warnings), ( + f"no WARNING naming {version} was logged on {_LOGGER}; got {warnings}" + ) + + +def test_a_babeldoc_without_a_version_is_treated_as_unverified( + stub_fonts, original_init, monkeypatch, caplog +): + monkeypatch.delattr(babeldoc, "__version__") + + with caplog.at_level(logging.WARNING, logger=_LOGGER): + font_cache.install() + + assert fontmap.FontMapper.__init__ is original_init + assert any( + record.name == _LOGGER and record.levelno == logging.WARNING for record in caplog.records + ), "no WARNING was logged on a BabelDOC that reports no version" + + +def test_the_installed_babeldoc_is_one_the_copy_was_checked_against(): + assert babeldoc.__version__ in font_cache.VERIFIED_BABELDOC, ( + f"BabelDOC {babeldoc.__version__} is not in font_cache.VERIFIED_BABELDOC, so " + "install() leaves its FontMapper alone and every mapper loads its own fonts " + "again. Re-check font_cache's copy of FontMapper.__init__ against this " + "version's babeldoc/format/pdf/document_il/utils/fontmap.py, then add the " + "version to the set." + ) + + +# --------------------------------------------------------------------------- +# The job path installs it +# --------------------------------------------------------------------------- + + +def test_the_job_path_installs_the_shared_fonts(monkeypatch): + """`_load_engine` is the only place that installs the shared fonts (the boot + warm thread installs just the hash memo), and every job runs it before + BabelDOC builds a mapper; without this call each stage would load every + font again.""" + import desktop_pdf_translator.processors as processors_pkg + from desktop_pdf_translator.api.routes import translation + + stub = types.ModuleType("processor") + stub.PDFProcessor = type("PDFProcessor", (), {}) + events = types.ModuleType("events") + events.EventType = type("EventType", (), {}) + monkeypatch.setattr(processors_pkg, "processor", stub, raising=False) + monkeypatch.setattr(processors_pkg, "events", events, raising=False) + monkeypatch.setitem(sys.modules, "desktop_pdf_translator.processors.processor", stub) + monkeypatch.setitem(sys.modules, "desktop_pdf_translator.processors.events", events) + calls: list[str] = [] + # Both replaced: this runs the job path for real, and neither install may + # reach BabelDOC from here. + monkeypatch.setattr(font_verify_cache, "install", lambda: calls.append("hash memo")) + monkeypatch.setattr(font_cache, "install", lambda: calls.append("fonts")) + + translation._load_engine() + + assert calls.count("fonts") == 1, calls diff --git a/tests/test_font_verify_cache.py b/tests/test_font_verify_cache.py index af964e8..2b5bd8c 100644 --- a/tests/test_font_verify_cache.py +++ b/tests/test_font_verify_cache.py @@ -1,7 +1,7 @@ """BabelDOC re-hashes the same cached fonts on every pipeline stage. Every stage builds a font mapper, and every mapper runs `verify_file` over all -34 fonts (266 MB, ~1 s) — 7-8 times per translated page. The fonts don't change +34 fonts (266 MB, ~1 s) — 6-7 times per translated page. The fonts don't change between those checks, so `font_verify_cache` remembers a pass. The risky half is remembering too much: a failure, a font rewritten by a re-download, or a file changing mid-hash must each be re-checked, or BabelDOC would trust bytes it @@ -23,7 +23,7 @@ import pytest -from desktop_pdf_translator.processors import font_verify_cache +from desktop_pdf_translator.processors import font_cache, font_verify_cache def _sha(data: bytes) -> str: @@ -380,6 +380,9 @@ def test_the_job_path_installs_the_memo(monkeypatch): monkeypatch.setitem(sys.modules, "desktop_pdf_translator.processors.events", events) calls: list[int] = [] monkeypatch.setattr(font_verify_cache, "install", lambda: calls.append(1)) + # The job path installs the shared fonts too; for real, that would patch + # BabelDOC's FontMapper for the rest of the run. + monkeypatch.setattr(font_cache, "install", lambda: None) translation._load_engine()