From 1776068a6668a6e2f31dbe8cceded0113c7998bf Mon Sep 17 00:00:00 2001 From: nammovuivui Date: Fri, 23 Oct 2026 16:52:34 +0700 Subject: [PATCH 1/5] fix(chat): join line-break hyphens without PyMuPDF's flag PyMuPDF 1.27, which the translation memory fix needs, ignores TEXT_DEHYPHENATE in every output mode. Without a replacement, chat indexed 1,129 words of a 346-page book as halves ("trans- action") that neither keyword nor semantic search matches. ingest._join_broken_words reproduces the flag's rule exactly: a line ending in "-" loses it and runs into the next, unless the line is only the dash. On PyMuPDF 1.25.5 the ingestor's chunks are byte-identical before and after this change on all seven text documents of the RAG benchmark, so CHUNKER_VERSION stays and no index is rebuilt. A variant that spared compounds (joining only lowercase continuations) was measured and lost two questions on BERT, whose tables use "-" for empty cells; the parity test's expected text is what 1.25.5 indexed. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 4 +++ docs/architecture/data-and-chat.md | 22 ++++++++++++-- src/desktop_pdf_translator/rag/ingest.py | 38 +++++++++++++++++++++--- tests/test_rag_ingest.py | 34 +++++++++++++++++++++ 4 files changed, 92 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 46934a8..05d5798 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -308,6 +308,10 @@ rule has, unless noted, a test. They are the things most easily undone by embeds means editing `rag/index_spec.py`. How a model is run — ONNX file, pooling, its `query: `/`passage: ` prefixes — is its `EmbeddingSpec` there; a wrong one gives worse vectors and no error. +- **Line-break hyphens are joined by `ingest._join_broken_words`, which + reproduces `TEXT_DEHYPHENATE` exactly** — PyMuPDF 1.27 ignores the flag. Keep + it exact: the parity test's expected text is what 1.25.5 indexed, and a + "smarter" rule sparing compounds lost two benchmark questions. - **A follow-up is searched for as its standalone rewrite, never fused with the question as typed** (the raw words outvote it). The rewrite prompt has no answer-language instruction; the history is the same document's last diff --git a/docs/architecture/data-and-chat.md b/docs/architecture/data-and-chat.md index f32a608..bcc37ba 100644 --- a/docs/architecture/data-and-chat.md +++ b/docs/architecture/data-and-chat.md @@ -124,8 +124,26 @@ index (once per document, rag/ingest.py) ask (rag/rag_chain.py) to be refused outright ("No text could be extracted"). OCR is slow — ~7 s a dense page on an i5 — and runs once, with per-page progress on the index job. - Text is NFKC-normalised (ligatures: "firmware" never matched "firmware") - and dehyphenated. `CHUNKER_VERSION` is "2", so every existing index is - rebuilt the next time chat opens its document. + and dehyphenated. Changing how a PDF becomes chunks means bumping + `CHUNKER_VERSION` (now "3"), which rebuilds every existing index the next + time chat opens its document. +- **Line-break hyphens are joined by `ingest._join_broken_words`, not by + PyMuPDF.** `TEXT_DEHYPHENATE` stopped working in PyMuPDF 1.27, which the + translation memory fix requires (translation-pipeline.md, "PyMuPDF is at + least 1.27.2"). Without a replacement, 1,129 words on 278 of a 346-page + book's pages were indexed as halves ("trans- action"), which neither search + matches. The helper is the flag's rule reproduced exactly: a line ending in + "-" loses it and runs into the next, unless the line is only the dash. + Checked chunk for chunk, the ingestor's output on 1.25.5 is identical + before and after the change on all seven text documents in the benchmark, + so `CHUNKER_VERSION` stays and no index is rebuilt. + - The rule glues compounds ("Bose-" / "Einstein" → "BoseEinstein"). A + variant that joined only lowercase continuations was measured and lost two + questions on BERT, whose tables use "-" for an empty cell, so the exact + rule stayed. + - Retrieval-only hit@1, PyMuPDF 1.25.5 → 1.27: 0.857 → 0.838 with no join, + 0.848 with the helper. The one question still lost (`p10`, ResNet) comes + from 1.27's table finder, not from the join. **Retrieval** (`EnhancedRAGChain.retrieve`) is local and costs one read of the index's chunks: diff --git a/src/desktop_pdf_translator/rag/ingest.py b/src/desktop_pdf_translator/rag/ingest.py index 7c3f76e..f568907 100644 --- a/src/desktop_pdf_translator/rag/ingest.py +++ b/src/desktop_pdf_translator/rag/ingest.py @@ -144,6 +144,35 @@ def _union(rects: Iterable[Rect]) -> Rect: ) +def _join_broken_words(lines: List[_Line]) -> List[_Line]: + """A word hyphenated across a line break, put back together. + + PyMuPDF's `TEXT_DEHYPHENATE` did this up to 1.26 and is ignored from 1.27, + which split ~1,100 words of a 346-page book into halves that neither + search matches. This is that flag's rule, reproduced exactly: a line + ending in "-" loses it and runs into the next, unless the line is nothing + but the dash, which stays. That glues compounds too ("BoseEinstein"), and + is kept anyway: a rule that spared them moved two benchmark questions off + their page, and matching the flag keeps every index built before the + upgrade the same text as one built after (CHUNKER_VERSION unchanged). + """ + merged: List[_Line] = [] + for line in lines: + if merged and merged[-1].text.endswith("-"): + prev = merged[-1] + head = prev.text if prev.text.strip() == "-" else prev.text[:-1] + merged[-1] = _Line( + text=head + line.text, + size=max(prev.size, line.size), + bbox=_union([prev.bbox, line.bbox]), + bold=prev.bold and line.bold, + horizontal=prev.horizontal and line.horizontal, + ) + continue + merged.append(line) + return merged + + def _clip(rect: Rect, page: fitz.Rect) -> Rect: return ( float(max(rect[0], page.x0)), @@ -159,9 +188,10 @@ def _inside(rect: Rect, area: Rect) -> bool: def _read_page(page: fitz.Page) -> _Page: - # Ligatures expanded and line-break hyphens joined by PyMuPDF itself: - # without `TEXT_PRESERVE_LIGATURES`, and with `TEXT_DEHYPHENATE`. - flags = fitz.TEXT_PRESERVE_WHITESPACE | fitz.TEXT_MEDIABOX_CLIP | fitz.TEXT_DEHYPHENATE + # Ligatures expanded by PyMuPDF itself: without `TEXT_PRESERVE_LIGATURES`. + # Not `TEXT_DEHYPHENATE`: 1.27 ignores it, and `_join_broken_words` is the + # one place its rule lives now. + flags = fitz.TEXT_PRESERVE_WHITESPACE | fitz.TEXT_MEDIABOX_CLIP data = page.get_text("dict", flags=flags) result = _Page(number=page.number, rect=page.rect) for block in data.get("blocks", []): @@ -183,7 +213,7 @@ def _read_page(page: fitz.Page) -> _Page: horizontal=abs(direction[0] - 1.0) < 0.01 and abs(direction[1]) < 0.01, )) if lines: - result.blocks.append(_Block(lines=lines, bbox=tuple(block["bbox"]))) + result.blocks.append(_Block(lines=_join_broken_words(lines), bbox=tuple(block["bbox"]))) result.text_chars = sum(len(b.text.replace(" ", "")) for b in result.blocks) area = abs(page.rect) or 1.0 diff --git a/tests/test_rag_ingest.py b/tests/test_rag_ingest.py index 7ff9e6e..017a9f9 100644 --- a/tests/test_rag_ingest.py +++ b/tests/test_rag_ingest.py @@ -196,6 +196,40 @@ def test_word_hyphenated_across_a_line_break_is_joined(tmp_path: Path) -> None: assert "perfor-" not in text +# What `TEXT_DEHYPHENATE` made of each line pair on PyMuPDF 1.25.5, read off the +# ingestor as it stood then: the text every index built before the upgrade holds. +_PYMUPDF_125_DEHYPHENATED = [ + ("Cooled atoms form a Bose-
Einstein condensate below a critical point.", + "Cooled atoms form a BoseEinstein condensate below a critical point."), + ("Cases rose in the COVID-
19 wave of that winter.", + "Cases rose in the COVID19 wave of that winter."), + ("Human - -
82.3 91.2 were the scores reported.", + "Human - 82.3 91.2 were the scores reported."), + ("The cost was 2.5k
-
Pre-OpenAI SOTA in that table.", + "The cost was 2.5k -Pre-OpenAI SOTA in that table."), + ("The pause ends with a dash—
and the next line starts here.", + "The pause ends with a dash— and the next line starts here."), +] + + +@pytest.mark.parametrize("html, indexed", _PYMUPDF_125_DEHYPHENATED) +def test_line_break_hyphens_are_joined_as_pymupdf_joined_them( + tmp_path: Path, html: str, indexed: str +) -> None: + """PyMuPDF 1.27 ignores `TEXT_DEHYPHENATE`, so the ingestor joins the + lines itself, exactly as the flag did — even where that glues a compound + ("BoseEinstein"). Keeping such compounds apart was measured, and it moved + two benchmark questions off their page; matching the old text also keeps + an index built before the upgrade the same as one built after.""" + doc = fitz.open() + _text_page(doc, [html]) + path = _save(doc, tmp_path) + + _, chunks = _ingest(path) + + assert _all_text(chunks) == indexed + + def test_runs_of_whitespace_collapse_to_one_space(tmp_path: Path) -> None: doc = fitz.open() _text_page(doc, ["Alpha beta
gamma
delta epsilon."]) From ed8e6a4eca7f4ed0f9eac846f74d579b263f8105 Mon Sep 17 00:00:00 2001 From: nammovuivui Date: Fri, 23 Oct 2026 16:54:25 +0700 Subject: [PATCH 2/5] fix(translate): require PyMuPDF 1.27.2, whose insert_pdf stops leaking Before 1.27.2, PyMuPDF's C++ helper behind annot_xrefs leaks a list on every call, and insert_pdf copies every page's links through it. rebuild_rolling_pdf re-inserts the whole document after each page, so a translation leaked ~2,120 lists (~150 KB) per page on a 346-page book: O(pages^2) for a whole document, and invisible to gc, since each list carries one extra reference and no referrer. Measured over 129 pages of that book after a ~40-page warm-up: +1.25 MB/page on 1.25.5, +0.18 on 1.27.2.3, live objects flat. 1.26.7 still leaks; 1.27.2.3 and 1.28.2 don't. <1.28 because 1.28 deprecates the `fitz` name every module imports. The GIL property the shared fonts rely on was re-checked on 1.27.2.3 (no PyEval_SaveThread import). The new test rebuilds a linked document's rolling PDF 20 times: 1,279 objects stay alive on 1.25.5; on 1.27.2.3 fewer than the threshold of one per rebuild. Pages without links never leaked, which is why the older rebuild tests could not catch it. 1.27 also ignores TEXT_DEHYPHENATE (handled in the previous commit) and its find_tables reads ResNet's architecture figure as tables; together with the dehyphenation fix, retrieval-only hit@1 is 0.848 against 0.857 on 1.25.5 (one question, McNemar p = 1.0). Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 7 ++- docs/architecture/testing-and-ci.md | 4 +- docs/architecture/translation-pipeline.md | 53 ++++++++++++++++++++--- pyproject.toml | 7 ++- requirements.txt | 7 ++- tests/test_pdf_pages.py | 52 ++++++++++++++++++++++ 6 files changed, 121 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 05d5798..e3506fe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -269,6 +269,10 @@ rule has, unless noted, a test. They are the things most easily undone by BabelDOC's, installed only on a babeldoc in `VERIFIED_BABELDOC`: re-check it against `fontmap.py` when bumping babeldoc. Sharing a font across pages relies on PyMuPDF never releasing the GIL; re-check that when bumping PyMuPDF. +- **PyMuPDF stays ≥ 1.27.2.** Older ones leak a list per link in `insert_pdf`, + and `rebuild_rolling_pdf` re-inserts the whole document after every page: + O(pages²) on a book, invisible to gc. `test_pdf_pages.py` fails on a version + that leaks. Also keep it `<1.28`, which deprecates `fitz`. **Data and chat** → `data-and-chat.md` - Three stores under one root: `pdfusion.db` (system of record), `vectors/` @@ -548,7 +552,8 @@ isolation (real ChromaDB under `tmp_path`), chat ingestion (real PDFs built with PyMuPDF), OCR respacing, rank fusion and page evidence, citation checking, page images, multimodal and streamed `generate()` per SDK, the answer chain's image, citation, fence and conversation rules, answer ratings, the benchmark's -interval statistics, chunking and rolling-PDF assembly, BabelDOC's shared +interval statistics, chunking and rolling-PDF assembly (and that rebuilding it +leaks nothing), BabelDOC's shared fonts, the data root per platform, packaging config agreement, and the viewer's pure half. diff --git a/docs/architecture/testing-and-ci.md b/docs/architecture/testing-and-ci.md index a2cc9b0..77ed312 100644 --- a/docs/architecture/testing-and-ci.md +++ b/docs/architecture/testing-and-ci.md @@ -14,7 +14,9 @@ CI gates. `generate()` per SDK and the answer chain's image and citation rules (`test_rag_*.py`, `test_multimodal_generate.py`). Splitting a document into chunks and assembling the rolling PDF are covered - against real PDFs (`test_pdf_pages.py`). + against real PDFs (`test_pdf_pages.py`), including that rebuilding a linked + document's rolling PDF again and again leaves no objects alive. The + PyMuPDF < 1.27.2 leak failed exactly that. Cross-platform (#69), `test_storage_paths.py` covers the data root per platform — the marked cases run only where they apply, the shell-override ones run everywhere — `test_config_security.py` covers the keystore scheme diff --git a/docs/architecture/translation-pipeline.md b/docs/architecture/translation-pipeline.md index a447fe5..a6ef954 100644 --- a/docs/architecture/translation-pipeline.md +++ b/docs/architecture/translation-pipeline.md @@ -192,10 +192,14 @@ a font BabelDOC's per-font attributes and memos once, at its first load. On the same book afterwards, its 301 pages with text went through in 23 minutes with the sidecar never above 2.4 GB — 1,806 mappers sharing one set of fonts — and a page translated before and after the change came out the same, every word in -the same place. Memory still drifts up by about 1.7 MB a page: most likely the -rolling PDF, rebuilt whole after every page (43.5 MB by the end), and per-page -garbage that still waits for a full collection, such as the source PDF's own -fonts, which `ILCreater` memoizes the same way. +the same place. Memory still drifted up afterwards, by +1.25 MB a page past a +~40-page warm-up. It was not garbage waiting for a full collection, which was +the first guess: 29 full collections ran on their own over 129 pages, and a +forced one at the end freed next to nothing. It had two causes: +- PyMuPDF leaking ~1,690 objects a page in the rolling-PDF rebuild. See the + next section. +- glibc keeping ~410 MB of freed heap. See "glibc's heap is trimmed after + every page". - **Sharing is safe because nothing writes to a font after construction.** The stages only read a mapper (`has_char`, `map`, `fontid2font`, `base_font`, @@ -215,12 +219,51 @@ fonts, which `ILCreater` memoizes the same way. the memory. `test_font_cache.py` compares the copy with the installed constructor attribute by attribute, and fails by name on an unverified version: re-check the copy against `fontmap.py` and extend the set when - bumping babeldoc. Re-check the GIL claim above when bumping PyMuPDF. + bumping babeldoc. Re-check the GIL claim above when bumping PyMuPDF (last + checked on 1.25.5 and 1.27.2.3). - **Installed by `routes/translation.py:_load_engine`**, which every job runs before BabelDOC builds anything. The class is patched, not the module's name for it: each stage imported `FontMapper` by name. - **A failed load is not remembered**, so a font BabelDOC re-downloads is loaded by the next mapper. +- **The shared memos fill slowly and stop.** Over 129 pages the 34 fonts held + ~8,300 `has_glyph` / `char_lengths` entries, out of a 696,320 cap. That is + the few thousand lists that still appear on PyMuPDF 1.27. They are not a + leak, so don't chase them. + +## PyMuPDF is at least 1.27.2 + +Before 1.27.2, PyMuPDF's C++ helper behind `annot_xrefs` +(`_extra.JM_get_annot_xref_list`) leaks a list on every call. `Link.next` +calls it twice per link, so `get_links()` on a page with 23 links leaked 46 +lists. `insert_pdf` copies every page's links through it. + +`rebuild_rolling_pdf` re-inserts the *whole* document after every page, so +each rebuild leaked one list per call for every link in the book: ~2,120 lists, +~150 KB, on the 346-page book. That is O(pages) per page and O(pages²) for a +whole document. The lists carry one extra reference and no referrer, so gc +never sees them as garbage. +- 1.25.5 and 1.26.7 leak. 1.27.2.3 and 1.28.2 don't. +- Measured after warm-up: +1.25 MB/page on 1.25.5, +0.18 on 1.27.2.3, and + live objects flat. +- `test_rebuilding_the_rolling_pdf_over_and_over_leaves_no_objects_behind` + guards it. Pages without links never leaked, which is why the older rebuild + tests could not catch it. +- `<1.28` because 1.28 deprecates the `fitz` name every module imports. + +What else the bump changed: +- `TEXT_DEHYPHENATE` is ignored, so chat joins line-break hyphens itself. + See data-and-chat.md. +- `find_tables` reads more as tables. It turns ResNet's architecture figure + into half-empty two-column grids that the ingestor's mostly-empty guard lets + through, which costs one benchmark question. +- `find_tables` prints a one-line `pymupdf_layout` notice to stdout once per + process. The shell drains stdout into shell.log. +- The translation output was checked over 129 pages: it differs from 1.25.5 + no more than two runs of 1.25.5 differ from each other. Argos's batching + depends on timing. +- Speed is unchanged: 4.33/4.57 s a page before, 4.47/4.38 after. That was + A-B-B-A, 46 pages each, and measured together with the trim below. ## Page ranges diff --git a/pyproject.toml b/pyproject.toml index 2e86c54..0780361 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,12 @@ dependencies = [ "cryptography>=41.0.0,<43.0.0", # PDF processing - "PyMuPDF>=1.25.1,<1.26.0", + # Before 1.27.2 the helper behind `get_links` leaks a list per call, and + # `insert_pdf` copies every link through it; the rolling PDF re-inserts the + # whole document after each page, so a 346-page book leaked ~2,100 lists a + # page, invisible to gc (tests/test_pdf_pages.py guards it). <1.28: 1.28 + # deprecates the `fitz` name every module imports. + "PyMuPDF>=1.27.2,<1.28", "Pillow>=9.5.0,<11.0.0", "babeldoc>=0.5.20,<0.6.0", # bitstring 5 removed `BitStream`, which babeldoc 0.5's PDF writer still imports diff --git a/requirements.txt b/requirements.txt index 8a38606..b7dd3a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,12 @@ tomlkit>=0.12.0,<1.0.0 python-dotenv>=1.0.0,<2.0.0 cryptography>=41.0.0,<43.0.0 -PyMuPDF>=1.25.1,<1.26.0 +# Before 1.27.2 the helper behind `get_links` leaks a list per call, and +# `insert_pdf` copies every link through it; the rolling PDF re-inserts the +# whole document after each page, so a 346-page book leaked ~2,100 lists a +# page, invisible to gc (tests/test_pdf_pages.py guards it). <1.28: 1.28 +# deprecates the `fitz` name every module imports. +PyMuPDF>=1.27.2,<1.28 Pillow>=9.5.0,<11.0.0 babeldoc>=0.5.20,<0.6.0 diff --git a/tests/test_pdf_pages.py b/tests/test_pdf_pages.py index 220ab6f..68c5f5a 100644 --- a/tests/test_pdf_pages.py +++ b/tests/test_pdf_pages.py @@ -8,6 +8,7 @@ from __future__ import annotations +import gc import struct import os from pathlib import Path @@ -224,6 +225,57 @@ def test_a_segment_without_its_translation_is_a_bug(source: Path, tmp_path: Path rebuild_rolling_pdf(tmp_path / "out.pdf", source, segments, [None]) +def _write_linked_pdf(path: Path, pages: int, links_per_page: int) -> Path: + with fitz.open() as doc: + for n in range(1, pages + 1): + page = doc.new_page() + page.insert_text((72, 72), f"original {n}") + for k in range(links_per_page): + top = 100 + 30 * k + page.insert_link( + { + "kind": fitz.LINK_URI, + "from": fitz.Rect(72, top, 300, top + 20), + "uri": f"https://example.com/{n}/{k}", + } + ) + doc.save(path) + # Guard the fixture: without links this test proves nothing about them. + with fitz.open(path) as doc: + assert all(len(page.get_links()) == links_per_page for page in doc) + return path + + +def test_rebuilding_the_rolling_pdf_over_and_over_leaves_no_objects_behind( + tmp_path: Path, +): + """A run rebuilds the rolling PDF after every chunk in one long-lived + process, so anything a rebuild leaves alive piles up with the page count. + Pages with links are the case that matters: a book is full of them + (contents, cross-references, URLs) and each one is an object per link.""" + linked = _write_linked_pdf(tmp_path / "linked.pdf", pages=10, links_per_page=4) + chunks = split_into_chunks(linked, tmp_path / "in", contiguous_runs([3, 4, 8], 1)) + results = [_translate(chunks[0][0], tmp_path), None, _translate(chunks[2][0], tmp_path)] + segments = rolling_segments( + 10, [r for _, r in chunks], [r is not None for r in results] + ) + rebuilds = 20 + + # The first call may fill caches that are meant to outlive it. + rebuild_rolling_pdf(tmp_path / "warmup.pdf", linked, segments, results) + gc.collect() + before = len(gc.get_objects()) + + for n in range(rebuilds): + rebuild_rolling_pdf(tmp_path / f"rolling_{n}.pdf", linked, segments, results) + gc.collect() + growth = len(gc.get_objects()) - before + + assert growth < rebuilds, ( + f"{growth} objects left alive after {rebuilds} rebuilds of a linked PDF" + ) + + # --------------------------------------------------------------------------- # subset_embedded_fonts # --------------------------------------------------------------------------- From 80c9fb4b5afc10b1c8054af476378cf5cba3137f Mon Sep 17 00:00:00 2001 From: nammovuivui Date: Fri, 23 Oct 2026 17:07:00 +0700 Subject: [PATCH 3/5] fix(translate): hand glibc's freed heap back to the OS after every page glibc keeps what Python frees inside its arenas and returns memory on its own only from the top of a heap. On PyMuPDF 1.27, a 129-page Argos run ended ~420 MB above what malloc_trim brings it back to; a forced gc.collect() freed next to nothing, since the memory was already free. The consumer loop in _process_with_babeldoc now ends each finished page with utils.memory.release_freed_memory, off the event loop: malloc_trim(0) on glibc, a no-op on Windows, macOS and musl. It runs after the rolling rebuild, so that document is freed too, and after chunk_ready, so the viewer's swap never waits on it. No gc.collect(): ~277 ms a page for nothing. Measured on this branch over the same 129 pages, run alone: -0.19 MB/page after warm-up (main: +1.25), 1,656 MB at the end (main: 2,118), a final forced trim frees 2 MB (main: 408), 4.50 s/page. The trim costs 8-22 ms a page. The helper and its tests are ported from fix/release-freed-memory (dc4c1be), whose prose blamed glibc for the 7 GB out-of-memory that was really BabelDOC's fonts (fixed by font_cache.py); the numbers here are re-measured. Windows is still to be measured (#134). Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 8 +- docs/architecture/testing-and-ci.md | 5 +- docs/architecture/translation-pipeline.md | 30 ++++++++ .../processors/processor.py | 8 ++ src/desktop_pdf_translator/utils/memory.py | 45 +++++++++++ tests/test_memory_release.py | 74 +++++++++++++++++++ 6 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 src/desktop_pdf_translator/utils/memory.py create mode 100644 tests/test_memory_release.py diff --git a/CLAUDE.md b/CLAUDE.md index e3506fe..1b39272 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -122,7 +122,7 @@ singleton with no job id, so it is polled via `GET /setup/status`. | `rag/` | `rag_chain.py` (`EnhancedRAGChain`), `conversation.py` (`ChatTurn`, stdlib-only), `ingest.py` (PDF → chunks, page kinds), `ocr.py`, `retrieval.py` (RRF, page evidence), `citations.py`, `page_images.py`, `vector_store.py`, `index_spec.py` (embedding model + `EmbeddingSpec`s), `onnx_embeddings.py`, `keyword_search.py` | | `benchmarks/rag/` | Corpus, questions with gold pages, harness (baseline from a `main` worktree), results — `README.md` there has the numbers | | `storage/` | `sqlite.py` (pragmas, per-thread connections), `migrations.py`, `records.py` (`pdfusion.db`) | -| `utils/` | `encryption.py`, `file_export.py`, `logging_setup.py`, `paths.py` | +| `utils/` | `encryption.py`, `file_export.py`, `logging_setup.py`, `memory.py` (glibc `malloc_trim`), `paths.py` | ### HTTP API @@ -273,6 +273,10 @@ rule has, unless noted, a test. They are the things most easily undone by and `rebuild_rolling_pdf` re-inserts the whole document after every page: O(pages²) on a book, invisible to gc. `test_pdf_pages.py` fails on a version that leaks. Also keep it `<1.28`, which deprecates `fitz`. +- **Each finished page ends with `release_freed_memory`** (`malloc_trim`, + glibc only), after its rebuild and `chunk_ready`. glibc kept ~420 MB of freed + heap otherwise. Never add `gc.collect()` there: ~277 ms a page, freed nothing. + (No unit test for the call site: it is in `processor.py`.) **Data and chat** → `data-and-chat.md` - Three stores under one root: `pdfusion.db` (system of record), `vectors/` @@ -554,7 +558,7 @@ page images, multimodal and streamed `generate()` per SDK, the answer chain's image, citation, fence and conversation rules, answer ratings, the benchmark's interval statistics, chunking and rolling-PDF assembly (and that rebuilding it leaks nothing), BabelDOC's shared -fonts, the data root per platform, packaging config agreement, and the viewer's +fonts, handing freed heap back to glibc, the data root per platform, packaging config agreement, and the viewer's pure half. Not covered: `processors/processor.py` proper, chat quality with the real diff --git a/docs/architecture/testing-and-ci.md b/docs/architecture/testing-and-ci.md index 77ed312..e820ab8 100644 --- a/docs/architecture/testing-and-ci.md +++ b/docs/architecture/testing-and-ci.md @@ -16,7 +16,10 @@ CI gates. Splitting a document into chunks and assembling the rolling PDF are covered against real PDFs (`test_pdf_pages.py`), including that rebuilding a linked document's rolling PDF again and again leaves no objects alive. The - PyMuPDF < 1.27.2 leak failed exactly that. + PyMuPDF < 1.27.2 leak failed exactly that. `test_memory_release.py` covers + the glibc trim helper: it trims where glibc is, and is a no-op off Linux, + on musl and without a loadable libc. The call after each page is in + `processor.py` and is checked by hand. Cross-platform (#69), `test_storage_paths.py` covers the data root per platform — the marked cases run only where they apply, the shell-override ones run everywhere — `test_config_security.py` covers the keystore scheme diff --git a/docs/architecture/translation-pipeline.md b/docs/architecture/translation-pipeline.md index a6ef954..1a06920 100644 --- a/docs/architecture/translation-pipeline.md +++ b/docs/architecture/translation-pipeline.md @@ -265,6 +265,36 @@ What else the bump changed: - Speed is unchanged: 4.33/4.57 s a page before, 4.47/4.38 after. That was A-B-B-A, 46 pages each, and measured together with the trim below. +## glibc's heap is trimmed after every page + +glibc keeps what Python frees inside its arenas, and gives memory back on its +own only from the very top of a heap. Untrimmed, on PyMuPDF 1.27, a 129-page +Argos run ended ~420 MB above what `malloc_trim` returned it to. A forced +`gc.collect()` freed next to nothing: the memory was already free, just never +returned. + +So the consumer loop in `_process_with_babeldoc` ends each finished page with +`utils/memory.release_freed_memory`, off the event loop. That is +`malloc_trim(0)` on glibc, and a no-op on Windows, macOS and musl. +- **Where it sits.** After the rolling rebuild, so the rebuild's document is + freed too. After `chunk_ready`, so the viewer's swap never waits on it. +- **What it buys.** The trim lowers the plateau by ~400 MB. It hardly changes + the slope: that was the PyMuPDF leak. +- **What it costs.** 8–22 ms a page; speed is in the section above. + `gc.collect()` is deliberately not called: it cost ~277 ms a page and freed + nothing. +- **What it does not lower.** The peak of two pages in flight. That is + `processing.max_parallel_chunks`. +- **Watch the trimmed baseline.** If it ever creeps across a long run, + something really leaks, and trimming only hides it. + +Recycling worker processes every N pages was weighed: it reclaims memory on +any OS. It was not taken, because concurrent chunks share paragraph batches, +the provider's rate limiter and the loaded layout model. Splitting them costs +API calls, risks 429s and reloads the model per worker. Windows' heap returns +large blocks as they are freed, so whether Windows needs anything at all is +being measured (#134), not assumed. + ## Page ranges A translation covers the pages the toolbar's Pages box names (#33) — `1-20, 35`, diff --git a/src/desktop_pdf_translator/processors/processor.py b/src/desktop_pdf_translator/processors/processor.py index e2d9a89..b7fb780 100644 --- a/src/desktop_pdf_translator/processors/processor.py +++ b/src/desktop_pdf_translator/processors/processor.py @@ -23,6 +23,7 @@ from ..translators.argos_translator import ArgosTranslator from ..translators.base import describe_fatal_error from ..translators.capabilities import resolve_effective_service, resolve_languages +from ..utils.memory import release_freed_memory from .events import ( ProcessingEvent, ProgressEvent, @@ -1080,6 +1081,13 @@ async def paragraph_ticker() -> None: progress_percent=global_progress, message=f"{pages_label} translated", ) + # glibc keeps freed heap in its arenas: untrimmed, a + # 129-page run ended ~420 MB above where malloc_trim + # returns it. Here, after the rebuild, so its document is + # freed too, and after chunk_ready, so the viewer's swap + # never waits on it. Never gc.collect(): ~277 ms a page, + # and it freed nothing (utils/memory.py). + await asyncio.to_thread(release_freed_memory) finally: # Stop the ticker; completion_task is already done by sentinel. ticker_task.cancel() diff --git a/src/desktop_pdf_translator/utils/memory.py b/src/desktop_pdf_translator/utils/memory.py new file mode 100644 index 0000000..fdb37a2 --- /dev/null +++ b/src/desktop_pdf_translator/utils/memory.py @@ -0,0 +1,45 @@ +"""Hand freed heap back to the OS between translated pages. + +A translation runs every page through BabelDOC inside the one long-lived +sidecar, and glibc keeps what Python frees inside its arenas: it gives memory +back on its own only from the very top of a heap. Measured on a 129-page Argos +run, the sidecar ended ~420 MB above the level `malloc_trim` brought it back to, +and a full `gc.collect()` freed next to nothing — the memory was already free, +just not returned. `malloc_trim` also releases free pages from the middle of a +heap, which is the part that matters here. + +Elsewhere this is a no-op. Windows' heap returns large blocks as they are +freed; whether it holds anything else over a long run is measured separately +(#134), not assumed. musl has no `malloc_trim`. Stdlib-only, so free to import. +""" + +from __future__ import annotations + +import ctypes +import functools +import sys +from typing import Callable, Optional + + +@functools.cache +def _malloc_trim() -> Optional[Callable[[int], int]]: + if not sys.platform.startswith("linux"): + return None + try: + libc = ctypes.CDLL("libc.so.6") + except OSError: + return None + return getattr(libc, "malloc_trim", None) + + +def release_freed_memory() -> bool: + """Return freed heap pages to the OS; True if glibc was asked to. + + Blocking (it walks every arena), so call it off the event loop. + """ + trim = _malloc_trim() + if trim is None: + return False + # 0 keeps no slack at the top of the heap; the next page grows it again. + trim(0) + return True diff --git a/tests/test_memory_release.py b/tests/test_memory_release.py new file mode 100644 index 0000000..cbd0e64 --- /dev/null +++ b/tests/test_memory_release.py @@ -0,0 +1,74 @@ +"""Handing freed heap back to the OS between translated pages. + +glibc keeps what Python frees inside its arenas: untrimmed, a 129-page Argos +run ended ~420 MB above the level `malloc_trim` brought it back to. The helper +must trim where glibc is, and be a silent no-op everywhere else. +""" + +from __future__ import annotations + +import sys + +import pytest + +from desktop_pdf_translator.utils import memory + + +@pytest.fixture(autouse=True) +def _fresh_lookup(): + memory._malloc_trim.cache_clear() + yield + memory._malloc_trim.cache_clear() + + +class _FakeLibc: + def __init__(self) -> None: + self.calls: list[int] = [] + + def malloc_trim(self, pad: int) -> int: + self.calls.append(pad) + return 1 + + +def test_off_linux_it_never_loads_a_libc(monkeypatch): + def refuse(*_args, **_kwargs): + raise AssertionError("loaded a libc off Linux") + + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr(memory.ctypes, "CDLL", refuse) + + assert memory.release_freed_memory() is False + + +def test_a_libc_without_malloc_trim_is_a_no_op(monkeypatch): + # musl has no malloc_trim. + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(memory.ctypes, "CDLL", lambda *_a, **_k: object()) + + assert memory.release_freed_memory() is False + + +def test_no_loadable_libc_is_a_no_op(monkeypatch): + def missing(*_args, **_kwargs): + raise OSError("libc.so.6: cannot open shared object file") + + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(memory.ctypes, "CDLL", missing) + + assert memory.release_freed_memory() is False + + +def test_glibc_is_trimmed_to_zero_padding(monkeypatch): + libc = _FakeLibc() + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(memory.ctypes, "CDLL", lambda *_a, **_k: libc) + + assert memory.release_freed_memory() is True + assert memory.release_freed_memory() is True + # 0 keeps no slack at the top of the heap; the next page re-grows it. + assert libc.calls == [0, 0] + + +@pytest.mark.skipif(not sys.platform.startswith("linux"), reason="glibc only") +def test_the_real_glibc_trims(): + assert memory.release_freed_memory() is True From a8870497537b1b3ef209b1095db60fd8f52628ea Mon Sep 17 00:00:00 2001 From: nammovuivui Date: Fri, 23 Oct 2026 17:26:40 +0700 Subject: [PATCH 4/5] fix(chat): join the Unicode hyphens TEXT_DEHYPHENATE joined too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The replacement for the flag only knew "-". Checked one by one on PyMuPDF 1.25.5, the flag also took U+00AD (soft hyphen), U+2010 and U+2011 for a line-break hyphen, and kept a lone U+2010 line the way it keeps a lone "-". Word processors export these; the benchmark corpus has none, which is why the chunk-for-chunk parity check missed them. Without this, "trans­" / "action" was indexed with the soft hyphen in the middle (NFKC keeps it). The U+2010/U+2011 cases join the parity table, their expected text read off main's ingestor on 1.25.5. U+00AD is tested on the helper itself: a base-14 font reads it back as "-" and PyMuPDF's built-in fonts drop it, so no portable PDF fixture can produce it; the expected text is what 1.25.5 indexed for a page set in DejaVu Sans. Chunks of all seven benchmark documents are still byte-identical to main on 1.25.5. Also says in data-and-chat.md why CHUNKER_VERSION stays although 1.27's find_tables changes some chunks. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/data-and-chat.md | 16 +++++++++----- src/desktop_pdf_translator/rag/ingest.py | 19 ++++++++++------ tests/test_rag_ingest.py | 28 +++++++++++++++++++++--- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/docs/architecture/data-and-chat.md b/docs/architecture/data-and-chat.md index bcc37ba..6fca2ec 100644 --- a/docs/architecture/data-and-chat.md +++ b/docs/architecture/data-and-chat.md @@ -132,11 +132,17 @@ index (once per document, rag/ingest.py) ask (rag/rag_chain.py) translation memory fix requires (translation-pipeline.md, "PyMuPDF is at least 1.27.2"). Without a replacement, 1,129 words on 278 of a 346-page book's pages were indexed as halves ("trans- action"), which neither search - matches. The helper is the flag's rule reproduced exactly: a line ending in - "-" loses it and runs into the next, unless the line is only the dash. - Checked chunk for chunk, the ingestor's output on 1.25.5 is identical - before and after the change on all seven text documents in the benchmark, - so `CHUNKER_VERSION` stays and no index is rebuilt. + matches. The helper is the flag's rule reproduced exactly: + - A line ending in a hyphen loses it and runs into the next, unless the + line is only the hyphen. + - A hyphen is "-", U+00AD, U+2010 or U+2011, each checked against 1.25.5. + Word processors export the Unicode ones, and the benchmark has none. + - Checked chunk for chunk, the ingestor's output on 1.25.5 is identical + before and after the change on all seven text documents in the benchmark. + - `CHUNKER_VERSION` stays, even though 1.27's `find_tables` changes some + chunks (ResNet, below). An index built on 1.25.5 keeps working, and it was + the better of the two on that document. Rebuilding every user's index, + OCR included, for a one-question difference would cost more than it buys. - The rule glues compounds ("Bose-" / "Einstein" → "BoseEinstein"). A variant that joined only lowercase continuations was measured and lost two questions on BERT, whose tables use "-" for an empty cell, so the exact diff --git a/src/desktop_pdf_translator/rag/ingest.py b/src/desktop_pdf_translator/rag/ingest.py index f568907..087b02d 100644 --- a/src/desktop_pdf_translator/rag/ingest.py +++ b/src/desktop_pdf_translator/rag/ingest.py @@ -144,23 +144,28 @@ def _union(rects: Iterable[Rect]) -> Rect: ) +# What `TEXT_DEHYPHENATE` took for a line-break hyphen, checked one by one on +# 1.25.5: hyphen-minus, soft hyphen, hyphen, non-breaking hyphen. Never a dash. +_LINE_BREAK_HYPHENS = ("-", "­", "‐", "‑") + + def _join_broken_words(lines: List[_Line]) -> List[_Line]: """A word hyphenated across a line break, put back together. PyMuPDF's `TEXT_DEHYPHENATE` did this up to 1.26 and is ignored from 1.27, which split ~1,100 words of a 346-page book into halves that neither search matches. This is that flag's rule, reproduced exactly: a line - ending in "-" loses it and runs into the next, unless the line is nothing - but the dash, which stays. That glues compounds too ("BoseEinstein"), and - is kept anyway: a rule that spared them moved two benchmark questions off - their page, and matching the flag keeps every index built before the - upgrade the same text as one built after (CHUNKER_VERSION unchanged). + ending in a hyphen loses it and runs into the next, unless the line is + nothing but the hyphen, which stays. That glues compounds too + ("BoseEinstein"), and is kept anyway: a rule that spared them moved two + benchmark questions off their page, and matching the flag keeps the + dehyphenated text of an index built before the upgrade the same as after. """ merged: List[_Line] = [] for line in lines: - if merged and merged[-1].text.endswith("-"): + if merged and merged[-1].text.endswith(_LINE_BREAK_HYPHENS): prev = merged[-1] - head = prev.text if prev.text.strip() == "-" else prev.text[:-1] + head = prev.text if prev.text.strip() in _LINE_BREAK_HYPHENS else prev.text[:-1] merged[-1] = _Line( text=head + line.text, size=max(prev.size, line.size), diff --git a/tests/test_rag_ingest.py b/tests/test_rag_ingest.py index 017a9f9..ec4cd47 100644 --- a/tests/test_rag_ingest.py +++ b/tests/test_rag_ingest.py @@ -18,7 +18,7 @@ import pytest from PIL import Image, ImageDraw -from desktop_pdf_translator.rag.ingest import PdfIngestor +from desktop_pdf_translator.rag.ingest import PdfIngestor, _join_broken_words, _Line PAGE_W, PAGE_H = 595.0, 842.0 @@ -207,6 +207,14 @@ def test_word_hyphenated_across_a_line_break_is_joined(tmp_path: Path) -> None: "Human - 82.3 91.2 were the scores reported."), ("The cost was 2.5k
-
Pre-OpenAI SOTA in that table.", "The cost was 2.5k -Pre-OpenAI SOTA in that table."), + # The flag knew the Unicode hyphens too, which word processors export. + ("Records survive the trans‐
action log of every node.", + "Records survive the transaction log of every node."), + ("Records survive the trans‑
action log of every node.", + "Records survive the transaction log of every node."), + ("The cost was 2.5k
‐
Pre-OpenAI SOTA in that table.", + "The cost was 2.5k ‐Pre-OpenAI SOTA in that table."), + # Joined by neither: guards against a rule that takes any dash for a hyphen. ("The pause ends with a dash—
and the next line starts here.", "The pause ends with a dash— and the next line starts here."), ] @@ -219,8 +227,8 @@ def test_line_break_hyphens_are_joined_as_pymupdf_joined_them( """PyMuPDF 1.27 ignores `TEXT_DEHYPHENATE`, so the ingestor joins the lines itself, exactly as the flag did — even where that glues a compound ("BoseEinstein"). Keeping such compounds apart was measured, and it moved - two benchmark questions off their page; matching the old text also keeps - an index built before the upgrade the same as one built after.""" + two benchmark questions off their page; matching the old text keeps the + dehyphenated text of an index built before the upgrade the same as after.""" doc = fitz.open() _text_page(doc, [html]) path = _save(doc, tmp_path) @@ -230,6 +238,20 @@ def test_line_break_hyphens_are_joined_as_pymupdf_joined_them( assert _all_text(chunks) == indexed +def test_a_soft_hyphen_at_a_line_break_joins_like_a_hyphen() -> None: + """No portable PDF puts U+00AD in the extracted text: base-14 fonts read + it back as "-", and PyMuPDF's built-in fonts drop it. These are the lines + 1.25.5 extracted from a page set in DejaVu Sans, and the text it indexed.""" + def line(text: str, top: float) -> _Line: + return _Line(text=text, size=11.0, bbox=(72.0, top, 300.0, top + 12.0)) + + merged = _join_broken_words( + [line("Records survive the trans­", 62.0), line("action log of every node.", 76.0)] + ) + + assert [m.text for m in merged] == ["Records survive the transaction log of every node."] + + def test_runs_of_whitespace_collapse_to_one_space(tmp_path: Path) -> None: doc = fitz.open() _text_page(doc, ["Alpha beta
gamma
delta epsilon."]) From bf48fe9aeb8ea57c23013d092411dd3dfc6d0fee Mon Sep 17 00:00:00 2001 From: nammovuivui Date: Fri, 23 Oct 2026 17:26:40 +0700 Subject: [PATCH 5/5] docs(translate): reconcile the memory figures across the notes Review found the same quantities quoted differently. Each now names what it was measured on: - leaked objects: ~2,120 lists per rebuild while the book is still untranslated, ~1,690 live objects a page averaged over the 129-page run (translated pages carry no links). Two lists per link, not one: the regression test leaves 1,278 objects after 20 rebuilds on 1.25.5, against 2 lists x 32 links (40, less the 8 on its two translated pages) x 20 = 1,280, and 0 on 1.27.2.3. - freed heap: ~410 MB at the end of the main run, ~420 MB on 1.27. - the slope is far larger than the lists' own ~150 KB; the notes now say the causal link is the before/after measurement of the bump, and that how each list pins memory around it is not measured. Also turns the two causes into prose, so Markdown no longer runs them into the font section's list. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- docs/architecture/translation-pipeline.md | 30 ++++++++++++++--------- pyproject.toml | 10 ++++---- requirements.txt | 10 ++++---- tests/test_pdf_pages.py | 3 ++- 5 files changed, 31 insertions(+), 24 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1b39272..14fcf8e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -269,7 +269,7 @@ rule has, unless noted, a test. They are the things most easily undone by BabelDOC's, installed only on a babeldoc in `VERIFIED_BABELDOC`: re-check it against `fontmap.py` when bumping babeldoc. Sharing a font across pages relies on PyMuPDF never releasing the GIL; re-check that when bumping PyMuPDF. -- **PyMuPDF stays ≥ 1.27.2.** Older ones leak a list per link in `insert_pdf`, +- **PyMuPDF stays ≥ 1.27.2.** Older ones leak two lists per link in `insert_pdf`, and `rebuild_rolling_pdf` re-inserts the whole document after every page: O(pages²) on a book, invisible to gc. `test_pdf_pages.py` fails on a version that leaks. Also keep it `<1.28`, which deprecates `fitz`. diff --git a/docs/architecture/translation-pipeline.md b/docs/architecture/translation-pipeline.md index 1a06920..d4c6e2d 100644 --- a/docs/architecture/translation-pipeline.md +++ b/docs/architecture/translation-pipeline.md @@ -195,11 +195,10 @@ page translated before and after the change came out the same, every word in the same place. Memory still drifted up afterwards, by +1.25 MB a page past a ~40-page warm-up. It was not garbage waiting for a full collection, which was the first guess: 29 full collections ran on their own over 129 pages, and a -forced one at the end freed next to nothing. It had two causes: -- PyMuPDF leaking ~1,690 objects a page in the rolling-PDF rebuild. See the - next section. -- glibc keeping ~410 MB of freed heap. See "glibc's heap is trimmed after - every page". +forced one at the end freed next to nothing. It had two causes: PyMuPDF +leaking ~1,690 objects a page in the rolling-PDF rebuild (the next section), +and glibc keeping ~410 MB of freed heap by the end of that run ("glibc's heap +is trimmed after every page"). - **Sharing is safe because nothing writes to a font after construction.** The stages only read a mapper (`has_char`, `map`, `fontid2font`, `base_font`, @@ -239,16 +238,23 @@ calls it twice per link, so `get_links()` on a page with 23 links leaked 46 lists. `insert_pdf` copies every page's links through it. `rebuild_rolling_pdf` re-inserts the *whole* document after every page, so -each rebuild leaked one list per call for every link in the book: ~2,120 lists, -~150 KB, on the 346-page book. That is O(pages) per page and O(pages²) for a -whole document. The lists carry one extra reference and no referrer, so gc -never sees them as garbage. +each rebuild leaked two lists for every link it copied. On the 346-page book +that was ~2,120 lists (~150 KB) per rebuild while the book was still +untranslated. Translated pages carry no links, so the count falls as they +replace linked ones: ~1,690 live objects a page, averaged over a 129-page +run. That is O(pages) per page and O(pages²) for a whole document. The lists +carry one extra reference and no referrer, so gc never sees them as garbage. - 1.25.5 and 1.26.7 leak. 1.27.2.3 and 1.28.2 don't. - Measured after warm-up: +1.25 MB/page on 1.25.5, +0.18 on 1.27.2.3, and live objects flat. +- The slope is far bigger than the lists' own ~150 KB. The link between them + is the before/after measurement of the bump, not their size. Each leaked + list most likely keeps the memory around it from being returned — that + part is not measured. - `test_rebuilding_the_rolling_pdf_over_and_over_leaves_no_objects_behind` - guards it. Pages without links never leaked, which is why the older rebuild - tests could not catch it. + guards it. 20 rebuilds of a 10-page, 40-link document left 1,278 objects on + 1.25.5 and 0 on 1.27.2.3. Pages without links never leaked, which is why + the older rebuild tests could not catch it. - `<1.28` because 1.28 deprecates the `fitz` name every module imports. What else the bump changed: @@ -278,7 +284,7 @@ So the consumer loop in `_process_with_babeldoc` ends each finished page with `malloc_trim(0)` on glibc, and a no-op on Windows, macOS and musl. - **Where it sits.** After the rolling rebuild, so the rebuild's document is freed too. After `chunk_ready`, so the viewer's swap never waits on it. -- **What it buys.** The trim lowers the plateau by ~400 MB. It hardly changes +- **What it buys.** The trim lowers the plateau by ~420 MB. It hardly changes the slope: that was the PyMuPDF leak. - **What it costs.** 8–22 ms a page; speed is in the section above. `gc.collect()` is deliberately not called: it cost ~277 ms a page and freed diff --git a/pyproject.toml b/pyproject.toml index 0780361..86e3ad7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,11 +38,11 @@ dependencies = [ "cryptography>=41.0.0,<43.0.0", # PDF processing - # Before 1.27.2 the helper behind `get_links` leaks a list per call, and - # `insert_pdf` copies every link through it; the rolling PDF re-inserts the - # whole document after each page, so a 346-page book leaked ~2,100 lists a - # page, invisible to gc (tests/test_pdf_pages.py guards it). <1.28: 1.28 - # deprecates the `fitz` name every module imports. + # Before 1.27.2 the helper behind `get_links` leaks two lists per link, + # and `insert_pdf` copies every link through it; the rolling PDF re-inserts + # the whole document after each page, so a 346-page book leaked ~1,700 + # objects a page, invisible to gc (tests/test_pdf_pages.py guards it). + # <1.28: 1.28 deprecates the `fitz` name every module imports. "PyMuPDF>=1.27.2,<1.28", "Pillow>=9.5.0,<11.0.0", "babeldoc>=0.5.20,<0.6.0", diff --git a/requirements.txt b/requirements.txt index b7dd3a1..bb83901 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,11 +6,11 @@ tomlkit>=0.12.0,<1.0.0 python-dotenv>=1.0.0,<2.0.0 cryptography>=41.0.0,<43.0.0 -# Before 1.27.2 the helper behind `get_links` leaks a list per call, and -# `insert_pdf` copies every link through it; the rolling PDF re-inserts the -# whole document after each page, so a 346-page book leaked ~2,100 lists a -# page, invisible to gc (tests/test_pdf_pages.py guards it). <1.28: 1.28 -# deprecates the `fitz` name every module imports. +# Before 1.27.2 the helper behind `get_links` leaks two lists per link, +# and `insert_pdf` copies every link through it; the rolling PDF re-inserts +# the whole document after each page, so a 346-page book leaked ~1,700 +# objects a page, invisible to gc (tests/test_pdf_pages.py guards it). +# <1.28: 1.28 deprecates the `fitz` name every module imports. PyMuPDF>=1.27.2,<1.28 Pillow>=9.5.0,<11.0.0 diff --git a/tests/test_pdf_pages.py b/tests/test_pdf_pages.py index 68c5f5a..09d95b7 100644 --- a/tests/test_pdf_pages.py +++ b/tests/test_pdf_pages.py @@ -252,7 +252,8 @@ def test_rebuilding_the_rolling_pdf_over_and_over_leaves_no_objects_behind( """A run rebuilds the rolling PDF after every chunk in one long-lived process, so anything a rebuild leaves alive piles up with the page count. Pages with links are the case that matters: a book is full of them - (contents, cross-references, URLs) and each one is an object per link.""" + (contents, cross-references, URLs), and a leaking `insert_pdf` leaves two + lists behind for every link it copies.""" linked = _write_linked_pdf(tmp_path / "linked.pdf", pages=10, links_per_page=4) chunks = split_into_chunks(linked, tmp_path / "in", contiguous_runs([3, 4, 8], 1)) results = [_translate(chunks[0][0], tmp_path), None, _translate(chunks[2][0], tmp_path)]