arXiv: quiet output, one line per skipped paper

docs/build-map.md

@@ -321,6 +321,7 @@ Before any feature work, dev and live are separated: the live worker, bot, datab
 - **Terms:** Effort label
 - **Status:** done
 - **Changed:** 2026-11-01 — Stage added at the owner's request after the build finished. Owner chose to fetch the full paper ("Fetch the full paper", over treating all arXiv as long or tagging only) and to keep the summary from the abstract ("Abstract, length from paper"): the safety scan and the model's input stay as they are.
+  2026-11-01 — After the live one-off fix (owner: "ok to the tidy up"): a paper whose PDF is skipped now prints one line naming it, its abstract link and the reason (not found, over 40 MB, can't be downloaded or read, no text), and the PDF reader's font and figure notes are hidden. Tests: `test_arxiv.py`. Local run: "arXiv 2608.00858 (https://arxiv.org/abs/2608.00858): PDF not found (HTTP 404); kept the abstract count", then the totals; nothing else printed.
 - **Tasks:**
   - [x] (a) A new arXiv link (abs, pdf or html form) is counted from the full paper's PDF; the title and summary still come from the abstract page.
     Detail: if the PDF can't be fetched or read, the abstract's count is kept, as today. Built: `ingest/arxiv.py` (`arxiv_id`, `paper_word_count`: PDF up to 40 MB, read with `pypdf`, words counted, text not kept); `ingest_url` reads any arXiv form from its abstract page. New dependency `pypdf`. Tests: `test_arxiv.py`.

src/readurlist/ingest/arxiv.py

@@ -17,6 +17,8 @@
 from readurlist.effort import word_count
 
 logger = logging.getLogger(__name__)
+# pypdf reports every odd font or figure it meets; the count doesn't need them.
+logging.getLogger("pypdf").setLevel(logging.CRITICAL)
 
 # New ids (2410.12345, optional version) and old ones (hep-th/9901001).
 _ARXIV = re.compile(
@@ -40,22 +42,35 @@ def abs_url(paper_id: str) -> str:
 def paper_word_count(paper_id: str) -> int | None:
     """Words in the paper's PDF, or None if it can't be fetched or read."""
     url = f"https://arxiv.org/pdf/{paper_id}"
+
+    def skip(reason: str) -> None:
+        logger.warning("arXiv %s (%s): %s; kept the abstract count", paper_id, abs_url(paper_id), reason)
+
     try:
         with httpx.Client(follow_redirects=True, timeout=_TIMEOUT, transport=_TRANSPORT) as client:
             with client.stream("GET", url) as response:
+                if response.status_code == 404:
+                    skip("PDF not found (HTTP 404)")
+                    return None
                 response.raise_for_status()
                 body = bytearray()
                 for chunk in response.iter_bytes():
                     body += chunk
                     if len(body) > _MAX_PDF_BYTES:
-                        logger.warning("arXiv PDF %s is over %s bytes; skipped", paper_id, _MAX_PDF_BYTES)
+                        skip(f"PDF over {_MAX_PDF_BYTES // (1024 * 1024)} MB")
                         return None
         reader = PdfReader(io.BytesIO(bytes(body)))
         words = sum(word_count(page.extract_text() or "") for page in reader.pages)
-    except Exception:
-        logger.warning("Could not count words in arXiv PDF %s", paper_id, exc_info=True)
+    except httpx.HTTPError as exc:
+        skip(f"PDF could not be downloaded ({exc.__class__.__name__})")
+        return None
+    except Exception as exc:
+        skip(f"PDF could not be read ({exc.__class__.__name__})")
+        return None
+    if not words:
+        skip("no text in the PDF")
         return None
-    return words or None
+    return words
 
 
 def fix_saved_papers(*, sleep=time.sleep) -> tuple[int, int]:

tests/test_arxiv.py

@@ -179,3 +179,29 @@ def fake_count(paper_id):
     assert words["https://arxiv.org/abs/2410.00001"] == 8000
     assert words["https://arxiv.org/abs/2410.00002"] == 120
     assert words["https://example.com/a"] == 120
+
+
+def test_a_missing_pdf_is_one_plain_line(monkeypatch, caplog):
+    monkeypatch.setattr(arxiv, "_TRANSPORT", _transport(None, []))
+    with caplog.at_level("WARNING"):
+        assert paper_word_count("2608.00858") is None
+    lines = [r for r in caplog.records if r.name.startswith("readurlist")]
+    assert [r.getMessage() for r in lines] == [
+        "arXiv 2608.00858 (https://arxiv.org/abs/2608.00858): PDF not found (HTTP 404); kept the abstract count"
+    ]
+    assert all(r.exc_info is None for r in lines)  # no traceback
+
+
+def test_a_pdf_without_text_says_so(monkeypatch, caplog):
+    monkeypatch.setattr(arxiv, "_TRANSPORT", _transport(_pdf(""), []))
+    with caplog.at_level("WARNING"):
+        assert paper_word_count("2410.12345") is None
+    assert [r.getMessage() for r in caplog.records if r.name.startswith("readurlist")] == [
+        "arXiv 2410.12345 (https://arxiv.org/abs/2410.12345): no text in the PDF; kept the abstract count"
+    ]
+
+
+def test_pdf_reader_notes_are_hidden():
+    import logging
+
+    assert logging.getLogger("pypdf").getEffectiveLevel() > logging.ERROR