From 907ba8f59956c7e939a5e9125f76bca11b680ae2 Mon Sep 17 00:00:00 2001 From: Alex Lee <13192723+leeaky@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:51:37 +1100 Subject: [PATCH] Stage 12: a paper whose PDF can't be read counts as 45 min Owner asked to assume long for pages with a citation PDF tag when the PDF fails; stand-in count (10,350 words), no database change. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++-- docs/build-map.md | 1 + src/readurlist/ingest/__init__.py | 4 ++-- src/readurlist/ingest/arxiv.py | 34 ++++++++++++++++++++----------- tests/test_paper_pdf.py | 15 ++++++++------ 5 files changed, 36 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 1b22aa4..7b1e291 100644 --- a/README.md +++ b/README.md @@ -172,13 +172,13 @@ readurlist-arxiv-lengths It pauses 3 seconds between papers, as arXiv asks. -**Papers from other sites:** any page that names its PDF in the standard `citation_pdf_url` tag (journals, ACL Anthology, bioRxiv and most paper sites) gets its word count from that PDF the same way. Sites that block downloads keep the page's count. Items saved before this change and shown as Short are rechecked once with: +**Papers from other sites:** any page that names its PDF in the standard `citation_pdf_url` tag (journals, ACL Anthology, bioRxiv and most paper sites) gets its word count from that PDF the same way. If that PDF can't be downloaded or read (some sites block it), the item is counted as 45 minutes, so it still lands in Long. Items saved before this change and shown as Short are rechecked once with: ```bash readurlist-paper-lengths ``` -It re-reads each Short item's page, 2 seconds apart, and recounts only those that name a PDF. +It re-reads each Short item's page, 2 seconds apart, and recounts only those that name a PDF. It is safe to run again. ### Migrate an old local SQLite file diff --git a/docs/build-map.md b/docs/build-map.md index f2608c4..f988664 100644 --- a/docs/build-map.md +++ b/docs/build-map.md @@ -342,6 +342,7 @@ Before any feature work, dev and live are separated: the live worker, bot, datab - **Terms:** Effort label - **Status:** in progress - **Changed:** 2026-11-01 — Stage added at the owner's request after stage 11. For papers already saved, the owner chose to recheck only items shown as Short ("Only items shown as Short"). + 2026-11-01 — After the live one-off fix (2 papers named a PDF that couldn't be read: Nature, MIT DSpace), owner: "for any url that has a citation URL, assume long and put '>45 min' as a placeholder length"; chose a stand-in count over a true ">45 min" label (no database change). A page whose PDF tag points to a PDF that can't be downloaded or read is now stored as 45 minutes' worth of words (10,350), so it shows Long · 45 min; the one-off fix does the same and prints a fourth count. arXiv is unchanged (a withdrawn paper really is short). Tests: `test_paper_pdf.py`. Local run, `https://www.nature.com/articles/s41562-017-0064`: page 1,646 words, PDF unreadable (PdfStreamError) → stored 10,350, long, 45 min; one-off fix after a reset: back to 10,350. `uv run pytest -q`: 154 passed. Deploy: merge → `scripts/live-worker-update.ps1` → rerun `readurlist-paper-lengths` in the live folder. - **Tasks:** - [x] (a) A new link whose page names its PDF in the standard citation tag is counted from that PDF; title and summary still come from the page. Detail: the tag is `citation_pdf_url`. Same rules as stage 11: PDF up to 40 MB, its text is counted and not kept, never shown to the model; if the PDF can't be fetched or read (some sites block downloads), the page's count is kept and one line says why. Detail: built: `extract.pdf_link` reads the tag (relative links resolved, http(s) only); `extract_article` now also returns it; `arxiv.pdf_word_count` counts any PDF with stage 11's rules; an arXiv link never falls through to its abstract page's tag. Tests: `test_paper_pdf.py`, `test_arxiv.py`. diff --git a/src/readurlist/ingest/__init__.py b/src/readurlist/ingest/__init__.py index af1a84f..01c53cb 100644 --- a/src/readurlist/ingest/__init__.py +++ b/src/readurlist/ingest/__init__.py @@ -3,7 +3,7 @@ from readurlist.config import Settings from readurlist.db import Item, get_session from readurlist.effort import word_count -from readurlist.ingest.arxiv import abs_url, arxiv_id, paper_word_count, pdf_word_count +from readurlist.ingest.arxiv import PAPER_STAND_IN_WORDS, abs_url, arxiv_id, paper_word_count, pdf_word_count from readurlist.ingest.extract import ExtractError, FetchError, extract_article from readurlist.ingest.summarize import summarize_article from readurlist.ingest.untrusted import assess @@ -67,7 +67,7 @@ def _paper_words(url: str, paper_id: str | None, pdf_url: str | None) -> int | N if paper_id: return paper_word_count(paper_id) if pdf_url: - return pdf_word_count(pdf_url, page_url=url) + return pdf_word_count(pdf_url, page_url=url) or PAPER_STAND_IN_WORDS return None diff --git a/src/readurlist/ingest/arxiv.py b/src/readurlist/ingest/arxiv.py index 4849bec..915e39e 100644 --- a/src/readurlist/ingest/arxiv.py +++ b/src/readurlist/ingest/arxiv.py @@ -16,7 +16,7 @@ import httpx from pypdf import PdfReader -from readurlist.effort import word_count +from readurlist.effort import WORDS_PER_MINUTE, word_count logger = logging.getLogger(__name__) # pypdf reports every odd font or figure it meets; the count doesn't need them. @@ -30,6 +30,10 @@ _MAX_PDF_BYTES = 40 * 1024 * 1024 _TIMEOUT = 60.0 _TRANSPORT: httpx.BaseTransport | None = None # tests swap in a fake +# A page names a paper but its PDF can't be read: assume long (owner, 2026-11-01: +# "assume long and put '>45 min' as a placeholder"; chose a stand-in count). +PAPER_STAND_IN_MINUTES = 45 +PAPER_STAND_IN_WORDS = PAPER_STAND_IN_MINUTES * WORDS_PER_MINUTE def arxiv_id(url: str) -> str | None: @@ -51,10 +55,12 @@ def skip(reason: str) -> None: def pdf_word_count(pdf_url: str, *, page_url: str) -> int | None: - """Words in the PDF a paper page names (build map stage 12), or None.""" + """Words in the PDF a paper page names (build map stage 12), or None. + + Callers then store PAPER_STAND_IN_WORDS: the page names a paper, so it is long.""" def skip(reason: str) -> None: - logger.warning("Paper %s (PDF %s): %s; kept the page count", page_url, pdf_url, reason) + logger.warning("Paper %s (PDF %s): %s; counted as %s min", page_url, pdf_url, reason, PAPER_STAND_IN_MINUTES) return _pdf_words(pdf_url, skip) @@ -117,10 +123,10 @@ def fix_saved_papers(*, sleep=time.sleep) -> tuple[int, int]: session.close() -def fix_saved_paper_pages(*, sleep=time.sleep) -> tuple[int, int, int]: +def fix_saved_paper_pages(*, sleep=time.sleep) -> tuple[int, int, int, int]: """One-off (stage 12): recheck saved non-arXiv items shown as Short for a PDF tag. - Returns (counted from their PDF, no PDF tag, kept the page count).""" + Returns (counted from their PDF, counted as 45 min, no PDF tag, kept the page count).""" from readurlist.db import Item, get_session from readurlist.effort import effort from readurlist.ingest import extract @@ -132,7 +138,7 @@ def fix_saved_paper_pages(*, sleep=time.sleep) -> tuple[int, int, int]: for row in session.query(Item).filter(Item.ingest_status == "ready").order_by(Item.id) if row.word_count is not None and effort(row.word_count)[0] == "short" and not arxiv_id(row.url) ] - fixed = no_tag = failed = 0 + fixed = stand_in = no_tag = failed = 0 for n, row in enumerate(rows): if n: sleep(2.0) # a polite pause between pages @@ -149,12 +155,13 @@ def fix_saved_paper_pages(*, sleep=time.sleep) -> tuple[int, int, int]: continue words = pdf_word_count(pdf_url, page_url=row.url) if words is None: - failed += 1 - continue + words = PAPER_STAND_IN_WORDS + stand_in += 1 + else: + fixed += 1 row.word_count = words session.commit() - fixed += 1 - return fixed, no_tag, failed + return fixed, stand_in, no_tag, failed except Exception: session.rollback() raise @@ -178,5 +185,8 @@ def paper_pages_main() -> None: from readurlist.db import init_db init_db(get_settings().database_url) - fixed, no_tag, failed = fix_saved_paper_pages() - print(f"Short items counted from their PDF: {fixed}; no PDF tag: {no_tag}; kept the page count: {failed}") + fixed, stand_in, no_tag, failed = fix_saved_paper_pages() + print( + f"Short items counted from their PDF: {fixed}; counted as {PAPER_STAND_IN_MINUTES} min " + f"(PDF unreadable): {stand_in}; no PDF tag: {no_tag}; kept the page count: {failed}" + ) diff --git a/tests/test_paper_pdf.py b/tests/test_paper_pdf.py index 40ad7d9..732b846 100644 --- a/tests/test_paper_pdf.py +++ b/tests/test_paper_pdf.py @@ -38,7 +38,7 @@ def test_a_blocked_pdf_is_one_plain_line(monkeypatch, caplog): with caplog.at_level("WARNING"): assert pdf_word_count(PDF, page_url=PAGE) is None assert [r.getMessage() for r in caplog.records if r.name.startswith("readurlist")] == [ - f"Paper {PAGE} (PDF {PDF}): PDF not found (HTTP 404); kept the page count" + f"Paper {PAGE} (PDF {PDF}): PDF not found (HTTP 404); counted as 45 min" ] @@ -80,9 +80,12 @@ def test_a_paper_page_takes_its_length_from_its_pdf(tmp_path, monkeypatch): assert "We study compost." in item.extracted_text # the page's text is kept, not the PDF's -def test_a_paper_page_keeps_its_count_when_the_pdf_fails(tmp_path, monkeypatch): +def test_a_paper_page_counts_as_45_min_when_the_pdf_fails(tmp_path, monkeypatch): + from readurlist.effort import effort + item, _ = _ingest(tmp_path, monkeypatch, PDF, None) - assert item.word_count == 120 + assert item.word_count == 45 * 230 # owner, 2026-11-01: a paper is long; stand-in length + assert effort(item.word_count) == ("long", 45) def test_a_page_without_the_tag_fetches_no_pdf(tmp_path, monkeypatch): @@ -103,7 +106,7 @@ def test_one_off_fix_rechecks_short_paper_pages(tmp_path, monkeypatch, caplog): for url, status, words in [ ("https://journal.org/paper", "ready", 120), # Short, has a tag: fixed ("https://blog.org/post", "ready", 900), # Short, no tag: left alone - ("https://journal.org/blocked", "ready", 150), # Short, PDF blocked: kept + ("https://journal.org/blocked", "ready", 150), # Short, PDF blocked: 45 min stand-in ("https://gone.org/page", "ready", 200), # Short, page gone: kept ("https://journal.org/long", "ready", 5000), # Long: not rechecked ("https://arxiv.org/abs/2410.00001", "ready", 120), # arXiv: stage 11's fix @@ -138,7 +141,7 @@ def fake_fetch(url, timeout): assert "Paper https://gone.org/page: page could not be fetched (HTTP 404); kept the page count" in [ r.getMessage() for r in caplog.records ] - assert result == (1, 1, 2) # counted from their PDF, no PDF tag, kept the page count + assert result == (1, 1, 1, 1) # counted from their PDF, counted as 45 min, no PDF tag, kept the page count assert fetched == ["https://journal.org/paper", "https://blog.org/post", "https://journal.org/blocked", "https://gone.org/page"] assert waits == [2.0, 2.0, 2.0] @@ -147,5 +150,5 @@ def fake_fetch(url, timeout): session.close() assert words["https://journal.org/paper"] == 8000 assert words["https://blog.org/post"] == 900 - assert words["https://journal.org/blocked"] == 150 + assert words["https://journal.org/blocked"] == 45 * 230 assert words["https://gone.org/page"] == 200