fix: improve external link verification

scripts/check_external_links.py

@@ -12,9 +12,10 @@
     python3 scripts/check_external_links.py --fail-on-broken # exit 1 when something is broken (off by default)
 
 Each URL is requested with a browser-like user agent and a timeout (HEAD first, then GET when the server does
-not allow HEAD). 2xx and 3xx (after redirects) are fine. 404, 410 and a name that does not resolve are
-broken. Everything else (403, 429, 5xx, timeouts, refused connections) is unknown: it is listed but never
-counted as broken, because it usually means the site blocks scripts or is down for a moment.
+not allow HEAD). A 404 gets one retry with a fuller browser-like user agent. For a DOI URL blocked with 403,
+the DOI handle API is used to distinguish a registered identifier from a missing one. Other 404, 410 and a
+name that does not resolve are broken. Everything else (403, 429, 5xx, timeouts, refused connections) is
+unknown: it is listed but never counted as broken, because it usually means the site blocks scripts or is down.
 """
 from __future__ import annotations
 
@@ -26,11 +27,16 @@
 import ssl
 import sys
 import urllib.error
+import urllib.parse
 import urllib.request
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parent.parent
 USER_AGENT = "Mozilla/5.0 (compatible; FairCodeLinkCheck/1.0; +https://github.com/yakew7/Fair-Code)"
+BROWSER_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 FairCodeLinkCheck/1.0"
+)
 LINK = re.compile(r"\]\((https?://[^)\s]+)\)")
 BARE = re.compile(r"<(https?://[^>\s]+)>")
 SKIP_HOSTS = {"localhost", "127.0.0.1", "example.com", "www.example.com"}
@@ -77,8 +83,37 @@ def check_url(url: str, timeout: float = 15.0) -> tuple[str, str]:
             with urllib.request.urlopen(request, timeout=timeout, context=_ssl_context()) as response:
                 return "ok", str(response.status)
         except urllib.error.HTTPError as error:
+            if error.code == 404:
+                retry_method = "GET" if method == "HEAD" else method
+                retry = urllib.request.Request(
+                    url, method=retry_method,
+                    headers={"User-Agent": BROWSER_USER_AGENT, "Accept": "*/*"},
+                )
+                try:
+                    with urllib.request.urlopen(retry, timeout=timeout, context=_ssl_context()) as response:
+                        return "ok", str(response.status)
+                except urllib.error.HTTPError as retry_error:
+                    if retry_error.code in BROKEN_STATUSES:
+                        return "broken", str(retry_error.code)
+                    if retry_error.code == 403:
+                        doi = _extract_doi(url)
+                        if doi:
+                            return _check_doi(doi, timeout)
+                    if retry_error.code in (405, 501) and method == "HEAD":
+                        continue
+                    return "unknown", str(retry_error.code)
+                except urllib.error.URLError as retry_error:
+                    if isinstance(retry_error.reason, socket.gaierror):
+                        return "broken", "name does not resolve"
+                    return "unknown", str(retry_error.reason)
+                except (TimeoutError, OSError) as retry_error:
+                    return "unknown", type(retry_error).__name__
             if error.code in BROKEN_STATUSES:
                 return "broken", str(error.code)
+            if error.code == 403:
+                doi = _extract_doi(url)
+                if doi:
+                    return _check_doi(doi, timeout)
             if error.code in (405, 501) and method == "HEAD":
                 continue                      # the server does not allow HEAD: try GET
             return "unknown", str(error.code)
@@ -91,6 +126,35 @@ def check_url(url: str, timeout: float = 15.0) -> tuple[str, str]:
     return "unknown", "no response"
 
 
+def _extract_doi(url: str) -> str | None:
+    path = urllib.parse.unquote(urllib.parse.urlsplit(url).path)
+    match = re.search(r"10\.\d{4,9}/[^\s?#]+", path, re.IGNORECASE)
+    return match.group().rstrip(".,;:)]}") if match else None
+
+
+def _check_doi(doi: str, timeout: float) -> tuple[str, str]:
+    endpoint = f"https://doi.org/api/handles/{urllib.parse.quote(doi, safe='/')}"
+    request = urllib.request.Request(endpoint, headers={"Accept": "application/json"})
+    try:
+        with urllib.request.urlopen(request, timeout=timeout, context=_ssl_context()) as response:
+            result = json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as error:
+        if error.code == 404:
+            return "broken", "DOI not found"
+        return "unknown", f"DOI resolver HTTP {error.code}"
+    except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as error:
+        return "unknown", f"DOI resolver {type(error).__name__}"
+
+    code = result.get("responseCode")
+    if code == 1:
+        return "ok", "DOI exists"
+    if code == 100:
+        return "broken", "DOI not found"
+    if code == 200:
+        return "broken", "DOI has no resolution values"
+    return "unknown", f"DOI resolver response {code}"
+
+
 def run(urls: dict[str, list[str]], limit: int | None = None, workers: int = 8, timeout: float = 15.0) -> dict:
     items = sorted(urls)[:limit] if limit else sorted(urls)
     results = {"ok": [], "broken": [], "unknown": []}

tests/test_check_external_links.py

@@ -2,18 +2,29 @@
 
 import http.server
 import importlib
+import io
 import json
 import threading
+import urllib.error
 
 import pytest
 
 script = importlib.import_module("scripts.check_external_links")
 
 
 class _Handler(http.server.BaseHTTPRequestHandler):
+    requests = []
+
     def _respond(self):
-        code = {"/ok": 200, "/gone": 404, "/removed": 410, "/blocked": 403, "/slow-down": 429, "/down": 503,
-                "/nohead": 200}.get(self.path, 404)
+        user_agent = self.headers.get("User-Agent", "")
+        self.requests.append((self.path, self.command, user_agent))
+        if self.path == "/always404":
+            code = 404
+        elif self.path == "/kaggle":
+            code = 200 if "Chrome/" in user_agent and "FairCodeLinkCheck/1.0" in user_agent else 404
+        else:
+            code = {"/ok": 200, "/gone": 404, "/removed": 410, "/blocked": 403, "/slow-down": 429, "/down": 503,
+                    "/nohead": 200}.get(self.path, 404)
         if self.path == "/nohead" and self.command == "HEAD":
             code = 405
         if self.path == "/redirect":
@@ -47,6 +58,78 @@ def test_each_status_class(server, path, kind):
     assert script.check_url(server + path, timeout=5)[0] == kind
 
 
+def test_404_retries_once_with_project_browser_user_agent(server):
+    _Handler.requests.clear()
+
+    assert script.check_url(server + "/kaggle", timeout=5) == ("ok", "200")
+
+    requests = [request for request in _Handler.requests if request[0] == "/kaggle"]
+    assert [(request[1], request[2]) for request in requests] == [
+        ("HEAD", script.USER_AGENT),
+        ("GET", script.BROWSER_USER_AGENT),
+    ]
+    assert len(requests) == 2
+
+
+def test_404_stays_broken_after_the_single_retry(server):
+    _Handler.requests.clear()
+
+    assert script.check_url(server + "/always404", timeout=5) == ("broken", "404")
+    assert len([request for request in _Handler.requests if request[0] == "/always404"]) == 2
+
+
+@pytest.mark.parametrize(
+    "resolver_body,expected",
+    [
+        (b'{"responseCode": 1}', ("ok", "DOI exists")),
+        (b'{"responseCode": 100}', ("broken", "DOI not found")),
+        (b'{"responseCode": 200}', ("broken", "DOI has no resolution values")),
+        (b'{"responseCode": 2}', ("unknown", "DOI resolver response 2")),
+    ],
+)
+def test_403_doi_page_uses_handle_resolver(monkeypatch, resolver_body, expected):
+    calls = []
+
+    class Response:
+        status = 200
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *_args):
+            return False
+
+        def read(self):
+            return resolver_body
+
+    def fake_urlopen(request, **_kwargs):
+        calls.append(request.full_url)
+        if request.full_url.startswith("https://doi.org/api/handles/"):
+            return Response()
+        raise urllib.error.HTTPError(request.full_url, 403, "blocked", {}, io.BytesIO())
+
+    monkeypatch.setattr(script.urllib.request, "urlopen", fake_urlopen)
+
+    assert script.check_url("https://publisher.example/doi/10.1234/example", timeout=1) == expected
+    assert calls == [
+        "https://publisher.example/doi/10.1234/example",
+        "https://doi.org/api/handles/10.1234/example",
+    ]
+
+
+def test_missing_doi_is_broken_when_resolver_returns_404(monkeypatch):
+    def fake_urlopen(request, **_kwargs):
+        if request.full_url.startswith("https://doi.org/api/handles/"):
+            raise urllib.error.HTTPError(request.full_url, 404, "missing", {}, io.BytesIO())
+        raise urllib.error.HTTPError(request.full_url, 403, "blocked", {}, io.BytesIO())
+
+    monkeypatch.setattr(script.urllib.request, "urlopen", fake_urlopen)
+
+    assert script.check_url("https://publisher.example/doi/10.1234/missing", timeout=1) == (
+        "broken", "DOI not found"
+    )
+
+
 def test_a_name_that_does_not_resolve_is_broken_and_a_refused_connection_is_unknown():
     assert script.check_url("http://this-host-does-not-exist.invalid/page", timeout=5)[0] == "broken"
     assert script.check_url("http://127.0.0.1:9/", timeout=5)[0] == "unknown"