diff --git a/scripts/check_external_links.py b/scripts/check_external_links.py index 53e07a1..e6aee2a 100644 --- a/scripts/check_external_links.py +++ b/scripts/check_external_links.py @@ -12,9 +12,10 @@ python3 scripts/check_external_links.py --fail-on-broken # exit 1 when something is broken (off by default) Each URL is requested with a browser-like user agent and a timeout (HEAD first, then GET when the server does -not allow HEAD). 2xx and 3xx (after redirects) are fine. 404, 410 and a name that does not resolve are -broken. Everything else (403, 429, 5xx, timeouts, refused connections) is unknown: it is listed but never -counted as broken, because it usually means the site blocks scripts or is down for a moment. +not allow HEAD). A 404 gets one retry with a fuller browser-like user agent. For a DOI URL blocked with 403, +the DOI handle API is used to distinguish a registered identifier from a missing one. Other 404, 410 and a +name that does not resolve are broken. Everything else (403, 429, 5xx, timeouts, refused connections) is +unknown: it is listed but never counted as broken, because it usually means the site blocks scripts or is down. """ from __future__ import annotations @@ -26,11 +27,16 @@ import ssl import sys import urllib.error +import urllib.parse import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parent.parent USER_AGENT = "Mozilla/5.0 (compatible; FairCodeLinkCheck/1.0; +https://github.com/yakew7/Fair-Code)" +BROWSER_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 FairCodeLinkCheck/1.0" +) LINK = re.compile(r"\]\((https?://[^)\s]+)\)") BARE = re.compile(r"<(https?://[^>\s]+)>") SKIP_HOSTS = {"localhost", "127.0.0.1", "example.com", "www.example.com"} @@ -77,8 +83,37 @@ def check_url(url: str, timeout: float = 15.0) -> tuple[str, str]: with urllib.request.urlopen(request, timeout=timeout, context=_ssl_context()) as response: return "ok", str(response.status) except urllib.error.HTTPError as error: + if error.code == 404: + retry_method = "GET" if method == "HEAD" else method + retry = urllib.request.Request( + url, method=retry_method, + headers={"User-Agent": BROWSER_USER_AGENT, "Accept": "*/*"}, + ) + try: + with urllib.request.urlopen(retry, timeout=timeout, context=_ssl_context()) as response: + return "ok", str(response.status) + except urllib.error.HTTPError as retry_error: + if retry_error.code in BROKEN_STATUSES: + return "broken", str(retry_error.code) + if retry_error.code == 403: + doi = _extract_doi(url) + if doi: + return _check_doi(doi, timeout) + if retry_error.code in (405, 501) and method == "HEAD": + continue + return "unknown", str(retry_error.code) + except urllib.error.URLError as retry_error: + if isinstance(retry_error.reason, socket.gaierror): + return "broken", "name does not resolve" + return "unknown", str(retry_error.reason) + except (TimeoutError, OSError) as retry_error: + return "unknown", type(retry_error).__name__ if error.code in BROKEN_STATUSES: return "broken", str(error.code) + if error.code == 403: + doi = _extract_doi(url) + if doi: + return _check_doi(doi, timeout) if error.code in (405, 501) and method == "HEAD": continue # the server does not allow HEAD: try GET return "unknown", str(error.code) @@ -91,6 +126,35 @@ def check_url(url: str, timeout: float = 15.0) -> tuple[str, str]: return "unknown", "no response" +def _extract_doi(url: str) -> str | None: + path = urllib.parse.unquote(urllib.parse.urlsplit(url).path) + match = re.search(r"10\.\d{4,9}/[^\s?#]+", path, re.IGNORECASE) + return match.group().rstrip(".,;:)]}") if match else None + + +def _check_doi(doi: str, timeout: float) -> tuple[str, str]: + endpoint = f"https://doi.org/api/handles/{urllib.parse.quote(doi, safe='/')}" + request = urllib.request.Request(endpoint, headers={"Accept": "application/json"}) + try: + with urllib.request.urlopen(request, timeout=timeout, context=_ssl_context()) as response: + result = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + if error.code == 404: + return "broken", "DOI not found" + return "unknown", f"DOI resolver HTTP {error.code}" + except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as error: + return "unknown", f"DOI resolver {type(error).__name__}" + + code = result.get("responseCode") + if code == 1: + return "ok", "DOI exists" + if code == 100: + return "broken", "DOI not found" + if code == 200: + return "broken", "DOI has no resolution values" + return "unknown", f"DOI resolver response {code}" + + def run(urls: dict[str, list[str]], limit: int | None = None, workers: int = 8, timeout: float = 15.0) -> dict: items = sorted(urls)[:limit] if limit else sorted(urls) results = {"ok": [], "broken": [], "unknown": []} diff --git a/tests/test_check_external_links.py b/tests/test_check_external_links.py index 320dcef..d65f73e 100644 --- a/tests/test_check_external_links.py +++ b/tests/test_check_external_links.py @@ -2,8 +2,10 @@ import http.server import importlib +import io import json import threading +import urllib.error import pytest @@ -11,9 +13,18 @@ class _Handler(http.server.BaseHTTPRequestHandler): + requests = [] + def _respond(self): - code = {"/ok": 200, "/gone": 404, "/removed": 410, "/blocked": 403, "/slow-down": 429, "/down": 503, - "/nohead": 200}.get(self.path, 404) + user_agent = self.headers.get("User-Agent", "") + self.requests.append((self.path, self.command, user_agent)) + if self.path == "/always404": + code = 404 + elif self.path == "/kaggle": + code = 200 if "Chrome/" in user_agent and "FairCodeLinkCheck/1.0" in user_agent else 404 + else: + code = {"/ok": 200, "/gone": 404, "/removed": 410, "/blocked": 403, "/slow-down": 429, "/down": 503, + "/nohead": 200}.get(self.path, 404) if self.path == "/nohead" and self.command == "HEAD": code = 405 if self.path == "/redirect": @@ -47,6 +58,78 @@ def test_each_status_class(server, path, kind): assert script.check_url(server + path, timeout=5)[0] == kind +def test_404_retries_once_with_project_browser_user_agent(server): + _Handler.requests.clear() + + assert script.check_url(server + "/kaggle", timeout=5) == ("ok", "200") + + requests = [request for request in _Handler.requests if request[0] == "/kaggle"] + assert [(request[1], request[2]) for request in requests] == [ + ("HEAD", script.USER_AGENT), + ("GET", script.BROWSER_USER_AGENT), + ] + assert len(requests) == 2 + + +def test_404_stays_broken_after_the_single_retry(server): + _Handler.requests.clear() + + assert script.check_url(server + "/always404", timeout=5) == ("broken", "404") + assert len([request for request in _Handler.requests if request[0] == "/always404"]) == 2 + + +@pytest.mark.parametrize( + "resolver_body,expected", + [ + (b'{"responseCode": 1}', ("ok", "DOI exists")), + (b'{"responseCode": 100}', ("broken", "DOI not found")), + (b'{"responseCode": 200}', ("broken", "DOI has no resolution values")), + (b'{"responseCode": 2}', ("unknown", "DOI resolver response 2")), + ], +) +def test_403_doi_page_uses_handle_resolver(monkeypatch, resolver_body, expected): + calls = [] + + class Response: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self): + return resolver_body + + def fake_urlopen(request, **_kwargs): + calls.append(request.full_url) + if request.full_url.startswith("https://doi.org/api/handles/"): + return Response() + raise urllib.error.HTTPError(request.full_url, 403, "blocked", {}, io.BytesIO()) + + monkeypatch.setattr(script.urllib.request, "urlopen", fake_urlopen) + + assert script.check_url("https://publisher.example/doi/10.1234/example", timeout=1) == expected + assert calls == [ + "https://publisher.example/doi/10.1234/example", + "https://doi.org/api/handles/10.1234/example", + ] + + +def test_missing_doi_is_broken_when_resolver_returns_404(monkeypatch): + def fake_urlopen(request, **_kwargs): + if request.full_url.startswith("https://doi.org/api/handles/"): + raise urllib.error.HTTPError(request.full_url, 404, "missing", {}, io.BytesIO()) + raise urllib.error.HTTPError(request.full_url, 403, "blocked", {}, io.BytesIO()) + + monkeypatch.setattr(script.urllib.request, "urlopen", fake_urlopen) + + assert script.check_url("https://publisher.example/doi/10.1234/missing", timeout=1) == ( + "broken", "DOI not found" + ) + + def test_a_name_that_does_not_resolve_is_broken_and_a_refused_connection_is_unknown(): assert script.check_url("http://this-host-does-not-exist.invalid/page", timeout=5)[0] == "broken" assert script.check_url("http://127.0.0.1:9/", timeout=5)[0] == "unknown"