diff --git a/Dockerfile b/Dockerfile index 922b993..03307dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,8 +19,12 @@ RUN pip install --no-cache-dir -r requirements.txt -r requirements-playwright.tx # not pip packages, so this has to run as a separate, explicit step. && playwright install --with-deps chromium +# Every module the entrypoint imports, transitively. This list had fallen +# behind: proxy_pool.py was missing while the engine imports it at module +# level, so the image died with ModuleNotFoundError on every invocation, +# `--help` included. Nothing noticed, because CI never built the image. COPY captcha_solver.py env_config.py fingerprint_client.py output_writer.py \ - page_flow.py playwright_scraper.py product_parser.py diff_runs.py ./ + page_flow.py playwright_scraper.py product_parser.py diff_runs.py proxy_pool.py ./ ENTRYPOINT ["python3", "playwright_scraper.py"] CMD ["--help"] diff --git a/captcha_solver.py b/captcha_solver.py index 56dbbd7..c8e3bb8 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -42,6 +42,24 @@ logger = logging.getLogger("captcha_solver") + +# An API key must never reach a log, and the easiest way for one to get there +# is an exception message. `requests` puts the FULL URL — query string and all +# — into the text of HTTPError and of every connection error, so any endpoint +# that takes its key as a query parameter leaks it the moment something goes +# wrong. That is not hypothetical: a 400 from the fingerprint endpoint printed +# a live key to a terminal, which is what prompted this. +# +# The endpoint and the status survive redaction, because which call failed is +# the useful half and is not the secret. +_KEY_IN_TEXT_RE = re.compile( + r"((?:client)?key|token|api[_-]?key)=([^&\s'\"]{6,})", re.IGNORECASE) + + +def _redact(text) -> str: + return _KEY_IN_TEXT_RE.sub(r"\1=***", str(text)) + + # 2captcha has two generations of solver API and both are live. # # v2 (current, documented at https://2captcha.com/api-docs): @@ -535,9 +553,15 @@ def _solve_with_2captcha_v1(api_key: str, challenge: CaptchaChallenge, while waited < max_wait: time.sleep(poll_interval) waited += poll_interval - result = requests.get(TWOCAPTCHA_RES_URL, params={ - "key": api_key, "action": "get", "id": task_id, "json": 1, - }, timeout=30).json() + try: + # The v1 result endpoint takes the key as a QUERY parameter, so a + # connection error here would otherwise put it in the log. + result = requests.get(TWOCAPTCHA_RES_URL, params={ + "key": api_key, "action": "get", "id": task_id, "json": 1, + }, timeout=30).json() + except requests.RequestException as exc: + raise RuntimeError("2captcha polling request failed: %s" + % _redact(exc)) from None if result.get("status") == 1: logger.info("2captcha.com solved the reCAPTCHA v3 challenge.") return result["request"] diff --git a/fingerprint_client.py b/fingerprint_client.py index b74dbd7..29c2ebc 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -18,7 +18,7 @@ GET https://api.2captcha.com/fingerprint/generate key your API key (also accepted as clientKey) format "chromium" (default) | "raw" - tags platform/OS/browser filters, e.g. "Windows,Chrome,Desktop" + tags ONE OS-family tag, e.g. "Windows". Not a list — see --tags country ISO 3166-1 alpha-2 min_browser_version / browser_version / force_browser_version build_version full version string — /generate only @@ -50,11 +50,11 @@ Usage ----- - python3 fingerprint_client.py --tags "Windows,Chrome,Desktop" --country us + python3 fingerprint_client.py --tags Windows --country de python3 fingerprint_client.py --generate --build-version 145.0.7632.162 # in code, with Playwright: - fp = get_fingerprint(api_key, tags="Windows,Chrome,Desktop", country="us") + fp = get_fingerprint(api_key, tags="Windows", country="de") context = browser.new_context(**playwright_context_kwargs(fp)) context.add_init_script(playwright_init_script(fp)) @@ -65,6 +65,7 @@ import hashlib import json import logging +import re import os import sys from typing import Optional @@ -74,6 +75,24 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("fingerprint_client") + +# An API key must never reach a log, and the easiest way for one to get there +# is an exception message. `requests` puts the FULL URL — query string and all +# — into the text of HTTPError and of every connection error, so any endpoint +# that takes its key as a query parameter leaks it the moment something goes +# wrong. That is not hypothetical: a 400 from the fingerprint endpoint printed +# a live key to the terminal. +# +# Everything raised or logged from this module goes through here first. The +# endpoint and the status survive, because which call failed is the useful +# half and is not the secret. +_KEY_IN_TEXT_RE = re.compile( + r"((?:client)?key|token|api[_-]?key)=([^&\s'\"]{6,})", re.IGNORECASE) + + +def _redact(text) -> str: + return _KEY_IN_TEXT_RE.sub(r"\1=***", str(text)) + API_BASE = "https://api.2captcha.com" RANDOM_URL = f"{API_BASE}/fingerprint/random" GENERATE_URL = f"{API_BASE}/fingerprint/generate" @@ -125,17 +144,43 @@ def get_fingerprint(api_key: str, *, tags: Optional[str] = None, url = GENERATE_URL if generate else RANDOM_URL logger.info("GET %s %s", url, {k: v for k, v in params.items()}) - resp = requests.get(url, params={**params, "key": api_key}, timeout=timeout) + try: + resp = requests.get(url, params={**params, "key": api_key}, timeout=timeout) + except requests.RequestException as exc: + # The key is a query parameter on this endpoint, so the URL inside a + # connection error carries it. Re-raised redacted. + raise RuntimeError("Fingerprint API request failed: %s" % _redact(exc)) from None if resp.status_code == 401: raise RuntimeError("Fingerprint API rejected the key (401). Note this is a " "separate subscription from captcha solving — a working " "solver key is not automatically enabled for fingerprints.") + if resp.status_code == 400: + # Say what the API said, and name the overwhelmingly likely cause. + # `tags` takes ONE OS-family value; a list — which the plural name and + # a fingerprint's own multi-valued `data.tags` both invite — is + # rejected outright. Measured 2026-09-09; see --tags. + try: + detail = resp.json().get("errorDescription") or resp.text[:200] + except ValueError: + detail = resp.text[:200] + hint = "" + if tags and any(sep in tags for sep in (",", "|", " ")): + hint = (" — `tags` takes ONE OS-family value (Windows, " + "Microsoft Windows, Android), not a list; you passed %r" + % tags) + raise RuntimeError("Fingerprint API rejected the request (400): %s%s" + % (_redact(detail), hint)) if resp.status_code == 429: raise RuntimeError("Fingerprint API rate limit hit (429). The per-minute cap " "depends on your plan (30/100/300/unlimited). Cache the " "result instead of fetching per request.") - resp.raise_for_status() + try: + resp.raise_for_status() + except requests.HTTPError as exc: + # requests puts resp.url — including `key=...` — into this message. + raise RuntimeError("Fingerprint API returned %s: %s" + % (resp.status_code, _redact(exc))) from None fp = resp.json() if cache_dir: @@ -143,26 +188,81 @@ def get_fingerprint(api_key: str, *, tags: Optional[str] = None, with open(_cache_path(cache_dir, params, generate), "w", encoding="utf-8") as f: json.dump(fp, f, indent=2) logger.info("Fingerprint %s (%s) — %s", fp.get("id"), fp.get("country"), - (fp.get("userAgent") or {}).get("value", "")[:70]) + (fingerprint_user_agent(fp) or "no user agent in response")[:70]) return fp +def fingerprint_user_agent(fp: dict) -> Optional[str]: + """The UA string, from wherever this response shape keeps it. + + `chromium` format nests it as `userAgent.userAgent`; `raw` format puts it + at `data.ua`. An earlier version read `userAgent.value`, which exists in + NEITHER — so `--fingerprint` silently never set a user agent at all, and + the browser kept its own. That is safe on its own but defeats the point + of the flag: the run then presented a German fingerprint's screen and + locale with a local Chromium's UA, which is the identity MISMATCH the + flag exists to avoid. + """ + ua = fp.get("userAgent") + if isinstance(ua, dict): + for key in ("userAgent", "value", "ua"): + if ua.get(key): + return ua[key] + if isinstance(ua, str) and ua: + return ua + data = fp.get("data") + if isinstance(data, dict) and data.get("ua"): + return data["ua"] + return None + + def playwright_context_kwargs(fp: dict) -> dict: - """The parts of a fingerprint Playwright can set natively on a context.""" + """The parts of a fingerprint Playwright can set natively on a context. + + Everything here comes from the fingerprint itself rather than being + derived from it. A derived value is a guess wearing the fingerprint's + authority, and the guesses this used to make were poor ones — see + `locale` below. + """ kwargs = {} - ua = (fp.get("userAgent") or {}).get("value") + ua = fingerprint_user_agent(fp) if ua: kwargs["user_agent"] = ua + screen = fp.get("screen") or {} if screen.get("width") and screen.get("height"): - # A real window is smaller than the screen; a viewport exactly equal to - # screen size is itself a signal. - kwargs["viewport"] = {"width": int(screen["width"]), - "height": max(400, int(screen["height"]) - 120)} - kwargs["screen"] = {"width": int(screen["width"]), "height": int(screen["height"])} - country = fp.get("country") - if country: - kwargs["locale"] = f"en-{country.upper()}" + width, height = int(screen["width"]), int(screen["height"]) + # The window, not the screen: a viewport exactly equal to screen size + # is itself a signal. The fingerprint states its own outer size, so + # use that when it is there and fall back to an estimate when it is + # not. + outer_w = int(screen.get("outerWidth") or width) + outer_h = int(screen.get("outerHeight") or max(400, height - 120)) + kwargs["viewport"] = {"width": outer_w, "height": max(400, outer_h)} + kwargs["screen"] = {"width": width, "height": height} + + intl = fp.get("intl") or {} + # The fingerprint's OWN locale. This used to be built as + # f"en-{country}", which produced "en-DE" for a German fingerprint — an + # English-speaking visitor in Germany is possible but it is not what the + # fingerprint describes, and a locale that contradicts the rest of the + # identity is exactly the mismatch this flag is meant to prevent. The + # real value here is "de-DE". + locale = intl.get("contentLocale") + if not locale: + languages = intl.get("languages") + if isinstance(languages, list) and languages: + locale = languages[0] + if not locale and fp.get("country"): + locale = "en-%s" % fp["country"].upper() + if locale: + kwargs["locale"] = locale + + # Playwright can set the timezone natively, and a fingerprint that says + # Europe/Berlin while the browser reports UTC contradicts itself in a way + # any script can read. + if intl.get("timeZone"): + kwargs["timezone_id"] = intl["timeZone"] return kwargs @@ -213,7 +313,18 @@ def main() -> int: p = argparse.ArgumentParser(description="Fetch a browser fingerprint from 2captcha") p.add_argument("--key", default=os.environ.get("TWOCAPTCHA_KEY"), help="API key. Defaults to $TWOCAPTCHA_KEY (safer than argv).") - p.add_argument("--tags", default=None, help='e.g. "Windows,Chrome,Desktop"') + # Measured against the live API on 2026-09-09, because the example this + # file used to carry ("Windows,Chrome,Desktop") returns 400 every time: + # accepted -> Windows, Microsoft Windows, Android + # rejected -> Chrome, Desktop, Mobile, Unknown, and EVERY combination, + # with any separator tried (comma, space, pipe) + # A returned fingerprint's own `data.tags` lists several values, which is + # what makes the plural form look plausible; the filter takes one. + p.add_argument("--tags", default=None, metavar="TAG", + help="ONE OS-family tag, not a list: Windows, " + "Microsoft Windows or Android. Chrome/Desktop/Mobile " + "are rejected by the API with 400, and no combination " + "is accepted. Use --country to narrow further.") p.add_argument("--country", default=None, help="ISO 3166-1 alpha-2, e.g. us") p.add_argument("--min-browser-version", type=int, default=None) p.add_argument("--browser-version", type=int, default=None)