Stop leaking the API key; make --fingerprint apply the fingerprint

Dockerfile

@@ -19,8 +19,12 @@ RUN pip install --no-cache-dir -r requirements.txt -r requirements-playwright.tx
     # not pip packages, so this has to run as a separate, explicit step.
     && playwright install --with-deps chromium
 
+# Every module the entrypoint imports, transitively. This list had fallen
+# behind: proxy_pool.py was missing while the engine imports it at module
+# level, so the image died with ModuleNotFoundError on every invocation,
+# `--help` included. Nothing noticed, because CI never built the image.
 COPY captcha_solver.py env_config.py fingerprint_client.py output_writer.py \
-     page_flow.py playwright_scraper.py product_parser.py diff_runs.py ./
+     page_flow.py playwright_scraper.py product_parser.py diff_runs.py proxy_pool.py ./
 
 ENTRYPOINT ["python3", "playwright_scraper.py"]
 CMD ["--help"]

captcha_solver.py

@@ -42,6 +42,24 @@
 
 logger = logging.getLogger("captcha_solver")
 
+
+# An API key must never reach a log, and the easiest way for one to get there
+# is an exception message. `requests` puts the FULL URL — query string and all
+# — into the text of HTTPError and of every connection error, so any endpoint
+# that takes its key as a query parameter leaks it the moment something goes
+# wrong. That is not hypothetical: a 400 from the fingerprint endpoint printed
+# a live key to a terminal, which is what prompted this.
+#
+# The endpoint and the status survive redaction, because which call failed is
+# the useful half and is not the secret.
+_KEY_IN_TEXT_RE = re.compile(
+    r"((?:client)?key|token|api[_-]?key)=([^&\s'\"]{6,})", re.IGNORECASE)
+
+
+def _redact(text) -> str:
+    return _KEY_IN_TEXT_RE.sub(r"\1=***", str(text))
+
+
 # 2captcha has two generations of solver API and both are live.
 #
 #   v2 (current, documented at https://2captcha.com/api-docs):
@@ -535,9 +553,15 @@ def _solve_with_2captcha_v1(api_key: str, challenge: CaptchaChallenge,
     while waited < max_wait:
         time.sleep(poll_interval)
         waited += poll_interval
-        result = requests.get(TWOCAPTCHA_RES_URL, params={
-            "key": api_key, "action": "get", "id": task_id, "json": 1,
-        }, timeout=30).json()
+        try:
+            # The v1 result endpoint takes the key as a QUERY parameter, so a
+            # connection error here would otherwise put it in the log.
+            result = requests.get(TWOCAPTCHA_RES_URL, params={
+                "key": api_key, "action": "get", "id": task_id, "json": 1,
+            }, timeout=30).json()
+        except requests.RequestException as exc:
+            raise RuntimeError("2captcha polling request failed: %s"
+                               % _redact(exc)) from None
         if result.get("status") == 1:
             logger.info("2captcha.com solved the reCAPTCHA v3 challenge.")
             return result["request"]

fingerprint_client.py

@@ -18,7 +18,7 @@
   GET https://api.2captcha.com/fingerprint/generate
     key           your API key (also accepted as clientKey)
     format        "chromium" (default) | "raw"
-    tags          platform/OS/browser filters, e.g. "Windows,Chrome,Desktop"
+    tags          ONE OS-family tag, e.g. "Windows". Not a list — see --tags
     country       ISO 3166-1 alpha-2
     min_browser_version / browser_version / force_browser_version
     build_version full version string — /generate only
@@ -50,11 +50,11 @@
 
 Usage
 -----
-    python3 fingerprint_client.py --tags "Windows,Chrome,Desktop" --country us
+    python3 fingerprint_client.py --tags Windows --country de
     python3 fingerprint_client.py --generate --build-version 145.0.7632.162
 
     # in code, with Playwright:
-    fp = get_fingerprint(api_key, tags="Windows,Chrome,Desktop", country="us")
+    fp = get_fingerprint(api_key, tags="Windows", country="de")
     context = browser.new_context(**playwright_context_kwargs(fp))
     context.add_init_script(playwright_init_script(fp))
 
@@ -65,6 +65,7 @@
 import hashlib
 import json
 import logging
+import re
 import os
 import sys
 from typing import Optional
@@ -74,6 +75,24 @@
 logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
 logger = logging.getLogger("fingerprint_client")
 
+
+# An API key must never reach a log, and the easiest way for one to get there
+# is an exception message. `requests` puts the FULL URL — query string and all
+# — into the text of HTTPError and of every connection error, so any endpoint
+# that takes its key as a query parameter leaks it the moment something goes
+# wrong. That is not hypothetical: a 400 from the fingerprint endpoint printed
+# a live key to the terminal.
+#
+# Everything raised or logged from this module goes through here first. The
+# endpoint and the status survive, because which call failed is the useful
+# half and is not the secret.
+_KEY_IN_TEXT_RE = re.compile(
+    r"((?:client)?key|token|api[_-]?key)=([^&\s'\"]{6,})", re.IGNORECASE)
+
+
+def _redact(text) -> str:
+    return _KEY_IN_TEXT_RE.sub(r"\1=***", str(text))
+
 API_BASE = "https://api.2captcha.com"
 RANDOM_URL = f"{API_BASE}/fingerprint/random"
 GENERATE_URL = f"{API_BASE}/fingerprint/generate"
@@ -125,44 +144,125 @@ def get_fingerprint(api_key: str, *, tags: Optional[str] = None,
 
     url = GENERATE_URL if generate else RANDOM_URL
     logger.info("GET %s %s", url, {k: v for k, v in params.items()})
-    resp = requests.get(url, params={**params, "key": api_key}, timeout=timeout)
+    try:
+        resp = requests.get(url, params={**params, "key": api_key}, timeout=timeout)
+    except requests.RequestException as exc:
+        # The key is a query parameter on this endpoint, so the URL inside a
+        # connection error carries it. Re-raised redacted.
+        raise RuntimeError("Fingerprint API request failed: %s" % _redact(exc)) from None
 
     if resp.status_code == 401:
         raise RuntimeError("Fingerprint API rejected the key (401). Note this is a "
                            "separate subscription from captcha solving — a working "
                            "solver key is not automatically enabled for fingerprints.")
+    if resp.status_code == 400:
+        # Say what the API said, and name the overwhelmingly likely cause.
+        # `tags` takes ONE OS-family value; a list — which the plural name and
+        # a fingerprint's own multi-valued `data.tags` both invite — is
+        # rejected outright. Measured 2026-09-09; see --tags.
+        try:
+            detail = resp.json().get("errorDescription") or resp.text[:200]
+        except ValueError:
+            detail = resp.text[:200]
+        hint = ""
+        if tags and any(sep in tags for sep in (",", "|", " ")):
+            hint = (" — `tags` takes ONE OS-family value (Windows, "
+                    "Microsoft Windows, Android), not a list; you passed %r"
+                    % tags)
+        raise RuntimeError("Fingerprint API rejected the request (400): %s%s"
+                           % (_redact(detail), hint))
     if resp.status_code == 429:
         raise RuntimeError("Fingerprint API rate limit hit (429). The per-minute cap "
                            "depends on your plan (30/100/300/unlimited). Cache the "
                            "result instead of fetching per request.")
-    resp.raise_for_status()
+    try:
+        resp.raise_for_status()
+    except requests.HTTPError as exc:
+        # requests puts resp.url — including `key=...` — into this message.
+        raise RuntimeError("Fingerprint API returned %s: %s"
+                           % (resp.status_code, _redact(exc))) from None
     fp = resp.json()
 
     if cache_dir:
         os.makedirs(cache_dir, exist_ok=True)
         with open(_cache_path(cache_dir, params, generate), "w", encoding="utf-8") as f:
             json.dump(fp, f, indent=2)
     logger.info("Fingerprint %s (%s) — %s", fp.get("id"), fp.get("country"),
-                (fp.get("userAgent") or {}).get("value", "")[:70])
+                (fingerprint_user_agent(fp) or "no user agent in response")[:70])
     return fp
 
 
+def fingerprint_user_agent(fp: dict) -> Optional[str]:
+    """The UA string, from wherever this response shape keeps it.
+
+    `chromium` format nests it as `userAgent.userAgent`; `raw` format puts it
+    at `data.ua`. An earlier version read `userAgent.value`, which exists in
+    NEITHER — so `--fingerprint` silently never set a user agent at all, and
+    the browser kept its own. That is safe on its own but defeats the point
+    of the flag: the run then presented a German fingerprint's screen and
+    locale with a local Chromium's UA, which is the identity MISMATCH the
+    flag exists to avoid.
+    """
+    ua = fp.get("userAgent")
+    if isinstance(ua, dict):
+        for key in ("userAgent", "value", "ua"):
+            if ua.get(key):
+                return ua[key]
+    if isinstance(ua, str) and ua:
+        return ua
+    data = fp.get("data")
+    if isinstance(data, dict) and data.get("ua"):
+        return data["ua"]
+    return None
+
+
 def playwright_context_kwargs(fp: dict) -> dict:
-    """The parts of a fingerprint Playwright can set natively on a context."""
+    """The parts of a fingerprint Playwright can set natively on a context.
+
+    Everything here comes from the fingerprint itself rather than being
+    derived from it. A derived value is a guess wearing the fingerprint's
+    authority, and the guesses this used to make were poor ones — see
+    `locale` below.
+    """
     kwargs = {}
-    ua = (fp.get("userAgent") or {}).get("value")
+    ua = fingerprint_user_agent(fp)
     if ua:
         kwargs["user_agent"] = ua
+
     screen = fp.get("screen") or {}
     if screen.get("width") and screen.get("height"):
-        # A real window is smaller than the screen; a viewport exactly equal to
-        # screen size is itself a signal.
-        kwargs["viewport"] = {"width": int(screen["width"]),
-                              "height": max(400, int(screen["height"]) - 120)}
-        kwargs["screen"] = {"width": int(screen["width"]), "height": int(screen["height"])}
-    country = fp.get("country")
-    if country:
-        kwargs["locale"] = f"en-{country.upper()}"
+        width, height = int(screen["width"]), int(screen["height"])
+        # The window, not the screen: a viewport exactly equal to screen size
+        # is itself a signal. The fingerprint states its own outer size, so
+        # use that when it is there and fall back to an estimate when it is
+        # not.
+        outer_w = int(screen.get("outerWidth") or width)
+        outer_h = int(screen.get("outerHeight") or max(400, height - 120))
+        kwargs["viewport"] = {"width": outer_w, "height": max(400, outer_h)}
+        kwargs["screen"] = {"width": width, "height": height}
+
+    intl = fp.get("intl") or {}
+    # The fingerprint's OWN locale. This used to be built as
+    # f"en-{country}", which produced "en-DE" for a German fingerprint — an
+    # English-speaking visitor in Germany is possible but it is not what the
+    # fingerprint describes, and a locale that contradicts the rest of the
+    # identity is exactly the mismatch this flag is meant to prevent. The
+    # real value here is "de-DE".
+    locale = intl.get("contentLocale")
+    if not locale:
+        languages = intl.get("languages")
+        if isinstance(languages, list) and languages:
+            locale = languages[0]
+    if not locale and fp.get("country"):
+        locale = "en-%s" % fp["country"].upper()
+    if locale:
+        kwargs["locale"] = locale
+
+    # Playwright can set the timezone natively, and a fingerprint that says
+    # Europe/Berlin while the browser reports UTC contradicts itself in a way
+    # any script can read.
+    if intl.get("timeZone"):
+        kwargs["timezone_id"] = intl["timeZone"]
     return kwargs
 
 
@@ -213,7 +313,18 @@ def main() -> int:
     p = argparse.ArgumentParser(description="Fetch a browser fingerprint from 2captcha")
     p.add_argument("--key", default=os.environ.get("TWOCAPTCHA_KEY"),
                    help="API key. Defaults to $TWOCAPTCHA_KEY (safer than argv).")
-    p.add_argument("--tags", default=None, help='e.g. "Windows,Chrome,Desktop"')
+    # Measured against the live API on 2026-09-09, because the example this
+    # file used to carry ("Windows,Chrome,Desktop") returns 400 every time:
+    #   accepted -> Windows, Microsoft Windows, Android
+    #   rejected -> Chrome, Desktop, Mobile, Unknown, and EVERY combination,
+    #               with any separator tried (comma, space, pipe)
+    # A returned fingerprint's own `data.tags` lists several values, which is
+    # what makes the plural form look plausible; the filter takes one.
+    p.add_argument("--tags", default=None, metavar="TAG",
+                   help="ONE OS-family tag, not a list: Windows, "
+                        "Microsoft Windows or Android. Chrome/Desktop/Mobile "
+                        "are rejected by the API with 400, and no combination "
+                        "is accepted. Use --country to narrow further.")
     p.add_argument("--country", default=None, help="ISO 3166-1 alpha-2, e.g. us")
     p.add_argument("--min-browser-version", type=int, default=None)
     p.add_argument("--browser-version", type=int, default=None)