From be78afc33559788990fe54c7aa1cbc370a0e51fd Mon Sep 17 00:00:00 2001 From: Petr Date: Wed, 9 Sep 2026 18:08:54 +0200 Subject: [PATCH] Stop leaking the API key; make --fingerprint apply the fingerprint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defects, all found while building the newest repo in this family against the same shared core, and all present here unchanged. The API key could reach a log ------------------------------ fingerprint_client.py sends the key as a QUERY parameter, and `requests` puts the full URL — query string included — into the text of HTTPError and of every connection error. A 400 from the fingerprint endpoint therefore prints a live key. It did, on a real call. Everything raised or logged from that module, and from captcha_solver.py's v1 polling GET (the other place the key rides in a URL rather than a body or a header), now goes through a redactor first. The endpoint and status survive: which call failed is the useful half and is not the secret. The other key-carrying calls were checked and are safe — createTask/getTaskResult put it in the JSON body. --fingerprint was applying almost none of the fingerprint ---------------------------------------------------------- * The USER AGENT was never set. It was read from `userAgent.value`, a key the API returns in NEITHER format — it is `userAgent.userAgent` in chromium and `data.ua` in raw. So the flag left the browser on its own UA while replacing the screen and locale around it: a fingerprint's identity wearing the local Chromium's user agent, which is the mismatch this family warns about when it says never to stack a second identity on a remote browser. * The LOCALE contradicted the fingerprint. Built as f"en-{country}", giving "en-DE" for a German fingerprint where the response says "de-DE". * The TIMEZONE was not applied at all, though the response states it and Playwright can set it. * The WINDOW SIZE was guessed as `screen height - 120` when the response states its own outerWidth/outerHeight. Verified against a live browser on the newest repo: with a German Windows fingerprint applied, the page reports that UA, de-DE, Europe/Berlin and a window smaller than the screen — every one matching the fingerprint rather than the host. The documented --tags example always returned 400 -------------------------------------------------- "Windows,Chrome,Desktop" is rejected every time. Measured against the live API: `tags` takes ONE OS-family value. Windows, Microsoft Windows and Android are accepted; Chrome, Desktop, Mobile and Unknown are not, and no combination is accepted with any separator. The plural name, and a returned fingerprint's own multi-valued data.tags, are what make the list form look plausible. The Docker image was broken on every invocation ------------------------------------------------ Including `--help`, its own CMD. The Dockerfile COPYs an explicit module list — correct, so the image carries no test suite and no stray .env — and the list had fallen behind: proxy_pool.py was missing while the entrypoint imports it at module level. Nothing noticed, because CI never builds the image. Verified by assembling the image's exact file set in a temporary directory and running the entrypoint from there; no container runtime was available. Not changed: the captcha detectors ----------------------------------- The newest repo in this family had to stop treating extension-injected markers as challenges, because the Scraping Browser's auto-solve extension puts `cf-turnstile` into every page it loads. This repo's marker set does not contain anything that extension injects, so the same guard here would be dead code. The hazard is recorded in the family CLAUDE.md instead, for whoever broadens a marker set next. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Wp7ctnpMGb5gi67Dco7C5V --- Dockerfile | 6 +- captcha_solver.py | 30 ++++++++- fingerprint_client.py | 145 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 160 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2b80643..2c007cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,8 +19,12 @@ RUN pip install --no-cache-dir -r requirements.txt -r requirements-playwright.tx # not pip packages, so this has to run as a separate, explicit step. && playwright install --with-deps chromium +# Every module the entrypoint imports, transitively. This list had fallen +# behind: proxy_pool.py was missing while the engine imports it at module +# level, so the image died with ModuleNotFoundError on every invocation, +# `--help` included. Nothing noticed, because CI never built the image. COPY captcha_solver.py env_config.py fingerprint_client.py output_writer.py \ - playwright_scraper.py product_parser.py diff_runs.py ./ + playwright_scraper.py product_parser.py diff_runs.py proxy_pool.py ./ ENTRYPOINT ["python3", "playwright_scraper.py"] CMD ["--help"] diff --git a/captcha_solver.py b/captcha_solver.py index eea5ff4..faef65d 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -39,6 +39,24 @@ logger = logging.getLogger("captcha_solver") + +# An API key must never reach a log, and the easiest way for one to get there +# is an exception message. `requests` puts the FULL URL — query string and all +# — into the text of HTTPError and of every connection error, so any endpoint +# that takes its key as a query parameter leaks it the moment something goes +# wrong. That is not hypothetical: a 400 from the fingerprint endpoint printed +# a live key to a terminal, which is what prompted this. +# +# The endpoint and the status survive redaction, because which call failed is +# the useful half and is not the secret. +_KEY_IN_TEXT_RE = re.compile( + r"((?:client)?key|token|api[_-]?key)=([^&\s'\"]{6,})", re.IGNORECASE) + + +def _redact(text) -> str: + return _KEY_IN_TEXT_RE.sub(r"\1=***", str(text)) + + # 2captcha has two generations of solver API and both are live. # # v2 (current, documented at https://2captcha.com/api-docs): @@ -530,9 +548,15 @@ def _solve_with_2captcha_v1(api_key: str, challenge: CaptchaChallenge, while waited < max_wait: time.sleep(poll_interval) waited += poll_interval - result = requests.get(TWOCAPTCHA_RES_URL, params={ - "key": api_key, "action": "get", "id": task_id, "json": 1, - }, timeout=30).json() + try: + # The v1 result endpoint takes the key as a QUERY parameter, so a + # connection error here would otherwise put it in the log. + result = requests.get(TWOCAPTCHA_RES_URL, params={ + "key": api_key, "action": "get", "id": task_id, "json": 1, + }, timeout=30).json() + except requests.RequestException as exc: + raise RuntimeError("2captcha polling request failed: %s" + % _redact(exc)) from None if result.get("status") == 1: logger.info("2captcha.com solved the reCAPTCHA v3 challenge.") return result["request"] diff --git a/fingerprint_client.py b/fingerprint_client.py index b74dbd7..29c2ebc 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -18,7 +18,7 @@ GET https://api.2captcha.com/fingerprint/generate key your API key (also accepted as clientKey) format "chromium" (default) | "raw" - tags platform/OS/browser filters, e.g. "Windows,Chrome,Desktop" + tags ONE OS-family tag, e.g. "Windows". Not a list — see --tags country ISO 3166-1 alpha-2 min_browser_version / browser_version / force_browser_version build_version full version string — /generate only @@ -50,11 +50,11 @@ Usage ----- - python3 fingerprint_client.py --tags "Windows,Chrome,Desktop" --country us + python3 fingerprint_client.py --tags Windows --country de python3 fingerprint_client.py --generate --build-version 145.0.7632.162 # in code, with Playwright: - fp = get_fingerprint(api_key, tags="Windows,Chrome,Desktop", country="us") + fp = get_fingerprint(api_key, tags="Windows", country="de") context = browser.new_context(**playwright_context_kwargs(fp)) context.add_init_script(playwright_init_script(fp)) @@ -65,6 +65,7 @@ import hashlib import json import logging +import re import os import sys from typing import Optional @@ -74,6 +75,24 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("fingerprint_client") + +# An API key must never reach a log, and the easiest way for one to get there +# is an exception message. `requests` puts the FULL URL — query string and all +# — into the text of HTTPError and of every connection error, so any endpoint +# that takes its key as a query parameter leaks it the moment something goes +# wrong. That is not hypothetical: a 400 from the fingerprint endpoint printed +# a live key to the terminal. +# +# Everything raised or logged from this module goes through here first. The +# endpoint and the status survive, because which call failed is the useful +# half and is not the secret. +_KEY_IN_TEXT_RE = re.compile( + r"((?:client)?key|token|api[_-]?key)=([^&\s'\"]{6,})", re.IGNORECASE) + + +def _redact(text) -> str: + return _KEY_IN_TEXT_RE.sub(r"\1=***", str(text)) + API_BASE = "https://api.2captcha.com" RANDOM_URL = f"{API_BASE}/fingerprint/random" GENERATE_URL = f"{API_BASE}/fingerprint/generate" @@ -125,17 +144,43 @@ def get_fingerprint(api_key: str, *, tags: Optional[str] = None, url = GENERATE_URL if generate else RANDOM_URL logger.info("GET %s %s", url, {k: v for k, v in params.items()}) - resp = requests.get(url, params={**params, "key": api_key}, timeout=timeout) + try: + resp = requests.get(url, params={**params, "key": api_key}, timeout=timeout) + except requests.RequestException as exc: + # The key is a query parameter on this endpoint, so the URL inside a + # connection error carries it. Re-raised redacted. + raise RuntimeError("Fingerprint API request failed: %s" % _redact(exc)) from None if resp.status_code == 401: raise RuntimeError("Fingerprint API rejected the key (401). Note this is a " "separate subscription from captcha solving — a working " "solver key is not automatically enabled for fingerprints.") + if resp.status_code == 400: + # Say what the API said, and name the overwhelmingly likely cause. + # `tags` takes ONE OS-family value; a list — which the plural name and + # a fingerprint's own multi-valued `data.tags` both invite — is + # rejected outright. Measured 2026-09-09; see --tags. + try: + detail = resp.json().get("errorDescription") or resp.text[:200] + except ValueError: + detail = resp.text[:200] + hint = "" + if tags and any(sep in tags for sep in (",", "|", " ")): + hint = (" — `tags` takes ONE OS-family value (Windows, " + "Microsoft Windows, Android), not a list; you passed %r" + % tags) + raise RuntimeError("Fingerprint API rejected the request (400): %s%s" + % (_redact(detail), hint)) if resp.status_code == 429: raise RuntimeError("Fingerprint API rate limit hit (429). The per-minute cap " "depends on your plan (30/100/300/unlimited). Cache the " "result instead of fetching per request.") - resp.raise_for_status() + try: + resp.raise_for_status() + except requests.HTTPError as exc: + # requests puts resp.url — including `key=...` — into this message. + raise RuntimeError("Fingerprint API returned %s: %s" + % (resp.status_code, _redact(exc))) from None fp = resp.json() if cache_dir: @@ -143,26 +188,81 @@ def get_fingerprint(api_key: str, *, tags: Optional[str] = None, with open(_cache_path(cache_dir, params, generate), "w", encoding="utf-8") as f: json.dump(fp, f, indent=2) logger.info("Fingerprint %s (%s) — %s", fp.get("id"), fp.get("country"), - (fp.get("userAgent") or {}).get("value", "")[:70]) + (fingerprint_user_agent(fp) or "no user agent in response")[:70]) return fp +def fingerprint_user_agent(fp: dict) -> Optional[str]: + """The UA string, from wherever this response shape keeps it. + + `chromium` format nests it as `userAgent.userAgent`; `raw` format puts it + at `data.ua`. An earlier version read `userAgent.value`, which exists in + NEITHER — so `--fingerprint` silently never set a user agent at all, and + the browser kept its own. That is safe on its own but defeats the point + of the flag: the run then presented a German fingerprint's screen and + locale with a local Chromium's UA, which is the identity MISMATCH the + flag exists to avoid. + """ + ua = fp.get("userAgent") + if isinstance(ua, dict): + for key in ("userAgent", "value", "ua"): + if ua.get(key): + return ua[key] + if isinstance(ua, str) and ua: + return ua + data = fp.get("data") + if isinstance(data, dict) and data.get("ua"): + return data["ua"] + return None + + def playwright_context_kwargs(fp: dict) -> dict: - """The parts of a fingerprint Playwright can set natively on a context.""" + """The parts of a fingerprint Playwright can set natively on a context. + + Everything here comes from the fingerprint itself rather than being + derived from it. A derived value is a guess wearing the fingerprint's + authority, and the guesses this used to make were poor ones — see + `locale` below. + """ kwargs = {} - ua = (fp.get("userAgent") or {}).get("value") + ua = fingerprint_user_agent(fp) if ua: kwargs["user_agent"] = ua + screen = fp.get("screen") or {} if screen.get("width") and screen.get("height"): - # A real window is smaller than the screen; a viewport exactly equal to - # screen size is itself a signal. - kwargs["viewport"] = {"width": int(screen["width"]), - "height": max(400, int(screen["height"]) - 120)} - kwargs["screen"] = {"width": int(screen["width"]), "height": int(screen["height"])} - country = fp.get("country") - if country: - kwargs["locale"] = f"en-{country.upper()}" + width, height = int(screen["width"]), int(screen["height"]) + # The window, not the screen: a viewport exactly equal to screen size + # is itself a signal. The fingerprint states its own outer size, so + # use that when it is there and fall back to an estimate when it is + # not. + outer_w = int(screen.get("outerWidth") or width) + outer_h = int(screen.get("outerHeight") or max(400, height - 120)) + kwargs["viewport"] = {"width": outer_w, "height": max(400, outer_h)} + kwargs["screen"] = {"width": width, "height": height} + + intl = fp.get("intl") or {} + # The fingerprint's OWN locale. This used to be built as + # f"en-{country}", which produced "en-DE" for a German fingerprint — an + # English-speaking visitor in Germany is possible but it is not what the + # fingerprint describes, and a locale that contradicts the rest of the + # identity is exactly the mismatch this flag is meant to prevent. The + # real value here is "de-DE". + locale = intl.get("contentLocale") + if not locale: + languages = intl.get("languages") + if isinstance(languages, list) and languages: + locale = languages[0] + if not locale and fp.get("country"): + locale = "en-%s" % fp["country"].upper() + if locale: + kwargs["locale"] = locale + + # Playwright can set the timezone natively, and a fingerprint that says + # Europe/Berlin while the browser reports UTC contradicts itself in a way + # any script can read. + if intl.get("timeZone"): + kwargs["timezone_id"] = intl["timeZone"] return kwargs @@ -213,7 +313,18 @@ def main() -> int: p = argparse.ArgumentParser(description="Fetch a browser fingerprint from 2captcha") p.add_argument("--key", default=os.environ.get("TWOCAPTCHA_KEY"), help="API key. Defaults to $TWOCAPTCHA_KEY (safer than argv).") - p.add_argument("--tags", default=None, help='e.g. "Windows,Chrome,Desktop"') + # Measured against the live API on 2026-09-09, because the example this + # file used to carry ("Windows,Chrome,Desktop") returns 400 every time: + # accepted -> Windows, Microsoft Windows, Android + # rejected -> Chrome, Desktop, Mobile, Unknown, and EVERY combination, + # with any separator tried (comma, space, pipe) + # A returned fingerprint's own `data.tags` lists several values, which is + # what makes the plural form look plausible; the filter takes one. + p.add_argument("--tags", default=None, metavar="TAG", + help="ONE OS-family tag, not a list: Windows, " + "Microsoft Windows or Android. Chrome/Desktop/Mobile " + "are rejected by the API with 400, and no combination " + "is accepted. Use --country to narrow further.") p.add_argument("--country", default=None, help="ISO 3166-1 alpha-2, e.g. us") p.add_argument("--min-browser-version", type=int, default=None) p.add_argument("--browser-version", type=int, default=None)