From f0049a151e756b89c9be2b2c73272fdd5a9fbecb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 10 Sep 2026 14:51:09 +0200 Subject: [PATCH] Two inherited defects: --fingerprint always 400'd, and the shipped credential check was dead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found while auditing tokopedia-scraper against the family notes, then checked here before being patched. Neither is Amazon-specific. 1. --fingerprint FAILED ON EVERY INVOCATION. The engines' --fp-tags default was "Windows,Chrome,Desktop", and the fingerprint API rejects it with HTTP 400 ("Request parameters are invalid") — while fingerprint_client.py's own --tags help has always said ONE OS-family tag, not a list, and that Chrome/Desktop/Mobile are each rejected. Measured against the live API on 2026-09-10: Windows OK Windows,Chrome,Desktop 400 Chrome 400 Desktop 400 Present in all four repos in this family; each is fixed on its own branch. 2. THE SHIPPED CREDENTIAL CHECK WAS DEAD, AND THE LIVE ONE HAD A HOLE. `.github/ci_checks.py` was in this repo and invoked by NOTHING — not CI, not the offline suite. `tests.yml` carried an inline grep doing a narrower version of the same job with its own allowlist, and the two disagreed in a way that mattered: * the inline version matched only `ws://` and `wss://`, so an `http://user:pass@` credential would have sailed past CI; * the shipped version, which DOES match `http://`, failed on this repo's own main — the documentation placeholder `http://login:password@host:port` in proxy_pool.py and the masking fixture in smoke_test.py were missing from its allowlist. A check that fails on its own repository is a check nobody can read; a check nothing runs is not a check. Now the allowlist covers this repo's real placeholders, `tests.yml` calls `ci_checks.py --secret-check`, and the offline suite runs `--all` so a failure shows up before a push. Both are pinned by checks, including that no second inline grep creeps back. Offline suite green. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0193w6TZpMqmbfERNZAGdtv3 --- .github/ci_checks.py | 8 +++++- .github/workflows/tests.yml | 24 +++++++---------- playwright_scraper.py | 17 ++++++++++-- selenium_scraper.py | 16 ++++++++++- smoke_test.py | 53 +++++++++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 18 deletions(-) diff --git a/.github/ci_checks.py b/.github/ci_checks.py index fccb77e..5e8f262 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -53,7 +53,13 @@ # Documented placeholders, which are supposed to look like the real thing. CREDENTIAL_ALLOWED = ("USER:PASS", "user:pass", "ACCOUNT:PASSWORD", "{login}", - "***", "password}@", "u:p@h", "LOGIN:PASSWORD") + "***", "password}@", "u:p@h", "LOGIN:PASSWORD", + # These were missing, which is why `--all` failed on + # this repo's own main: this file matches `http://` as + # well as `ws://`, and the documentation and masking + # fixtures use `http://` placeholders. A check that + # fails on its own repository is one nobody can read. + "{user}", "user:secret@", "login:password@host:port", "myuser:s3cr3t@", "u:supersecret@", "login:supersecret@", "u:pass@h1", "u:pass@h2") # A 2captcha API key is a 32-character hex string. HEX32 = re.compile(r"\b[0-9a-f]{32}\b") diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index da22063..98c11e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -132,20 +132,16 @@ jobs: - name: No credentials committed # A key pasted into a file during debugging is the one mistake in this # repo that cannot be undone by a later commit. - run: | - set -e - if grep -rInE '(ws|wss)://[^ "'"'"']+:[^ "'"'"']+@' \ - --include='*.py' --include='*.md' --include='*.txt' --include='*.yml' . \ - | grep -vE 'USER:PASS|user:pass|ACCOUNT:PASSWORD|\{login\}|\*\*\*|password\}@'; then - echo "FAILED: what looks like a real credentialled URL is committed" - exit 1 - fi - if grep -rIn --include='*.py' --include='*.md' -E '\b[0-9a-f]{32}\b' . \ - | grep -viE 'sha|hash|nonce|example|md5'; then - echo "FAILED: what looks like a 2captcha API key is committed" - exit 1 - fi - echo "clean" + # + # THIS RUNS THE SHIPPED CHECK, not a copy of it. There used to be an + # inline grep here with its own allowlist, and the two disagreed in a + # way that mattered: the inline version matched only `ws://` and + # `wss://`, so an `http://user:pass@` credential would have sailed + # past CI, while `.github/ci_checks.py` — which does match `http://` + # — was in the repo and invoked by nothing at all. Two sources of + # truth, one of them dead, and the live one narrower than the dead + # one. + run: python3 .github/ci_checks.py --secret-check engine-smoke: # The offline job above installs NO engine library on purpose (that is what diff --git a/playwright_scraper.py b/playwright_scraper.py index bfe321c..d42adf0 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -1302,8 +1302,21 @@ def parse_args(): "API and apply it to the launched browser. Needs " "--twocaptcha-key. Ignored with --cdp-endpoint, where the " "Scraping Browser supplies its own.") - p.add_argument("--fp-tags", default="Windows,Chrome,Desktop", - help="Fingerprint filter tags (default: Windows,Chrome,Desktop)") + # ONE OS-family tag, not a list — and this default is what makes + # --fingerprint work at all. It shipped as "Windows,Chrome,Desktop", + # which the fingerprint API rejects with HTTP 400 ("Request parameters + # are invalid"), so --fingerprint failed on every invocation. + # + # fingerprint_client.py's own --tags help has said so all along; the + # engines' default contradicted it. Measured against the live API on + # 2026-09-10: `Windows` succeeds, and `Windows,Chrome,Desktop`, + # `Chrome` and `Desktop` each 400. + p.add_argument("--fp-tags", default="Windows", + help="ONE OS-family tag for the fingerprint filter: " + "Windows, Microsoft Windows or Android. NOT a list — " + "Chrome, Desktop and Mobile are each rejected by the " + "API with 400, and no combination is accepted. Use " + "--fp-country to narrow further. (default: Windows)") p.add_argument("--fp-country", default=None, help="Fingerprint country, ISO 3166-1 alpha-2. Match it to " "your proxy's exit country — a US fingerprint on a " diff --git a/selenium_scraper.py b/selenium_scraper.py index fb54ffe..94cb7ca 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -830,7 +830,21 @@ def parse_args(): help="Fetch a fingerprint from 2captcha's Fingerprint API " "and apply it over CDP. Needs --twocaptcha-key. " "Ignored with --cdp-endpoint.") - p.add_argument("--fp-tags", default="Windows,Chrome,Desktop") + # ONE OS-family tag, not a list — and this default is what makes + # --fingerprint work at all. It shipped as "Windows,Chrome,Desktop", + # which the fingerprint API rejects with HTTP 400 ("Request parameters + # are invalid"), so --fingerprint failed on every invocation. + # + # fingerprint_client.py's own --tags help has said so all along; the + # engines' default contradicted it. Measured against the live API on + # 2026-09-10: `Windows` succeeds, and `Windows,Chrome,Desktop`, + # `Chrome` and `Desktop` each 400. + p.add_argument("--fp-tags", default="Windows", + help="ONE OS-family tag for the fingerprint filter: " + "Windows, Microsoft Windows or Android. NOT a list — " + "Chrome, Desktop and Mobile are each rejected by the " + "API with 400, and no combination is accepted. Use " + "--fp-country to narrow further. (default: Windows)") p.add_argument("--fp-country", default=None, help="Fingerprint country, ISO 3166-1 alpha-2. Match it to " "your proxy's exit country.") diff --git a/smoke_test.py b/smoke_test.py index 0e7f02d..19ea65a 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1385,6 +1385,58 @@ def test_sample_output(): # --------------------------------------------------------------------------- +def test_ci_checks_is_actually_wired_up(): + group("the shipped CI checks run, and pass on this repo") + ok = True + # `.github/ci_checks.py` was in this repo and invoked by NOTHING — not + # CI, not this suite — while `tests.yml` carried an inline grep doing a + # narrower version of the same job with its own allowlist. The inline one + # matched only ws:// and wss://, so an `http://user:pass@` credential + # would have sailed past CI; the shipped one, which does match http://, + # meanwhile failed on this repo's own main because its allowlist was + # missing the documentation and masking placeholders the repo really uses. + # + # Two sources of truth, one dead and one with a hole. Running the shipped + # one here as well means a failure shows up locally, before a push. + script = os.path.join(REPO_ROOT, ".github", "ci_checks.py") + ok &= check("ci_checks.py is present", os.path.exists(script)) + if not os.path.exists(script): + return False + proc = subprocess.run([sys.executable, script, "--all"], + cwd=REPO_ROOT, capture_output=True, text=True) + ok &= check("ci_checks.py --all passes on this repo (exit %d)" + % proc.returncode, proc.returncode == 0) + if proc.returncode != 0: + for line in (proc.stdout + proc.stderr).strip().split("\n")[-12:]: + print(" %s" % line) + wf = open(os.path.join(REPO_ROOT, ".github", "workflows", "tests.yml"), + encoding="utf-8").read() + ok &= check("tests.yml runs the shipped check rather than an inline copy", + "ci_checks.py --secret-check" in wf + or "ci_checks.py --all" in wf) + ok &= check("...and carries no second, narrower inline credential grep", + "(ws|wss)://[^ " not in wf) + + # `--fp-tags` MUST DEFAULT TO ONE OS-FAMILY TAG. It shipped as + # "Windows,Chrome,Desktop", which the fingerprint API rejects with HTTP + # 400 — so --fingerprint failed on every invocation, while + # fingerprint_client.py's own --tags help said ONE tag all along. + # Measured against the live API on 2026-09-10: `Windows` succeeds, and + # `Windows,Chrome,Desktop`, `Chrome` and `Desktop` each 400. + import glob as _glob + for path in sorted(_glob.glob(os.path.join(REPO_ROOT, "*_scraper.py"))): + m = re.search(r'--fp-tags"\s*,\s*default="([^"]*)"', + open(path, encoding="utf-8").read()) + if m is None: + continue + ok &= check("%s's --fp-tags default is ONE tag the API accepts" + % os.path.basename(path), + "," not in m.group(1) + and m.group(1) in ("Windows", "Microsoft Windows", + "Android")) + return ok + + def main() -> int: ok = True # Checks that could not run because an optional engine library is absent. @@ -1412,6 +1464,7 @@ def main() -> int: ok &= test_proxy_pool() ok &= test_engines(skips) ok &= test_no_capture_leaks() + ok &= test_ci_checks_is_actually_wired_up() ok &= test_wording() ok &= test_sample_output()