diff --git a/.github/ci_checks.py b/.github/ci_checks.py index fccb77e..5e8f262 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -53,7 +53,13 @@ # Documented placeholders, which are supposed to look like the real thing. CREDENTIAL_ALLOWED = ("USER:PASS", "user:pass", "ACCOUNT:PASSWORD", "{login}", - "***", "password}@", "u:p@h", "LOGIN:PASSWORD") + "***", "password}@", "u:p@h", "LOGIN:PASSWORD", + # These were missing, which is why `--all` failed on + # this repo's own main: this file matches `http://` as + # well as `ws://`, and the documentation and masking + # fixtures use `http://` placeholders. A check that + # fails on its own repository is one nobody can read. + "{user}", "user:secret@", "login:password@host:port", "myuser:s3cr3t@", "u:supersecret@", "login:supersecret@", "u:pass@h1", "u:pass@h2") # A 2captcha API key is a 32-character hex string. HEX32 = re.compile(r"\b[0-9a-f]{32}\b") diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1f71d94..1ee4ca2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -132,20 +132,16 @@ jobs: - name: No credentials committed # A key pasted into a file during debugging is the one mistake in this # repo that cannot be undone by a later commit. - run: | - set -e - if grep -rInE '(ws|wss)://[^ "'"'"']+:[^ "'"'"']+@' \ - --include='*.py' --include='*.md' --include='*.txt' --include='*.yml' . \ - | grep -vE 'USER:PASS|user:pass|ACCOUNT:PASSWORD|\{login\}|\*\*\*|password\}@'; then - echo "FAILED: what looks like a real credentialled URL is committed" - exit 1 - fi - if grep -rIn --include='*.py' --include='*.md' -E '\b[0-9a-f]{32}\b' . \ - | grep -viE 'sha|hash|nonce|example|md5'; then - echo "FAILED: what looks like a 2captcha API key is committed" - exit 1 - fi - echo "clean" + # + # THIS RUNS THE SHIPPED CHECK, not a copy of it. There used to be an + # inline grep here with its own allowlist, and the two disagreed in a + # way that mattered: the inline version matched only `ws://` and + # `wss://`, so an `http://user:pass@` credential would have sailed + # past CI, while `.github/ci_checks.py` — which does match `http://` + # — was in the repo and invoked by nothing at all. Two sources of + # truth, one of them dead, and the live one narrower than the dead + # one. + run: python3 .github/ci_checks.py --secret-check engine-smoke: # The offline job above installs NO engine library on purpose (that's diff --git a/playwright_scraper.py b/playwright_scraper.py index 9b7db71..e90641f 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -949,8 +949,21 @@ def parse_args(): help="Fetch a browser fingerprint from 2captcha's Fingerprint API and " "apply it to the launched browser. Needs --twocaptcha-key. Ignored " "with --cdp-endpoint, where the Scraping Browser supplies its own.") - p.add_argument("--fp-tags", default="Windows,Chrome,Desktop", - help="Fingerprint filter tags (default: Windows,Chrome,Desktop)") + # ONE OS-family tag, not a list — and this default is what makes + # --fingerprint work at all. It shipped as "Windows,Chrome,Desktop", + # which the fingerprint API rejects with HTTP 400 ("Request parameters + # are invalid"), so --fingerprint failed on every invocation. + # + # fingerprint_client.py's own --tags help has said so all along; the + # engines' default contradicted it. Measured against the live API on + # 2026-09-10: `Windows` succeeds, and `Windows,Chrome,Desktop`, + # `Chrome` and `Desktop` each 400. + p.add_argument("--fp-tags", default="Windows", + help="ONE OS-family tag for the fingerprint filter: " + "Windows, Microsoft Windows or Android. NOT a list — " + "Chrome, Desktop and Mobile are each rejected by the " + "API with 400, and no combination is accepted. Use " + "--fp-country to narrow further. (default: Windows)") p.add_argument("--fp-country", default=None, help="Fingerprint country, ISO 3166-1 alpha-2. Match it to your proxy's " "exit country — a US fingerprint on a German IP is a contradiction.") diff --git a/smoke_test.py b/smoke_test.py index 83c65e3..db22703 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -18,6 +18,7 @@ import re import builtins import inspect +import subprocess import sys import tempfile @@ -2070,6 +2071,57 @@ class _NoProxyArgs: ok &= check("the placeholder antidetect endpoint constant is gone", not hasattr(_cs, "ANTIDETECT_LOCAL_API")) + # THE SHIPPED CI CHECKS RUN, AND PASS ON THIS REPO. + # + # `.github/ci_checks.py` was in this repo and invoked by NOTHING — not + # CI, not this suite — while `tests.yml` carried an inline grep doing a + # narrower version of the same job with its own allowlist. The inline one + # matched only ws:// and wss://, so an `http://user:pass@` credential + # would have sailed past CI; the shipped one, which does match http://, + # meanwhile failed on this repo's own main because the documentation + # placeholder in proxy_pool.py and the masking fixtures here were missing + # from its allowlist. + # + # Two sources of truth, one dead and one with a hole. Running the shipped + # one here as well means a failure shows up locally, before a push. + _repo_root = os.path.dirname(os.path.abspath(__file__)) + _script = os.path.join(_repo_root, ".github", "ci_checks.py") + ok &= check("ci_checks.py is present", os.path.exists(_script)) + if os.path.exists(_script): + _proc = subprocess.run([sys.executable, _script, "--all"], + cwd=_repo_root, capture_output=True, text=True) + ok &= check(f"ci_checks.py --all passes on this repo " + f"(exit {_proc.returncode})", _proc.returncode == 0) + if _proc.returncode != 0: + for _line in (_proc.stdout + _proc.stderr).strip().split("\n")[-12:]: + print(f" {_line}") + _wf = open(os.path.join(_repo_root, ".github", "workflows", + "tests.yml"), encoding="utf-8").read() + ok &= check("tests.yml runs the shipped check rather than an inline " + "copy", + "ci_checks.py --secret-check" in _wf + or "ci_checks.py --all" in _wf) + ok &= check("...and carries no second, narrower inline credential " + "grep", "(ws|wss)://[^ " not in _wf) + + # `--fp-tags` MUST DEFAULT TO ONE OS-FAMILY TAG. It shipped as + # "Windows,Chrome,Desktop", which the fingerprint API rejects with HTTP + # 400 — so --fingerprint failed on every invocation, while + # fingerprint_client.py's own --tags help said ONE tag all along. + # Measured against the live API on 2026-09-10: `Windows` succeeds, and + # `Windows,Chrome,Desktop`, `Chrome` and `Desktop` each 400. + import glob as _glob + for _path in sorted(_glob.glob(os.path.join(_repo_root, "*_scraper.py"))): + _m = re.search(r'--fp-tags"\s*,\s*default="([^"]*)"', + open(_path, encoding="utf-8").read()) + if _m is None: + continue + ok &= check(f"{os.path.basename(_path)}'s --fp-tags default is ONE " + f"tag the API accepts", + "," not in _m.group(1) + and _m.group(1) in ("Windows", "Microsoft Windows", + "Android")) + print() if _skips: print(f"{len(_skips)} group(s) of checks SKIPPED — an optional engine "