diff --git a/.github/ci_checks.py b/.github/ci_checks.py index fccb77e..5e8f262 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -53,7 +53,13 @@ # Documented placeholders, which are supposed to look like the real thing. CREDENTIAL_ALLOWED = ("USER:PASS", "user:pass", "ACCOUNT:PASSWORD", "{login}", - "***", "password}@", "u:p@h", "LOGIN:PASSWORD") + "***", "password}@", "u:p@h", "LOGIN:PASSWORD", + # These were missing, which is why `--all` failed on + # this repo's own main: this file matches `http://` as + # well as `ws://`, and the documentation and masking + # fixtures use `http://` placeholders. A check that + # fails on its own repository is one nobody can read. + "{user}", "user:secret@", "login:password@host:port", "myuser:s3cr3t@", "u:supersecret@", "login:supersecret@", "u:pass@h1", "u:pass@h2") # A 2captcha API key is a 32-character hex string. HEX32 = re.compile(r"\b[0-9a-f]{32}\b") diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index da22063..98c11e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -132,20 +132,16 @@ jobs: - name: No credentials committed # A key pasted into a file during debugging is the one mistake in this # repo that cannot be undone by a later commit. - run: | - set -e - if grep -rInE '(ws|wss)://[^ "'"'"']+:[^ "'"'"']+@' \ - --include='*.py' --include='*.md' --include='*.txt' --include='*.yml' . \ - | grep -vE 'USER:PASS|user:pass|ACCOUNT:PASSWORD|\{login\}|\*\*\*|password\}@'; then - echo "FAILED: what looks like a real credentialled URL is committed" - exit 1 - fi - if grep -rIn --include='*.py' --include='*.md' -E '\b[0-9a-f]{32}\b' . \ - | grep -viE 'sha|hash|nonce|example|md5'; then - echo "FAILED: what looks like a 2captcha API key is committed" - exit 1 - fi - echo "clean" + # + # THIS RUNS THE SHIPPED CHECK, not a copy of it. There used to be an + # inline grep here with its own allowlist, and the two disagreed in a + # way that mattered: the inline version matched only `ws://` and + # `wss://`, so an `http://user:pass@` credential would have sailed + # past CI, while `.github/ci_checks.py` — which does match `http://` + # — was in the repo and invoked by nothing at all. Two sources of + # truth, one of them dead, and the live one narrower than the dead + # one. + run: python3 .github/ci_checks.py --secret-check engine-smoke: # The offline job above installs NO engine library on purpose (that is what diff --git a/playwright_scraper.py b/playwright_scraper.py index bfe321c..d42adf0 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -1302,8 +1302,21 @@ def parse_args(): "API and apply it to the launched browser. Needs " "--twocaptcha-key. Ignored with --cdp-endpoint, where the " "Scraping Browser supplies its own.") - p.add_argument("--fp-tags", default="Windows,Chrome,Desktop", - help="Fingerprint filter tags (default: Windows,Chrome,Desktop)") + # ONE OS-family tag, not a list — and this default is what makes + # --fingerprint work at all. It shipped as "Windows,Chrome,Desktop", + # which the fingerprint API rejects with HTTP 400 ("Request parameters + # are invalid"), so --fingerprint failed on every invocation. + # + # fingerprint_client.py's own --tags help has said so all along; the + # engines' default contradicted it. Measured against the live API on + # 2026-09-10: `Windows` succeeds, and `Windows,Chrome,Desktop`, + # `Chrome` and `Desktop` each 400. + p.add_argument("--fp-tags", default="Windows", + help="ONE OS-family tag for the fingerprint filter: " + "Windows, Microsoft Windows or Android. NOT a list — " + "Chrome, Desktop and Mobile are each rejected by the " + "API with 400, and no combination is accepted. Use " + "--fp-country to narrow further. (default: Windows)") p.add_argument("--fp-country", default=None, help="Fingerprint country, ISO 3166-1 alpha-2. Match it to " "your proxy's exit country — a US fingerprint on a " diff --git a/selenium_scraper.py b/selenium_scraper.py index fb54ffe..94cb7ca 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -830,7 +830,21 @@ def parse_args(): help="Fetch a fingerprint from 2captcha's Fingerprint API " "and apply it over CDP. Needs --twocaptcha-key. " "Ignored with --cdp-endpoint.") - p.add_argument("--fp-tags", default="Windows,Chrome,Desktop") + # ONE OS-family tag, not a list — and this default is what makes + # --fingerprint work at all. It shipped as "Windows,Chrome,Desktop", + # which the fingerprint API rejects with HTTP 400 ("Request parameters + # are invalid"), so --fingerprint failed on every invocation. + # + # fingerprint_client.py's own --tags help has said so all along; the + # engines' default contradicted it. Measured against the live API on + # 2026-09-10: `Windows` succeeds, and `Windows,Chrome,Desktop`, + # `Chrome` and `Desktop` each 400. + p.add_argument("--fp-tags", default="Windows", + help="ONE OS-family tag for the fingerprint filter: " + "Windows, Microsoft Windows or Android. NOT a list — " + "Chrome, Desktop and Mobile are each rejected by the " + "API with 400, and no combination is accepted. Use " + "--fp-country to narrow further. (default: Windows)") p.add_argument("--fp-country", default=None, help="Fingerprint country, ISO 3166-1 alpha-2. Match it to " "your proxy's exit country.") diff --git a/smoke_test.py b/smoke_test.py index 0e7f02d..19ea65a 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -1385,6 +1385,58 @@ def test_sample_output(): # --------------------------------------------------------------------------- +def test_ci_checks_is_actually_wired_up(): + group("the shipped CI checks run, and pass on this repo") + ok = True + # `.github/ci_checks.py` was in this repo and invoked by NOTHING — not + # CI, not this suite — while `tests.yml` carried an inline grep doing a + # narrower version of the same job with its own allowlist. The inline one + # matched only ws:// and wss://, so an `http://user:pass@` credential + # would have sailed past CI; the shipped one, which does match http://, + # meanwhile failed on this repo's own main because its allowlist was + # missing the documentation and masking placeholders the repo really uses. + # + # Two sources of truth, one dead and one with a hole. Running the shipped + # one here as well means a failure shows up locally, before a push. + script = os.path.join(REPO_ROOT, ".github", "ci_checks.py") + ok &= check("ci_checks.py is present", os.path.exists(script)) + if not os.path.exists(script): + return False + proc = subprocess.run([sys.executable, script, "--all"], + cwd=REPO_ROOT, capture_output=True, text=True) + ok &= check("ci_checks.py --all passes on this repo (exit %d)" + % proc.returncode, proc.returncode == 0) + if proc.returncode != 0: + for line in (proc.stdout + proc.stderr).strip().split("\n")[-12:]: + print(" %s" % line) + wf = open(os.path.join(REPO_ROOT, ".github", "workflows", "tests.yml"), + encoding="utf-8").read() + ok &= check("tests.yml runs the shipped check rather than an inline copy", + "ci_checks.py --secret-check" in wf + or "ci_checks.py --all" in wf) + ok &= check("...and carries no second, narrower inline credential grep", + "(ws|wss)://[^ " not in wf) + + # `--fp-tags` MUST DEFAULT TO ONE OS-FAMILY TAG. It shipped as + # "Windows,Chrome,Desktop", which the fingerprint API rejects with HTTP + # 400 — so --fingerprint failed on every invocation, while + # fingerprint_client.py's own --tags help said ONE tag all along. + # Measured against the live API on 2026-09-10: `Windows` succeeds, and + # `Windows,Chrome,Desktop`, `Chrome` and `Desktop` each 400. + import glob as _glob + for path in sorted(_glob.glob(os.path.join(REPO_ROOT, "*_scraper.py"))): + m = re.search(r'--fp-tags"\s*,\s*default="([^"]*)"', + open(path, encoding="utf-8").read()) + if m is None: + continue + ok &= check("%s's --fp-tags default is ONE tag the API accepts" + % os.path.basename(path), + "," not in m.group(1) + and m.group(1) in ("Windows", "Microsoft Windows", + "Android")) + return ok + + def main() -> int: ok = True # Checks that could not run because an optional engine library is absent. @@ -1412,6 +1464,7 @@ def main() -> int: ok &= test_proxy_pool() ok &= test_engines(skips) ok &= test_no_capture_leaks() + ok &= test_ci_checks_is_actually_wired_up() ok &= test_wording() ok &= test_sample_output()