From 5604b45ce07501aacf6280ce090c0f52ec8bf277 Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Mon, 26 Oct 2026 20:40:14 +0000 Subject: [PATCH 1/2] Stop carrying the crawler's name in a field called is_crawler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ipapi.is worker returned the crawler's NAME through `is_crawler` — "BaiduSpider", "GoogleBot" — while every consumer typed it `boolean`. Measured on production over an hour: `false` on 29,869 sessions, then "BaiduSpider" 1,637, "FacebookBot" 35, "GoogleAdsBot" 8, "AhrefsBot" 6, "GoogleBot" 1. So `ipInfo.isCrawler === true` matched nothing, and two files in captcha-private carry long comments working around it. The reputation sidecar that replaced it answers a real boolean plus a separate `crawler` field, so `isCrawler` is now genuinely a boolean and the name lives in `crawlerName`. No accommodation for the old worker's shape: it is being retired, not run alongside. Part of prosopo/captcha-private#5548. Co-Authored-By: Claude Opus 5 --- packages/ipinfo/src/backends/ipapi.ts | 1 + packages/ipinfo/src/tests/ipapi.unit.test.ts | 37 ++++++++++++++++++++ packages/types/src/api/ipapi.ts | 11 ++++++ 3 files changed, 49 insertions(+) diff --git a/packages/ipinfo/src/backends/ipapi.ts b/packages/ipinfo/src/backends/ipapi.ts index 1a6e448409..5ad4fea157 100644 --- a/packages/ipinfo/src/backends/ipapi.ts +++ b/packages/ipinfo/src/backends/ipapi.ts @@ -150,6 +150,7 @@ export class IpapiBackend { isMobile: data.is_mobile, isSatellite: data.is_satellite, isCrawler: data.is_crawler, + crawlerName: data.crawler, providerName: data.company?.name || data.datacenter?.datacenter, providerType: data.company?.type || data.asn?.type, asnNumber: data.asn?.asn, diff --git a/packages/ipinfo/src/tests/ipapi.unit.test.ts b/packages/ipinfo/src/tests/ipapi.unit.test.ts index 67a37d54cd..8a06dba69d 100644 --- a/packages/ipinfo/src/tests/ipapi.unit.test.ts +++ b/packages/ipinfo/src/tests/ipapi.unit.test.ts @@ -443,6 +443,43 @@ describe("IpapiBackend.lookup responses", () => { }); }); +describe("the crawler name", () => { + // The worker this replaced returned the crawler's name through + // `is_crawler`, a field typed boolean, so `isCrawler === true` matched + // nothing and the name had nowhere to live. Ours answers both separately. + it("keeps the boolean and the name apart", async () => { + const result = expectValid( + await backend( + respondWith(baseResponse({ is_crawler: true, crawler: "GoogleBot" })), + ).lookup(IP), + ); + + expect(result.isCrawler).toBe(true); + expect(result.crawlerName).toBe("GoogleBot"); + }); + + // Some ranges are flagged without a name. + it("reports a crawler with no name as a crawler", async () => { + const result = expectValid( + await backend(respondWith(baseResponse({ is_crawler: true }))).lookup(IP), + ); + + expect(result.isCrawler).toBe(true); + expect(result.crawlerName).toBeUndefined(); + }); + + it("leaves a non-crawler false and unnamed", async () => { + const result = expectValid( + await backend(respondWith(baseResponse({ is_crawler: false }))).lookup( + IP, + ), + ); + + expect(result.isCrawler).toBe(false); + expect(result.crawlerName).toBeUndefined(); + }); +}); + describe("IpapiBackend.lookup timeout", () => { it("gives up on a hanging upstream and says so", async () => { // An IP lookup sits in the request path: a backend that never answers diff --git a/packages/types/src/api/ipapi.ts b/packages/types/src/api/ipapi.ts index 55408f338c..c08a51607b 100644 --- a/packages/types/src/api/ipapi.ts +++ b/packages/types/src/api/ipapi.ts @@ -100,7 +100,15 @@ export interface IPApiResponse { is_bogon: boolean; is_mobile: boolean; is_satellite: boolean; + /** + * A real boolean, unlike the ipapi.is worker this replaced: that returned + * the crawler's NAME through this field — "GoogleBot", "BaiduSpider" — + * while every consumer typed it `boolean`, so `=== true` matched nothing. + * The name is in `crawler`. + */ is_crawler: boolean; + /** The declared crawler's name, present exactly when `is_crawler`. */ + crawler?: string; is_datacenter: boolean; is_tor: boolean; is_proxy: boolean; @@ -135,6 +143,9 @@ export interface IPInfoResult { isMobile: boolean; isSatellite: boolean; isCrawler: boolean; + /** The declared crawler's name where one is known: "GoogleBot". Set from + * the sidecar's `crawler`, or from the vendor's misnamed `is_crawler`. */ + crawlerName?: string; // Provider information providerName?: string; From bf2800da55027f9c558b31748cc13236a0352b3c Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Mon, 26 Oct 2026 21:08:34 +0000 Subject: [PATCH 2/2] Add a changeset Co-Authored-By: Claude Opus 5 --- .changeset/crawler-name-beside-the-boolean.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/crawler-name-beside-the-boolean.md diff --git a/.changeset/crawler-name-beside-the-boolean.md b/.changeset/crawler-name-beside-the-boolean.md new file mode 100644 index 0000000000..932121b0ac --- /dev/null +++ b/.changeset/crawler-name-beside-the-boolean.md @@ -0,0 +1,8 @@ +--- +--- + +Return the crawler's name beside `isCrawler` instead of inside it. + +The ipapi.is worker returned the crawler's name — "GoogleBot", "BaiduSpider" — through a field called `is_crawler` that every consumer declared a boolean, so checking it for `true` matched nothing. The reputation service that replaced it answers a real boolean and a separate name, so `isCrawler` is now genuinely a boolean and the name is in `crawlerName`. + +Nothing that makes a decision changes: a name and `true` are both truthy, which is how every caller already reads it. What comes back is the ability to say which crawler was seen.