diff --git a/.changeset/traffic-filter-country-exceptions.md b/.changeset/traffic-filter-country-exceptions.md new file mode 100644 index 0000000000..00c8c211d7 --- /dev/null +++ b/.changeset/traffic-filter-country-exceptions.md @@ -0,0 +1,13 @@ +--- +"@prosopo/types": minor +"@prosopo/types-database": minor +"@prosopo/database": minor +"@prosopo/provider": minor +"@prosopo/captcha-severity": minor +--- + +Traffic filter categories can now have exceptions. An exception matches visitors by country, by whether their country is inside or outside the account's market, by a minimum abuser score, or any mix of these. It then replaces the category's block or challenge action. When several exceptions match, the harshest wins: a block beats any challenge, and a stricter captcha beats a milder one. So the order of the list does not matter. For example, block proxies, but challenge those from the account's market unless their abuser score is at least 0.5. + +The account's market countries now travel to providers inside each site's traffic filter, including when a provider pulls updated sites from the portal database. An exception that depends on the market never matches while the account has no market set. + +`@prosopo/captcha-severity` gains `policyHarshness` and `harshestPolicy`. They rank a policy that may block, and the access rules now use them too, with the same ordering as before. diff --git a/packages/captcha-severity/src/index.test.ts b/packages/captcha-severity/src/index.test.ts index 37833707ad..bb318c71f8 100644 --- a/packages/captcha-severity/src/index.test.ts +++ b/packages/captcha-severity/src/index.test.ts @@ -15,8 +15,10 @@ import { describe, expect, it } from "vitest"; import { captchaPolicySeverity, + harshestPolicy, isStricterCaptchaPolicy, isStricterCaptchaType, + policyHarshness, rankCaptchaType, } from "./index.js"; @@ -284,3 +286,44 @@ describe("isStricterCaptchaPolicy — type then settings", () => { expect(isStricterCaptchaPolicy(harder, milder)).toBe(true); }); }); + +describe("policyHarshness — blocks above every challenge", () => { + it("ranks a block above the strictest possible challenge", () => { + expect(policyHarshness({}, true)).toBeGreaterThan( + policyHarshness({ captchaType: "image", solvedImagesCount: 1e9 }, false), + ); + }); + + it("ranks a challenge by its captcha policy severity", () => { + const policy = { captchaType: "pow", powDifficulty: 7 }; + expect(policyHarshness(policy, false)).toBe(captchaPolicySeverity(policy)); + }); +}); + +describe("harshestPolicy", () => { + type Rule = { name: string; block: boolean; captchaType?: string }; + const blocks = (rule: Rule): boolean => rule.block; + + it("picks a block over any challenge, wherever it sits", () => { + const block: Rule = { name: "block", block: true }; + const image: Rule = { name: "image", block: false, captchaType: "image" }; + expect(harshestPolicy([image, block], blocks)).toBe(block); + expect(harshestPolicy([block, image], blocks)).toBe(block); + }); + + it("picks the stricter challenge", () => { + const pow: Rule = { name: "pow", block: false, captchaType: "pow" }; + const image: Rule = { name: "image", block: false, captchaType: "image" }; + expect(harshestPolicy([pow, image], blocks)).toBe(image); + }); + + it("keeps the first of equally harsh policies", () => { + const first: Rule = { name: "first", block: false, captchaType: "pow" }; + const second: Rule = { name: "second", block: false, captchaType: "pow" }; + expect(harshestPolicy([first, second], blocks)).toBe(first); + }); + + it("returns undefined for no policies", () => { + expect(harshestPolicy([], blocks)).toBeUndefined(); + }); +}); diff --git a/packages/captcha-severity/src/index.ts b/packages/captcha-severity/src/index.ts index fe83bb552b..3ee9620cc2 100644 --- a/packages/captcha-severity/src/index.ts +++ b/packages/captcha-severity/src/index.ts @@ -239,3 +239,37 @@ export const isStricterCaptchaPolicy = ( incumbent: CaptchaPolicySeverityInput, ): boolean => captchaPolicySeverity(candidate) > captchaPolicySeverity(incumbent); + +/** + * Severity of a policy that may block instead of challenging. A block + * outranks every challenge; challenges rank by `captchaPolicySeverity`. + * + * Whether the policy blocks is passed in rather than read off it, because + * each caller encodes that with its own enum (`AccessPolicyType.Block`, + * `TrafficFilterAction.Block`) and this package imports neither. + */ +export const policyHarshness = ( + policy: CaptchaPolicySeverityInput, + blocks: boolean, +): number => (blocks ? Number.MAX_SAFE_INTEGER : captchaPolicySeverity(policy)); + +/** + * The harshest of `policies` by `policyHarshness`, or undefined when there + * are none. A tie keeps the earlier policy, so the result never depends on + * anything but the policies and their order. + */ +export const harshestPolicy = ( + policies: readonly T[], + blocks: (policy: T) => boolean, +): T | undefined => { + let harshest: T | undefined; + let harshestSeverity = Number.NEGATIVE_INFINITY; + for (const policy of policies) { + const severity = policyHarshness(policy, blocks(policy)); + if (severity > harshestSeverity) { + harshest = policy; + harshestSeverity = severity; + } + } + return harshest; +}; diff --git a/packages/database/src/databases/client.ts b/packages/database/src/databases/client.ts index 2fbd2c5d52..464617887f 100644 --- a/packages/database/src/databases/client.ts +++ b/packages/database/src/databases/client.ts @@ -14,7 +14,12 @@ import { ProsopoError } from "@prosopo/common"; import type { Logger } from "@prosopo/logger"; -import type { IUserSettings, Tier, Timestamp } from "@prosopo/types"; +import { + type IUserSettings, + type Tier, + type Timestamp, + withMarketCountries, +} from "@prosopo/types"; import { AccountSchema, type IClientDatabase, @@ -29,6 +34,7 @@ import { MongoDatabase } from "../base/index.js"; // fields read below cannot drift apart. interface UpdatedClientAccount { tier: Tier; + marketCountries?: string[]; sites?: { siteKey?: string; settings?: IUserSettings; @@ -99,6 +105,7 @@ export class ClientDatabase extends MongoDatabase implements IClientDatabase { }, { tier: 1, + marketCountries: 1, "sites.siteKey": 1, "sites.settings": 1, "sites.updatedAt": 1, @@ -119,7 +126,7 @@ export class ClientDatabase extends MongoDatabase implements IClientDatabase { if (!isNew) continue; newClientRecords.push({ account: site.siteKey, - settings: site.settings, + settings: withMarketCountries(site.settings, account.marketCountries), tier: account.tier, }); } diff --git a/packages/database/src/tests/integration/getUpdatedClients.integration.test.ts b/packages/database/src/tests/integration/getUpdatedClients.integration.test.ts index b53eb1a545..ccf195b54d 100644 --- a/packages/database/src/tests/integration/getUpdatedClients.integration.test.ts +++ b/packages/database/src/tests/integration/getUpdatedClients.integration.test.ts @@ -13,7 +13,11 @@ // limitations under the License. import { LogLevel, getLogger } from "@prosopo/logger"; -import { ClientSettingsSchema, Tier } from "@prosopo/types"; +import { + ClientSettingsSchema, + Tier, + TrafficFilterAction, +} from "@prosopo/types"; import { MongoMemoryServer } from "mongodb-memory-server"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { ClientDatabase } from "../../databases/client.js"; @@ -154,4 +158,47 @@ describe("ClientDatabase.getUpdatedClients", () => { expect(await db.getUpdatedClients(100)).toHaveLength(0); }); + + it("puts the account's market countries into the site's traffic filter", async () => { + const db = await seed(); + // Inserted raw: this read-side schema declares only some settings + // fields, and the portal writes the full settings object. + await db.tables.accounts.collection.insertOne({ + signupEmail: "market@example.com", + tier: Tier.Professional, + marketCountries: ["GB", "IE"], + users: [{ email: "market@example.com", status: "active" }], + sites: [ + { + name: "m", + siteKey: "marketKey", + secretKey: "s", + settings: ClientSettingsSchema.parse({ + domains: ["m.example.com"], + trafficFilter: { + proxy: { + action: TrafficFilterAction.Block, + exceptions: [ + { inMarket: true, action: TrafficFilterAction.Challenge }, + ], + }, + }, + }), + createdAt: 1, + updatedAt: 900, + active: true, + }, + ], + }); + + const [record] = await db.getUpdatedClients(100); + + expect(record?.settings.trafficFilter?.marketCountries).toEqual([ + "GB", + "IE", + ]); + expect(record?.settings.trafficFilter?.proxy?.exceptions).toEqual([ + { inMarket: true, action: TrafficFilterAction.Challenge }, + ]); + }); }); diff --git a/packages/provider/src/api/blacklistRequestInspector.ts b/packages/provider/src/api/blacklistRequestInspector.ts index 3262493083..41fca71762 100644 --- a/packages/provider/src/api/blacklistRequestInspector.ts +++ b/packages/provider/src/api/blacklistRequestInspector.ts @@ -13,7 +13,7 @@ // limitations under the License. import { randomUUID } from "node:crypto"; -import { captchaPolicySeverity } from "@prosopo/captcha-severity"; +import { policyHarshness } from "@prosopo/captcha-severity"; import { ProsopoError, toPublicError } from "@prosopo/common"; import type { Logger } from "@prosopo/logger"; import { @@ -309,12 +309,8 @@ const ruleSpecificity = ( // fires (request-time vs verify-time), not how severe it is. The flag // rides on the chosen rule and downstream consumers (`findHardBlockPolicy`, // blockMiddleware's `enforceable` filter) read it after ranking. -const ruleHarshness = (rule: AccessRule): number => { - if (rule.type === AccessPolicyType.Block) { - return Number.MAX_SAFE_INTEGER; - } - return captchaPolicySeverity(rule); -}; +const ruleHarshness = (rule: AccessRule): number => + policyHarshness(rule, rule.type === AccessPolicyType.Block); /** * Rank the candidate rules a single Redis query returned. A rule "applies" iff diff --git a/packages/provider/src/tasks/spam/checkTrafficFilter.ts b/packages/provider/src/tasks/spam/checkTrafficFilter.ts index e532108cb2..8df1838c34 100644 --- a/packages/provider/src/tasks/spam/checkTrafficFilter.ts +++ b/packages/provider/src/tasks/spam/checkTrafficFilter.ts @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { isStricterCaptchaType } from "@prosopo/captcha-severity"; +import { + harshestPolicy, + isStricterCaptchaType, +} from "@prosopo/captcha-severity"; import { type CaptchaType, type IAudioSettings, @@ -20,6 +23,7 @@ import { type IPInfoResponse, type IPInfoResult, type IPuzzleSettings, + type ITrafficCategoryException, type ITrafficCategoryPolicy, type ITrafficFilter, ResultReason, @@ -164,12 +168,47 @@ export const isEffectivelyDatacenter = ( ); }; -const push = ( - matches: TrafficFilterMatch[], - category: TrafficCategory, - policy: ITrafficCategoryPolicy | undefined, -): void => { - if (policy) matches.push({ category, policy }); +const maxAbuserScore = (ipInfo: IPInfoResult): number => + Math.max(ipInfo.abuserScore ?? 0, ipInfo.companyAbuserScore ?? 0); + +const exceptionApplies = ( + exception: ITrafficCategoryException, + ipInfo: IPInfoResult, + marketCountries: ReadonlyArray, +): boolean => { + const countryCode = ipInfo.countryCode?.trim().toUpperCase(); + if ( + exception.countries && + !(countryCode && exception.countries.includes(countryCode)) + ) { + return false; + } + if (exception.inMarket !== undefined) { + if (!countryCode || marketCountries.length === 0) return false; + if (marketCountries.includes(countryCode) !== exception.inMarket) { + return false; + } + } + return ( + exception.minAbuserScore === undefined || + maxAbuserScore(ipInfo) >= exception.minAbuserScore + ); +}; + +const policyForIp = ( + policy: ITrafficCategoryPolicy, + ipInfo: IPInfoResult, + marketCountries: ReadonlyArray, +): ITrafficCategoryPolicy => { + const exception = harshestPolicy( + (policy.exceptions ?? []).filter((candidate) => + exceptionApplies(candidate, ipInfo, marketCountries), + ), + (candidate) => candidate.action === TrafficFilterAction.Block, + ); + if (!exception) return policy; + const { countries, inMarket, minAbuserScore, ...exceptionPolicy } = exception; + return exceptionPolicy; }; // Highest-precedence flag set on the IP wins. Only that category's policy @@ -192,23 +231,35 @@ const evaluateIpInfo = ( const matches: TrafficFilterMatch[] = []; if (!ipInfo || !ipInfo.isValid) return matches; + const push = ( + category: TrafficCategory, + policy: ITrafficCategoryPolicy | undefined, + ): void => { + if (!policy) return; + const marketCountries = trafficFilter.marketCountries ?? []; + matches.push({ + category, + policy: policyForIp(policy, ipInfo, marketCountries), + }); + }; + if (ipInfo.isTor) { - push(matches, "tor", trafficFilter.tor); + push("tor", trafficFilter.tor); return matches; } if (ipInfo.isVPN) { - push(matches, "vpn", trafficFilter.vpn); + push("vpn", trafficFilter.vpn); return matches; } if (ipInfo.isProxy) { - push(matches, "proxy", trafficFilter.proxy); + push("proxy", trafficFilter.proxy); return matches; } if (isEffectivelyDatacenter(ipInfo, trafficFilter)) { - push(matches, "datacenter", trafficFilter.datacenter); + push("datacenter", trafficFilter.datacenter); return matches; } @@ -223,29 +274,25 @@ const evaluateIpInfo = ( const threshold = trafficFilter.abuserScoreThreshold ?? trafficFilterAbuserScoreThresholdDefault; - const maxScore = Math.max( - ipInfo.abuserScore ?? 0, - ipInfo.companyAbuserScore ?? 0, - ); - if (maxScore >= threshold) { - push(matches, "abuser", trafficFilter.abuser); + if (maxAbuserScore(ipInfo) >= threshold) { + push("abuser", trafficFilter.abuser); return matches; } } // Public DNS resolvers share IP space with search crawlers. if (ipInfo.isCrawler && !isDnsExtra) { - push(matches, "crawler", trafficFilter.crawler); + push("crawler", trafficFilter.crawler); return matches; } if (ipInfo.isSatellite) { - push(matches, "satellite", trafficFilter.satellite); + push("satellite", trafficFilter.satellite); return matches; } if (ipInfo.isMobile) { - push(matches, "mobile", trafficFilter.mobile); + push("mobile", trafficFilter.mobile); return matches; } @@ -265,6 +312,13 @@ const evaluateIpInfo = ( * passes cleanly — lower flags are not consulted. So an IP that's flagged * as both VPN and proxy is only acted on if the operator has a VPN policy. * + * Exceptions: once a category owns the IP, its `exceptions` whose conditions + * (country, in or out of the account's market, minimum abuser score) the IP + * meets replace the category's policy. When several match, the harshest wins + * by the same ranking the access rules use, so their order does not matter: + * e.g. proxies blocked, but challenged in the market unless their abuser + * score is high. + * * Qualifying conditions: a flag that fails its qualifier does not "own" * the IP and evaluation falls through to the next flag. The abuser flag * needs `abuserScore` (or `companyAbuserScore`) at or above diff --git a/packages/provider/src/tests/unit/tasks/spam/checkTrafficFilter.unit.test.ts b/packages/provider/src/tests/unit/tasks/spam/checkTrafficFilter.unit.test.ts index 67d44051c5..13ec93c175 100644 --- a/packages/provider/src/tests/unit/tasks/spam/checkTrafficFilter.unit.test.ts +++ b/packages/provider/src/tests/unit/tasks/spam/checkTrafficFilter.unit.test.ts @@ -15,6 +15,7 @@ import { CaptchaType, type IPInfoResult, + type ITrafficCategoryException, type ITrafficFilter, TrafficFilterAction, } from "@prosopo/types"; @@ -1145,6 +1146,286 @@ describe("checkTrafficFilter", () => { }); }); }); + + describe("exceptions", () => { + const challenge = { action: TrafficFilterAction.Challenge } as const; + + const proxyChallengedInGbOnly: ITrafficFilter = { + proxy: { + action: TrafficFilterAction.Block, + exceptions: [ + { + countries: ["GB"], + action: TrafficFilterAction.Challenge, + captchaType: CaptchaType.image, + }, + ], + }, + abuserScoreThreshold: 0.2, + skipExtrasOnValidDnsPath: true, + }; + + it("challenges a GB proxy with the exception's captcha settings", () => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, countryCode: "GB" }), + proxyChallengedInGbOnly, + ); + expect(result.isBlocked).toBe(false); + expect(result.matches).toEqual([ + { + category: "proxy", + policy: { + action: TrafficFilterAction.Challenge, + captchaType: CaptchaType.image, + }, + }, + ]); + }); + + it("matches the country code case-insensitively", () => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, countryCode: "gb" }), + proxyChallengedInGbOnly, + ); + expect(result.isBlocked).toBe(false); + }); + + it.each([["FR"], [undefined]])( + "blocks a proxy whose country is %s", + (countryCode: string | undefined) => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, countryCode }), + proxyChallengedInGbOnly, + ); + expect(result).toMatchObject({ + isBlocked: true, + reason: "API.PROXY_BLOCKED", + }); + }, + ); + + const gbHighScoreBlocked: ITrafficFilter = { + proxy: { + ...challenge, + exceptions: [ + { + countries: ["GB"], + minAbuserScore: 0.5, + action: TrafficFilterAction.Block, + }, + ], + }, + abuserScoreThreshold: 0.2, + skipExtrasOnValidDnsPath: true, + }; + + it.each([ + [{ abuserScore: 0.5 }], + [{ abuserScore: 0.1, companyAbuserScore: 0.7 }], + ])( + "blocks a GB proxy whose abuser score reaches the minimum (%o)", + (scores: Partial) => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, countryCode: "GB", ...scores }), + gbHighScoreBlocked, + ); + expect(result).toMatchObject({ + isBlocked: true, + reason: "API.PROXY_BLOCKED", + }); + }, + ); + + it.each([ + [{ countryCode: "GB", abuserScore: 0.49 }], + [{ countryCode: "GB" }], + [{ countryCode: "US", abuserScore: 0.9 }], + ])( + "challenges a proxy that misses either condition (%o)", + (info: Partial) => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, ...info }), + gbHighScoreBlocked, + ); + expect(result.isBlocked).toBe(false); + expect(result.matches).toEqual([ + { category: "proxy", policy: gbHighScoreBlocked.proxy }, + ]); + }, + ); + + describe("when several exceptions match", () => { + // Proxies blocked; in the market, challenged unless the abuser + // score is high. Both exceptions match a high-scoring GB proxy. + const highScoreBlock = { + inMarket: true, + minAbuserScore: 0.5, + action: TrafficFilterAction.Block, + }; + const inMarketChallenge = { + inMarket: true, + action: TrafficFilterAction.Challenge, + captchaType: CaptchaType.puzzle, + }; + const filterWith = ( + exceptions: ITrafficCategoryException[], + ): ITrafficFilter => ({ + proxy: { action: TrafficFilterAction.Block, exceptions }, + marketCountries: ["GB"], + abuserScoreThreshold: 0.2, + skipExtrasOnValidDnsPath: true, + }); + + it.each([ + ["block listed first", [highScoreBlock, inMarketChallenge]], + ["challenge listed first", [inMarketChallenge, highScoreBlock]], + ])( + "blocks with %s, so the order of the list does not matter", + (_order: string, exceptions: ITrafficCategoryException[]) => { + const highScore = baseInfo({ + isProxy: true, + countryCode: "GB", + abuserScore: 0.7, + }); + const lowScore = baseInfo({ + isProxy: true, + countryCode: "GB", + abuserScore: 0.1, + }); + expect( + checkTrafficFilter(highScore, filterWith(exceptions)).isBlocked, + ).toBe(true); + expect( + checkTrafficFilter(lowScore, filterWith(exceptions)).matches, + ).toEqual([ + { + category: "proxy", + policy: { + action: TrafficFilterAction.Challenge, + captchaType: CaptchaType.puzzle, + }, + }, + ]); + }, + ); + + it("takes the stricter captcha between two matching challenges", () => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, countryCode: "GB" }), + filterWith([ + { + countries: ["GB"], + action: TrafficFilterAction.Challenge, + captchaType: CaptchaType.pow, + powDifficulty: 10, + }, + { + inMarket: true, + action: TrafficFilterAction.Challenge, + captchaType: CaptchaType.image, + }, + ]), + ); + expect(result.matches[0]?.policy.captchaType).toBe(CaptchaType.image); + }); + + it("keeps the earlier of two equally harsh exceptions", () => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, countryCode: "GB" }), + filterWith([ + { + countries: ["GB"], + action: TrafficFilterAction.Challenge, + padBytes: 1024, + }, + { + inMarket: true, + action: TrafficFilterAction.Challenge, + padBytes: 2048, + }, + ]), + ); + expect(result.matches[0]?.policy.padBytes).toBe(1024); + }); + }); + + describe("inMarket", () => { + const marketFilter = ( + inMarket: boolean, + marketCountries?: string[], + ): ITrafficFilter => ({ + proxy: { + action: TrafficFilterAction.Block, + exceptions: [{ inMarket, action: TrafficFilterAction.Challenge }], + }, + marketCountries, + abuserScoreThreshold: 0.2, + skipExtrasOnValidDnsPath: true, + }); + + it.each([ + [true, "GB", false], + [true, "FR", true], + [false, "FR", false], + [false, "GB", true], + ])( + "with inMarket %s, a proxy from %s is blocked: %s", + (inMarket: boolean, countryCode: string, blocked: boolean) => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, countryCode }), + marketFilter(inMarket, ["GB", "IE"]), + ); + expect(result.isBlocked).toBe(blocked); + }, + ); + + it.each([[true], [false]])( + "never matches inMarket %s while the account has no market", + (inMarket: boolean) => { + for (const marketCountries of [undefined, []]) { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, countryCode: "GB" }), + marketFilter(inMarket, marketCountries), + ); + expect(result.isBlocked).toBe(true); + } + }, + ); + + it("never matches an IP with no country", () => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true }), + marketFilter(false, ["GB"]), + ); + expect(result.isBlocked).toBe(true); + }); + + it("strips the condition from the policy it returns", () => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, countryCode: "IE" }), + marketFilter(true, ["GB", "IE"]), + ); + expect(result.matches).toEqual([ + { + category: "proxy", + policy: { action: TrafficFilterAction.Challenge }, + }, + ]); + }); + }); + + it("judges a DNS extra IP by its own country", () => { + const result = checkTrafficFilter( + baseInfo({ isProxy: true, countryCode: "GB" }), + proxyChallengedInGbOnly, + [baseInfo({ ip: "5.6.7.8", isProxy: true, countryCode: "DE" })], + ); + expect(result).toMatchObject({ + isBlocked: true, + reason: "API.PROXY_BLOCKED", + }); + }); + }); }); describe("resolveChallengePolicy", () => { diff --git a/packages/types-database/src/types/client.ts b/packages/types-database/src/types/client.ts index 8b9c249ae1..36b73356b9 100644 --- a/packages/types-database/src/types/client.ts +++ b/packages/types-database/src/types/client.ts @@ -167,7 +167,7 @@ const SELECTABLE_CAPTCHA_TYPES: CaptchaType[] = Object.values( // Sub-schema for one trafficFilter category's policy. `_id: false` prevents // Mongoose from stamping an implicit ObjectId onto each subdoc. -export const TrafficCategoryPolicySchema = new Schema( +const TrafficCategoryActionSchema = new Schema( { action: { type: String, @@ -200,6 +200,21 @@ export const TrafficCategoryPolicySchema = new Schema( { _id: false }, ); +const TrafficCategoryExceptionSchema = TrafficCategoryActionSchema.clone().add({ + countries: { type: [String], default: undefined, required: false }, + inMarket: { type: Boolean, required: false }, + minAbuserScore: { type: Number, min: 0, max: 1, required: false }, +}); + +export const TrafficCategoryPolicySchema = + TrafficCategoryActionSchema.clone().add({ + exceptions: { + type: [TrafficCategoryExceptionSchema], + default: undefined, + required: false, + }, + }); + export const UserSettingsSchema = new Schema({ captchaType: { type: String, @@ -364,6 +379,7 @@ export const UserSettingsSchema = new Schema({ mobile: { type: TrafficCategoryPolicySchema, required: false }, satellite: { type: TrafficCategoryPolicySchema, required: false }, crawler: { type: TrafficCategoryPolicySchema, required: false }, + marketCountries: { type: [String], default: undefined, required: false }, }, storeMetadata: { type: Boolean, @@ -415,6 +431,7 @@ type AccountRecord = mongoose.Document & { signupEmail: string; tier: string; tierRequestQuota: number; + marketCountries?: string[]; marketingPreferences: boolean; users: User[]; sites: { @@ -436,6 +453,7 @@ export const AccountSchema = new Schema({ signupEmail: String, tier: String, tierRequestQuota: Number, + marketCountries: { type: [String], default: undefined }, marketingPreferences: Boolean, users: [ { diff --git a/packages/types-database/src/types/schemas.test.ts b/packages/types-database/src/types/schemas.test.ts index c8c0f81ae0..6848b10751 100644 --- a/packages/types-database/src/types/schemas.test.ts +++ b/packages/types-database/src/types/schemas.test.ts @@ -688,6 +688,51 @@ describe("UserSettingsSchema", () => { expect(doc.trafficFilter.vpn.puzzle.pieceScale.max).toBe(0.2); }); + it("persists exceptions on a traffic filter category", () => { + const doc = settings({ + trafficFilter: { + proxy: { + action: TrafficFilterAction.Challenge, + exceptions: [ + { + countries: ["GB"], + minAbuserScore: 0.5, + action: TrafficFilterAction.Block, + }, + ], + }, + }, + }); + const [exception] = doc.trafficFilter.proxy.exceptions; + expect([...exception.countries]).toEqual(["GB"]); + expect(exception.minAbuserScore).toBe(0.5); + expect(exception.action).toBe(TrafficFilterAction.Block); + }); + + it("leaves exceptions absent when none are configured", () => { + const doc = settings({ + trafficFilter: { proxy: { action: TrafficFilterAction.Block } }, + }); + expect(doc.trafficFilter.proxy.exceptions).toBeUndefined(); + }); + + it("rejects an exception abuser score outside the unit interval", () => { + expect( + errorPaths( + settings({ + trafficFilter: { + proxy: { + action: TrafficFilterAction.Challenge, + exceptions: [ + { minAbuserScore: 2, action: TrafficFilterAction.Block }, + ], + }, + }, + }).validateSync(), + ), + ).toEqual(["trafficFilter.proxy.exceptions.0.minAbuserScore"]); + }); + it("persists the tarpit padBytes on a challenge category", () => { const doc = settings({ trafficFilter: { diff --git a/packages/types/src/client/settings.test.ts b/packages/types/src/client/settings.test.ts index 7bcf9ca7f6..dc1756e62e 100644 --- a/packages/types/src/client/settings.test.ts +++ b/packages/types/src/client/settings.test.ts @@ -58,6 +58,7 @@ import { resolveImageRoundsBounds, trafficFilterAbuserScoreThresholdDefault, widgetFeatureFlagDefaults, + withMarketCountries, } from "./settings.js"; type Settings = output; @@ -646,6 +647,45 @@ describe("TrafficFilterSchema", () => { ).toBe(false); }); + it("upper-cases exception country codes", () => { + const filter = TrafficFilterSchema.parse({ + proxy: { + action: TrafficFilterAction.Block, + exceptions: [ + { countries: [" gb "], action: TrafficFilterAction.Challenge }, + ], + }, + }); + expect(filter.proxy?.exceptions?.[0]?.countries).toEqual(["GB"]); + }); + + it("accepts an exception conditioned only on the market", () => { + const filter = TrafficFilterSchema.parse({ + proxy: { + action: TrafficFilterAction.Block, + exceptions: [ + { inMarket: false, action: TrafficFilterAction.Challenge }, + ], + }, + marketCountries: ["gb"], + }); + expect(filter.proxy?.exceptions?.[0]?.inMarket).toBe(false); + expect(filter.marketCountries).toEqual(["GB"]); + }); + + it.each([ + [{ action: TrafficFilterAction.Challenge }], + [{ countries: [], action: TrafficFilterAction.Challenge }], + [{ countries: ["GBR"], action: TrafficFilterAction.Challenge }], + [{ minAbuserScore: 1.5, action: TrafficFilterAction.Block }], + ])("rejects an invalid exception %o", (exception: object) => { + expect( + TrafficFilterSchema.safeParse({ + proxy: { action: TrafficFilterAction.Block, exceptions: [exception] }, + }).success, + ).toBe(false); + }); + it("leaves the allow and deny lists absent by default", () => { const filter = TrafficFilterSchema.parse({}); expect(filter.datacenterNameAllowlist).toBeUndefined(); @@ -1008,3 +1048,31 @@ describe("widget feature flags", () => { ).toEqual({ puzzleImageSwitch: true }); }); }); + +describe("withMarketCountries", () => { + const settings = (trafficFilter?: object): Settings => + ClientSettingsSchema.parse({ domains: ["example.com"], trafficFilter }); + + it("sets the account's market on the traffic filter", () => { + const result = withMarketCountries(settings({}), ["GB", "IE"]); + expect(result.trafficFilter?.marketCountries).toEqual(["GB", "IE"]); + }); + + it("replaces a stale copy saved on the site", () => { + const stale = settings({ marketCountries: ["FR"] }); + expect( + withMarketCountries(stale, ["GB"]).trafficFilter?.marketCountries, + ).toEqual(["GB"]); + expect( + withMarketCountries(stale, []).trafficFilter?.marketCountries, + ).toBeUndefined(); + expect( + withMarketCountries(stale, undefined).trafficFilter?.marketCountries, + ).toBeUndefined(); + }); + + it("leaves a site without a traffic filter alone", () => { + const bare = settings(); + expect(withMarketCountries(bare, ["GB"])).toBe(bare); + }); +}); diff --git a/packages/types/src/client/settings.ts b/packages/types/src/client/settings.ts index 4f040ddac5..71244f17cd 100644 --- a/packages/types/src/client/settings.ts +++ b/packages/types/src/client/settings.ts @@ -724,7 +724,7 @@ export const TrafficFilterActionSchema = z.nativeEnum(TrafficFilterAction); // fields override the site-wide `ClientSettingsSchema` defaults for that // request (same override semantics as `AccessPolicy`). Field validators // are reused verbatim from the site-wide settings so bounds stay in sync. -export const TrafficCategoryPolicySchema = object({ +const TrafficCategoryActionSchema = object({ action: TrafficFilterActionSchema, captchaType: SelectableCaptchaTypeSpec.optional(), powDifficulty: powDifficultyFieldSchema.optional(), @@ -743,6 +743,51 @@ export const TrafficCategoryPolicySchema = object({ iconOrder: IconOrderSettingsSchema.optional(), }); +const MAX_TRAFFIC_CATEGORY_EXCEPTIONS = 20; +const MAX_COUNTRY_CODES = 250; + +// ISO 3166-1 alpha-2, so the UK is "GB". +const countryCodesSchema = array( + string() + .trim() + .length(2) + .transform((code) => code.toUpperCase()), +).max(MAX_COUNTRY_CODES); + +// Replaces the category's action (and captcha overrides) for IPs meeting +// every condition set on it. `inMarket` tests the IP's country against the +// filter's `marketCountries`: true matches inside the market, false outside +// it, and neither matches while the market is unknown. `minAbuserScore` +// compares against the higher of the IP's own and its company's abuser +// score, inclusive. +export const TrafficCategoryExceptionSchema = + TrafficCategoryActionSchema.extend({ + countries: countryCodesSchema.min(1).optional(), + inMarket: boolean().optional(), + minAbuserScore: number().min(0).max(1).optional(), + }).refine( + (exception) => + exception.countries !== undefined || + exception.inMarket !== undefined || + exception.minAbuserScore !== undefined, + { + message: "An exception needs countries, inMarket or minAbuserScore", + }, + ); + +export type ITrafficCategoryException = output< + typeof TrafficCategoryExceptionSchema +>; + +// When several `exceptions` match an IP the harshest wins (a block beats any +// challenge, then by captcha severity); an IP matching none gets the +// category's own action. +export const TrafficCategoryPolicySchema = TrafficCategoryActionSchema.extend({ + exceptions: array(TrafficCategoryExceptionSchema) + .max(MAX_TRAFFIC_CATEGORY_EXCEPTIONS) + .optional(), +}); + export type ITrafficCategoryPolicy = output; export const TrafficFilterSchema = object({ @@ -786,6 +831,10 @@ export const TrafficFilterSchema = object({ mobile: TrafficCategoryPolicySchema.optional(), satellite: TrafficCategoryPolicySchema.optional(), crawler: TrafficCategoryPolicySchema.optional(), + // The account's market, which the portal copies in when it pushes the site + // to providers. Account-owned: values saved from the site settings form + // are overwritten on the next push. + marketCountries: countryCodesSchema.optional(), }); export type IEmailSpamRules = output; @@ -935,6 +984,24 @@ export const ClientSettingsSchema = object({ export type IUserSettings = output; +// The market is account-owned, so whatever the site's own copy says is +// replaced on the way to a provider, and dropped once the account has none. +export const withMarketCountries = ( + settings: IUserSettings, + marketCountries: ReadonlyArray | undefined, +): IUserSettings => { + if (!settings.trafficFilter) return settings; + const { marketCountries: _siteCopy, ...trafficFilter } = + settings.trafficFilter; + return { + ...settings, + trafficFilter: + marketCountries && marketCountries.length > 0 + ? { ...trafficFilter, marketCountries: [...marketCountries] } + : trafficFilter, + }; +}; + /** * The subset of a client record's settings that bounds how many image rounds * a challenge may carry. Accepts a whole `IUserSettings` — every call site