Traffic filter: per-country, market and abuser-score exceptions

.changeset/traffic-filter-country-exceptions.md

@@ -0,0 +1,13 @@
+---
+"@prosopo/types": minor
+"@prosopo/types-database": minor
+"@prosopo/database": minor
+"@prosopo/provider": minor
+"@prosopo/captcha-severity": minor
+---
+
+Traffic filter categories can now have exceptions. An exception matches visitors by country, by whether their country is inside or outside the account's market, by a minimum abuser score, or any mix of these. It then replaces the category's block or challenge action. When several exceptions match, the harshest wins: a block beats any challenge, and a stricter captcha beats a milder one. So the order of the list does not matter. For example, block proxies, but challenge those from the account's market unless their abuser score is at least 0.5.
+
+The account's market countries now travel to providers inside each site's traffic filter, including when a provider pulls updated sites from the portal database. An exception that depends on the market never matches while the account has no market set.
+
+`@prosopo/captcha-severity` gains `policyHarshness` and `harshestPolicy`. They rank a policy that may block, and the access rules now use them too, with the same ordering as before.

packages/captcha-severity/src/index.test.ts

@@ -15,8 +15,10 @@
 import { describe, expect, it } from "vitest";
 import {
 	captchaPolicySeverity,
+	harshestPolicy,
 	isStricterCaptchaPolicy,
 	isStricterCaptchaType,
+	policyHarshness,
 	rankCaptchaType,
 } from "./index.js";
 
@@ -284,3 +286,44 @@ describe("isStricterCaptchaPolicy — type then settings", () => {
 		expect(isStricterCaptchaPolicy(harder, milder)).toBe(true);
 	});
 });
+
+describe("policyHarshness — blocks above every challenge", () => {
+	it("ranks a block above the strictest possible challenge", () => {
+		expect(policyHarshness({}, true)).toBeGreaterThan(
+			policyHarshness({ captchaType: "image", solvedImagesCount: 1e9 }, false),
+		);
+	});
+
+	it("ranks a challenge by its captcha policy severity", () => {
+		const policy = { captchaType: "pow", powDifficulty: 7 };
+		expect(policyHarshness(policy, false)).toBe(captchaPolicySeverity(policy));
+	});
+});
+
+describe("harshestPolicy", () => {
+	type Rule = { name: string; block: boolean; captchaType?: string };
+	const blocks = (rule: Rule): boolean => rule.block;
+
+	it("picks a block over any challenge, wherever it sits", () => {
+		const block: Rule = { name: "block", block: true };
+		const image: Rule = { name: "image", block: false, captchaType: "image" };
+		expect(harshestPolicy([image, block], blocks)).toBe(block);
+		expect(harshestPolicy([block, image], blocks)).toBe(block);
+	});
+
+	it("picks the stricter challenge", () => {
+		const pow: Rule = { name: "pow", block: false, captchaType: "pow" };
+		const image: Rule = { name: "image", block: false, captchaType: "image" };
+		expect(harshestPolicy([pow, image], blocks)).toBe(image);
+	});
+
+	it("keeps the first of equally harsh policies", () => {
+		const first: Rule = { name: "first", block: false, captchaType: "pow" };
+		const second: Rule = { name: "second", block: false, captchaType: "pow" };
+		expect(harshestPolicy([first, second], blocks)).toBe(first);
+	});
+
+	it("returns undefined for no policies", () => {
+		expect(harshestPolicy<Rule>([], blocks)).toBeUndefined();
+	});
+});

packages/captcha-severity/src/index.ts

@@ -239,3 +239,37 @@ export const isStricterCaptchaPolicy = (
 	incumbent: CaptchaPolicySeverityInput,
 ): boolean =>
 	captchaPolicySeverity(candidate) > captchaPolicySeverity(incumbent);
+
+/**
+ * Severity of a policy that may block instead of challenging. A block
+ * outranks every challenge; challenges rank by `captchaPolicySeverity`.
+ *
+ * Whether the policy blocks is passed in rather than read off it, because
+ * each caller encodes that with its own enum (`AccessPolicyType.Block`,
+ * `TrafficFilterAction.Block`) and this package imports neither.
+ */
+export const policyHarshness = (
+	policy: CaptchaPolicySeverityInput,
+	blocks: boolean,
+): number => (blocks ? Number.MAX_SAFE_INTEGER : captchaPolicySeverity(policy));
+
+/**
+ * The harshest of `policies` by `policyHarshness`, or undefined when there
+ * are none. A tie keeps the earlier policy, so the result never depends on
+ * anything but the policies and their order.
+ */
+export const harshestPolicy = <T extends CaptchaPolicySeverityInput>(
+	policies: readonly T[],
+	blocks: (policy: T) => boolean,
+): T | undefined => {
+	let harshest: T | undefined;
+	let harshestSeverity = Number.NEGATIVE_INFINITY;
+	for (const policy of policies) {
+		const severity = policyHarshness(policy, blocks(policy));
+		if (severity > harshestSeverity) {
+			harshest = policy;
+			harshestSeverity = severity;
+		}
+	}
+	return harshest;
+};

packages/database/src/databases/client.ts

@@ -14,7 +14,12 @@
 
 import { ProsopoError } from "@prosopo/common";
 import type { Logger } from "@prosopo/logger";
-import type { IUserSettings, Tier, Timestamp } from "@prosopo/types";
+import {
+	type IUserSettings,
+	type Tier,
+	type Timestamp,
+	withMarketCountries,
+} from "@prosopo/types";
 import {
 	AccountSchema,
 	type IClientDatabase,
@@ -29,6 +34,7 @@ import { MongoDatabase } from "../base/index.js";
 // fields read below cannot drift apart.
 interface UpdatedClientAccount {
 	tier: Tier;
+	marketCountries?: string[];
 	sites?: {
 		siteKey?: string;
 		settings?: IUserSettings;
@@ -99,6 +105,7 @@ export class ClientDatabase extends MongoDatabase implements IClientDatabase {
 				},
 				{
 					tier: 1,
+					marketCountries: 1,
 					"sites.siteKey": 1,
 					"sites.settings": 1,
 					"sites.updatedAt": 1,
@@ -119,7 +126,7 @@ export class ClientDatabase extends MongoDatabase implements IClientDatabase {
 				if (!isNew) continue;
 				newClientRecords.push({
 					account: site.siteKey,
-					settings: site.settings,
+					settings: withMarketCountries(site.settings, account.marketCountries),
 					tier: account.tier,
 				});
 			}

packages/database/src/tests/integration/getUpdatedClients.integration.test.ts

@@ -13,7 +13,11 @@
 // limitations under the License.
 
 import { LogLevel, getLogger } from "@prosopo/logger";
-import { ClientSettingsSchema, Tier } from "@prosopo/types";
+import {
+	ClientSettingsSchema,
+	Tier,
+	TrafficFilterAction,
+} from "@prosopo/types";
 import { MongoMemoryServer } from "mongodb-memory-server";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import { ClientDatabase } from "../../databases/client.js";
@@ -154,4 +158,47 @@ describe("ClientDatabase.getUpdatedClients", () => {
 
 		expect(await db.getUpdatedClients(100)).toHaveLength(0);
 	});
+
+	it("puts the account's market countries into the site's traffic filter", async () => {
+		const db = await seed();
+		// Inserted raw: this read-side schema declares only some settings
+		// fields, and the portal writes the full settings object.
+		await db.tables.accounts.collection.insertOne({
+			signupEmail: "market@example.com",
+			tier: Tier.Professional,
+			marketCountries: ["GB", "IE"],
+			users: [{ email: "market@example.com", status: "active" }],
+			sites: [
+				{
+					name: "m",
+					siteKey: "marketKey",
+					secretKey: "s",
+					settings: ClientSettingsSchema.parse({
+						domains: ["m.example.com"],
+						trafficFilter: {
+							proxy: {
+								action: TrafficFilterAction.Block,
+								exceptions: [
+									{ inMarket: true, action: TrafficFilterAction.Challenge },
+								],
+							},
+						},
+					}),
+					createdAt: 1,
+					updatedAt: 900,
+					active: true,
+				},
+			],
+		});
+
+		const [record] = await db.getUpdatedClients(100);
+
+		expect(record?.settings.trafficFilter?.marketCountries).toEqual([
+			"GB",
+			"IE",
+		]);
+		expect(record?.settings.trafficFilter?.proxy?.exceptions).toEqual([
+			{ inMarket: true, action: TrafficFilterAction.Challenge },
+		]);
+	});
 });

packages/provider/src/api/blacklistRequestInspector.ts

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 import { randomUUID } from "node:crypto";
-import { captchaPolicySeverity } from "@prosopo/captcha-severity";
+import { policyHarshness } from "@prosopo/captcha-severity";
 import { ProsopoError, toPublicError } from "@prosopo/common";
 import type { Logger } from "@prosopo/logger";
 import {
@@ -309,12 +309,8 @@ const ruleSpecificity = (
 // fires (request-time vs verify-time), not how severe it is. The flag
 // rides on the chosen rule and downstream consumers (`findHardBlockPolicy`,
 // blockMiddleware's `enforceable` filter) read it after ranking.
-const ruleHarshness = (rule: AccessRule): number => {
-	if (rule.type === AccessPolicyType.Block) {
-		return Number.MAX_SAFE_INTEGER;
-	}
-	return captchaPolicySeverity(rule);
-};
+const ruleHarshness = (rule: AccessRule): number =>
+	policyHarshness(rule, rule.type === AccessPolicyType.Block);
 
 /**
  * Rank the candidate rules a single Redis query returned. A rule "applies" iff

packages/provider/src/tasks/spam/checkTrafficFilter.ts

@@ -12,14 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { isStricterCaptchaType } from "@prosopo/captcha-severity";
+import {
+	harshestPolicy,
+	isStricterCaptchaType,
+} from "@prosopo/captcha-severity";
 import {
 	type CaptchaType,
 	type IAudioSettings,
 	type IIconOrderSettings,
 	type IPInfoResponse,
 	type IPInfoResult,
 	type IPuzzleSettings,
+	type ITrafficCategoryException,
 	type ITrafficCategoryPolicy,
 	type ITrafficFilter,
 	ResultReason,
@@ -164,12 +168,47 @@ export const isEffectivelyDatacenter = (
 	);
 };
 
-const push = (
-	matches: TrafficFilterMatch[],
-	category: TrafficCategory,
-	policy: ITrafficCategoryPolicy | undefined,
-): void => {
-	if (policy) matches.push({ category, policy });
+const maxAbuserScore = (ipInfo: IPInfoResult): number =>
+	Math.max(ipInfo.abuserScore ?? 0, ipInfo.companyAbuserScore ?? 0);
+
+const exceptionApplies = (
+	exception: ITrafficCategoryException,
+	ipInfo: IPInfoResult,
+	marketCountries: ReadonlyArray<string>,
+): boolean => {
+	const countryCode = ipInfo.countryCode?.trim().toUpperCase();
+	if (
+		exception.countries &&
+		!(countryCode && exception.countries.includes(countryCode))
+	) {
+		return false;
+	}
+	if (exception.inMarket !== undefined) {
+		if (!countryCode || marketCountries.length === 0) return false;
+		if (marketCountries.includes(countryCode) !== exception.inMarket) {
+			return false;
+		}
+	}
+	return (
+		exception.minAbuserScore === undefined ||
+		maxAbuserScore(ipInfo) >= exception.minAbuserScore
+	);
+};
+
+const policyForIp = (
+	policy: ITrafficCategoryPolicy,
+	ipInfo: IPInfoResult,
+	marketCountries: ReadonlyArray<string>,
+): ITrafficCategoryPolicy => {
+	const exception = harshestPolicy(
+		(policy.exceptions ?? []).filter((candidate) =>
+			exceptionApplies(candidate, ipInfo, marketCountries),
+		),
+		(candidate) => candidate.action === TrafficFilterAction.Block,
+	);
+	if (!exception) return policy;
+	const { countries, inMarket, minAbuserScore, ...exceptionPolicy } = exception;
+	return exceptionPolicy;
 };
 
 // Highest-precedence flag set on the IP wins. Only that category's policy
@@ -192,23 +231,35 @@ const evaluateIpInfo = (
 	const matches: TrafficFilterMatch[] = [];
 	if (!ipInfo || !ipInfo.isValid) return matches;
 
+	const push = (
+		category: TrafficCategory,
+		policy: ITrafficCategoryPolicy | undefined,
+	): void => {
+		if (!policy) return;
+		const marketCountries = trafficFilter.marketCountries ?? [];
+		matches.push({
+			category,
+			policy: policyForIp(policy, ipInfo, marketCountries),
+		});
+	};
+
 	if (ipInfo.isTor) {
-		push(matches, "tor", trafficFilter.tor);
+		push("tor", trafficFilter.tor);
 		return matches;
 	}
 
 	if (ipInfo.isVPN) {
-		push(matches, "vpn", trafficFilter.vpn);
+		push("vpn", trafficFilter.vpn);
 		return matches;
 	}
 
 	if (ipInfo.isProxy) {
-		push(matches, "proxy", trafficFilter.proxy);
+		push("proxy", trafficFilter.proxy);
 		return matches;
 	}
 
 	if (isEffectivelyDatacenter(ipInfo, trafficFilter)) {
-		push(matches, "datacenter", trafficFilter.datacenter);
+		push("datacenter", trafficFilter.datacenter);
 		return matches;
 	}
 
@@ -223,29 +274,25 @@ const evaluateIpInfo = (
 		const threshold =
 			trafficFilter.abuserScoreThreshold ??
 			trafficFilterAbuserScoreThresholdDefault;
-		const maxScore = Math.max(
-			ipInfo.abuserScore ?? 0,
-			ipInfo.companyAbuserScore ?? 0,
-		);
-		if (maxScore >= threshold) {
-			push(matches, "abuser", trafficFilter.abuser);
+		if (maxAbuserScore(ipInfo) >= threshold) {
+			push("abuser", trafficFilter.abuser);
 			return matches;
 		}
 	}
 
 	// Public DNS resolvers share IP space with search crawlers.
 	if (ipInfo.isCrawler && !isDnsExtra) {
-		push(matches, "crawler", trafficFilter.crawler);
+		push("crawler", trafficFilter.crawler);
 		return matches;
 	}
 
 	if (ipInfo.isSatellite) {
-		push(matches, "satellite", trafficFilter.satellite);
+		push("satellite", trafficFilter.satellite);
 		return matches;
 	}
 
 	if (ipInfo.isMobile) {
-		push(matches, "mobile", trafficFilter.mobile);
+		push("mobile", trafficFilter.mobile);
 		return matches;
 	}
 
@@ -265,6 +312,13 @@ const evaluateIpInfo = (
  * passes cleanly — lower flags are not consulted. So an IP that's flagged
  * as both VPN and proxy is only acted on if the operator has a VPN policy.
  *
+ * Exceptions: once a category owns the IP, its `exceptions` whose conditions
+ * (country, in or out of the account's market, minimum abuser score) the IP
+ * meets replace the category's policy. When several match, the harshest wins
+ * by the same ranking the access rules use, so their order does not matter:
+ * e.g. proxies blocked, but challenged in the market unless their abuser
+ * score is high.
+ *
  * Qualifying conditions: a flag that fails its qualifier does not "own"
  * the IP and evaluation falls through to the next flag. The abuser flag
  * needs `abuserScore` (or `companyAbuserScore`) at or above