From 9860f85aab30875b5b810765fb6d0d17eb63fa49 Mon Sep 17 00:00:00 2001 From: madars Date: Mon, 31 Aug 2026 14:49:35 -0400 Subject: [PATCH 1/4] feat(hosted): authenticate hosted AI fetchers against vendor-published source ranges Co-Authored-By: Claude Fable 5 --- internal/hosted/chatgpt_user.json | 617 ++++++++++++++++++ internal/hosted/claude.go | 19 + internal/hosted/claude_bots.json | 83 +++ internal/hosted/claude_outbound.json | 7 + internal/hosted/google.go | 11 + .../hosted/google_user_triggered_agents.json | 65 ++ internal/hosted/hosted.go | 147 +++++ internal/hosted/hosted_test.go | 88 +++ internal/hosted/openai.go | 10 + 9 files changed, 1047 insertions(+) create mode 100644 internal/hosted/chatgpt_user.json create mode 100644 internal/hosted/claude.go create mode 100644 internal/hosted/claude_bots.json create mode 100644 internal/hosted/claude_outbound.json create mode 100644 internal/hosted/google.go create mode 100644 internal/hosted/google_user_triggered_agents.json create mode 100644 internal/hosted/hosted.go create mode 100644 internal/hosted/hosted_test.go create mode 100644 internal/hosted/openai.go diff --git a/internal/hosted/chatgpt_user.json b/internal/hosted/chatgpt_user.json new file mode 100644 index 0000000..dba8964 --- /dev/null +++ b/internal/hosted/chatgpt_user.json @@ -0,0 +1,617 @@ +{ + "creationTime": "2026-08-14T20:03:38.055838", + "prefixes": [ + { + "ipv4Prefix": "4.151.71.176/28" + }, + { + "ipv4Prefix": "4.151.119.48/28" + }, + { + "ipv4Prefix": "4.189.118.208/28" + }, + { + "ipv4Prefix": "4.189.119.48/28" + }, + { + "ipv4Prefix": "4.197.22.112/28" + }, + { + "ipv4Prefix": "4.197.64.0/28" + }, + { + "ipv4Prefix": "4.197.64.16/28" + }, + { + "ipv4Prefix": "4.197.64.48/28" + }, + { + "ipv4Prefix": "4.197.64.64/28" + }, + { + "ipv4Prefix": "4.197.115.112/28" + }, + { + "ipv4Prefix": "4.201.232.64/28" + }, + { + "ipv4Prefix": "4.205.128.176/28" + }, + { + "ipv4Prefix": "4.218.24.64/28" + }, + { + "ipv4Prefix": "4.226.200.16/28" + }, + { + "ipv4Prefix": "4.226.226.32/28" + }, + { + "ipv4Prefix": "9.129.0.0/17" + }, + { + "ipv4Prefix": "9.160.34.144/28" + }, + { + "ipv4Prefix": "9.160.36.16/28" + }, + { + "ipv4Prefix": "9.160.96.16/28" + }, + { + "ipv4Prefix": "9.160.128.16/28" + }, + { + "ipv4Prefix": "9.160.128.64/28" + }, + { + "ipv4Prefix": "9.160.163.128/28" + }, + { + "ipv4Prefix": "9.163.101.48/28" + }, + { + "ipv4Prefix": "9.205.8.48/28" + }, + { + "ipv4Prefix": "9.205.8.64/28" + }, + { + "ipv4Prefix": "9.205.25.128/28" + }, + { + "ipv4Prefix": "9.205.30.128/28" + }, + { + "ipv4Prefix": "9.205.30.144/28" + }, + { + "ipv4Prefix": "9.205.30.176/28" + }, + { + "ipv4Prefix": "9.205.128.32/28" + }, + { + "ipv4Prefix": "9.205.128.48/28" + }, + { + "ipv4Prefix": "9.223.181.208/28" + }, + { + "ipv4Prefix": "9.234.96.192/28" + }, + { + "ipv4Prefix": "9.234.97.96/28" + }, + { + "ipv4Prefix": "13.65.138.112/28" + }, + { + "ipv4Prefix": "13.67.72.16/28" + }, + { + "ipv4Prefix": "13.71.2.208/28" + }, + { + "ipv4Prefix": "13.76.32.208/28" + }, + { + "ipv4Prefix": "13.76.116.80/28" + }, + { + "ipv4Prefix": "13.83.167.128/28" + }, + { + "ipv4Prefix": "13.83.237.176/28" + }, + { + "ipv4Prefix": "20.45.178.144/28" + }, + { + "ipv4Prefix": "20.55.229.144/28" + }, + { + "ipv4Prefix": "20.57.199.192/28" + }, + { + "ipv4Prefix": "20.63.180.96/28" + }, + { + "ipv4Prefix": "20.63.221.64/28" + }, + { + "ipv4Prefix": "20.83.243.176/28" + }, + { + "ipv4Prefix": "20.102.212.144/28" + }, + { + "ipv4Prefix": "20.113.211.112/28" + }, + { + "ipv4Prefix": "20.113.225.112/28" + }, + { + "ipv4Prefix": "20.125.112.224/28" + }, + { + "ipv4Prefix": "20.125.144.144/28" + }, + { + "ipv4Prefix": "20.161.75.208/28" + }, + { + "ipv4Prefix": "20.168.7.192/28" + }, + { + "ipv4Prefix": "20.169.73.32/28" + }, + { + "ipv4Prefix": "20.169.73.64/28" + }, + { + "ipv4Prefix": "20.169.78.48/28" + }, + { + "ipv4Prefix": "20.169.78.64/28" + }, + { + "ipv4Prefix": "20.169.78.128/28" + }, + { + "ipv4Prefix": "20.169.78.160/28" + }, + { + "ipv4Prefix": "20.169.78.192/28" + }, + { + "ipv4Prefix": "20.169.86.224/28" + }, + { + "ipv4Prefix": "20.169.86.240/28" + }, + { + "ipv4Prefix": "20.170.184.16/28" + }, + { + "ipv4Prefix": "20.170.184.32/28" + }, + { + "ipv4Prefix": "20.170.184.48/28" + }, + { + "ipv4Prefix": "20.170.184.64/28" + }, + { + "ipv4Prefix": "20.170.184.80/28" + }, + { + "ipv4Prefix": "20.172.29.32/28" + }, + { + "ipv4Prefix": "20.199.211.160/28" + }, + { + "ipv4Prefix": "20.199.242.0/28" + }, + { + "ipv4Prefix": "20.200.212.240/28" + }, + { + "ipv4Prefix": "20.210.211.192/28" + }, + { + "ipv4Prefix": "20.215.187.208/28" + }, + { + "ipv4Prefix": "20.215.219.128/28" + }, + { + "ipv4Prefix": "20.215.219.160/28" + }, + { + "ipv4Prefix": "20.215.219.208/28" + }, + { + "ipv4Prefix": "20.218.30.240/28" + }, + { + "ipv4Prefix": "20.219.71.192/28" + }, + { + "ipv4Prefix": "20.222.36.192/28" + }, + { + "ipv4Prefix": "20.227.140.32/28" + }, + { + "ipv4Prefix": "20.228.106.176/28" + }, + { + "ipv4Prefix": "20.235.87.224/28" + }, + { + "ipv4Prefix": "20.249.63.208/28" + }, + { + "ipv4Prefix": "20.250.6.128/28" + }, + { + "ipv4Prefix": "20.250.136.64/28" + }, + { + "ipv4Prefix": "23.98.142.176/28" + }, + { + "ipv4Prefix": "23.98.186.64/28" + }, + { + "ipv4Prefix": "23.98.186.96/28" + }, + { + "ipv4Prefix": "23.98.186.176/28" + }, + { + "ipv4Prefix": "23.98.186.192/28" + }, + { + "ipv4Prefix": "23.102.140.144/28" + }, + { + "ipv4Prefix": "23.102.141.32/28" + }, + { + "ipv4Prefix": "40.74.200.208/28" + }, + { + "ipv4Prefix": "40.81.67.96/28" + }, + { + "ipv4Prefix": "40.81.234.144/28" + }, + { + "ipv4Prefix": "40.84.221.208/28" + }, + { + "ipv4Prefix": "40.84.221.224/28" + }, + { + "ipv4Prefix": "40.116.73.208/28" + }, + { + "ipv4Prefix": "48.221.40.176/28" + }, + { + "ipv4Prefix": "48.221.184.80/28" + }, + { + "ipv4Prefix": "48.221.184.96/28" + }, + { + "ipv4Prefix": "51.8.155.48/28" + }, + { + "ipv4Prefix": "51.8.155.112/28" + }, + { + "ipv4Prefix": "51.57.0.96/28" + }, + { + "ipv4Prefix": "51.59.24.64/28" + }, + { + "ipv4Prefix": "51.59.24.80/28" + }, + { + "ipv4Prefix": "51.59.48.80/28" + }, + { + "ipv4Prefix": "51.116.2.80/28" + }, + { + "ipv4Prefix": "51.116.221.96/28" + }, + { + "ipv4Prefix": "52.148.129.32/28" + }, + { + "ipv4Prefix": "52.156.77.144/28" + }, + { + "ipv4Prefix": "52.159.227.32/28" + }, + { + "ipv4Prefix": "52.159.249.96/28" + }, + { + "ipv4Prefix": "52.161.49.96/28" + }, + { + "ipv4Prefix": "52.165.212.48/28" + }, + { + "ipv4Prefix": "52.172.129.160/28" + }, + { + "ipv4Prefix": "52.173.219.96/28" + }, + { + "ipv4Prefix": "52.173.219.112/28" + }, + { + "ipv4Prefix": "52.173.221.16/28" + }, + { + "ipv4Prefix": "52.173.221.176/28" + }, + { + "ipv4Prefix": "52.173.221.208/28" + }, + { + "ipv4Prefix": "52.173.234.16/28" + }, + { + "ipv4Prefix": "52.173.234.80/28" + }, + { + "ipv4Prefix": "52.183.217.240/28" + }, + { + "ipv4Prefix": "52.190.137.16/28" + }, + { + "ipv4Prefix": "52.190.137.144/28" + }, + { + "ipv4Prefix": "52.190.139.48/28" + }, + { + "ipv4Prefix": "52.190.142.64/28" + }, + { + "ipv4Prefix": "52.190.190.16/28" + }, + { + "ipv4Prefix": "52.225.75.208/28" + }, + { + "ipv4Prefix": "52.231.30.48/28" + }, + { + "ipv4Prefix": "52.231.34.176/28" + }, + { + "ipv4Prefix": "52.231.39.144/28" + }, + { + "ipv4Prefix": "52.231.50.64/28" + }, + { + "ipv4Prefix": "52.236.94.144/28" + }, + { + "ipv4Prefix": "52.241.146.208/28" + }, + { + "ipv4Prefix": "52.242.132.224/28" + }, + { + "ipv4Prefix": "52.242.132.240/28" + }, + { + "ipv4Prefix": "52.255.109.80/28" + }, + { + "ipv4Prefix": "52.255.109.96/28" + }, + { + "ipv4Prefix": "52.255.109.112/28" + }, + { + "ipv4Prefix": "52.255.109.144/28" + }, + { + "ipv4Prefix": "52.255.111.0/28" + }, + { + "ipv4Prefix": "52.255.111.80/28" + }, + { + "ipv4Prefix": "57.154.174.112/28" + }, + { + "ipv4Prefix": "57.154.175.0/28" + }, + { + "ipv4Prefix": "57.154.187.32/28" + }, + { + "ipv4Prefix": "68.154.28.96/28" + }, + { + "ipv4Prefix": "68.220.57.64/28" + }, + { + "ipv4Prefix": "70.153.32.16/28" + }, + { + "ipv4Prefix": "70.153.32.32/28" + }, + { + "ipv4Prefix": "70.153.87.224/28" + }, + { + "ipv4Prefix": "70.153.189.192/28" + }, + { + "ipv4Prefix": "70.156.152.96/28" + }, + { + "ipv4Prefix": "74.7.35.48/28" + }, + { + "ipv4Prefix": "74.7.35.112/28" + }, + { + "ipv4Prefix": "74.7.36.64/28" + }, + { + "ipv4Prefix": "74.7.36.80/28" + }, + { + "ipv4Prefix": "74.7.36.96/28" + }, + { + "ipv4Prefix": "74.161.200.96/28" + }, + { + "ipv4Prefix": "74.224.217.64/28" + }, + { + "ipv4Prefix": "74.226.253.160/28" + }, + { + "ipv4Prefix": "85.211.128.16/28" + }, + { + "ipv4Prefix": "85.211.128.32/28" + }, + { + "ipv4Prefix": "104.208.184.192/28" + }, + { + "ipv4Prefix": "104.210.139.192/28" + }, + { + "ipv4Prefix": "104.210.139.224/28" + }, + { + "ipv4Prefix": "128.85.198.32/28" + }, + { + "ipv4Prefix": "132.196.82.48/28" + }, + { + "ipv4Prefix": "134.149.233.80/28" + }, + { + "ipv4Prefix": "135.13.64.240/28" + }, + { + "ipv4Prefix": "135.116.136.160/28" + }, + { + "ipv4Prefix": "135.220.73.208/28" + }, + { + "ipv4Prefix": "135.220.73.240/28" + }, + { + "ipv4Prefix": "135.237.131.208/28" + }, + { + "ipv4Prefix": "135.237.133.48/28" + }, + { + "ipv4Prefix": "137.135.191.176/28" + }, + { + "ipv4Prefix": "138.91.46.96/28" + }, + { + "ipv4Prefix": "145.132.136.96/28" + }, + { + "ipv4Prefix": "158.158.5.32/28" + }, + { + "ipv4Prefix": "168.63.252.240/28" + }, + { + "ipv4Prefix": "172.162.248.64/28" + }, + { + "ipv4Prefix": "172.170.1.80/28" + }, + { + "ipv4Prefix": "172.170.225.0/28" + }, + { + "ipv4Prefix": "172.170.241.80/28" + }, + { + "ipv4Prefix": "172.175.152.224/28" + }, + { + "ipv4Prefix": "172.178.140.144/28" + }, + { + "ipv4Prefix": "172.178.141.112/28" + }, + { + "ipv4Prefix": "172.178.141.128/28" + }, + { + "ipv4Prefix": "172.183.143.224/28" + }, + { + "ipv4Prefix": "172.183.222.128/28" + }, + { + "ipv4Prefix": "172.192.112.208/28" + }, + { + "ipv4Prefix": "172.197.160.192/28" + }, + { + "ipv4Prefix": "172.197.203.16/28" + }, + { + "ipv4Prefix": "172.199.137.80/28" + }, + { + "ipv4Prefix": "172.204.28.224/28" + }, + { + "ipv4Prefix": "172.204.96.80/28" + }, + { + "ipv4Prefix": "172.205.189.192/28" + }, + { + "ipv4Prefix": "172.207.1.32/28" + }, + { + "ipv4Prefix": "172.212.172.160/28" + }, + { + "ipv4Prefix": "172.215.215.32/28" + }, + { + "ipv4Prefix": "191.233.199.160/28" + }, + { + "ipv4Prefix": "191.237.249.64/28" + } + ] +} diff --git a/internal/hosted/claude.go b/internal/hosted/claude.go new file mode 100644 index 0000000..edf4d2b --- /dev/null +++ b/internal/hosted/claude.go @@ -0,0 +1,19 @@ +package hosted + +import _ "embed" + +const claudeBotsURL = "https://claude.com/crawling/bots.json" +const claudeOutboundURL = "https://platform.claude.com/docs/en/api/ip-addresses" + +// Anteroom accepts Claude-User from both Anthropic's published bot ranges and +// its stable outbound service range. The latter covers web-fetch and MCP tool +// calls and has no machine-readable feed. +// Sources of truth: +// - https://support.claude.com/en/articles/8896518 +// - https://platform.claude.com/docs/en/api/ip-addresses + +//go:embed claude_bots.json +var claudeBotsJSON []byte + +//go:embed claude_outbound.json +var claudeOutboundJSON []byte diff --git a/internal/hosted/claude_bots.json b/internal/hosted/claude_bots.json new file mode 100644 index 0000000..8c45d96 --- /dev/null +++ b/internal/hosted/claude_bots.json @@ -0,0 +1,83 @@ +{ + "creationTime": "2026-08-18T23:56:36Z", + "prefixes": [ + { + "ipv4Prefix": "16.58.26.69/32" + }, + { + "ipv4Prefix": "18.225.238.228/32" + }, + { + "ipv4Prefix": "18.227.226.255/32" + }, + { + "ipv4Prefix": "20.64.57.208/28" + }, + { + "ipv4Prefix": "20.102.46.224/28" + }, + { + "ipv4Prefix": "34.11.34.31/32" + }, + { + "ipv4Prefix": "34.85.172.162/32" + }, + { + "ipv4Prefix": "34.150.241.79/32" + }, + { + "ipv4Prefix": "34.162.191.81/32" + }, + { + "ipv4Prefix": "34.162.230.222/32" + }, + { + "ipv4Prefix": "34.162.244.71/32" + }, + { + "ipv4Prefix": "34.182.140.95/32" + }, + { + "ipv4Prefix": "34.182.161.143/32" + }, + { + "ipv4Prefix": "34.182.218.27/32" + }, + { + "ipv4Prefix": "34.182.220.85/32" + }, + { + "ipv4Prefix": "34.182.222.37/32" + }, + { + "ipv4Prefix": "34.182.225.167/32" + }, + { + "ipv4Prefix": "34.182.226.151/32" + }, + { + "ipv4Prefix": "34.182.226.221/32" + }, + { + "ipv4Prefix": "34.186.108.163/32" + }, + { + "ipv4Prefix": "35.221.29.174/32" + }, + { + "ipv4Prefix": "35.245.89.239/32" + }, + { + "ipv4Prefix": "35.245.175.129/32" + }, + { + "ipv4Prefix": "40.124.101.48/28" + }, + { + "ipv4Prefix": "136.107.176.208/32" + }, + { + "ipv4Prefix": "216.73.216.0/22" + } + ] +} diff --git a/internal/hosted/claude_outbound.json b/internal/hosted/claude_outbound.json new file mode 100644 index 0000000..eae4d4c --- /dev/null +++ b/internal/hosted/claude_outbound.json @@ -0,0 +1,7 @@ +{ + "prefixes": [ + { + "ipv4Prefix": "160.79.104.0/21" + } + ] +} diff --git a/internal/hosted/google.go b/internal/hosted/google.go new file mode 100644 index 0000000..fdaf90f --- /dev/null +++ b/internal/hosted/google.go @@ -0,0 +1,11 @@ +package hosted + +import _ "embed" + +const googleAgentURL = "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json" + +// Source of truth: +// https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers + +//go:embed google_user_triggered_agents.json +var googleAgentJSON []byte diff --git a/internal/hosted/google_user_triggered_agents.json b/internal/hosted/google_user_triggered_agents.json new file mode 100644 index 0000000..0a0a439 --- /dev/null +++ b/internal/hosted/google_user_triggered_agents.json @@ -0,0 +1,65 @@ +{ + "creationTime": "2026-08-28T14:46:09.000000", + "prefixes": [ + { + "ipv4Prefix": "74.125.232.0/28" + }, + { + "ipv4Prefix": "74.125.232.16/28" + }, + { + "ipv4Prefix": "74.125.232.32/28" + }, + { + "ipv4Prefix": "74.125.232.48/28" + }, + { + "ipv4Prefix": "74.125.232.64/28" + }, + { + "ipv4Prefix": "74.125.232.80/28" + }, + { + "ipv4Prefix": "74.125.232.96/28" + }, + { + "ipv4Prefix": "74.125.232.112/28" + }, + { + "ipv4Prefix": "136.121.16.0/24" + }, + { + "ipv4Prefix": "136.121.24.0/21" + }, + { + "ipv4Prefix": "136.121.40.0/21" + }, + { + "ipv4Prefix": "136.122.0.0/16" + }, + { + "ipv6Prefix": "2001:4860:c::/124" + }, + { + "ipv6Prefix": "2001:4860:c::10/124" + }, + { + "ipv6Prefix": "2001:4860:c::20/124" + }, + { + "ipv6Prefix": "2001:4860:c::30/124" + }, + { + "ipv6Prefix": "2001:4860:c::40/124" + }, + { + "ipv6Prefix": "2001:4860:c::50/124" + }, + { + "ipv6Prefix": "2001:4860:c::60/124" + }, + { + "ipv6Prefix": "2001:4860:c::70/124" + } + ] +} diff --git a/internal/hosted/hosted.go b/internal/hosted/hosted.go new file mode 100644 index 0000000..37ee894 --- /dev/null +++ b/internal/hosted/hosted.go @@ -0,0 +1,147 @@ +// Package hosted authenticates user-triggered fetchers operated by AI vendors. +// It identifies a claim from the User-Agent, then verifies the source address +// against that vendor's published ranges. It does not decide whether the +// request is admitted; that policy belongs to the gate. +package hosted + +import ( + "encoding/json" + "fmt" + "net/netip" + "strings" + + "github.com/radiustechsystems/anteroom/internal/useragent" +) + +// Provider is a stable hosted-fetcher identity claimed in a User-Agent. +type Provider string + +const ( + None Provider = "" + Claude Provider = "claude" + ChatGPT Provider = "chatgpt" + GoogleAgent Provider = "google-agent" +) + +type definition struct { + provider Provider + token string + sources []rangeSource +} + +type rangeSource struct { + raw []byte + url string +} + +var definitions = [...]definition{ + { + provider: Claude, + token: "Claude-User/", + sources: []rangeSource{ + {raw: claudeBotsJSON, url: claudeBotsURL}, + {raw: claudeOutboundJSON, url: claudeOutboundURL}, + }, + }, + { + provider: ChatGPT, + token: "ChatGPT-User/", + sources: []rangeSource{{raw: chatGPTUserJSON, url: chatGPTUserURL}}, + }, + { + provider: GoogleAgent, + token: "Google-Agent;", + sources: []rangeSource{{raw: googleAgentJSON, url: googleAgentURL}}, + }, +} + +// Set is the immutable collection of published hosted-fetcher ranges. +type Set struct { + prefixes map[Provider][]netip.Prefix +} + +// New parses every embedded range snapshot. A malformed generated file is a +// startup error rather than an identity check that silently stops working. +func New() (*Set, error) { + s := &Set{prefixes: make(map[Provider][]netip.Prefix, len(definitions))} + for _, d := range definitions { + for _, source := range d.sources { + prefixes, err := parsePrefixes(source.raw, source.url) + if err != nil { + return nil, err + } + s.prefixes[d.provider] = append(s.prefixes[d.provider], prefixes...) + } + } + return s, nil +} + +// Claim identifies a known hosted-fetcher User-Agent. It is deliberately +// unauthoritative: a claim changes presentation, never access, until Verify +// authenticates its source address. +func Claim(userAgent string) Provider { + for _, d := range definitions { + // Claude Code is a command-line agent that can follow Anteroom's x402 + // instructions. Never let a future UA variant containing Claude-User/ + // move it onto the hosted fetcher's strict non-x402 path. + if d.provider == Claude && strings.Contains(userAgent, "claude-code/") { + continue + } + if useragent.ContainsProduct(userAgent, d.token) { + return d.provider + } + } + return None +} + +func (p Provider) String() string { + return string(p) +} + +// Verify reports whether addr belongs to the published ranges for claim. +func (s *Set) Verify(claim Provider, addr netip.Addr) bool { + if s == nil || claim == None || !addr.IsValid() { + return false + } + addr = addr.Unmap() + for _, prefix := range s.prefixes[claim] { + if prefix.Contains(addr) { + return true + } + } + return false +} + +func parsePrefixes(raw []byte, source string) ([]netip.Prefix, error) { + var document struct { + Prefixes []struct { + IPv4 string `json:"ipv4Prefix"` + IPv6 string `json:"ipv6Prefix"` + } `json:"prefixes"` + } + if err := json.Unmarshal(raw, &document); err != nil { + return nil, fmt.Errorf("hosted: parsing %s: %w", source, err) + } + if len(document.Prefixes) == 0 { + return nil, fmt.Errorf("hosted: %s contains no prefixes", source) + } + prefixes := make([]netip.Prefix, 0, len(document.Prefixes)) + for _, item := range document.Prefixes { + if (item.IPv4 == "") == (item.IPv6 == "") { + return nil, fmt.Errorf("hosted: %s prefix entry must contain exactly one address family", source) + } + rawPrefix := item.IPv4 + if rawPrefix == "" { + rawPrefix = item.IPv6 + } + prefix, err := netip.ParsePrefix(rawPrefix) + if err != nil { + return nil, fmt.Errorf("hosted: bad prefix %q from %s: %w", rawPrefix, source, err) + } + if (item.IPv4 != "") != prefix.Addr().Is4() { + return nil, fmt.Errorf("hosted: prefix %q from %s is under the wrong address family", rawPrefix, source) + } + prefixes = append(prefixes, prefix.Masked()) + } + return prefixes, nil +} diff --git a/internal/hosted/hosted_test.go b/internal/hosted/hosted_test.go new file mode 100644 index 0000000..898cb3b --- /dev/null +++ b/internal/hosted/hosted_test.go @@ -0,0 +1,88 @@ +package hosted + +import ( + "net/netip" + "testing" +) + +func TestClaim(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + ua string + want Provider + }{ + {"Claude web", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +claude-user@anthropic.com)", Claude}, + {"ChatGPT web", "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot", ChatGPT}, + {"Google user-triggered agent", "Mozilla/5.0 (compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent)", GoogleAgent}, + {"Claude Code", "Claude-User (claude-code/2.1.250; +https://support.anthropic.com/)", None}, + {"future Claude Code form", "Claude-User/1.0 (claude-code/2.2.0; +https://support.anthropic.com/)", None}, + {"embedded Claude token", "NotClaude-User/1.0", None}, + {"embedded ChatGPT token", "NotChatGPT-User/1.0", None}, + {"embedded Google token", "NotGoogle-Agent;", None}, + {"wrong-case Claude", "Mozilla/5.0 (compatible; claude-user/1.0)", None}, + {"wrong-case ChatGPT", "Mozilla/5.0 (compatible; Chatgpt-User/1.0)", None}, + {"wrong-case Google", "Mozilla/5.0 (compatible; google-agent)", None}, + {"Googlebot", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", None}, + {"ordinary browser", "Mozilla/5.0 Chrome/152.0", None}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := Claim(tc.ua); got != tc.want { + t.Fatalf("Claim(%q) = %q, want %q", tc.ua, got, tc.want) + } + }) + } +} + +func TestPublishedRangesAuthenticateTheirProvider(t *testing.T) { + t.Parallel() + set, err := New() + if err != nil { + t.Fatal(err) + } + for _, definition := range definitions { + provider := definition.provider + t.Run(provider.String(), func(t *testing.T) { + t.Parallel() + if len(set.prefixes[provider]) == 0 { + t.Fatal("embedded manifest has no ranges") + } + ip := set.prefixes[provider][0].Addr() + if !set.Verify(provider, ip) { + t.Fatalf("%s did not verify for %s", ip, provider) + } + if ip.Is4() && !set.Verify(provider, netip.AddrFrom16(ip.As16())) { + t.Fatalf("IPv4-mapped %s did not verify for %s", ip, provider) + } + for other := range set.prefixes { + if other != provider && set.Verify(other, ip) { + t.Fatalf("%s for %s also verified as %s", ip, provider, other) + } + } + }) + } + if set.Verify(Claude, netip.MustParseAddr("192.0.2.1")) { + t.Fatal("TEST-NET address verified as Claude") + } +} + +func TestClaudeUsesBotAndStableOutboundRanges(t *testing.T) { + t.Parallel() + set, err := New() + if err != nil { + t.Fatal(err) + } + botRanges, err := parsePrefixes(claudeBotsJSON, claudeBotsURL) + if err != nil { + t.Fatal(err) + } + for _, ip := range []string{botRanges[0].Addr().String(), "160.79.104.1", "160.79.111.254"} { + if !set.Verify(Claude, netip.MustParseAddr(ip)) { + t.Errorf("published Claude address %s did not verify", ip) + } + } + if set.Verify(Claude, netip.MustParseAddr("160.79.112.1")) { + t.Error("address adjacent to Claude's stable outbound range verified") + } +} diff --git a/internal/hosted/openai.go b/internal/hosted/openai.go new file mode 100644 index 0000000..1a37904 --- /dev/null +++ b/internal/hosted/openai.go @@ -0,0 +1,10 @@ +package hosted + +import _ "embed" + +const chatGPTUserURL = "https://openai.com/chatgpt-user.json" + +// Source of truth: https://developers.openai.com/api/docs/bots + +//go:embed chatgpt_user.json +var chatGPTUserJSON []byte From a33a11910c4a92890e6ac01468aee9bb72697864 Mon Sep 17 00:00:00 2001 From: madars Date: Mon, 31 Aug 2026 14:49:46 -0400 Subject: [PATCH 2/4] fix(gate): serve spoofed identity claims a strict 403, never the ok_body_agents 200 Co-Authored-By: Claude Fable 5 --- docs/operating.md | 3 +++ internal/gate/identity.go | 2 +- internal/gate/waitpage.go | 10 +++++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/operating.md b/docs/operating.md index 78a93ec..a379539 100644 --- a/docs/operating.md +++ b/docs/operating.md @@ -294,6 +294,9 @@ verification uses the same resolved client address as CIDR bypasses. If that address cannot be resolved, the request follows the ordinary ladder rather than being reported to the crawler as a permanent DNS outage. +Claimed-but-unverified machine identities always receive a strict `403`; the +`ok_body_agents` compatibility downgrade cannot turn a spoof into a 200. + Refresh the embedded snapshots with `scripts/update-crawler-ips.py`. The scheduled workflow reports a semantic range change for human review rather than committing generated data automatically. diff --git a/internal/gate/identity.go b/internal/gate/identity.go index bb58855..f4cdeca 100644 --- a/internal/gate/identity.go +++ b/internal/gate/identity.go @@ -37,7 +37,7 @@ func (g *Gate) serveClaimedIdentity(w http.ResponseWriter, r *gateRequest) (deci serveCrawlerVerificationUnavailable(w) return decisionCrawlerVerificationUnavailable, true default: - g.serveRefusal(w, r.Request) + g.serveStrictRefusal(w, r.Request) return decisionCrawlerUnverified, true } } diff --git a/internal/gate/waitpage.go b/internal/gate/waitpage.go index 009ab34..fab9a8e 100644 --- a/internal/gate/waitpage.go +++ b/internal/gate/waitpage.go @@ -251,11 +251,19 @@ func (g *Gate) serveInstructions(w http.ResponseWriter, r *http.Request) { // Deliberately omit WWW-Authenticate: 403 does not require it, and advertising // a Payment scheme there makes some agent clients mistake x402 for HTTP auth. func (g *Gate) serveRefusal(w http.ResponseWriter, r *http.Request) { - challengeRequired(w) status := http.StatusForbidden if g.okBodyAgent(r) { status = http.StatusOK } + g.renderRefusal(w, r, status) +} + +func (g *Gate) serveStrictRefusal(w http.ResponseWriter, r *http.Request) { + g.renderRefusal(w, r, http.StatusForbidden) +} + +func (g *Gate) renderRefusal(w http.ResponseWriter, r *http.Request, status int) { + challengeRequired(w) if g.cfg.Triage.JSONAccept && wantsJSON(r) { w.Header().Set("Content-Type", "application/json") From aa194cdc26dbbc266519d47c6002bc1e25e5f65c Mon Sep 17 00:00:00 2001 From: madars Date: Mon, 31 Aug 2026 14:49:55 -0400 Subject: [PATCH 3/4] feat(gate): pass source-verified hosted fetchers, bypassing PoW and x402 (allow_hosted_fetchers, default on) Co-Authored-By: Claude Fable 5 --- README.md | 6 +++ anteroom.example.toml | 12 +++-- docs/operating.md | 28 +++++++++- internal/config/config.go | 17 +++--- internal/config/config_test.go | 14 ++++- internal/gate/decision.go | 6 +++ internal/gate/gate.go | 7 +++ internal/gate/gate_test.go | 96 +++++++++++++++++++++++++++++++--- internal/gate/identity.go | 52 ++++++++++++------ internal/gate/pay_test.go | 23 ++++++++ internal/gate/request.go | 4 ++ 11 files changed, 225 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 5cb3fc7..e7c89c7 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,12 @@ after a restart. The goal is not to tell humans from bots. It is that high-volume automated access is either cheap to tolerate, cheap to discourage, or paid for. +One explicit exception is enabled by default: source-verified `Claude-User`, +`ChatGPT-User`, and `Google-Agent` hosted fetchers pass through because they can +complete neither the browser proof nor x402. This also bypasses paid routes; it +authenticates vendor infrastructure, not a human or application user. Set +`triage.allow_hosted_fetchers = false` to give those requests a strict `403`. + ## Running it Every release attaches a static binary for Linux (x86-64, ARM64, ARMv7), macOS diff --git a/anteroom.example.toml b/anteroom.example.toml index 0c3fb1e..f7de02d 100644 --- a/anteroom.example.toml +++ b/anteroom.example.toml @@ -254,11 +254,13 @@ verified_crawlers = ["googlebot", "bingbot", "yandexbot", "ccbot"] # Default: 402 status + PAYMENT-REQUIRED header + markdown body. # --------------------------------------------------------------------------- [triage] -json_accept = true # Accept: application/json → JSON 402 body -ok_body_agents = ["claude-user"] # user agents of fetch tools that discard non-2xx - # bodies (measured); these receive the markdown - # with status 200, the only status they surface. - # Matched case-insensitively as a UA substring. +json_accept = true # Accept: application/json → JSON 402 body +ok_body_agents = ["claude-user"] # case-insensitive UA substrings for CLI + # agents that discard non-2xx bodies +# This defaults true and bypasses proof-of-work AND x402, including paid routes, +# for source-verified vendor-hosted fetchers. They cannot complete either +# protocol. Set false to return a strict 403 to them instead. +allow_hosted_fetchers = true # Claude-User, ChatGPT-User, Google-Agent # The binary also serves /.anteroom/healthz (liveness), /.anteroom/renew.js (the # injected renewal script) and /.anteroom/uninstall (removes the renewal service diff --git a/docs/operating.md b/docs/operating.md index a379539..bfa19aa 100644 --- a/docs/operating.md +++ b/docs/operating.md @@ -301,6 +301,30 @@ Refresh the embedded snapshots with `scripts/update-crawler-ips.py`. The scheduled workflow reports a semantic range change for human review rather than committing generated data automatically. +### Hosted user-triggered fetchers + +`Claude-User/`, `ChatGPT-User/`, and `Google-Agent;` identify fetches made on a +user's behalf by vendor-hosted tools. They are not command-line agents and +cannot complete proof of work or x402. Anteroom checks the advertised +case-sensitive User-Agent token, then requires the source address to appear in +that vendor's embedded published ranges. A verified request passes through by +default, including on x402-paid routes; an unverified claim receives a +machine-readable `403`, never a payment offer. This authenticates vendor +infrastructure, not application authorization. + +If the client address cannot be resolved, Anteroom warns once and sends the +request through the ordinary ladder rather than labeling it a spoof or telling +the vendor the site is temporarily unavailable forever. + +Claude uses the union of `claude.com/crawling/bots.json` and Anthropic's stable +outbound `160.79.104.0/21`. This authenticates Anthropic infrastructure, not a +human or a particular Claude product. Claude Code's different +`Claude-User (claude-code/...)` form remains on the ordinary agent path and can +receive x402. `triage.allow_hosted_fetchers` defaults to `true` because these +hosted clients cannot complete either PoW or x402. Set it to `false` to remove +the free exception: verified hosted fetchers then receive a strict `403` rather +than falling through to a payment offer they cannot use. + Percent-encoding is *not* restricted: `/repos/owner%2Frepo`, `/file%20name.txt`, and friends pass through untouched. Only paths whose decoded form is non-canonical — dot-segments (`..`, `.`), doubled slashes, or a backslash — are @@ -439,7 +463,9 @@ keeps settlement retry separate from replaying an upstream mutation. `anteroom -v` logs one line per request naming the rung of the ladder that answered it — `own-endpoint`, `bypass-path`, `bypass-ip`, `bypass-crawler`, -`crawler-verification-unavailable`, `crawler-unverified`, `pass-pow`, `pass-paid`, +`crawler-verification-unavailable`, `crawler-unverified`, `bypass-hosted`, +`hosted-refusal`, `hosted-unverified`, +`pass-pow`, `pass-paid`, `wait-page`, `refusal`, `non-canonical-path` — with the status, response size, and duration: diff --git a/internal/config/config.go b/internal/config/config.go index 847947f..e4f252f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -255,8 +255,9 @@ func (a *Activity) validate() error { } type Triage struct { - JSONAccept bool `toml:"json_accept"` - OKBodyAgents []string `toml:"ok_body_agents"` + JSONAccept bool `toml:"json_accept"` + OKBodyAgents []string `toml:"ok_body_agents"` + AllowHostedFetchers bool `toml:"allow_hosted_fetchers"` } // defaults returns a Config carrying every default the contract documents. @@ -284,12 +285,12 @@ func defaults() Config { Inject: true, Triage: Triage{ JSONAccept: true, - // Some agentic fetch tools discard the body of any non-2xx - // response, so a 401 carrying instructions is invisible to them — - // measured behavior, not a guess. For those clients the - // instructions are served with status 200 instead, which is the - // only way they see anything at all. Matched case-insensitively as - // a substring of the User-Agent. + // Verified vendor-hosted user fetchers cannot complete PoW or x402, + // so the current policy admits them even on paid routes. Operators + // can disable that exception and refuse them instead. + AllowHostedFetchers: true, + // Claude Code's fetch output omits non-2xx bodies. Matching its + // User-Agent here keeps Anteroom's instructions visible. OKBodyAgents: []string{"claude-user"}, }, } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f96dda3..e0fc6c9 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -48,8 +48,8 @@ func TestLoadMinimal(t *testing.T) { if cfg.Difficulty != 14 || cfg.RenewDifficulty != 6 { t.Errorf("difficulty defaults wrong: %+v", cfg) } - if !cfg.Inject || !cfg.Triage.JSONAccept { - t.Error("inject/json_accept should default true") + if !cfg.Inject || !cfg.Triage.JSONAccept || !cfg.Triage.AllowHostedFetchers { + t.Error("inject, json_accept, and allow_hosted_fetchers should default true") } // No admin listener unless asked for: it is unauthenticated, so silently // opening a port the operator never configured would be a surprise surface. @@ -103,6 +103,16 @@ verified_crawlers = ["googlebot"] } } +func TestHostedFetcherBypassCanBeDisabled(t *testing.T) { + cfg, err := Load(write(t, minimal+"\n[triage]\nallow_hosted_fetchers = false\n")) + if err != nil { + t.Fatal(err) + } + if cfg.Triage.AllowHostedFetchers { + t.Fatal("allow_hosted_fetchers remained enabled") + } +} + func TestLoadFullPayments(t *testing.T) { cfg, err := Load(write(t, minimal+paymentsHeader+` [[payments.rules]] diff --git a/internal/gate/decision.go b/internal/gate/decision.go index 1054763..5c4ec13 100644 --- a/internal/gate/decision.go +++ b/internal/gate/decision.go @@ -17,6 +17,9 @@ const ( decisionBypassCrawler decisionCrawlerVerificationUnavailable decisionCrawlerUnverified + decisionBypassHosted + decisionHostedRefusal + decisionHostedUnverified decisionPaymentRequired decisionPayMethodRefused decisionPayUpgradeRefused @@ -60,6 +63,9 @@ var decisionInfos = [decisionCount]decisionInfo{ decisionBypassCrawler: {name: "bypass-crawler", flags: decisionUpstream}, decisionCrawlerVerificationUnavailable: {name: "crawler-verification-unavailable"}, decisionCrawlerUnverified: {name: "crawler-unverified", flags: decisionWalled}, + decisionBypassHosted: {name: "bypass-hosted", flags: decisionUpstream}, + decisionHostedRefusal: {name: "hosted-refusal", flags: decisionWalled}, + decisionHostedUnverified: {name: "hosted-unverified", flags: decisionWalled}, decisionPaymentRequired: {name: "payment-required", flags: decisionWalled}, decisionPayMethodRefused: {name: "pay-method-refused"}, decisionPayUpgradeRefused: {name: "pay-upgrade-refused"}, diff --git a/internal/gate/gate.go b/internal/gate/gate.go index be3cf28..7d73cb0 100644 --- a/internal/gate/gate.go +++ b/internal/gate/gate.go @@ -28,6 +28,7 @@ import ( "github.com/radiustechsystems/anteroom/internal/challenge" "github.com/radiustechsystems/anteroom/internal/config" "github.com/radiustechsystems/anteroom/internal/crawler" + "github.com/radiustechsystems/anteroom/internal/hosted" "github.com/radiustechsystems/anteroom/internal/metrics" "github.com/radiustechsystems/anteroom/internal/payment" "github.com/radiustechsystems/anteroom/internal/token" @@ -57,6 +58,7 @@ type Gate struct { now func() time.Time met *gateMetrics crawlers crawlerVerifier + hosted hostedVerifier // The challenge-activity log for external ban tooling. Nil when the // [activity] section is unconfigured — every Record call no-ops on nil, @@ -146,6 +148,10 @@ func New(cfg *config.Config, lg *slog.Logger) (*Gate, error) { return nil, err } crawlers.RegisterMetrics(met.registry) + hostedFetchers, err := hosted.New() + if err != nil { + return nil, err + } g := &Gate{ cfg: cfg, lg: lg, @@ -161,6 +167,7 @@ func New(cfg *config.Config, lg *slog.Logger) (*Gate, error) { now: time.Now, met: met, crawlers: crawlers, + hosted: hostedFetchers, } g.solverJS, g.solverURL = buildSolver(cfg.AllowInsecureContext) if cfg.Payments != nil { diff --git a/internal/gate/gate_test.go b/internal/gate/gate_test.go index 78eaf24..290db97 100644 --- a/internal/gate/gate_test.go +++ b/internal/gate/gate_test.go @@ -22,6 +22,7 @@ import ( "github.com/radiustechsystems/anteroom/internal/challenge" "github.com/radiustechsystems/anteroom/internal/config" "github.com/radiustechsystems/anteroom/internal/crawler" + "github.com/radiustechsystems/anteroom/internal/hosted" "github.com/radiustechsystems/anteroom/internal/payment" "github.com/radiustechsystems/anteroom/internal/token" ) @@ -42,6 +43,17 @@ func (v *testCrawlerVerifier) Claim(userAgent string) string { } return "" } + +type testHostedVerifier struct { + verified netip.Addr + calls int +} + +func (v *testHostedVerifier) Verify(_ hosted.Provider, addr netip.Addr) bool { + v.calls++ + return addr == v.verified +} + func (v *testCrawlerVerifier) Verify(_ context.Context, _ string, addr netip.Addr) crawler.Verdict { v.calls++ if v.verdict != 0 { @@ -755,6 +767,82 @@ verified_crawlers = ["googlebot"] } } +func TestHostedFetcherRequiresPublishedSource(t *testing.T) { + g, _ := newTestGate(t, fastCfg) + verifier := &testHostedVerifier{verified: netip.MustParseAddr("192.0.2.2")} + g.hosted = verifier + + for _, ua := range []string{ + "Mozilla/5.0 (compatible; Claude-User/1.0; +claude-user@anthropic.com)", + "Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)", + "Mozilla/5.0 (compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent)", + } { + r := agentReq("/article") + r.RemoteAddr = "192.0.2.2:1234" + r.Header.Set("User-Agent", ua) + w := do(g, r) + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "UPSTREAM:") { + t.Errorf("verified %q was not proxied: %d %q", ua, w.Code, w.Body.String()) + } + } + if verifier.calls != 3 { + t.Fatalf("verified fetchers caused %d verifier calls, want 3", verifier.calls) + } + + spoof := agentReq("/article") + spoof.RemoteAddr = "192.0.2.3:1234" + spoof.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Claude-User/1.0; +claude-user@anthropic.com)") + w := do(g, spoof) + if w.Code != http.StatusForbidden || w.Header().Get(actionHeader) != "challenge" { + t.Fatalf("unverified hosted fetcher response = %d, marker %q", w.Code, w.Header().Get(actionHeader)) + } +} + +func TestHostedFetcherBypassCanBeDisabled(t *testing.T) { + g, _ := newTestGate(t, fastCfg+"[triage]\nallow_hosted_fetchers = false\n") + g.hosted = &testHostedVerifier{verified: netip.MustParseAddr("192.0.2.2")} + r := agentReq("/article") + r.RemoteAddr = "192.0.2.2:1234" + r.Header.Set("User-Agent", "Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)") + if w := do(g, r); w.Code != http.StatusForbidden || strings.Contains(w.Body.String(), "UPSTREAM:") { + t.Fatalf("disabled hosted bypass returned %d %q", w.Code, w.Body.String()) + } +} + +func TestHostedFetcherWithUnresolvedClientUsesTheOrdinaryLadder(t *testing.T) { + g, _ := newTestGate(t, fastCfg) + g.hosted = &testHostedVerifier{} + var logs bytes.Buffer + g.lg = slog.New(slog.NewTextHandler(&logs, nil)) + r := agentReq("/article") + r.RemoteAddr = "not-an-address" + r.Header.Set("User-Agent", "Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)") + for range 2 { + w := do(g, r) + if w.Code != http.StatusForbidden || w.Header().Get(actionHeader) != "challenge" { + t.Fatalf("unresolved hosted fetcher = %d, marker %q", w.Code, w.Header().Get(actionHeader)) + } + } + if got := strings.Count(logs.String(), "machine identity verification skipped"); got != 1 { + t.Fatalf("warning count = %d, want one; logs: %s", got, logs.String()) + } +} + +func TestCrawlerClaimPrecedesHostedClaim(t *testing.T) { + g, _ := newTestGate(t, fastCfg+"\n[bypass]\nverified_crawlers = [\"googlebot\"]\n") + crawlers := &testCrawlerVerifier{verified: netip.MustParseAddr("192.0.2.2")} + hostedVerifier := &testHostedVerifier{verified: netip.MustParseAddr("192.0.2.2")} + g.crawlers = crawlers + g.hosted = hostedVerifier + r := agentReq("/article") + r.RemoteAddr = "192.0.2.2:1234" + r.Header.Set("User-Agent", "Googlebot/2.1 Google-Agent;") + w := do(g, r) + if w.Code != http.StatusOK || crawlers.calls != 1 || hostedVerifier.calls != 0 { + t.Fatalf("crawler calls = %d, hosted calls = %d, status = %d", crawlers.calls, hostedVerifier.calls, w.Code) + } +} + func TestPrefersMarkdown(t *testing.T) { for _, tc := range []struct { accept string @@ -1750,14 +1838,6 @@ func TestPublicHostsPrecedeEveryAdmissionPath(t *testing.T) { } } -func TestRefusalCarriesChallengeMarker(t *testing.T) { - g, _ := newTestGate(t, fastCfg) - w := do(g, agentReq("/x")) - if got := w.Header().Get(actionHeader); got != "challenge" { - t.Errorf("%s = %q, want challenge", actionHeader, got) - } -} - func TestVendorClientIPHeadersStripped(t *testing.T) { var seen http.Header up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/gate/identity.go b/internal/gate/identity.go index f4cdeca..ca8d21e 100644 --- a/internal/gate/identity.go +++ b/internal/gate/identity.go @@ -6,6 +6,7 @@ import ( "net/netip" "github.com/radiustechsystems/anteroom/internal/crawler" + "github.com/radiustechsystems/anteroom/internal/hosted" ) type crawlerVerifier interface { @@ -13,31 +14,50 @@ type crawlerVerifier interface { Verify(context.Context, string, netip.Addr) crawler.Verdict } +type hostedVerifier interface { + Verify(hosted.Provider, netip.Addr) bool +} + // serveClaimedIdentity handles requests that claim a machine identity whose // protocol is known not to be x402. A claim chooses verification and response // shape; only an authenticated source can bypass the gate. func (g *Gate) serveClaimedIdentity(w http.ResponseWriter, r *gateRequest) (decision, bool) { - claim := r.facts.crawlerClaim - if claim == "" { - return 0, false - } - if !r.clientIP.IsValid() { + if !r.clientIP.IsValid() && (r.facts.crawlerClaim != "" || r.facts.hostedClaim != hosted.None) { // Verification cannot run without a resolved peer. Fall back to the - // ordinary ladder rather than telling a real crawler the site is + // ordinary ladder rather than telling a real provider the site is // temporarily unavailable forever because of local proxy config. g.warnIdentityIP(r.Request) return 0, false } - switch g.crawlers.Verify(r.Context(), claim, r.clientIP) { - case crawler.Verified: - r.Header.Set("X-Anteroom-Status", "bypass-crawler-"+claim) - g.forward(w, r.Request) - return decisionBypassCrawler, true - case crawler.Indeterminate: - serveCrawlerVerificationUnavailable(w) - return decisionCrawlerVerificationUnavailable, true - default: + + if claim := r.facts.crawlerClaim; claim != "" { + switch g.crawlers.Verify(r.Context(), claim, r.clientIP) { + case crawler.Verified: + r.Header.Set("X-Anteroom-Status", "bypass-crawler-"+claim) + g.forward(w, r.Request) + return decisionBypassCrawler, true + case crawler.Indeterminate: + serveCrawlerVerificationUnavailable(w) + return decisionCrawlerVerificationUnavailable, true + default: + g.serveStrictRefusal(w, r.Request) + return decisionCrawlerUnverified, true + } + } + + if claim := r.facts.hostedClaim; claim != hosted.None { + verified := g.hosted.Verify(claim, r.clientIP) + if verified && g.cfg.Triage.AllowHostedFetchers { + r.Header.Set("X-Anteroom-Status", "bypass-hosted-"+claim.String()) + g.forward(w, r.Request) + return decisionBypassHosted, true + } g.serveStrictRefusal(w, r.Request) - return decisionCrawlerUnverified, true + if verified { + return decisionHostedRefusal, true + } + return decisionHostedUnverified, true } + + return 0, false } diff --git a/internal/gate/pay_test.go b/internal/gate/pay_test.go index f4787b0..368a521 100644 --- a/internal/gate/pay_test.go +++ b/internal/gate/pay_test.go @@ -192,6 +192,29 @@ verified_crawlers = ["googlebot"] } } +func TestHostedFetcherTriagePrecedesPayment(t *testing.T) { + g, _ := payGate(t, oneRule, nil) + g.hosted = &testHostedVerifier{verified: netip.MustParseAddr("192.0.2.2")} + + web := agentReq("/report") + web.RemoteAddr = "192.0.2.2:1234" + web.Header.Set("User-Agent", "Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)") + w := do(g, web) + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "UPSTREAM:") { + t.Fatalf("verified hosted fetcher did not bypass paid route: %d %q", w.Code, w.Body.String()) + } + if got := w.Header().Get(payment.HeaderRequired); got != "" { + t.Fatalf("verified hosted fetcher received a payment offer: %q", got) + } + + cli := agentReq("/report") + cli.Header.Set("User-Agent", "Claude-User (claude-code/2.1.250; +https://support.anthropic.com/)") + w = do(g, cli) + if got := w.Header().Get(payment.HeaderRequired); got == "" { + t.Fatalf("Claude Code did not receive an x402 offer: status %d body %q", w.Code, w.Body.String()) + } +} + // facFake is one fake facilitator with per-endpoint counters, for tests that // need to know WHICH facilitator was called. type facFake struct { diff --git a/internal/gate/request.go b/internal/gate/request.go index 0da1db5..53a377b 100644 --- a/internal/gate/request.go +++ b/internal/gate/request.go @@ -5,6 +5,8 @@ import ( "net/netip" "strconv" "strings" + + "github.com/radiustechsystems/anteroom/internal/hosted" ) // requestFacts is the immutable, header-derived view of a request. Parsing it @@ -15,6 +17,7 @@ type requestFacts struct { navigation bool preflight bool crawlerClaim string + hostedClaim hosted.Provider } // gateRequest adds facts that need Gate state to resolve. Embedding the ordinary @@ -53,6 +56,7 @@ func inspectRequest(r *http.Request, crawlerClaim string) requestFacts { userAgent: r.Header.Get("User-Agent"), preflight: isCORSPreflight(r), crawlerClaim: crawlerClaim, + hostedClaim: hosted.Claim(r.Header.Get("User-Agent")), } facts.navigation = classifyNavigation(r, facts, accept, isFragmentRequest(r)) return facts From fc1954c239f614a53c2b03c6b58f735214e79248 Mon Sep 17 00:00:00 2001 From: madars Date: Mon, 31 Aug 2026 14:50:06 -0400 Subject: [PATCH 4/4] chore(ranges): extend the IP-range updater and freshness check to the hosted vendor feeds Co-Authored-By: Claude Fable 5 --- ...wler-ranges.yaml => published-ranges.yaml} | 4 ++-- docs/operating.md | 2 +- ...crawler-ips.py => update-published-ips.py} | 19 ++++++++++++++++++- 3 files changed, 21 insertions(+), 4 deletions(-) rename .github/workflows/{crawler-ranges.yaml => published-ranges.yaml} (67%) rename scripts/{update-crawler-ips.py => update-published-ips.py} (84%) diff --git a/.github/workflows/crawler-ranges.yaml b/.github/workflows/published-ranges.yaml similarity index 67% rename from .github/workflows/crawler-ranges.yaml rename to .github/workflows/published-ranges.yaml index eba9ec9..94003e7 100644 --- a/.github/workflows/crawler-ranges.yaml +++ b/.github/workflows/published-ranges.yaml @@ -1,4 +1,4 @@ -name: crawler ranges +name: published IP ranges on: schedule: @@ -13,4 +13,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: python3 scripts/update-crawler-ips.py --check + - run: python3 scripts/update-published-ips.py --check diff --git a/docs/operating.md b/docs/operating.md index bfa19aa..55c8461 100644 --- a/docs/operating.md +++ b/docs/operating.md @@ -297,7 +297,7 @@ being reported to the crawler as a permanent DNS outage. Claimed-but-unverified machine identities always receive a strict `403`; the `ok_body_agents` compatibility downgrade cannot turn a spoof into a 200. -Refresh the embedded snapshots with `scripts/update-crawler-ips.py`. The +Refresh the embedded snapshots with `scripts/update-published-ips.py`. The scheduled workflow reports a semantic range change for human review rather than committing generated data automatically. diff --git a/scripts/update-crawler-ips.py b/scripts/update-published-ips.py similarity index 84% rename from scripts/update-crawler-ips.py rename to scripts/update-published-ips.py index e6d021b..0521afe 100755 --- a/scripts/update-crawler-ips.py +++ b/scripts/update-published-ips.py @@ -1,5 +1,10 @@ #!/usr/bin/env python3 -"""Refresh Anteroom's embedded crawler IP ranges.""" +"""Refresh Anteroom's machine-readable machine-identity IP ranges. + +Anthropic's stable outbound 160.79.104.0/21 has no JSON feed. Its embedded +claude_outbound.json is intentionally reviewed by hand against +https://platform.claude.com/docs/en/api/ip-addresses. +""" import argparse import ipaddress @@ -23,6 +28,18 @@ "https://index.commoncrawl.org/ccbot.json", ROOT / "internal/crawler/ccbot.json", ), + "claude-user": ( + "https://claude.com/crawling/bots.json", + ROOT / "internal/hosted/claude_bots.json", + ), + "chatgpt-user": ( + "https://openai.com/chatgpt-user.json", + ROOT / "internal/hosted/chatgpt_user.json", + ), + "google-agent": ( + "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json", + ROOT / "internal/hosted/google_user_triggered_agents.json", + ), } MAX_BYTES = 1 << 20