feat(gate): triage verified hosted fetchers

.github/workflows/crawler-ranges.yaml → .github/workflows/published-ranges.yaml

@@ -1,4 +1,4 @@
-name: crawler ranges
+name: published IP ranges
 
 on:
   schedule:
@@ -13,4 +13,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - run: python3 scripts/update-crawler-ips.py --check
+      - run: python3 scripts/update-published-ips.py --check

README.md

@@ -43,6 +43,12 @@ after a restart.
 The goal is not to tell humans from bots. It is that high-volume automated access
 is either cheap to tolerate, cheap to discourage, or paid for.
 
+One explicit exception is enabled by default: source-verified `Claude-User`,
+`ChatGPT-User`, and `Google-Agent` hosted fetchers pass through because they can
+complete neither the browser proof nor x402. This also bypasses paid routes; it
+authenticates vendor infrastructure, not a human or application user. Set
+`triage.allow_hosted_fetchers = false` to give those requests a strict `403`.
+
 ## Running it
 
 Every release attaches a static binary for Linux (x86-64, ARM64, ARMv7), macOS

anteroom.example.toml

@@ -254,11 +254,13 @@ verified_crawlers = ["googlebot", "bingbot", "yandexbot", "ccbot"]
 # Default: 402 status + PAYMENT-REQUIRED header + markdown body.
 # ---------------------------------------------------------------------------
 [triage]
-json_accept    = true             # Accept: application/json → JSON 402 body
-ok_body_agents = ["claude-user"]  # user agents of fetch tools that discard non-2xx
-                                  # bodies (measured); these receive the markdown
-                                  # with status 200, the only status they surface.
-                                  # Matched case-insensitively as a UA substring.
+json_accept           = true             # Accept: application/json → JSON 402 body
+ok_body_agents        = ["claude-user"]  # case-insensitive UA substrings for CLI
+                                         # agents that discard non-2xx bodies
+# This defaults true and bypasses proof-of-work AND x402, including paid routes,
+# for source-verified vendor-hosted fetchers. They cannot complete either
+# protocol. Set false to return a strict 403 to them instead.
+allow_hosted_fetchers = true             # Claude-User, ChatGPT-User, Google-Agent
 
 # The binary also serves /.anteroom/healthz (liveness), /.anteroom/renew.js (the
 # injected renewal script) and /.anteroom/uninstall (removes the renewal service

docs/operating.md

@@ -294,10 +294,37 @@ verification uses the same resolved client address as CIDR bypasses. If that
 address cannot be resolved, the request follows the ordinary ladder rather than
 being reported to the crawler as a permanent DNS outage.
 
-Refresh the embedded snapshots with `scripts/update-crawler-ips.py`. The
+Claimed-but-unverified machine identities always receive a strict `403`; the
+`ok_body_agents` compatibility downgrade cannot turn a spoof into a 200.
+
+Refresh the embedded snapshots with `scripts/update-published-ips.py`. The
 scheduled workflow reports a semantic range change for human review rather
 than committing generated data automatically.
 
+### Hosted user-triggered fetchers
+
+`Claude-User/`, `ChatGPT-User/`, and `Google-Agent;` identify fetches made on a
+user's behalf by vendor-hosted tools. They are not command-line agents and
+cannot complete proof of work or x402. Anteroom checks the advertised
+case-sensitive User-Agent token, then requires the source address to appear in
+that vendor's embedded published ranges. A verified request passes through by
+default, including on x402-paid routes; an unverified claim receives a
+machine-readable `403`, never a payment offer. This authenticates vendor
+infrastructure, not application authorization.
+
+If the client address cannot be resolved, Anteroom warns once and sends the
+request through the ordinary ladder rather than labeling it a spoof or telling
+the vendor the site is temporarily unavailable forever.
+
+Claude uses the union of `claude.com/crawling/bots.json` and Anthropic's stable
+outbound `160.79.104.0/21`. This authenticates Anthropic infrastructure, not a
+human or a particular Claude product. Claude Code's different
+`Claude-User (claude-code/...)` form remains on the ordinary agent path and can
+receive x402. `triage.allow_hosted_fetchers` defaults to `true` because these
+hosted clients cannot complete either PoW or x402. Set it to `false` to remove
+the free exception: verified hosted fetchers then receive a strict `403` rather
+than falling through to a payment offer they cannot use.
+
 Percent-encoding is *not* restricted: `/repos/owner%2Frepo`, `/file%20name.txt`,
 and friends pass through untouched. Only paths whose decoded form is
 non-canonical — dot-segments (`..`, `.`), doubled slashes, or a backslash — are
@@ -436,7 +463,9 @@ keeps settlement retry separate from replaying an upstream mutation.
 
 `anteroom -v` logs one line per request naming the rung of the ladder that
 answered it — `own-endpoint`, `bypass-path`, `bypass-ip`, `bypass-crawler`,
-`crawler-verification-unavailable`, `crawler-unverified`, `pass-pow`, `pass-paid`,
+`crawler-verification-unavailable`, `crawler-unverified`, `bypass-hosted`,
+`hosted-refusal`, `hosted-unverified`,
+`pass-pow`, `pass-paid`,
 `wait-page`, `refusal`, `non-canonical-path` — with the status, response size, and
 duration:
 

internal/config/config.go

@@ -255,8 +255,9 @@ func (a *Activity) validate() error {
 }
 
 type Triage struct {
-	JSONAccept   bool     `toml:"json_accept"`
-	OKBodyAgents []string `toml:"ok_body_agents"`
+	JSONAccept          bool     `toml:"json_accept"`
+	OKBodyAgents        []string `toml:"ok_body_agents"`
+	AllowHostedFetchers bool     `toml:"allow_hosted_fetchers"`
 }
 
 // defaults returns a Config carrying every default the contract documents.
@@ -284,12 +285,12 @@ func defaults() Config {
 		Inject:          true,
 		Triage: Triage{
 			JSONAccept: true,
-			// Some agentic fetch tools discard the body of any non-2xx
-			// response, so a 401 carrying instructions is invisible to them —
-			// measured behavior, not a guess. For those clients the
-			// instructions are served with status 200 instead, which is the
-			// only way they see anything at all. Matched case-insensitively as
-			// a substring of the User-Agent.
+			// Verified vendor-hosted user fetchers cannot complete PoW or x402,
+			// so the current policy admits them even on paid routes. Operators
+			// can disable that exception and refuse them instead.
+			AllowHostedFetchers: true,
+			// Claude Code's fetch output omits non-2xx bodies. Matching its
+			// User-Agent here keeps Anteroom's instructions visible.
 			OKBodyAgents: []string{"claude-user"},
 		},
 	}

internal/config/config_test.go

@@ -48,8 +48,8 @@ func TestLoadMinimal(t *testing.T) {
 	if cfg.Difficulty != 14 || cfg.RenewDifficulty != 6 {
 		t.Errorf("difficulty defaults wrong: %+v", cfg)
 	}
-	if !cfg.Inject || !cfg.Triage.JSONAccept {
-		t.Error("inject/json_accept should default true")
+	if !cfg.Inject || !cfg.Triage.JSONAccept || !cfg.Triage.AllowHostedFetchers {
+		t.Error("inject, json_accept, and allow_hosted_fetchers should default true")
 	}
 	// No admin listener unless asked for: it is unauthenticated, so silently
 	// opening a port the operator never configured would be a surprise surface.
@@ -103,6 +103,16 @@ verified_crawlers = ["googlebot"]
 	}
 }
 
+func TestHostedFetcherBypassCanBeDisabled(t *testing.T) {
+	cfg, err := Load(write(t, minimal+"\n[triage]\nallow_hosted_fetchers = false\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cfg.Triage.AllowHostedFetchers {
+		t.Fatal("allow_hosted_fetchers remained enabled")
+	}
+}
+
 func TestLoadFullPayments(t *testing.T) {
 	cfg, err := Load(write(t, minimal+paymentsHeader+`
 [[payments.rules]]

internal/gate/decision.go

@@ -17,6 +17,9 @@ const (
 	decisionBypassCrawler
 	decisionCrawlerVerificationUnavailable
 	decisionCrawlerUnverified
+	decisionBypassHosted
+	decisionHostedRefusal
+	decisionHostedUnverified
 	decisionPaymentRequired
 	decisionPayMethodRefused
 	decisionPayUpgradeRefused
@@ -60,6 +63,9 @@ var decisionInfos = [decisionCount]decisionInfo{
 	decisionBypassCrawler:                  {name: "bypass-crawler", flags: decisionUpstream},
 	decisionCrawlerVerificationUnavailable: {name: "crawler-verification-unavailable"},
 	decisionCrawlerUnverified:              {name: "crawler-unverified", flags: decisionWalled},
+	decisionBypassHosted:                   {name: "bypass-hosted", flags: decisionUpstream},
+	decisionHostedRefusal:                  {name: "hosted-refusal", flags: decisionWalled},
+	decisionHostedUnverified:               {name: "hosted-unverified", flags: decisionWalled},
 	decisionPaymentRequired:                {name: "payment-required", flags: decisionWalled},
 	decisionPayMethodRefused:               {name: "pay-method-refused"},
 	decisionPayUpgradeRefused:              {name: "pay-upgrade-refused"},

internal/gate/gate.go

@@ -28,6 +28,7 @@ import (
 	"github.com/radiustechsystems/anteroom/internal/challenge"
 	"github.com/radiustechsystems/anteroom/internal/config"
 	"github.com/radiustechsystems/anteroom/internal/crawler"
+	"github.com/radiustechsystems/anteroom/internal/hosted"
 	"github.com/radiustechsystems/anteroom/internal/metrics"
 	"github.com/radiustechsystems/anteroom/internal/payment"
 	"github.com/radiustechsystems/anteroom/internal/token"
@@ -57,6 +58,7 @@ type Gate struct {
 	now         func() time.Time
 	met         *gateMetrics
 	crawlers    crawlerVerifier
+	hosted      hostedVerifier
 
 	// The challenge-activity log for external ban tooling. Nil when the
 	// [activity] section is unconfigured — every Record call no-ops on nil,
@@ -146,6 +148,10 @@ func New(cfg *config.Config, lg *slog.Logger) (*Gate, error) {
 		return nil, err
 	}
 	crawlers.RegisterMetrics(met.registry)
+	hostedFetchers, err := hosted.New()
+	if err != nil {
+		return nil, err
+	}
 	g := &Gate{
 		cfg:         cfg,
 		lg:          lg,
@@ -161,6 +167,7 @@ func New(cfg *config.Config, lg *slog.Logger) (*Gate, error) {
 		now:         time.Now,
 		met:         met,
 		crawlers:    crawlers,
+		hosted:      hostedFetchers,
 	}
 	g.solverJS, g.solverURL = buildSolver(cfg.AllowInsecureContext)
 	if cfg.Payments != nil {

internal/gate/gate_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/radiustechsystems/anteroom/internal/challenge"
 	"github.com/radiustechsystems/anteroom/internal/config"
 	"github.com/radiustechsystems/anteroom/internal/crawler"
+	"github.com/radiustechsystems/anteroom/internal/hosted"
 	"github.com/radiustechsystems/anteroom/internal/payment"
 	"github.com/radiustechsystems/anteroom/internal/token"
 )
@@ -42,6 +43,17 @@ func (v *testCrawlerVerifier) Claim(userAgent string) string {
 	}
 	return ""
 }
+
+type testHostedVerifier struct {
+	verified netip.Addr
+	calls    int
+}
+
+func (v *testHostedVerifier) Verify(_ hosted.Provider, addr netip.Addr) bool {
+	v.calls++
+	return addr == v.verified
+}
+
 func (v *testCrawlerVerifier) Verify(_ context.Context, _ string, addr netip.Addr) crawler.Verdict {
 	v.calls++
 	if v.verdict != 0 {
@@ -755,6 +767,82 @@ verified_crawlers = ["googlebot"]
 	}
 }
 
+func TestHostedFetcherRequiresPublishedSource(t *testing.T) {
+	g, _ := newTestGate(t, fastCfg)
+	verifier := &testHostedVerifier{verified: netip.MustParseAddr("192.0.2.2")}
+	g.hosted = verifier
+
+	for _, ua := range []string{
+		"Mozilla/5.0 (compatible; Claude-User/1.0; +claude-user@anthropic.com)",
+		"Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)",
+		"Mozilla/5.0 (compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent)",
+	} {
+		r := agentReq("/article")
+		r.RemoteAddr = "192.0.2.2:1234"
+		r.Header.Set("User-Agent", ua)
+		w := do(g, r)
+		if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "UPSTREAM:") {
+			t.Errorf("verified %q was not proxied: %d %q", ua, w.Code, w.Body.String())
+		}
+	}
+	if verifier.calls != 3 {
+		t.Fatalf("verified fetchers caused %d verifier calls, want 3", verifier.calls)
+	}
+
+	spoof := agentReq("/article")
+	spoof.RemoteAddr = "192.0.2.3:1234"
+	spoof.Header.Set("User-Agent", "Mozilla/5.0 (compatible; Claude-User/1.0; +claude-user@anthropic.com)")
+	w := do(g, spoof)
+	if w.Code != http.StatusForbidden || w.Header().Get(actionHeader) != "challenge" {
+		t.Fatalf("unverified hosted fetcher response = %d, marker %q", w.Code, w.Header().Get(actionHeader))
+	}
+}
+
+func TestHostedFetcherBypassCanBeDisabled(t *testing.T) {
+	g, _ := newTestGate(t, fastCfg+"[triage]\nallow_hosted_fetchers = false\n")
+	g.hosted = &testHostedVerifier{verified: netip.MustParseAddr("192.0.2.2")}
+	r := agentReq("/article")
+	r.RemoteAddr = "192.0.2.2:1234"
+	r.Header.Set("User-Agent", "Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)")
+	if w := do(g, r); w.Code != http.StatusForbidden || strings.Contains(w.Body.String(), "UPSTREAM:") {
+		t.Fatalf("disabled hosted bypass returned %d %q", w.Code, w.Body.String())
+	}
+}
+
+func TestHostedFetcherWithUnresolvedClientUsesTheOrdinaryLadder(t *testing.T) {
+	g, _ := newTestGate(t, fastCfg)
+	g.hosted = &testHostedVerifier{}
+	var logs bytes.Buffer
+	g.lg = slog.New(slog.NewTextHandler(&logs, nil))
+	r := agentReq("/article")
+	r.RemoteAddr = "not-an-address"
+	r.Header.Set("User-Agent", "Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)")
+	for range 2 {
+		w := do(g, r)
+		if w.Code != http.StatusForbidden || w.Header().Get(actionHeader) != "challenge" {
+			t.Fatalf("unresolved hosted fetcher = %d, marker %q", w.Code, w.Header().Get(actionHeader))
+		}
+	}
+	if got := strings.Count(logs.String(), "machine identity verification skipped"); got != 1 {
+		t.Fatalf("warning count = %d, want one; logs: %s", got, logs.String())
+	}
+}
+
+func TestCrawlerClaimPrecedesHostedClaim(t *testing.T) {
+	g, _ := newTestGate(t, fastCfg+"\n[bypass]\nverified_crawlers = [\"googlebot\"]\n")
+	crawlers := &testCrawlerVerifier{verified: netip.MustParseAddr("192.0.2.2")}
+	hostedVerifier := &testHostedVerifier{verified: netip.MustParseAddr("192.0.2.2")}
+	g.crawlers = crawlers
+	g.hosted = hostedVerifier
+	r := agentReq("/article")
+	r.RemoteAddr = "192.0.2.2:1234"
+	r.Header.Set("User-Agent", "Googlebot/2.1 Google-Agent;")
+	w := do(g, r)
+	if w.Code != http.StatusOK || crawlers.calls != 1 || hostedVerifier.calls != 0 {
+		t.Fatalf("crawler calls = %d, hosted calls = %d, status = %d", crawlers.calls, hostedVerifier.calls, w.Code)
+	}
+}
+
 func TestPrefersMarkdown(t *testing.T) {
 	for _, tc := range []struct {
 		accept string
@@ -1750,14 +1838,6 @@ func TestPublicHostsPrecedeEveryAdmissionPath(t *testing.T) {
 	}
 }
 
-func TestRefusalCarriesChallengeMarker(t *testing.T) {
-	g, _ := newTestGate(t, fastCfg)
-	w := do(g, agentReq("/x"))
-	if got := w.Header().Get(actionHeader); got != "challenge" {
-		t.Errorf("%s = %q, want challenge", actionHeader, got)
-	}
-}
-
 func TestVendorClientIPHeadersStripped(t *testing.T) {
 	var seen http.Header
 	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {