lsb11 · Oct 10, 2026 · Oct 10, 2026
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -118,6 +118,30 @@ now feeds an "Every guide and tool" section on the homepage and the whole of
 `tests/site-index.test.js` holds it equal to the built sitemap. **A new
 indexable page must be added to `siteIndex.ts`, or `npm test` fails.**
 
+**Cookieless page and crawler counts (10 Oct 2026).** GA4 sees only visitors
+who press Accept, and nothing recorded whether Googlebot, Bingbot or the AI
+crawlers fetched any page. `functions/_middleware.js` now adds one to a daily
+D1 counter (`pageviews`, `schema/006-pageviews.sql`, apply it once) per page,
+status, client kind, referring host and country, inside `waitUntil` and never
+throwing. No cookie, IP or User-Agent is stored, so it needs no consent;
+`/privacy/` says so. `npm run traffic` prints: who fetched pages, which pages
+each search/AI crawler fetched and when it was last seen, human referrers
+(AI assistants flagged), views vs affiliate clicks per page, and crawlers
+still requesting retired URLs. Crawler names are claimed, not verified.
+
+**`npm run cf:audit` reads what the dashboard hides.** Without a token it
+fetches pages as each crawler; with `CF_API_TOKEN` (read-only) it also reads
+Bot Fight Mode, "Block AI bots", managed robots.txt, security level, Crawler
+Hints, WAF/redirect/cache/transform rules and Page Rules, and ranks findings.
+Run it before concluding anything about crawling: two of this site's worst
+faults (a zone redirect rule, Crawler Hints) lived only in the dashboard.
+
+**Content is not duplicated (measured 10 Oct 2026).** Six-word shingles over
+`<main>` across the 51 sitemap pages: median 5% of a page's shingles appear
+on 3+ other pages, the highest pair overlap is 16% (two calculators), and the
+only sitewide repeat is the author box. Do not rewrite pages to fix
+"duplication"; there is none to fix.
+
 **Build-time date branches need a scheduled build.** `isPostShutdown()` in
 `src/utils/stockyDeadline.ts` resolves at build time — six files branch on it: `Nav.astro` plus
 five page files (`index`, `pro`, `pro/[slug]` — four routes — `stocky-swap`

diff --git a/functions/_middleware.js b/functions/_middleware.js
@@ -46,11 +46,37 @@
 // scripts/gen-legacy-redirects.mjs; tests/legacy-redirects.test.js fails if the
 // two drift.
 import { resolveLegacy } from './_legacy-redirects.js';
+import { rowFor, recordPageview } from './_pageviews.js';
 
 const PRIMARY_HOST = 'stackarchitect.xyz';
 const FILE_RE = /\.[a-zA-Z0-9]{2,5}$/;
 
+// Every response, redirect or page, is counted on the way out: one daily
+// counter per page and kind of client, no cookie, no IP, no User-Agent stored.
+// Why, and what is never kept: schema/006-pageviews.sql. The count runs in
+// waitUntil after the response is decided, and recordPageview never throws, so
+// a missing table or binding costs a statistic and never a page.
 export async function onRequest(context) {
+  const response = await route(context);
+  try {
+    const req = context.request;
+    const row = rowFor({
+      url: req.url,
+      method: req.method,
+      status: response.status,
+      contentType: response.headers.get('content-type'),
+      userAgent: req.headers.get('user-agent'),
+      referer: req.headers.get('referer'),
+      country: req.cf?.country,
+    });
+    if (row && typeof context.waitUntil === 'function') context.waitUntil(recordPageview(context.env?.DB, row));
+  } catch {
+    // Counting must never cost the response.
+  }
+  return response;
+}
+
+async function route(context) {
   const url = new URL(context.request.url);
   let changed = false;
 

diff --git a/functions/_pageviews.js b/functions/_pageviews.js
@@ -0,0 +1,109 @@
+/**
+ * _pageviews.js — cookieless daily counts of who fetched which page.
+ *
+ * Underscore-prefixed, so Pages does not route it. Called from
+ * functions/_middleware.js inside waitUntil, after the response is decided,
+ * so it can never slow down or break a page. Why it exists and what is never
+ * stored: schema/006-pageviews.sql.
+ */
+
+// Crawlers that matter to this site, named. First match wins, so the more
+// specific tokens come before the ones they contain.
+const NAMED = [
+  ['Google-InspectionTool', /Google-InspectionTool/i],
+  ['GoogleOther', /GoogleOther/i],
+  ['Storebot-Google', /Storebot-Google/i],
+  ['Googlebot', /Googlebot/i],
+  ['Bingbot', /bingbot|BingPreview/i],
+  ['OAI-SearchBot', /OAI-SearchBot/i],
+  ['ChatGPT-User', /ChatGPT-User/i],
+  ['GPTBot', /GPTBot/i],
+  ['Claude-SearchBot', /Claude-SearchBot/i],
+  ['Claude-User', /Claude-User/i],
+  ['ClaudeBot', /ClaudeBot|anthropic-ai/i],
+  ['Perplexity-User', /Perplexity-User/i],
+  ['PerplexityBot', /PerplexityBot/i],
+  ['Applebot', /Applebot/i],
+  ['DuckDuckBot', /DuckDuckBot|DuckAssistBot/i],
+  ['meta-externalagent', /meta-external/i],
+  ['Bytespider', /Bytespider/i],
+  ['AhrefsBot', /AhrefsBot/i],
+  ['SemrushBot', /SemrushBot/i],
+  ['YandexBot', /YandexBot/i],
+];
+const OTHER_BOT = /bot\b|bot\/|crawl|spider|slurp|curl\/|wget|python-|go-http|headless|facebookexternalhit|preview/i;
+
+/** Name the client, or bucket it. Only this label is stored, never the UA. */
+export function agentOf(userAgent) {
+  const ua = typeof userAgent === 'string' ? userAgent.trim() : '';
+  if (!ua) return 'no-ua';
+  for (const [name, re] of NAMED) if (re.test(ua)) return name;
+  if (OTHER_BOT.test(ua)) return 'other-bot';
+  return /^Mozilla\/5\.0 /.test(ua) ? 'human' : 'other';
+}
+
+/** The referring site's host only: never the path, which can carry anything. */
+export function refOf(referer, selfHost = 'stackarchitect.xyz') {
+  if (!referer) return '(none)';
+  try {
+    const h = new URL(referer).hostname.toLowerCase().replace(/^www\./, '');
+    if (h === selfHost) return '(internal)';
+    return h.slice(0, 80) || '(none)';
+  } catch {
+    return '(none)';
+  }
+}
+
+const KEEP_REDIRECT_PATHS = new Set(['Googlebot', 'Bingbot', 'GPTBot', 'OAI-SearchBot', 'ClaudeBot', 'PerplexityBot', 'Applebot']);
+
+/**
+ * Which row this request belongs in, or null when it is not worth counting.
+ * Pages and their redirects are counted; assets, API calls and /go/ (which has
+ * its own counter) are not. A 404 is bucketed, because a probe for
+ * /wp-login.php should not add a row per guess. A redirect keeps its path only
+ * for the named search and AI crawlers: whether Google still crawls a retired
+ * URL is worth knowing, and every other client's would be noise.
+ */
+export function rowFor({ url, method, status, contentType, userAgent, referer, country, day }) {
+  if (method !== 'GET') return null;
+  const u = new URL(url);
+  const p = u.pathname;
+  if (p.startsWith('/api/') || p.startsWith('/go/') || p.startsWith('/_astro/') || p.startsWith('/cdn-cgi/')) return null;
+  const agent = agentOf(userAgent);
+  const isFile = /\.[a-z0-9]{2,5}$/i.test(p);
+  let path;
+  if (status === 200) {
+    const wanted = !isFile ? /text\/html/i.test(contentType || '') : /^\/(llms(-full)?\.txt|robots\.txt|sitemap[^/]*\.xml)$/.test(p);
+    if (!wanted) return null;
+    path = p.slice(0, 160);
+  } else if (status >= 300 && status < 400) {
+    path = KEEP_REDIRECT_PATHS.has(agent) ? p.slice(0, 160) : '(redirect)';
+  } else if (status === 404) {
+    path = '(404)';
+  } else {
+    path = '(other)';
+  }
+  return {
+    day: day ?? new Date().toISOString().slice(0, 10),
+    path,
+    status: String(status),
+    agent,
+    ref: refOf(referer, u.hostname.replace(/^www\./, '')),
+    country: /^[A-Z]{2}$/.test(country || '') ? country : 'XX',
+  };
+}
+
+const SQL = `INSERT INTO pageviews (day, path, status, agent, ref, country, n)
+VALUES (?, ?, ?, ?, ?, ?, 1)
+ON CONFLICT (day, path, status, agent, ref, country) DO UPDATE SET n = n + 1`;
+
+/** Count one request. Never throws; a missing table or binding costs a count, never a page. */
+export async function recordPageview(db, row) {
+  if (!row || !db || typeof db.prepare !== 'function') return false;
+  try {
+    await db.prepare(SQL).bind(row.day, row.path, row.status, row.agent, row.ref, row.country).run();
+    return true;
+  } catch {
+    return false;
+  }
+}
diff --git a/package.json b/package.json
@@ -30,7 +30,9 @@
     "aff:check": "node scripts/aff-check.mjs",
     "syndication:build": "node scripts/syndication-build.mjs",
     "syndication:sync": "node scripts/syndication-sync.mjs",
-    "edge:check": "node scripts/edge-freshness.mjs"
+    "edge:check": "node scripts/edge-freshness.mjs",
+    "traffic": "node scripts/traffic-report.mjs",
+    "cf:audit": "node scripts/cf-audit.mjs"
   },
   "dependencies": {
     "@astrojs/cloudflare": "^13.1.7",

diff --git a/schema/006-pageviews.sql b/schema/006-pageviews.sql
@@ -0,0 +1,38 @@
+-- Migration 006 — cookieless page view and crawler counts
+--
+-- Apply once:
+--
+--   npx wrangler d1 execute attribution-gap --remote --file schema/006-pageviews.sql
+--
+-- Safe to re-run: everything here is IF NOT EXISTS.
+--
+-- WHY (10 Oct 2026)
+-- GA4 only sees visitors who press Accept, so the site had no count of its own
+-- traffic, and no record at all of whether Googlebot, Bingbot or the AI
+-- crawlers were fetching its pages while both engines showed one page indexed.
+-- functions/_middleware.js already runs on every request; it now adds one to a
+-- daily counter per (page, kind of client, referring site, country).
+--
+-- WHAT IS NOT STORED
+-- No cookie, no IP address, no User-Agent string, no time finer than the day,
+-- no identifier of any kind. `agent` is a named crawler or a coarse bucket,
+-- decided from the User-Agent and then discarded. Nothing is read from or
+-- written to the visitor's device, so this needs no consent, the same as the
+-- /go/ click counts (see /privacy/).
+--
+-- `agent` names a crawler by what it CLAIMS to be. A scraper can send
+-- "Googlebot"; Googlebot's own figures are in Search Console → Settings →
+-- Crawl stats. Read crawler rows as "something calling itself X".
+
+CREATE TABLE IF NOT EXISTS pageviews (
+  day     TEXT    NOT NULL,          -- UTC day, yyyy-mm-dd
+  path    TEXT    NOT NULL,          -- page path, or '(404)' / '(other)'
+  status  TEXT    NOT NULL,          -- '200', '301', '404', ...
+  agent   TEXT    NOT NULL,          -- 'human', 'Googlebot', 'GPTBot', 'other-bot', ...
+  ref     TEXT    NOT NULL,          -- referring host, '(none)' or '(internal)'
+  country TEXT    NOT NULL,          -- two-letter code from Cloudflare, or 'XX'
+  n       INTEGER NOT NULL DEFAULT 0,
+  PRIMARY KEY (day, path, status, agent, ref, country)
+);
+
+CREATE INDEX IF NOT EXISTS pageviews_agent ON pageviews (agent, day);
diff --git a/scripts/cf-audit.mjs b/scripts/cf-audit.mjs
@@ -0,0 +1,125 @@
+#!/usr/bin/env node
+// scripts/cf-audit.mjs — read-only check of everything outside this repo that
+// decides whether search engines and AI crawlers can reach the site.
+//
+//   node scripts/cf-audit.mjs                 # part 1 only: probe as each crawler
+//   CF_API_TOKEN=… node scripts/cf-audit.mjs  # parts 1 and 2: plus Cloudflare settings
+//
+// Token: Cloudflare dashboard → My Profile → API Tokens → Create Token →
+// template "Read all resources" → limit Zone Resources to stackarchitect.xyz.
+// Read-only: this script never changes a setting. Do not commit the token.
+//
+// WHY (10 Oct 2026)
+// Google and Bing each show one indexed page and Ahrefs Brand Radar shows no
+// AI citations. Every check inside the repo is clean, and several settings
+// that can block crawlers live only in the Cloudflare dashboard: "Block AI
+// bots", Bot Fight Mode, managed robots.txt, WAF custom rules, Cache Rules,
+// Redirect Rules. CLAUDE.md already records two of them causing harm
+// unseen (a zone trailing-slash rule; Crawler Hints sending ~55,000 IndexNow
+// URLs). This reads them all at once.
+const SITE = 'https://stackarchitect.xyz';
+const ZONE = 'stackarchitect.xyz';
+const problems = [];
+const flag = (level, msg) => { problems.push([level, msg]); };
+
+// ── Part 1: fetch as each crawler ─────────────────────────────────────────
+// A request from this machine carries the crawler's User-Agent but not its IP,
+// so a block of *spoofed* Googlebot/Bingbot is expected and harmless (Cloudflare
+// verifies those by IP). A block of GPTBot, ClaudeBot or PerplexityBot here is
+// what "Block AI bots" does by User-Agent, and real ones get the same.
+const UAS = {
+  browser: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
+  Googlebot: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+  Bingbot: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/116.0.1938.76 Safari/537.36',
+  GPTBot: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot',
+  'OAI-SearchBot': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot',
+  ClaudeBot: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)',
+  PerplexityBot: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)',
+};
+console.log('\n1. Fetching pages as each crawler (from this computer)\n');
+for (const path of ['/', '/capi-shield/', '/robots.txt']) {
+  for (const [name, ua] of Object.entries(UAS)) {
+    let line;
+    try {
+      const r = await fetch(SITE + path, { headers: { 'user-agent': ua }, redirect: 'manual' });
+      const body = await r.text();
+      const challenged = r.headers.get('cf-mitigated') || /Just a moment|Attention Required|cf-chl|challenge-platform\/h\//i.test(body) && r.status !== 200;
+      const robots = r.headers.get('x-robots-tag');
+      line = `${String(r.status).padEnd(4)}${challenged ? ' CHALLENGED/BLOCKED' : ''}${robots ? ` x-robots-tag: ${robots}` : ''}`;
+      if (r.status !== 200 && !['Googlebot', 'Bingbot'].includes(name)) flag('CRITICAL', `${path} answers ${r.status} to ${name}${challenged ? ' (Cloudflare challenge/block)' : ''}`);
+      if (robots && /noindex/i.test(robots)) flag('CRITICAL', `${path} sends X-Robots-Tag: ${robots} to ${name}`);
+    } catch (e) { line = `ERROR ${e.message}`; }
+    console.log(`  ${path.padEnd(15)} ${name.padEnd(14)} ${line}`);
+  }
+}
+const robotsTxt = await fetch(`${SITE}/robots.txt`).then((r) => r.text()).catch(() => '');
+if (/Content-Signal|BEGIN Cloudflare Managed/i.test(robotsTxt)) flag('WARN', 'robots.txt carries Cloudflare-managed content (Content-Signal / managed block): check it does not disallow AI crawlers');
+if (/User-agent:\s*(GPTBot|ClaudeBot|PerplexityBot|OAI-SearchBot)[\s\S]{0,200}?Disallow:\s*\/\s*$/im.test(robotsTxt)) flag('CRITICAL', 'robots.txt disallows an AI crawler from the whole site');
+
+// ── Part 2: Cloudflare settings ───────────────────────────────────────────
+const token = process.env.CF_API_TOKEN;
+if (!token) {
+  console.log('\n2. Cloudflare settings: skipped (set CF_API_TOKEN to read them; see the top of this file)');
+} else {
+  const api = async (p) => {
+    const r = await fetch(`https://api.cloudflare.com/client/v4${p}`, { headers: { authorization: `Bearer ${token}` } });
+    const j = await r.json().catch(() => ({}));
+    if (!j.success) return { error: (j.errors || []).map((e) => e.message).join('; ') || `HTTP ${r.status}` };
+    return j.result;
+  };
+  console.log('\n2. Cloudflare settings (read-only)\n');
+  const zones = await api(`/zones?name=${ZONE}`);
+  const zone = Array.isArray(zones) ? zones[0] : null;
+  if (!zone) { console.log(`  ✗ cannot read the zone: ${zones?.error ?? 'not found'}`); }
+  else {
+    const z = `/zones/${zone.id}`;
+    console.log(`  plan: ${zone.plan?.name}   status: ${zone.status}`);
+
+    const bm = await api(`${z}/bot_management`);
+    console.log('\n  Bots:', JSON.stringify(bm));
+    if (bm && !bm.error) {
+      if (bm.ai_bots_protection === 'block') flag('CRITICAL', '"Block AI bots" is ON: GPTBot, ClaudeBot, PerplexityBot etc. are refused at the edge, so no AI assistant can read or cite the site. Security → Bots → turn off "Block AI bots".');
+      if (bm.fight_mode) flag('HIGH', 'Bot Fight Mode is ON: it challenges automated clients that are not on Cloudflare\'s verified list, including some AI and SEO crawlers, and it cannot be bypassed with rules on the Free plan. Security → Bots → turn it off for a content site.');
+      if (bm.is_robots_txt_managed) flag('WARN', 'Managed robots.txt is ON: Cloudflare prepends its own rules to robots.txt. Check what it adds.');
+      if (bm.crawler_protection && bm.crawler_protection !== 'disabled') flag('WARN', `Crawler protection is "${bm.crawler_protection}".`);
+      if (bm.enable_js) flag('INFO', 'JavaScript detections are ON: Cloudflare injects a script into every page (the per-request bytes edge:check saw). Harmless to crawlers.');
+    }
+
+    const settings = await api(`${z}/settings`);
+    if (Array.isArray(settings)) {
+      const s = Object.fromEntries(settings.map((x) => [x.id, x.value]));
+      const show = ['security_level', 'browser_check', 'challenge_ttl', 'always_use_https', 'ssl', 'min_tls_version', 'email_obfuscation', 'rocket_loader', 'early_hints', 'development_mode', 'cache_level', 'browser_cache_ttl'];
+      console.log('\n  Settings:'); for (const k of show) if (k in s) console.log(`    ${k.padEnd(20)} ${JSON.stringify(s[k])}`);
+      if (['high', 'under_attack'].includes(s.security_level)) flag('HIGH', `Security level is "${s.security_level}": crawlers get challenged. Set it to "medium" or lower.`);
+      if (s.rocket_loader === 'on') flag('WARN', 'Rocket Loader is ON: it rewrites script loading and can break the inline consent/GA4 script and the calculators.');
+      if (s.development_mode === 'on') flag('INFO', 'Development mode is ON (caching bypassed).');
+    }
+
+    const hints = await api(`${z}/flags/products/cache/changes`);
+    if (hints?.crawlhints_enabled) flag('HIGH', 'Crawler Hints is ON again: it sends IndexNow pings on every cache change (CLAUDE.md: ~55,000 URLs). Caching → Configuration → turn off.');
+
+    for (const phase of ['http_request_firewall_custom', 'http_ratelimit', 'http_request_dynamic_redirect', 'http_request_cache_settings', 'http_request_transform', 'http_response_headers_transform', 'http_request_late_transform']) {
+      const rs = await api(`${z}/rulesets/phases/${phase}/entrypoint`);
+      const rules = rs?.rules ?? [];
+      if (!rules.length) continue;
+      console.log(`\n  ${phase}:`);
+      for (const r of rules) {
+        console.log(`    [${r.enabled === false ? 'off' : 'on '}] ${r.action.padEnd(16)} ${r.description || ''}\n          ${r.expression}`);
+        if (r.enabled !== false && ['block', 'challenge', 'managed_challenge', 'js_challenge'].includes(r.action) && !/cf\.client\.bot|verified_bot/.test(r.expression)) {
+          flag('HIGH', `WAF rule "${r.description || r.expression}" ${r.action}s without exempting verified bots (add "and not cf.client.bot").`);
+        }
+        if (r.enabled !== false && phase === 'http_request_cache_settings' && /"cache":\s*true|eligible/i.test(JSON.stringify(r.action_parameters || {}))) {
+          flag('WARN', `Cache Rule "${r.description}" makes requests cache-eligible: HTML could be served stale after a deploy.`);
+        }
+      }
+    }
+    const pr = await api(`${z}/pagerules`);
+    if (Array.isArray(pr) && pr.length) { console.log('\n  Page Rules:'); for (const r of pr) console.log(`    ${r.status} ${JSON.stringify(r.targets?.[0]?.constraint?.value)} → ${JSON.stringify(r.actions)}`); }
+  }
+}
+
+console.log('\n3. Findings\n');
+if (!problems.length) console.log('  Nothing found that stops a crawler.');
+for (const [level, msg] of problems.sort((a, b) => ['CRITICAL', 'HIGH', 'WARN', 'INFO'].indexOf(a[0]) - ['CRITICAL', 'HIGH', 'WARN', 'INFO'].indexOf(b[0]))) console.log(`  ${level.padEnd(8)} ${msg}`);
+console.log('');
+process.exit(problems.some(([l]) => l === 'CRITICAL') ? 1 : 0);