From 45d2fccb71c9fe72704c8d608006dc4cc8282ded Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 17:16:02 +0100 Subject: [PATCH] Count every page view and crawler fetch without cookies; audit Cloudflare GA4 only sees visitors who accept cookies, and nothing recorded whether Googlebot, Bingbot or AI crawlers were fetching pages while both engines show one page indexed. The middleware now adds one to a daily D1 counter per page, status, client kind (named crawler or bucket), referring host and country, in waitUntil, never throwing. No cookie, IP or User-Agent stored, so no consent is needed; /privacy/ says so. npm run traffic reports crawlers per page, human referrers (AI assistants flagged), and views vs affiliate clicks per page. npm run cf:audit fetches pages as each crawler and, with a read-only token, reads Bot Fight Mode, Block AI bots, managed robots.txt, security level, Crawler Hints and every zone rule, then ranks what could block a crawler. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CBB8KT6Paf7tCUjTvaanBB --- CLAUDE.md | 24 +++++++ functions/_middleware.js | 26 ++++++++ functions/_pageviews.js | 109 ++++++++++++++++++++++++++++++++ package.json | 4 +- schema/006-pageviews.sql | 38 +++++++++++ scripts/cf-audit.mjs | 125 +++++++++++++++++++++++++++++++++++++ scripts/traffic-report.mjs | 64 +++++++++++++++++++ src/pages/privacy.astro | 3 +- tests/pageviews.test.js | 79 +++++++++++++++++++++++ 9 files changed, 470 insertions(+), 2 deletions(-) create mode 100644 functions/_pageviews.js create mode 100644 schema/006-pageviews.sql create mode 100644 scripts/cf-audit.mjs create mode 100644 scripts/traffic-report.mjs create mode 100644 tests/pageviews.test.js diff --git a/CLAUDE.md b/CLAUDE.md index dded252..14f524e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,6 +118,30 @@ now feeds an "Every guide and tool" section on the homepage and the whole of `tests/site-index.test.js` holds it equal to the built sitemap. **A new indexable page must be added to `siteIndex.ts`, or `npm test` fails.** +**Cookieless page and crawler counts (10 Oct 2026).** GA4 sees only visitors +who press Accept, and nothing recorded whether Googlebot, Bingbot or the AI +crawlers fetched any page. `functions/_middleware.js` now adds one to a daily +D1 counter (`pageviews`, `schema/006-pageviews.sql`, apply it once) per page, +status, client kind, referring host and country, inside `waitUntil` and never +throwing. No cookie, IP or User-Agent is stored, so it needs no consent; +`/privacy/` says so. `npm run traffic` prints: who fetched pages, which pages +each search/AI crawler fetched and when it was last seen, human referrers +(AI assistants flagged), views vs affiliate clicks per page, and crawlers +still requesting retired URLs. Crawler names are claimed, not verified. + +**`npm run cf:audit` reads what the dashboard hides.** Without a token it +fetches pages as each crawler; with `CF_API_TOKEN` (read-only) it also reads +Bot Fight Mode, "Block AI bots", managed robots.txt, security level, Crawler +Hints, WAF/redirect/cache/transform rules and Page Rules, and ranks findings. +Run it before concluding anything about crawling: two of this site's worst +faults (a zone redirect rule, Crawler Hints) lived only in the dashboard. + +**Content is not duplicated (measured 10 Oct 2026).** Six-word shingles over +`
` across the 51 sitemap pages: median 5% of a page's shingles appear +on 3+ other pages, the highest pair overlap is 16% (two calculators), and the +only sitewide repeat is the author box. Do not rewrite pages to fix +"duplication"; there is none to fix. + **Build-time date branches need a scheduled build.** `isPostShutdown()` in `src/utils/stockyDeadline.ts` resolves at build time — six files branch on it: `Nav.astro` plus five page files (`index`, `pro`, `pro/[slug]` — four routes — `stocky-swap` diff --git a/functions/_middleware.js b/functions/_middleware.js index da040f4..bcf5fe5 100644 --- a/functions/_middleware.js +++ b/functions/_middleware.js @@ -46,11 +46,37 @@ // scripts/gen-legacy-redirects.mjs; tests/legacy-redirects.test.js fails if the // two drift. import { resolveLegacy } from './_legacy-redirects.js'; +import { rowFor, recordPageview } from './_pageviews.js'; const PRIMARY_HOST = 'stackarchitect.xyz'; const FILE_RE = /\.[a-zA-Z0-9]{2,5}$/; +// Every response, redirect or page, is counted on the way out: one daily +// counter per page and kind of client, no cookie, no IP, no User-Agent stored. +// Why, and what is never kept: schema/006-pageviews.sql. The count runs in +// waitUntil after the response is decided, and recordPageview never throws, so +// a missing table or binding costs a statistic and never a page. export async function onRequest(context) { + const response = await route(context); + try { + const req = context.request; + const row = rowFor({ + url: req.url, + method: req.method, + status: response.status, + contentType: response.headers.get('content-type'), + userAgent: req.headers.get('user-agent'), + referer: req.headers.get('referer'), + country: req.cf?.country, + }); + if (row && typeof context.waitUntil === 'function') context.waitUntil(recordPageview(context.env?.DB, row)); + } catch { + // Counting must never cost the response. + } + return response; +} + +async function route(context) { const url = new URL(context.request.url); let changed = false; diff --git a/functions/_pageviews.js b/functions/_pageviews.js new file mode 100644 index 0000000..696a5be --- /dev/null +++ b/functions/_pageviews.js @@ -0,0 +1,109 @@ +/** + * _pageviews.js — cookieless daily counts of who fetched which page. + * + * Underscore-prefixed, so Pages does not route it. Called from + * functions/_middleware.js inside waitUntil, after the response is decided, + * so it can never slow down or break a page. Why it exists and what is never + * stored: schema/006-pageviews.sql. + */ + +// Crawlers that matter to this site, named. First match wins, so the more +// specific tokens come before the ones they contain. +const NAMED = [ + ['Google-InspectionTool', /Google-InspectionTool/i], + ['GoogleOther', /GoogleOther/i], + ['Storebot-Google', /Storebot-Google/i], + ['Googlebot', /Googlebot/i], + ['Bingbot', /bingbot|BingPreview/i], + ['OAI-SearchBot', /OAI-SearchBot/i], + ['ChatGPT-User', /ChatGPT-User/i], + ['GPTBot', /GPTBot/i], + ['Claude-SearchBot', /Claude-SearchBot/i], + ['Claude-User', /Claude-User/i], + ['ClaudeBot', /ClaudeBot|anthropic-ai/i], + ['Perplexity-User', /Perplexity-User/i], + ['PerplexityBot', /PerplexityBot/i], + ['Applebot', /Applebot/i], + ['DuckDuckBot', /DuckDuckBot|DuckAssistBot/i], + ['meta-externalagent', /meta-external/i], + ['Bytespider', /Bytespider/i], + ['AhrefsBot', /AhrefsBot/i], + ['SemrushBot', /SemrushBot/i], + ['YandexBot', /YandexBot/i], +]; +const OTHER_BOT = /bot\b|bot\/|crawl|spider|slurp|curl\/|wget|python-|go-http|headless|facebookexternalhit|preview/i; + +/** Name the client, or bucket it. Only this label is stored, never the UA. */ +export function agentOf(userAgent) { + const ua = typeof userAgent === 'string' ? userAgent.trim() : ''; + if (!ua) return 'no-ua'; + for (const [name, re] of NAMED) if (re.test(ua)) return name; + if (OTHER_BOT.test(ua)) return 'other-bot'; + return /^Mozilla\/5\.0 /.test(ua) ? 'human' : 'other'; +} + +/** The referring site's host only: never the path, which can carry anything. */ +export function refOf(referer, selfHost = 'stackarchitect.xyz') { + if (!referer) return '(none)'; + try { + const h = new URL(referer).hostname.toLowerCase().replace(/^www\./, ''); + if (h === selfHost) return '(internal)'; + return h.slice(0, 80) || '(none)'; + } catch { + return '(none)'; + } +} + +const KEEP_REDIRECT_PATHS = new Set(['Googlebot', 'Bingbot', 'GPTBot', 'OAI-SearchBot', 'ClaudeBot', 'PerplexityBot', 'Applebot']); + +/** + * Which row this request belongs in, or null when it is not worth counting. + * Pages and their redirects are counted; assets, API calls and /go/ (which has + * its own counter) are not. A 404 is bucketed, because a probe for + * /wp-login.php should not add a row per guess. A redirect keeps its path only + * for the named search and AI crawlers: whether Google still crawls a retired + * URL is worth knowing, and every other client's would be noise. + */ +export function rowFor({ url, method, status, contentType, userAgent, referer, country, day }) { + if (method !== 'GET') return null; + const u = new URL(url); + const p = u.pathname; + if (p.startsWith('/api/') || p.startsWith('/go/') || p.startsWith('/_astro/') || p.startsWith('/cdn-cgi/')) return null; + const agent = agentOf(userAgent); + const isFile = /\.[a-z0-9]{2,5}$/i.test(p); + let path; + if (status === 200) { + const wanted = !isFile ? /text\/html/i.test(contentType || '') : /^\/(llms(-full)?\.txt|robots\.txt|sitemap[^/]*\.xml)$/.test(p); + if (!wanted) return null; + path = p.slice(0, 160); + } else if (status >= 300 && status < 400) { + path = KEEP_REDIRECT_PATHS.has(agent) ? p.slice(0, 160) : '(redirect)'; + } else if (status === 404) { + path = '(404)'; + } else { + path = '(other)'; + } + return { + day: day ?? new Date().toISOString().slice(0, 10), + path, + status: String(status), + agent, + ref: refOf(referer, u.hostname.replace(/^www\./, '')), + country: /^[A-Z]{2}$/.test(country || '') ? country : 'XX', + }; +} + +const SQL = `INSERT INTO pageviews (day, path, status, agent, ref, country, n) +VALUES (?, ?, ?, ?, ?, ?, 1) +ON CONFLICT (day, path, status, agent, ref, country) DO UPDATE SET n = n + 1`; + +/** Count one request. Never throws; a missing table or binding costs a count, never a page. */ +export async function recordPageview(db, row) { + if (!row || !db || typeof db.prepare !== 'function') return false; + try { + await db.prepare(SQL).bind(row.day, row.path, row.status, row.agent, row.ref, row.country).run(); + return true; + } catch { + return false; + } +} diff --git a/package.json b/package.json index f880c82..29c9b1f 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,9 @@ "aff:check": "node scripts/aff-check.mjs", "syndication:build": "node scripts/syndication-build.mjs", "syndication:sync": "node scripts/syndication-sync.mjs", - "edge:check": "node scripts/edge-freshness.mjs" + "edge:check": "node scripts/edge-freshness.mjs", + "traffic": "node scripts/traffic-report.mjs", + "cf:audit": "node scripts/cf-audit.mjs" }, "dependencies": { "@astrojs/cloudflare": "^13.1.7", diff --git a/schema/006-pageviews.sql b/schema/006-pageviews.sql new file mode 100644 index 0000000..699efc4 --- /dev/null +++ b/schema/006-pageviews.sql @@ -0,0 +1,38 @@ +-- Migration 006 — cookieless page view and crawler counts +-- +-- Apply once: +-- +-- npx wrangler d1 execute attribution-gap --remote --file schema/006-pageviews.sql +-- +-- Safe to re-run: everything here is IF NOT EXISTS. +-- +-- WHY (10 Oct 2026) +-- GA4 only sees visitors who press Accept, so the site had no count of its own +-- traffic, and no record at all of whether Googlebot, Bingbot or the AI +-- crawlers were fetching its pages while both engines showed one page indexed. +-- functions/_middleware.js already runs on every request; it now adds one to a +-- daily counter per (page, kind of client, referring site, country). +-- +-- WHAT IS NOT STORED +-- No cookie, no IP address, no User-Agent string, no time finer than the day, +-- no identifier of any kind. `agent` is a named crawler or a coarse bucket, +-- decided from the User-Agent and then discarded. Nothing is read from or +-- written to the visitor's device, so this needs no consent, the same as the +-- /go/ click counts (see /privacy/). +-- +-- `agent` names a crawler by what it CLAIMS to be. A scraper can send +-- "Googlebot"; Googlebot's own figures are in Search Console → Settings → +-- Crawl stats. Read crawler rows as "something calling itself X". + +CREATE TABLE IF NOT EXISTS pageviews ( + day TEXT NOT NULL, -- UTC day, yyyy-mm-dd + path TEXT NOT NULL, -- page path, or '(404)' / '(other)' + status TEXT NOT NULL, -- '200', '301', '404', ... + agent TEXT NOT NULL, -- 'human', 'Googlebot', 'GPTBot', 'other-bot', ... + ref TEXT NOT NULL, -- referring host, '(none)' or '(internal)' + country TEXT NOT NULL, -- two-letter code from Cloudflare, or 'XX' + n INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (day, path, status, agent, ref, country) +); + +CREATE INDEX IF NOT EXISTS pageviews_agent ON pageviews (agent, day); diff --git a/scripts/cf-audit.mjs b/scripts/cf-audit.mjs new file mode 100644 index 0000000..7f84170 --- /dev/null +++ b/scripts/cf-audit.mjs @@ -0,0 +1,125 @@ +#!/usr/bin/env node +// scripts/cf-audit.mjs — read-only check of everything outside this repo that +// decides whether search engines and AI crawlers can reach the site. +// +// node scripts/cf-audit.mjs # part 1 only: probe as each crawler +// CF_API_TOKEN=… node scripts/cf-audit.mjs # parts 1 and 2: plus Cloudflare settings +// +// Token: Cloudflare dashboard → My Profile → API Tokens → Create Token → +// template "Read all resources" → limit Zone Resources to stackarchitect.xyz. +// Read-only: this script never changes a setting. Do not commit the token. +// +// WHY (10 Oct 2026) +// Google and Bing each show one indexed page and Ahrefs Brand Radar shows no +// AI citations. Every check inside the repo is clean, and several settings +// that can block crawlers live only in the Cloudflare dashboard: "Block AI +// bots", Bot Fight Mode, managed robots.txt, WAF custom rules, Cache Rules, +// Redirect Rules. CLAUDE.md already records two of them causing harm +// unseen (a zone trailing-slash rule; Crawler Hints sending ~55,000 IndexNow +// URLs). This reads them all at once. +const SITE = 'https://stackarchitect.xyz'; +const ZONE = 'stackarchitect.xyz'; +const problems = []; +const flag = (level, msg) => { problems.push([level, msg]); }; + +// ── Part 1: fetch as each crawler ───────────────────────────────────────── +// A request from this machine carries the crawler's User-Agent but not its IP, +// so a block of *spoofed* Googlebot/Bingbot is expected and harmless (Cloudflare +// verifies those by IP). A block of GPTBot, ClaudeBot or PerplexityBot here is +// what "Block AI bots" does by User-Agent, and real ones get the same. +const UAS = { + browser: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36', + Googlebot: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', + Bingbot: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/116.0.1938.76 Safari/537.36', + GPTBot: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot', + 'OAI-SearchBot': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot', + ClaudeBot: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)', + PerplexityBot: 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)', +}; +console.log('\n1. Fetching pages as each crawler (from this computer)\n'); +for (const path of ['/', '/capi-shield/', '/robots.txt']) { + for (const [name, ua] of Object.entries(UAS)) { + let line; + try { + const r = await fetch(SITE + path, { headers: { 'user-agent': ua }, redirect: 'manual' }); + const body = await r.text(); + const challenged = r.headers.get('cf-mitigated') || /Just a moment|Attention Required|cf-chl|challenge-platform\/h\//i.test(body) && r.status !== 200; + const robots = r.headers.get('x-robots-tag'); + line = `${String(r.status).padEnd(4)}${challenged ? ' CHALLENGED/BLOCKED' : ''}${robots ? ` x-robots-tag: ${robots}` : ''}`; + if (r.status !== 200 && !['Googlebot', 'Bingbot'].includes(name)) flag('CRITICAL', `${path} answers ${r.status} to ${name}${challenged ? ' (Cloudflare challenge/block)' : ''}`); + if (robots && /noindex/i.test(robots)) flag('CRITICAL', `${path} sends X-Robots-Tag: ${robots} to ${name}`); + } catch (e) { line = `ERROR ${e.message}`; } + console.log(` ${path.padEnd(15)} ${name.padEnd(14)} ${line}`); + } +} +const robotsTxt = await fetch(`${SITE}/robots.txt`).then((r) => r.text()).catch(() => ''); +if (/Content-Signal|BEGIN Cloudflare Managed/i.test(robotsTxt)) flag('WARN', 'robots.txt carries Cloudflare-managed content (Content-Signal / managed block): check it does not disallow AI crawlers'); +if (/User-agent:\s*(GPTBot|ClaudeBot|PerplexityBot|OAI-SearchBot)[\s\S]{0,200}?Disallow:\s*\/\s*$/im.test(robotsTxt)) flag('CRITICAL', 'robots.txt disallows an AI crawler from the whole site'); + +// ── Part 2: Cloudflare settings ─────────────────────────────────────────── +const token = process.env.CF_API_TOKEN; +if (!token) { + console.log('\n2. Cloudflare settings: skipped (set CF_API_TOKEN to read them; see the top of this file)'); +} else { + const api = async (p) => { + const r = await fetch(`https://api.cloudflare.com/client/v4${p}`, { headers: { authorization: `Bearer ${token}` } }); + const j = await r.json().catch(() => ({})); + if (!j.success) return { error: (j.errors || []).map((e) => e.message).join('; ') || `HTTP ${r.status}` }; + return j.result; + }; + console.log('\n2. Cloudflare settings (read-only)\n'); + const zones = await api(`/zones?name=${ZONE}`); + const zone = Array.isArray(zones) ? zones[0] : null; + if (!zone) { console.log(` ✗ cannot read the zone: ${zones?.error ?? 'not found'}`); } + else { + const z = `/zones/${zone.id}`; + console.log(` plan: ${zone.plan?.name} status: ${zone.status}`); + + const bm = await api(`${z}/bot_management`); + console.log('\n Bots:', JSON.stringify(bm)); + if (bm && !bm.error) { + if (bm.ai_bots_protection === 'block') flag('CRITICAL', '"Block AI bots" is ON: GPTBot, ClaudeBot, PerplexityBot etc. are refused at the edge, so no AI assistant can read or cite the site. Security → Bots → turn off "Block AI bots".'); + if (bm.fight_mode) flag('HIGH', 'Bot Fight Mode is ON: it challenges automated clients that are not on Cloudflare\'s verified list, including some AI and SEO crawlers, and it cannot be bypassed with rules on the Free plan. Security → Bots → turn it off for a content site.'); + if (bm.is_robots_txt_managed) flag('WARN', 'Managed robots.txt is ON: Cloudflare prepends its own rules to robots.txt. Check what it adds.'); + if (bm.crawler_protection && bm.crawler_protection !== 'disabled') flag('WARN', `Crawler protection is "${bm.crawler_protection}".`); + if (bm.enable_js) flag('INFO', 'JavaScript detections are ON: Cloudflare injects a script into every page (the per-request bytes edge:check saw). Harmless to crawlers.'); + } + + const settings = await api(`${z}/settings`); + if (Array.isArray(settings)) { + const s = Object.fromEntries(settings.map((x) => [x.id, x.value])); + const show = ['security_level', 'browser_check', 'challenge_ttl', 'always_use_https', 'ssl', 'min_tls_version', 'email_obfuscation', 'rocket_loader', 'early_hints', 'development_mode', 'cache_level', 'browser_cache_ttl']; + console.log('\n Settings:'); for (const k of show) if (k in s) console.log(` ${k.padEnd(20)} ${JSON.stringify(s[k])}`); + if (['high', 'under_attack'].includes(s.security_level)) flag('HIGH', `Security level is "${s.security_level}": crawlers get challenged. Set it to "medium" or lower.`); + if (s.rocket_loader === 'on') flag('WARN', 'Rocket Loader is ON: it rewrites script loading and can break the inline consent/GA4 script and the calculators.'); + if (s.development_mode === 'on') flag('INFO', 'Development mode is ON (caching bypassed).'); + } + + const hints = await api(`${z}/flags/products/cache/changes`); + if (hints?.crawlhints_enabled) flag('HIGH', 'Crawler Hints is ON again: it sends IndexNow pings on every cache change (CLAUDE.md: ~55,000 URLs). Caching → Configuration → turn off.'); + + for (const phase of ['http_request_firewall_custom', 'http_ratelimit', 'http_request_dynamic_redirect', 'http_request_cache_settings', 'http_request_transform', 'http_response_headers_transform', 'http_request_late_transform']) { + const rs = await api(`${z}/rulesets/phases/${phase}/entrypoint`); + const rules = rs?.rules ?? []; + if (!rules.length) continue; + console.log(`\n ${phase}:`); + for (const r of rules) { + console.log(` [${r.enabled === false ? 'off' : 'on '}] ${r.action.padEnd(16)} ${r.description || ''}\n ${r.expression}`); + if (r.enabled !== false && ['block', 'challenge', 'managed_challenge', 'js_challenge'].includes(r.action) && !/cf\.client\.bot|verified_bot/.test(r.expression)) { + flag('HIGH', `WAF rule "${r.description || r.expression}" ${r.action}s without exempting verified bots (add "and not cf.client.bot").`); + } + if (r.enabled !== false && phase === 'http_request_cache_settings' && /"cache":\s*true|eligible/i.test(JSON.stringify(r.action_parameters || {}))) { + flag('WARN', `Cache Rule "${r.description}" makes requests cache-eligible: HTML could be served stale after a deploy.`); + } + } + } + const pr = await api(`${z}/pagerules`); + if (Array.isArray(pr) && pr.length) { console.log('\n Page Rules:'); for (const r of pr) console.log(` ${r.status} ${JSON.stringify(r.targets?.[0]?.constraint?.value)} → ${JSON.stringify(r.actions)}`); } + } +} + +console.log('\n3. Findings\n'); +if (!problems.length) console.log(' Nothing found that stops a crawler.'); +for (const [level, msg] of problems.sort((a, b) => ['CRITICAL', 'HIGH', 'WARN', 'INFO'].indexOf(a[0]) - ['CRITICAL', 'HIGH', 'WARN', 'INFO'].indexOf(b[0]))) console.log(` ${level.padEnd(8)} ${msg}`); +console.log(''); +process.exit(problems.some(([l]) => l === 'CRITICAL') ? 1 : 0); diff --git a/scripts/traffic-report.mjs b/scripts/traffic-report.mjs new file mode 100644 index 0000000..a17ebe5 --- /dev/null +++ b/scripts/traffic-report.mjs @@ -0,0 +1,64 @@ +#!/usr/bin/env node +// scripts/traffic-report.mjs — what the cookieless counters say, in one screen. +// +// npm run traffic # last 7 days +// npm run traffic -- 28 # last 28 days +// +// Reads D1 through wrangler (you must be logged in: `npx wrangler login`). +// Tables: pageviews (schema/006-pageviews.sql) and clicks (schema/003, /005). +// Unlike GA4, these count every visitor whether or not they accepted cookies, +// because they store nothing about the visitor. Crawler names are what the +// client claims to be; Search Console's Crawl stats are Google's own figures. +import { execFileSync } from 'node:child_process'; + +const days = Number(process.argv[2] || 7); +const since = new Date(Date.now() - days * 864e5).toISOString().slice(0, 10); + +function q(sql) { + try { + const out = execFileSync('npx', ['wrangler', 'd1', 'execute', 'attribution-gap', '--remote', '--json', '--command', sql], + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + return JSON.parse(out)[0]?.results ?? []; + } catch (e) { + const msg = String(e.stderr || e.message); + if (/no such table: pageviews/.test(msg)) { + console.error('✗ The pageviews table does not exist yet. Run:\n npx wrangler d1 execute attribution-gap --remote --file schema/006-pageviews.sql'); + process.exit(1); + } + throw new Error(msg.split('\n').slice(-6).join('\n')); + } +} +const table = (rows, cols) => { + if (!rows.length) { console.log(' (none yet)'); return; } + const w = cols.map((c) => Math.max(c.length, ...rows.map((r) => String(r[c] ?? '').length))); + console.log(' ' + cols.map((c, i) => c.padEnd(w[i])).join(' ')); + for (const r of rows) console.log(' ' + cols.map((c, i) => String(r[c] ?? '').padEnd(w[i])).join(' ')); +}; +const AI = "('chatgpt.com','chat.openai.com','perplexity.ai','claude.ai','gemini.google.com','copilot.microsoft.com','bing.com','you.com','phind.com','meta.ai')"; + +console.log(`\nTraffic since ${since} (cookieless counters; every visitor, no consent needed)\n`); + +console.log('1. Who fetched pages (HTTP 200 pages only)'); +table(q(`SELECT agent, SUM(n) AS fetches, COUNT(DISTINCT path) AS pages FROM pageviews WHERE day >= '${since}' AND status = '200' GROUP BY agent ORDER BY fetches DESC`), ['agent', 'fetches', 'pages']); + +console.log('\n2. Search and AI crawlers: pages fetched, and the last day each was seen'); +table(q(`SELECT agent, path, SUM(n) AS fetches, MAX(day) AS last_seen FROM pageviews WHERE day >= '${since}' AND agent IN ('Googlebot','Bingbot','GPTBot','OAI-SearchBot','ChatGPT-User','ClaudeBot','Claude-User','Claude-SearchBot','PerplexityBot','Perplexity-User','Applebot') GROUP BY agent, path ORDER BY agent, fetches DESC LIMIT 120`), ['agent', 'path', 'fetches', 'last_seen']); + +console.log('\n3. Where human readers came from'); +table(q(`SELECT ref, SUM(n) AS views, CASE WHEN ref IN ${AI} THEN 'AI assistant' ELSE '' END AS note FROM pageviews WHERE day >= '${since}' AND agent = 'human' AND status = '200' GROUP BY ref ORDER BY views DESC LIMIT 25`), ['ref', 'views', 'note']); + +console.log('\n4. Human views and partner clicks per page (click-through = clicks / views)'); +let clicks; +try { + clicks = q(`SELECT landing AS path, SUM(n) AS clicks FROM clicks WHERE day >= '${since}' AND ua_class = 'browser-like' AND sec_fetch_mode = 'navigate' GROUP BY landing`); +} catch { + clicks = q(`SELECT landing AS path, SUM(n) AS clicks FROM clicks WHERE day >= '${since}' GROUP BY landing`); + console.log(' (clicks are unfiltered: schema/005 is not applied, so bots are not separated from readers)'); +} +const byPath = Object.fromEntries(clicks.map((c) => [c.path, c.clicks])); +const views = q(`SELECT path, SUM(n) AS views FROM pageviews WHERE day >= '${since}' AND agent = 'human' AND status = '200' GROUP BY path ORDER BY views DESC LIMIT 40`); +table(views.map((v) => ({ ...v, clicks: byPath[v.path] ?? 0, ctr: v.views ? `${(((byPath[v.path] ?? 0) / v.views) * 100).toFixed(1)}%` : '' })), ['path', 'views', 'clicks', 'ctr']); + +console.log('\n5. Crawlers still requesting retired URLs (each costs a redirect hop)'); +table(q(`SELECT agent, path, SUM(n) AS hits FROM pageviews WHERE day >= '${since}' AND status IN ('301','302','308') AND path NOT IN ('(redirect)') GROUP BY agent, path ORDER BY hits DESC LIMIT 20`), ['agent', 'path', 'hits']); +console.log(''); diff --git a/src/pages/privacy.astro b/src/pages/privacy.astro index 009a11d..d429bde 100644 --- a/src/pages/privacy.astro +++ b/src/pages/privacy.astro @@ -65,6 +65,7 @@ const tocHeadings = [
  • Email address: when you sign up for guides or deployment plans. Depending on the form, your address is sent to Systeme.io or Beehiiv, which store it and send the emails.
  • Analytics data, only if you accept: page views and interaction events collected by Google Analytics 4. It is off unless you choose to turn it on. See Cookies and Analytics Consent for exactly what is recorded.
  • Purchase data: payments for the Complete Kit and the single blueprints are taken by Stripe on its own checkout pages. We never see your card details. Stripe shares your email address and the purchase with us so we can deliver the product.
    • +
  • Page counts: when any page is requested, our server adds one to a daily count for that page, the kind of client that asked for it (a browser, or a search engine crawler that names itself), the website that linked to it (the site name only, never the address of the page) and the country Cloudflare reports. No cookie, IP address, browser details or other identifier is stored, so this count cannot tell one visitor from another. It happens whatever your analytics choice, because it records nothing about you.
  • Partner link counts: when you follow one of our partner links (addresses starting /go/), our server adds one to a daily count for that link, the place on the page it sat in, and the page you clicked from. No cookie, IP address, browser details or other identifier is stored, so this count cannot tell one visitor from another. It happens whatever your analytics choice, because it records nothing about you.
    • @@ -89,7 +90,7 @@ const tocHeadings = [

    Your Rights (GDPR)

    If you are in the UK or EU, you have the right to access, correct, or delete any personal data we hold. To exercise these rights, email support@stackarchitect.xyz. Email subscribers can unsubscribe at any time via the link in any email we send.

    Data Retention

    -

    Email subscriber data is retained while your subscription is active. GA4 keeps event-level data for the retention period set in our Google Analytics property; Google offers 2 or 14 months. Daily partner link counts hold no personal data and are kept indefinitely. Purchase records are retained as required for legal and accounting purposes.

    +

    Email subscriber data is retained while your subscription is active. GA4 keeps event-level data for the retention period set in our Google Analytics property; Google offers 2 or 14 months. Daily page and partner link counts hold no personal data and are kept indefinitely. Purchase records are retained as required for legal and accounting purposes.

    Changes to This Policy

    We may update this policy from time to time. The "last updated" date at the top reflects the most recent revision. Continued use of the site constitutes acceptance of the updated policy.

    Contact

    diff --git a/tests/pageviews.test.js b/tests/pageviews.test.js new file mode 100644 index 0000000..7764bb3 --- /dev/null +++ b/tests/pageviews.test.js @@ -0,0 +1,79 @@ +// tests/pageviews.test.js — the cookieless page and crawler counter. +// Why it exists: schema/006-pageviews.sql. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { agentOf, refOf, rowFor, recordPageview } from '../functions/_pageviews.js'; +import { onRequest } from '../functions/_middleware.js'; + +const CHROME = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0 Safari/537.36'; + +test('crawlers are named, the rest bucketed, and no UA is kept', () => { + assert.equal(agentOf('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'), 'Googlebot'); + assert.equal(agentOf('Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0 Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0;)'), 'Google-InspectionTool'); + assert.equal(agentOf('Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/116.0 Safari/537.36'), 'Bingbot'); + assert.equal(agentOf('Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot'), 'OAI-SearchBot'); + assert.equal(agentOf('Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot'), 'GPTBot'); + assert.equal(agentOf('Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'), 'ClaudeBot'); + assert.equal(agentOf('Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)'), 'PerplexityBot'); + assert.equal(agentOf(CHROME), 'human'); + assert.equal(agentOf('curl/8.4.0'), 'other-bot'); + assert.equal(agentOf(''), 'no-ua'); +}); + +test('only the referring host is kept', () => { + assert.equal(refOf('https://chatgpt.com/c/abc-private-conversation'), 'chatgpt.com'); + assert.equal(refOf('https://www.google.com/'), 'google.com'); + assert.equal(refOf('https://stackarchitect.xyz/tools/'), '(internal)'); + assert.equal(refOf(null), '(none)'); +}); + +const base = { method: 'GET', contentType: 'text/html; charset=utf-8', userAgent: CHROME, referer: null, country: 'GB', day: '2026-10-10' }; + +test('pages are counted; assets, API, /go/ and POSTs are not', () => { + assert.deepEqual(rowFor({ ...base, url: 'https://stackarchitect.xyz/capi-shield/?utm_source=x', status: 200 }), + { day: '2026-10-10', path: '/capi-shield/', status: '200', agent: 'human', ref: '(none)', country: 'GB' }); + for (const url of ['https://stackarchitect.xyz/_astro/a.css', 'https://stackarchitect.xyz/api/gap-stats', 'https://stackarchitect.xyz/go/make']) { + assert.equal(rowFor({ ...base, url, status: 200 }), null, url); + } + assert.equal(rowFor({ ...base, url: 'https://stackarchitect.xyz/og/home.png', status: 200, contentType: 'image/png' }), null); + assert.equal(rowFor({ ...base, method: 'POST', url: 'https://stackarchitect.xyz/', status: 200 }), null); + assert.equal(rowFor({ ...base, url: 'https://stackarchitect.xyz/llms.txt', status: 200, contentType: 'text/plain' }).path, '/llms.txt'); +}); + +test('404s are bucketed; redirects keep their path only for named search and AI crawlers', () => { + assert.equal(rowFor({ ...base, url: 'https://stackarchitect.xyz/wp-login.php', status: 404 }).path, '(404)'); + assert.equal(rowFor({ ...base, url: 'https://stackarchitect.xyz/stocky-shutdown/', status: 301 }).path, '(redirect)'); + assert.equal(rowFor({ ...base, userAgent: 'Mozilla/5.0 (compatible; Googlebot/2.1)', url: 'https://stackarchitect.xyz/stocky-shutdown/', status: 301 }).path, '/stocky-shutdown/'); +}); + +test('a failing database never throws', async () => { + const db = { prepare() { throw new Error('no such table: pageviews'); } }; + assert.equal(await recordPageview(db, { day: 'd', path: '/', status: '200', agent: 'human', ref: '(none)', country: 'GB' }), false); + assert.equal(await recordPageview(undefined, {}), false); +}); + +test('the middleware counts a page in waitUntil and still serves it when the DB throws', async () => { + const waits = []; + const ctx = { + request: new Request('https://stackarchitect.xyz/capi-shield/', { headers: { 'user-agent': CHROME } }), + env: { DB: { prepare() { throw new Error('down'); } } }, + next: async () => new Response('
    ok
    ', { status: 200, headers: { 'content-type': 'text/html' } }), + waitUntil: (p) => waits.push(p), + }; + const res = await onRequest(ctx); + assert.equal(res.status, 200); + assert.equal(await res.text(), '
    ok
    '); + assert.equal(waits.length, 1); + assert.equal(await waits[0], false); +}); + +test('a redirect is still a single 301 and is counted', async () => { + const waits = []; + const res = await onRequest({ + request: new Request('https://stackarchitect.xyz/capi-shield', { headers: { 'user-agent': 'Mozilla/5.0 (compatible; bingbot/2.0)' } }), + env: {}, next: async () => new Response('x'), waitUntil: (p) => waits.push(p), + }); + assert.equal(res.status, 301); + assert.equal(res.headers.get('location'), 'https://stackarchitect.xyz/capi-shield/'); + assert.equal(waits.length, 1); +});