feat: name hosted collectors, allow reporting-only beacons

CHANGELOG.md

@@ -2,6 +2,45 @@
 
 Both packages are released together and share a version.
 
+## 0.1.8
+
+### Added
+
+- **Hosted collectors have names.** `analytics.collector` takes
+  `"snow-analytics"` or `"snowseo"` and fills in the ingest URL from the new
+  exported `COLLECTORS` map. The endpoint default only ever named SnowSEO, so
+  every other collector had to be spelled out as a full URL — and getting that
+  URL subtly wrong (a bare origin, or the wrong mount path) fails as an opaque
+  `401` rather than a 404. `endpoint` still wins, for self-hosted collectors.
+
+- **Reporting-only beacons.** `createBeacon({ analytics })` with no `dir` and
+  no `resolve` is now a valid, analytics-only install instead of a thrown
+  error. It used to need a `resolve: () => null` whose only purpose was to
+  satisfy that check — boilerplate that read like it did something.
+
+### Changed
+
+- **`siteUrl` is optional.** It is required only by the methods that mint
+  absolute URLs (`markdownUrlFor`, `sitemap`, `llmsTxt`, `robotsDirective`, and
+  twin responses), which now throw an error naming it. A reporting-only beacon
+  mints no URLs, so it does not need one.
+
+- **`strictNegotiation` defaults to `false` for reporting-only beacons** and
+  stays `true` when the beacon serves twins. A beacon that will never answer a
+  request has no business turning the site's own JSON endpoints into `406`s.
+  Passing the option explicitly still overrides both defaults.
+
+- **`host` is optional on the wire.** A key belongs to one site, so a collector
+  can resolve it; `@snowseo/beacon` omits `host` only when the caller gives
+  neither `siteUrl` nor `analytics.host`, and `@snowseo/beacon-server` falls
+  back to the key's host when the key names exactly one (and answers `400` when
+  it names several, where guessing would file hits under the wrong site). A
+  present-but-empty `host` is still rejected — that is a client that failed to
+  compute a value, not one deliberately leaving it out.
+
+  Existing installs are unaffected: anything passing `siteUrl` sends `host`
+  exactly as before.
+
 ## 0.1.3
 
 ### Fixed

README.md

@@ -39,6 +39,28 @@ protocol: one `auto_prepend_file` line, no application code touched. It lives
 with the [WordPress plugin](https://github.com/Snow-SEO/snowseo-wordpress-plugin),
 which is what it is built into.
 
+## Reporting only
+
+Measuring AI crawlers and serving them Markdown are separate jobs. If you only
+want the measurement, that is the whole config:
+
+```ts
+import { createBeacon } from "@snowseo/beacon";
+
+export const beacon = createBeacon({
+  analytics: { key: process.env.SNOW_API_KEY, collector: "snow-analytics" },
+});
+```
+
+No `siteUrl` — a key belongs to one site, so the collector resolves it. No
+`resolve`, because there are no twins to serve. And `strictNegotiation`
+defaults to `false` here, so a beacon that answers nothing never turns your own
+JSON endpoints into `406`s.
+
+`collector` takes a name from `COLLECTORS`: `"snow-analytics"` or `"snowseo"`.
+Naming it beats spelling out a URL, which is easy to get subtly wrong — a bare
+origin, or the wrong mount path, fails as an opaque `401` rather than a 404.
+
 ## Running your own collector
 
 The analytics half is not tied to any hosted service. Point it wherever you

packages/beacon-server/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@snowseo/beacon-server",
-	"version": "0.1.7",
+	"version": "0.1.8",
 	"description": "Server side of the beacon protocol: AI crawler verification (provider IP ranges, datacenter detection, reverse DNS, Web Bot Auth) plus a reference ingest server.",
 	"keywords": [
 		"ai",

packages/beacon-server/src/protocol.ts

@@ -29,7 +29,12 @@ export interface IncomingHit {
 }
 
 export interface IngestRequestBody {
-	host: string;
+	/**
+	 * The site the hits belong to. Optional on the wire: a key identifies one
+	 * site, so a server that knows its keys can resolve the host itself. Servers
+	 * that cannot must reject a batch without one.
+	 */
+	host?: string;
 	hits: IncomingHit[];
 }
 
@@ -70,11 +75,16 @@ export function validateIngestBody(value: unknown): string | null {
 		return "body must be a JSON object";
 	}
 	const body = value as Partial<IngestRequestBody>;
-	if (typeof body.host !== "string" || body.host.length === 0) {
-		return "`host` must be a non-empty string";
-	}
-	if (body.host.length > MAX_HOST_LENGTH) {
-		return `\`host\` exceeds ${MAX_HOST_LENGTH} characters`;
+	// Optional: a key belongs to a known set of hosts, so a server can resolve
+	// the site itself. Present-but-empty is still an error — that is a client
+	// sending a value it failed to compute, not one deliberately omitting it.
+	if (body.host !== undefined) {
+		if (typeof body.host !== "string" || body.host.length === 0) {
+			return "`host` must be a non-empty string when present";
+		}
+		if (body.host.length > MAX_HOST_LENGTH) {
+			return `\`host\` exceeds ${MAX_HOST_LENGTH} characters`;
+		}
 	}
 	if (!Array.isArray(body.hits) || body.hits.length === 0) {
 		return "`hits` must be a non-empty array";

packages/beacon-server/src/server/app.ts

@@ -23,6 +23,16 @@ import { type BeaconKey, KeyRegistry } from "./auth.js";
 
 const DEFAULT_MAX_BODY_BYTES = 4 * 1024 * 1024;
 
+/**
+ * The one host a key is scoped to, or undefined when it is scoped to several
+ * (or to none, meaning "any host" — which is equally unresolvable).
+ */
+function soleAllowedHost(
+	allowedHosts: readonly string[] | undefined,
+): string | undefined {
+	return allowedHosts?.length === 1 ? allowedHosts[0] : undefined;
+}
+
 export interface BeaconServerOptions {
 	store: HitStore;
 	keys: readonly BeaconKey[];
@@ -147,7 +157,22 @@ export function createBeaconServer(options: BeaconServerOptions): Server {
 			sendError(res, 400, "BAD_REQUEST", invalid);
 			return;
 		}
-		const { host, hits } = parsed as IngestRequestBody;
+		const body = parsed as IngestRequestBody;
+		// A batch may omit `host` and let the key say which site it is for, but
+		// only when the key names exactly one — with several allowed hosts there
+		// is nothing to fall back to, and guessing would file hits under the wrong
+		// site.
+		const host = body.host ?? soleAllowedHost(key.allowedHosts);
+		if (!host) {
+			sendError(
+				res,
+				400,
+				"BAD_REQUEST",
+				"`host` is required: this key is valid for more than one host, so the site cannot be inferred",
+			);
+			return;
+		}
+		const { hits } = body;
 		if (
 			key.allowedHosts &&
 			!hostMatchesAllowList(normalizeHost(host), key.allowedHosts)

packages/beacon-server/test/protocol.test.ts

@@ -25,8 +25,17 @@ describe("validateIngestBody", () => {
 			null,
 		);
 	});
-	it("rejects a missing host", () => {
-		assert.match(validateIngestBody({ hits: [{}] }) ?? "", HOST_REQUIRED_RE);
+	// A key identifies one site, so an omitted host is a valid batch: the server
+	// resolves it. Present-but-empty stays an error — that is a client that
+	// failed to compute a value, not one deliberately leaving it out.
+	it("accepts a batch that omits host", () => {
+		assert.equal(validateIngestBody({ hits: [{}] }), null);
+	});
+	it("rejects an empty host", () => {
+		assert.match(
+			validateIngestBody({ host: "", hits: [{}] }) ?? "",
+			HOST_REQUIRED_RE,
+		);
 	});
 	it("rejects an empty hits array", () => {
 		assert.match(

packages/beacon/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@snowseo/beacon",
-	"version": "0.1.7",
+	"version": "0.1.8",
 	"description": "Serve Markdown twins to AI crawlers, make them discoverable, and measure which crawlers actually arrive.",
 	"keywords": [
 		"ai",

packages/beacon/src/analytics.ts

@@ -6,7 +6,23 @@ import {
 
 export const INGEST_PATH = "/beacon/hits";
 
-export const DEFAULT_INGEST_ENDPOINT = "https://api.snowseo.com/v3/beacon/hits";
+/**
+ * Hosted collectors that speak this protocol, by name.
+ *
+ * Beacon is a protocol client, not one product's SDK, but the endpoint default
+ * only ever named SnowSEO — so pointing it anywhere else meant spelling out a
+ * full URL, and getting that URL subtly wrong (a bare origin, or the wrong
+ * mount path) fails as an opaque `401` rather than a 404. Naming the collectors
+ * removes the URL from the install entirely.
+ */
+export const COLLECTORS = {
+	"snow-analytics": "https://api.snowanalytics.app/api/beacon/hits",
+	snowseo: "https://api.snowseo.com/v3/beacon/hits",
+} as const;
+
+export type CollectorName = keyof typeof COLLECTORS;
+
+export const DEFAULT_INGEST_ENDPOINT = COLLECTORS.snowseo;
 
 const DEFAULT_BATCH_SIZE = 25;
 
@@ -29,6 +45,30 @@ export function normalizeIngestEndpoint(endpoint: string): string {
 	return url.toString();
 }
 
+/**
+ * `endpoint` (self-hosted) beats `collector` (hosted by name) beats the
+ * historical default, which stays SnowSEO so existing installs are unaffected.
+ */
+function resolveEndpoint(
+	config: Pick<AnalyticsConfig, "collector" | "endpoint">,
+): string {
+	if (config.endpoint) {
+		return normalizeIngestEndpoint(config.endpoint);
+	}
+	if (config.collector) {
+		const endpoint = COLLECTORS[config.collector];
+		if (!endpoint) {
+			throw new Error(
+				`[beacon] unknown collector "${config.collector}". Known collectors: ${Object.keys(
+					COLLECTORS,
+				).join(", ")}. Pass \`endpoint\` for a self-hosted one.`,
+			);
+		}
+		return endpoint;
+	}
+	return DEFAULT_INGEST_ENDPOINT;
+}
+
 export interface BeaconHit {
 	path: string;
 	userAgent: string;
@@ -48,7 +88,20 @@ export interface BeaconHit {
 
 export interface AnalyticsConfig {
 	key: string;
-	host: string;
+	/**
+	 * Which site the hits belong to.
+	 *
+	 * Optional: a key belongs to exactly one site, so a collector can resolve it
+	 * server-side. Send it when one key covers several hosts, or when the
+	 * collector requires it — an omitted host is simply left out of the body.
+	 */
+	host?: string;
+	/**
+	 * A hosted collector by name — the endpoint is filled in from
+	 * {@link COLLECTORS}. Ignored when `endpoint` is given.
+	 */
+	collector?: CollectorName;
+	/** A full ingest URL, for self-hosted collectors. Wins over `collector`. */
 	endpoint?: string;
 	batchSize?: number;
 	flushIntervalMs?: number;
@@ -109,8 +162,12 @@ function looksAutomated(userAgent: string | undefined): boolean {
 
 export class BeaconAnalytics {
 	private readonly config: Required<
-		Omit<AnalyticsConfig, "onError" | "fetch" | "onHit" | "disableCategories">
+		Omit<
+			AnalyticsConfig,
+			"onError" | "fetch" | "onHit" | "disableCategories" | "host" | "collector"
+		>
 	> & {
+		host?: string;
 		onError?: (error: unknown) => void;
 		onHit?: (hit: BeaconHit, match: AICrawlerMatch) => void;
 		fetch: typeof fetch;
@@ -122,9 +179,7 @@ export class BeaconAnalytics {
 		this.config = {
 			key: config.key,
 			host: config.host,
-			endpoint: config.endpoint
-				? normalizeIngestEndpoint(config.endpoint)
-				: DEFAULT_INGEST_ENDPOINT,
+			endpoint: resolveEndpoint(config),
 			batchSize: config.batchSize ?? DEFAULT_BATCH_SIZE,
 			flushIntervalMs: config.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS,
 			timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS,
@@ -201,7 +256,9 @@ export class BeaconAnalytics {
 					"Content-Type": "application/json",
 					"X-Beacon-Key": this.config.key,
 				},
-				body: JSON.stringify({ host: this.config.host, hits }),
+				body: JSON.stringify(
+					this.config.host ? { host: this.config.host, hits } : { hits },
+				),
 				signal: controller.signal,
 				keepalive: true,
 			});