From db8a62bb495cb2ac9914ba6b6b2e04c2f4be4f1a Mon Sep 17 00:00:00 2001 From: Pablo Abella Date: Tue, 13 Oct 2026 16:30:33 +0200 Subject: [PATCH 1/4] Tidy the design system, docs, and dependencies for forkers - Fix the /?range=toString crash and cap console route bodies at 64 KB. - Add a same-origin check to the console storage route. - Sanitize the server SDK pageUrl like the browser client (new allowedQueryParams option). - Update outdated dependencies and remove unused ones. - Share one Button, card, input, and focus style. Replace arbitrary values with scale values. - Raise badge colors to WCAG AA contrast and remove unused tokens. - Move branding to lib/site.ts and derive engagement labels from the query constants. - Rewrite the README, SDK README, CONTRIBUTING, and AGENTS.md so each fact lives in one place. Co-Authored-By: Claude Opus 5.5 (1M context) --- .env.example | 23 ++-- .github/pull_request_template.md | 2 +- AGENTS.md | 33 ++--- CONTRIBUTING.md | 17 +-- README.md | 161 ++++++++++++++--------- app/(dashboard)/layout.tsx | 38 +++--- app/(dashboard)/page.tsx | 28 ++-- app/(dashboard)/recordings/[id]/page.tsx | 17 +-- app/api/console/server-event/route.ts | 6 +- app/api/console/stored/route.ts | 7 +- app/globals.css | 118 ++++++++--------- app/layout.tsx | 6 +- app/sign-in/page.tsx | 85 ++++++------ components/button.tsx | 34 +++++ components/charts.tsx | 30 ++--- components/console/console-app.tsx | 47 ++++--- components/date/button.tsx | 59 --------- components/date/calendar.tsx | 41 +++--- components/date/date-filter.tsx | 4 +- components/date/date-range-picker.tsx | 31 ++--- components/date/popover.tsx | 2 +- components/recording-player.tsx | 30 ++--- components/sidebar-nav.tsx | 36 +++-- components/sign-in/brand-lines.tsx | 5 +- components/sign-in/search-select.tsx | 10 +- components/sign-in/workspace-picker.tsx | 16 ++- components/ui.tsx | 104 ++++++++------- examples/recipes/RECIPES.md | 12 +- lib/access.ts | 7 +- lib/collect.ts | 12 +- lib/crawlers.ts | 5 +- lib/dashboard.ts | 1 + lib/format.ts | 2 +- lib/queries.ts | 10 +- lib/range.ts | 10 +- lib/site.ts | 14 ++ package-lock.json | 133 ++++++++----------- package.json | 12 +- packages/analytics/README.md | 95 +++++++++---- packages/analytics/src/recorder.ts | 7 +- packages/analytics/src/server.ts | 9 +- packages/analytics/test/server.test.ts | 13 ++ test/dashboard-counts.test.ts | 2 +- 43 files changed, 714 insertions(+), 620 deletions(-) create mode 100644 components/button.tsx delete mode 100644 components/date/button.tsx create mode 100644 lib/site.ts diff --git a/.env.example b/.env.example index 233131a..6ca7114 100644 --- a/.env.example +++ b/.env.example @@ -3,26 +3,30 @@ # RawTree API base URL. Optional: defaults to https://api.rawtree.com. RAWTREE_API_URL=https://api.rawtree.com -# Database that holds the `events` and `recordings` tables. +# Database that holds the `events` and `recordings` tables. Required for the collector. RAWTREE_DATABASE=web_analytics # Insert-only key for event and recording ingestion (rtree key create --permission write_only). -# Tables are created on first insert. A key bound to database_roles with only -# `GRANT INSERT ON .*` is tighter, but then create the tables first. +# Required for the collector in both modes. Tables are created on first insert. A key bound +# to database_roles with only `GRANT INSERT ON .*` is tighter, but then create the +# tables first. RAWTREE_INGEST_KEY=rt_... # Read-only key for the dashboard and replay (rtree key create --permission read_only), # or a key bound to database_roles with only `GRANT SELECT ON .*`. +# Mode: with RAWTREE_QUERY_KEY and RAWTREE_INGEST_KEY both set, the dashboard needs no sign-in +# (env mode). Leave RAWTREE_QUERY_KEY empty for sign-in mode: visitors use their own RawTree +# API key or, when RAWTREE_CONNECTOR is set, Connect with RawTree through Vercel Connect. +# The collector keeps using RAWTREE_INGEST_KEY and RAWTREE_DATABASE in both modes. RAWTREE_QUERY_KEY=rt_... -# Mode: with RAWTREE_QUERY_KEY and RAWTREE_INGEST_KEY set, the dashboard needs no sign-in -# (env mode). Leave both empty for sign-in mode: visitors use their own RawTree API key or, -# when RAWTREE_CONNECTOR is set, Connect with RawTree through Vercel Connect. # Optional, sign-in mode only. Vercel Connect connector UID, e.g. rawtree/web-analytics -# (vercel connect create rawtree --target api --name web-analytics; run `vercel env pull` locally). +# (vercel connect create rawtree --target api --name web-analytics). Locally, also set +# VERCEL_OIDC_TOKEN: run `vercel env pull .env.vercel` and copy it (plain `vercel env pull` +# overwrites this file). RAWTREE_CONNECTOR= -# Collector (app/api/collect) +# Collector (app/api/collect). Required: without it the collector answers 500. # Comma-separated origins (scheme://host[:port]) of the sites that send events to /api/collect, # or "*" to accept any origin. Origin checks reject unwanted browsers; they are not authentication. # The dashboard's own Console doesn't need an entry here. @@ -33,5 +37,6 @@ ANALYTICS_ALLOWED_ORIGINS=https://example.com # Leave empty to disable server events. Generate with: openssl rand -hex 32 ANALYTICS_SERVER_TOKEN= -# Optional. Prefix for table names, used for disposable test tables (e.g. e2e_). Empty in production. +# Optional. Prefix for table names, for manual runs against disposable tables (e.g. e2e_). +# Empty in production. The automated tests never reach RawTree. RAWTREE_TABLE_PREFIX= diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 58bf83d..c7a4f55 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,7 +8,7 @@ ## Test plan -- [ ] `npm run typecheck`, `npm test`, and `npm run build` pass +- [ ] `npm run typecheck`, `npm test`, and `npm run build` pass (see CONTRIBUTING.md) - [ ] Verified locally with `npm run dev` (dashboard, recordings, or console, as relevant) - [ ] New or changed dashboard queries count each event once (`uniqExact` or `LIMIT 1 BY`) diff --git a/AGENTS.md b/AGENTS.md index 4ab41a5..94533ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # rawtree-web-analytics -Read `README.md` for what the project does and how it's set up, and `CONTRIBUTING.md` for the checks every change must pass. +Read `README.md` for what the project does and how it's set up. Follow `CONTRIBUTING.md`: its guidelines are the project's rules, and every change must pass its three checks (`npm run typecheck`, `npm test`, `npm run build`; install with `npm ci`). ## Scope @@ -9,30 +9,19 @@ Keep it simple. The project is only: an SDK to send events, examples, and a dash ## Layout - `packages/analytics/`: the publishable SDK (`@rawtree/analytics`). `src/protocol.ts` is the event contract; its types and validator are the source of truth for the SDK and the collector. -- `app/api/collect/route.ts` + `lib/collect.ts`, `lib/rawtree.ts`: the Next.js collector. -- `lib/queries.ts`: every dashboard SQL query (one documented template per widget; aliases are the fields `app/(dashboard)/page.tsx` reads). `lib/dashboard.ts` runs them; pages import only `lib/dashboard.ts`. Bot detection lives in `lib/crawlers.ts`, date ranges in `lib/range.ts`. -- `app/(dashboard)/console/` + `components/console/`: the test console page (`/console`). It sends through the workspace SDK to `app/api/console/collect` (same `handleCollect`, with the visitor's `access.ingest`); `app/api/console/server-event` runs the server SDK in-process and `app/api/console/stored` is the storage check (`access.query`). +- `app/api/collect/route.ts` + `lib/collect.ts`, `lib/rawtree.ts`: the Next.js collector. It always writes with the env ingest key. `lib/access.ts` decides env mode vs sign-in mode for the dashboard and console. +- `lib/queries.ts`: every dashboard SQL query (one documented template per widget; aliases are the fields `app/(dashboard)/page.tsx` reads). `lib/dashboard.ts` runs them; pages import only `lib/dashboard.ts`. Bot detection lives in `lib/crawlers.ts`, date ranges in `lib/range.ts`, branding in `lib/site.ts`. +- `app/(dashboard)/console/` + `components/console/`: the test console page (`/console`). It sends through the workspace SDK to `app/api/console/collect` (same `handleCollect`, with the visitor's `access.ingest`); `app/api/console/server-event` runs the server SDK in-process and `app/api/console/stored` is the storage check (`access.query`). Both accept same-origin requests only and cap bodies at 64 KB. - `examples/recipes/`: tracking recipes (`RECIPES.md` + `.ts` modules), typechecked by `npm run typecheck`. -- `test/`: collector tests. +- `test/`: collector, access, dashboard query, and recording reassembly tests. Every test mocks `fetch`, so none reaches RawTree. -## Commands +## Agent notes -```sh -npm install -npm test # builds the SDK, runs SDK and collector tests -npm run typecheck -npm run build # SDK + Next.js -``` - -Node โ‰ฅ 24 runs `.ts` directly: use erasable syntax only (no enums, namespaces, or parameter properties) and `.ts` extensions in relative imports. - -## Invariants - -- The browser SDK must stay free of rrweb (`./recorder` is a separate entry) and the server entry free of browser globals. `packages/analytics/test/entries.test.ts` and `server.test.ts` check it (part of `npm test`). -- The collector acknowledges only after RawTree accepted every row. `/api/collect` takes credentials and table destinations only from configuration, never from requests. In sign-in mode the dashboard and `/api/console/*` use the visitor's key or Connect grant (`lib/access.ts`), but SQL and table names still come only from the server. -- Delivery is at least once: keep IDs stable across retries and deduplicate in every query (`uniqExact(event_id)` or `LIMIT 1 BY`). -- RawTree reads lag writes by about 0.1โ€“0.4 s and replicas can disagree briefly. Poll in tests; never assume read-after-write. -- Write synthetic data to the real `events`/`recordings` tables only when the owner asks (since 2026-10-08 they hold example test data at the owner's request). For automated tests use temporary prefixed tables (`RAWTREE_TABLE_PREFIX`) and drop them afterwards. Ask the owner before any other write to shared RawTree resources. +- Never run SQL or table names that come from a request. Credentials come from configuration or `lib/access.ts`. +- Delivery is at least once: deduplicate every count (`uniqExact(event_id)` or `LIMIT 1 BY`). +- Node โ‰ฅ 24 runs `.ts` directly: erasable syntax only and `.ts` extensions in relative imports. +- RawTree reads lag writes by about 0.1โ€“0.4 s and replicas can disagree briefly. Poll, and never assume read-after-write. +- Write synthetic data to the real `events`/`recordings` tables only when the owner asks (since 2026-10-08 they hold example test data at the owner's request). For live or automated runs against RawTree use temporary prefixed tables (`RAWTREE_TABLE_PREFIX`) and drop them afterwards. Ask the owner before any other write to shared RawTree resources. - Secrets live only in `.env.local` (ignored). Keep `.env.example` in sync when configuration changes. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7df8357..11102f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,8 +17,8 @@ npm run dev # http://localhost:3000 ## Before opening a pull request ```sh -npm run typecheck -npm test +npm run typecheck # also typechecks the recipes in examples/recipes/ +npm test # offline: RawTree is mocked npm run build ``` @@ -28,16 +28,17 @@ CI runs the same three checks on every pull request. Every pull request also get - **Keep pull requests small and focused**, with a test plan and screenshots for UI changes. - **The event contract is `packages/analytics/src/protocol.ts`.** Keep it backward compatible: existing events must keep validating and storing the same way. -- **Dashboard SQL lives in `lib/queries.ts`.** Count each event once (`uniqExact(toString(event_id))` or `LIMIT 1 BY`), never `count()` over raw rows. Add new counting queries to `EVENT_QUERIES` so the dedup test covers them. -- **Never accept SQL or table names from the browser.** Credentials and destinations come from server configuration or the signed-in viewer's access (`lib/access.ts`). -- **Tests never write to real tables.** Use temporary tables with `RAWTREE_TABLE_PREFIX` and drop them afterwards. -- **The browser SDK stays free of rrweb** (`./recorder` is a separate entry) and the server entry free of browser globals. `packages/analytics/test/entries.test.ts` checks this. +- **Dashboard SQL lives in `lib/queries.ts`** and follows the [query rules](README.md#-ask-your-ai-agent): count each event once (`uniqExact(toString(event_id))` or `LIMIT 1 BY`), never `count()` over raw rows. Add new counting queries to `EVENT_QUERIES` so the dedup test covers them. The README's [add a widget](README.md#-customize-it) steps cover the rest. +- **The collector acknowledges only after RawTree accepted every row.** Keep event and chunk IDs stable across retries. +- **Never accept SQL or table names from the browser.** Credentials and destinations come from server configuration or the signed-in visitor's access (`lib/access.ts`). +- **Tests stay offline.** Mock `fetch` instead of calling RawTree. For a manual run against real RawTree, set `RAWTREE_TABLE_PREFIX` (e.g. `e2e_`) to use disposable tables, and drop them afterwards. +- **RawTree reads lag writes** by a fraction of a second, and replicas can briefly disagree. Poll, and never assume read-after-write. +- **The browser SDK stays free of rrweb** (`./recorder` is a separate entry): `packages/analytics/test/entries.test.ts` checks it. **The server entry stays free of browser globals:** `server.test.ts` checks it. - **Node runs the `.ts` sources directly:** erasable TypeScript only (no enums, namespaces, or parameter properties) and `.ts` extensions in relative imports. - **Next.js 16 differs from older versions.** Check `node_modules/next/dist/docs/` before using its APIs. +- **Configuration changes** go in `.env.example` too. Secrets live only in `.env.local`. - **No secrets, keys, or private event data** in code, issues, or pull requests. -[AGENTS.md](AGENTS.md) has the same rules in more detail, for people and coding agents. - ## Releasing the SDK (maintainers) `@rawtree/analytics` is published to npm by the [Release SDK](.github/workflows/release-sdk.yml) workflow, with provenance. diff --git a/README.md b/README.md index 8416ff2..69014e0 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Web analytics on [RawTree](https://rawtree.com): an SDK for events and session recordings, a Next.js collector, and a dashboard with bot detection and session replay. Your events land in your own RawTree tables, so every number on the dashboard is a SQL query you can run yourself. +![RawTree Web Analytics dashboard](public/overview.jpg) + ## ๐Ÿ“ฆ What's inside - **SDK** ([`@rawtree/analytics`](packages/analytics)): send product-defined events from the browser and your backend, plus rrweb session recordings with masking on by default. Delivery is batched, retried, and at least once. @@ -36,7 +38,7 @@ flowchart LR The collector and the dashboard are the same Next.js app. Its Console page plays "Your app", sending through the same SDK to its own collect route. - Events and recording chunks are stored as rows. Sessions, recordings, and their completeness are derived in SQL, not kept in a mutable table. -- Delivery is at least once, so event IDs stay stable across retries and every query deduplicates (`uniqExact(event_id)` or `LIMIT 1 BY`). +- Delivery is at least once, so event IDs stay stable across retries and every query deduplicates. - `/api/collect` writes only with the insert key from server configuration. The dashboard and console use either the server's keys or the signed-in visitor's own access, and only ever run the dashboard's own SQL: never SQL or table names from the browser. ## ๐ŸŒณ Set up RawTree @@ -51,41 +53,35 @@ rtree key create --name web-analytics-query --permission read_only --database we That's it. The `events` and `recordings` tables are created on the first ingestion, and their columns appear as events arrive. -These keys apply to every database in your cluster. To limit them to the analytics database, use keys bound to [database roles](https://rawtree.com/docs/reference/authentication) instead. Role-bound inserts don't create tables, so create `events` and `recordings` first in that case. +`--database` only sets the key's default database. The permission itself (`write_only`, `read_only`) applies to every database in your cluster. To limit a key to the analytics database, use a key bound to [database roles](https://rawtree.com/docs/reference/authentication) instead. Role-bound inserts don't create tables, so create `events` and `recordings` first in that case. ## ๐Ÿš€ Run it locally You need Node.js 24 or later. ```sh -cp .env.example .env.local # set RAWTREE_DATABASE, RAWTREE_INGEST_KEY, RAWTREE_QUERY_KEY +cp .env.example .env.local # set RAWTREE_DATABASE, RAWTREE_INGEST_KEY, RAWTREE_QUERY_KEY, ANALYTICS_ALLOWED_ORIGINS npm install npm run dev # dashboard + collector on http://localhost:3000 ``` -[`.env.example`](.env.example) documents every variable. With both keys set, the dashboard opens without signing in. Leave `RAWTREE_QUERY_KEY` and `RAWTREE_INGEST_KEY` empty to try sign-in mode instead (see "Deploy the dashboard" below). - -## ๐Ÿงช Use the console +[`.env.example`](.env.example) documents every variable. With both keys set, the dashboard opens without signing in. Leave `RAWTREE_QUERY_KEY` empty to try sign-in mode instead (see [Sign-in mode](#sign-in-mode-a-public-dashboard)). -Open **Console** in the dashboard's sidebar (`/console`). It sends data through `@rawtree/analytics` exactly as a real app would, to `/api/console/collect`: the same validation and insert path as `/api/collect`, but with your own credentials, so test data only lands in your database. It needs no allowed origin or server token. +## โœ๏ธ Add it to your app -1. **Consent:** turn on Analytics. Nothing is sent before that. Turn on Session recording too if you want a replay. -2. **Browser events:** send recipe-shaped events (`page_view`, `cta_click`, `scroll_depth`, and more) or a custom event with your own JSON. -3. **Server events:** send backend events like `signup_completed` with a stable ID, through `@rawtree/analytics/server` on the app's server. Resending keeps the same ID, and the dashboard still counts it once. -4. **Simulated traffic:** send page views as crawlers (GPTBot, ClaudeBot, Googlebot, curl, and more) or as campaign visits (Google ad, newsletter, Hacker News). Each click is a new visitor. This is how you fill the Bots and Acquisition sections. -5. **Recording playground:** change the page while recording, and check that the input, the `rr-block` region, and the sample secrets stay hidden. "Mask all text" switches to fully masked recordings. -6. **Event viewer:** every row moves from Queued to Sent (collector 200) to Stored (found in RawTree). Reads lag writes by a moment, so Stored can take a few seconds. +```sh +npm install @rawtree/analytics +``` -Everything you send goes to the database you're signed in to (or `RAWTREE_DATABASE` when the keys come from the environment). Use a separate test database (same setup, another name) if you don't want test data next to real traffic: RawTree can't delete individual rows yet. +```ts +import { createAnalytics } from "@rawtree/analytics"; -## ๐Ÿ“Š Use the dashboard +// Create the client only after the user allows analytics. +const analytics = createAnalytics({ endpoint: "https:///api/collect" }); +analytics.sendEvent("checkout_started", { plan: "pro" }); +``` -- **Sign in:** only in sign-in mode. Connect with RawTree or paste an API key, and the sidebar shows which database you're looking at and a Sign out button. -- **Date range:** the picker in the header has presets (Today, This week, Last 7 days, Last month, and more) and a calendar for custom ranges. Days are UTC. Every number is compared with the previous period of the same length. -- **Humans only:** once events carry a user agent, every section except Bots excludes them. Before that, the sections show an "All traffic" badge. -- **Sections:** Overview has the headline numbers. Traffic has the daily trend and breakdown. Acquisition shows channels, referrers, and UTM campaigns by each session's first touch. Content lists top pages. Engagement covers time on page, scroll depth, and CTA clicks. Bots shows bot page requests, crawler types, top crawlers, and the most crawled paths. -- **Recordings:** pick a session from the list on the left and replay it. Pause, seek, and change speed from the controls under the player. Incomplete recordings replay only their complete stretches and say what was skipped. -- **Tune a widget:** every dashboard query lives in [`lib/queries.ts`](lib/queries.ts), one documented template per widget. Edit the SQL there (for example add a channel host to `SEARCH_HOSTS` or change `ENGAGED_MS`). The aliases are the field names the page reads: if you add or rename one, update that query's row type in `lib/dashboard.ts` and its use in `app/(dashboard)/page.tsx`. +Add your site's origin to `ANALYTICS_ALLOWED_ORIGINS`. Nothing is tracked automatically: event names and properties are yours. The [SDK README](packages/analytics/README.md) covers options, backend events, session recording, and the stored columns. The [tracking recipes](examples/recipes/RECIPES.md) show page views, CTA clicks, scroll depth, web vitals, signups, and their queries. ## โ˜๏ธ Deploy the dashboard @@ -94,14 +90,14 @@ Deploy the Next.js app (for example on Vercel) in one of two modes, chosen by it | Variable | Value | | --- | --- | | `RAWTREE_API_URL` | Optional. Defaults to `https://api.rawtree.com` | -| `RAWTREE_DATABASE` | Your analytics database (used with the env keys) | -| `RAWTREE_INGEST_KEY` | The insert-only key, used by the collector (and the console in env mode) | -| `RAWTREE_QUERY_KEY` | The read-only key, used by the dashboard and replay in env mode | +| `RAWTREE_DATABASE` | Required for the collector. Your analytics database | +| `RAWTREE_INGEST_KEY` | Required for the collector. The insert-only key | +| `ANALYTICS_ALLOWED_ORIGINS` | Required for the collector. The origins of the sites you track, e.g. `https://example.com` | +| `RAWTREE_QUERY_KEY` | The read-only key. Set it for env mode, leave it empty for sign-in mode | | `RAWTREE_CONNECTOR` | Optional, sign-in mode only. The Vercel Connect connector, e.g. `rawtree/web-analytics` | -| `ANALYTICS_ALLOWED_ORIGINS` | The origins of the sites you track, e.g. `https://example.com` | | `ANALYTICS_SERVER_TOKEN` | Optional. A random secret for backend events | -Then point the SDK at `https:///api/collect`. The collector always uses `RAWTREE_INGEST_KEY` and `RAWTREE_DATABASE`, in both modes. Without them it rejects events as not configured. +Then point the SDK at `https:///api/collect`. The collector works the same in both modes: it always writes to `RAWTREE_DATABASE` with `RAWTREE_INGEST_KEY`, and accepts browsers only from `ANALYTICS_ALLOWED_ORIGINS`. If any of the three is missing, it answers `500 collector_not_configured`. ### Env mode: a private dashboard @@ -109,10 +105,10 @@ Set both `RAWTREE_QUERY_KEY` and `RAWTREE_INGEST_KEY`. There is no sign-in: **an ### Sign-in mode: a public dashboard -Leave `RAWTREE_QUERY_KEY` and `RAWTREE_INGEST_KEY` unset. Every page asks visitors to sign in with their own RawTree access, and shows only their own database: +Leave `RAWTREE_QUERY_KEY` empty. Keep `RAWTREE_INGEST_KEY`, `RAWTREE_DATABASE`, and `ANALYTICS_ALLOWED_ORIGINS` set if the deployment also collects your sites' events. Every page asks visitors to sign in with their own RawTree access, and shows only their own database. The dashboard and the console use the visitor's access, never the server's keys. Your tracked sites still write to `RAWTREE_DATABASE`, whoever signs in. -- **Connect with RawTree** (when `RAWTREE_CONNECTOR` is set): the visitor approves access with their RawTree account through Vercel Connect, then picks the organization, cluster, and database. The browser only holds an opaque, HTTP-only session cookie; Vercel Connect stores and refreshes the grant. Sign out revokes it. -- **Use an API key:** the visitor pastes one key and a database name (default `web_analytics`). The key must **read and write** that database: the dashboard reads it and the console writes test events to it. Create one with `rtree key create --name web-analytics --permission read_write --database web_analytics`. Permission keys cover the whole cluster; a key limited to one database needs a role with `GRANT SELECT, INSERT ON web_analytics.*`, and then create the tables first. The key is kept in an HTTP-only session cookie until sign-out or the browser closes. +- **Connect with RawTree** (when `RAWTREE_CONNECTOR` is set): the visitor approves access with their RawTree account through Vercel Connect, then picks the organization, cluster, and database. The browser only holds an opaque, HTTP-only session cookie. Vercel Connect stores and refreshes the grant. Sign out revokes it. +- **Use an API key:** the visitor pastes one key and a database name (default `web_analytics`). The key must **read and write** that database: the dashboard reads it and the console writes test events to it. Create one with `rtree key create --name web-analytics --permission read_write --database web_analytics`. Like the keys above, it covers the whole cluster. A key limited to one database needs a role with `GRANT SELECT, INSERT ON web_analytics.*`, and then the tables must exist first. The key is kept in an HTTP-only session cookie until sign-out or the browser closes. RawTree OAuth grants are not read-only, so the server only runs the dashboard's own queries ([`lib/queries.ts`](lib/queries.ts)) and collector-validated inserts with them. It never accepts SQL or table names from the browser. @@ -122,84 +118,125 @@ To enable Connect, create a Vercel Connect connector for the RawTree API from th vercel link vercel connect create rawtree --target api --name vercel env add RAWTREE_CONNECTOR # e.g. rawtree/ -vercel env pull # for local development: gives the SDK the project's OIDC token +vercel env pull .env.vercel # for local development: copy VERCEL_OIDC_TOKEN into .env.local ``` -Connect needs a Vercel deployment (or `vercel env pull` locally). Elsewhere, visitors sign in with an API key. +Plain `vercel env pull` overwrites `.env.local`, including your RawTree keys, so pull into another file. The OIDC token expires: pull it again when Connect stops working locally. Connect needs a Vercel deployment (or that token locally). Elsewhere, visitors sign in with an API key. ### Before you deploy - **The collector accepts events from the allowed origins** and doesn't rate limit. Origin checks stop other websites' browsers, not scripts, so add rate limiting for `/api/collect` on your platform if you need it (for example a firewall rule on Vercel). - **Recordings replay what was recorded.** Keep recordings masked (the default) unless you're sure about what your pages show. -## โœ๏ธ Add it to your app +## ๐Ÿ“Š Use the dashboard -```sh -npm install @rawtree/analytics -npm install rrweb@^2.1.7 # only if you use session recording -``` +- **Sign in:** only in sign-in mode. Connect with RawTree or paste an API key, and the sidebar shows which database you're looking at and a Sign out button. +- **Date range:** the picker in the header has presets (Today, This week, Last 7 days, Last month, and more) and a calendar for custom ranges. Days are UTC. Every number is compared with the previous period of the same length. +- **Humans only:** once the events table has a `user_agent` column, every section except Bots excludes bots. Before that, the sections show an "All traffic" badge. +- **Sections:** Overview has the headline numbers. Traffic has the daily trend and breakdown. Acquisition shows channels, referrers, and UTM campaigns by each session's first touch. Content lists top pages. Engagement covers time on page, scroll depth, and CTA clicks. Bots shows bot page requests, crawler types, top crawlers, and the most crawled paths. +- **Recordings:** pick a session from the list on the left and replay it. Pause, seek, and change speed from the controls under the player. Incomplete recordings replay only their complete stretches and say what was skipped. -```ts -import { createAnalytics } from "@rawtree/analytics"; +## ๐Ÿงช Use the console -// Create the client only after the user allows analytics. -const analytics = createAnalytics({ endpoint: "https:///api/collect" }); -analytics.sendEvent("checkout_started", { plan: "pro" }); -``` +Open **Console** in the dashboard's sidebar (`/console`). It sends data through `@rawtree/analytics` exactly as a real app would, to `/api/console/collect`: the same validation and insert path as `/api/collect`, but with your own access (the env keys in env mode, your sign-in in sign-in mode), so test data only lands in your database. It needs no allowed origin or server token. + +1. **Consent:** turn on Analytics. Nothing is sent before that. Turn on Session recording too if you want a replay. +2. **Browser events:** send recipe-shaped events (`page_view`, `cta_click`, `scroll_depth`, and more) or a custom event with your own JSON. +3. **Server events:** send backend events like `signup_completed` with a stable ID, through `@rawtree/analytics/server` on the app's server. Resending keeps the same ID, and the dashboard still counts it once. +4. **Simulated traffic:** send page views as crawlers (GPTBot, ClaudeBot, Googlebot, curl, and more) or as campaign visits (Google ad, newsletter, Hacker News). Each click is a new visitor. This is how you fill the Bots and Acquisition sections. +5. **Recording playground:** change the page while recording, and check that the input, the `rr-block` region, and the sample secrets stay hidden. "Mask all text" switches to fully masked recordings. +6. **Event viewer:** every row moves from Queued to Sent (collector 200) to Stored (found in RawTree). Reads lag writes by a moment, so Stored can take a few seconds. -Then query it, counting each event once: +Everything you send goes to the database you're signed in to (or `RAWTREE_DATABASE` in env mode). Use a separate test database (same setup, another name) if you don't want test data next to real traffic: RawTree can't delete individual rows yet. -```sql -SELECT toString(properties.plan) AS plan, uniqExact(toString(event_id)) AS checkouts -FROM events -WHERE toString(event_name) = 'checkout_started' -GROUP BY plan -``` +## ๐ŸŽจ Customize it + +**Branding.** The name, tagline, description, and docs/GitHub links live in [`lib/site.ts`](lib/site.ts). The images are `app/icon.svg`, `app/favicon.ico`, `app/apple-icon.png`, `app/opengraph-image.png`, `components/rawtree-logo.svg`, and `public/overview.jpg`. Colors and other design tokens are CSS variables in `app/globals.css`. -Nothing is tracked automatically: event names and properties are yours. The [SDK README](packages/analytics/README.md) covers options, backend events, and recording privacy. The [tracking recipes](examples/recipes/RECIPES.md) show page views, CTA clicks, scroll depth, web vitals, signups, and their queries. +**Tune a widget.** Every dashboard query lives in [`lib/queries.ts`](lib/queries.ts), one documented template per widget. Edit the SQL there, for example add a channel host to `SEARCH_HOSTS` or change `ENGAGED_MS` (the dashboard's labels follow it). The SQL aliases are the field names the page reads. + +**Add a widget.** + +1. Write its SQL template in `lib/queries.ts`, reading `FROM ${events(s)}` and filtering with `inWindow(s)`. Follow the [query rules](#-ask-your-ai-agent). +2. If it counts events, add it to `EVENT_QUERIES`, so the dedup test checks it. +3. Add every column it reads that a young table may lack (for example `properties.plan`) to `OPTIONAL_COLUMNS`. A missing column then reads as NULL instead of failing the dashboard. +4. Run it in `getDashboard` in `lib/dashboard.ts`, with a row type that matches the aliases. +5. Render it in `app/(dashboard)/page.tsx`. +6. For a new section, also add its `id` to the `sections` list in `components/sidebar-nav.tsx`, so the sidebar links to it. + +Date presets and the previous-period comparison live in `lib/range.ts`. Every day is a UTC day. ## ๐Ÿค– Ask your AI agent Any AI agent or LLM client that speaks MCP can query your analytics through the [RawTree MCP server](https://rawtree.com/docs/reference/mcp). Follow the docs to connect your client of choice, and [use an API key](https://rawtree.com/docs/reference/mcp#hosted-mcp-with-an-api-key) instead of OAuth: give it the **read-only key** from the setup. OAuth grants broad access to your RawTree account, including destructive operations, while the read-only key can only read data. -Give the agent these rules so its numbers match the dashboard: +These query rules make any number match the dashboard. Give them to your agent, and follow them in your own SQL: -- Tables: `events` (one row per event) and `recordings` (one row per recording chunk part). -- Delivery is at least once. Count with `uniqExact(toString(event_id))`, never `count()`. Sessions are `uniqExact(toString(session_id))`, visitors are `uniqExact(toString(anonymous_id))`. +- Tables: `events` (one row per event) and `recordings` (one row per recording chunk part). Their columns are listed in the [SDK README](packages/analytics/README.md#stored-tables). +- Delivery is at least once. Count with `uniqExact(toString(event_id))`, never `count()`, or deduplicate with `LIMIT 1 BY event_id` before aggregating values. Sessions are `uniqExact(toString(session_id))`, visitors are `uniqExact(toString(anonymous_id))`. - Fields are dynamic JSON. Cast them: `toString(event_name)`, `CAST(occurred_at_ms AS Int64)`, `toString(properties.plan)`. - Time is `occurred_at_ms` (epoch milliseconds, UTC). Always filter on an explicit window. - Page views are `event_name = 'page_view'`, with `page_path`, `page_url`, and `referrer` on the row. - Bots are told apart by `user_agent`. The dashboard's pattern is `BOT_UA_PATTERN` in [`lib/crawlers.ts`](lib/crawlers.ts). +For example, checkouts per plan: + +```sql +SELECT toString(properties.plan) AS plan, uniqExact(toString(event_id)) AS checkouts +FROM events +WHERE toString(event_name) = 'checkout_started' + AND CAST(occurred_at_ms AS Int64) >= toUnixTimestamp64Milli(now64() - INTERVAL 7 DAY) +GROUP BY plan +``` + The [tracking recipes](examples/recipes/RECIPES.md) and [`lib/queries.ts`](lib/queries.ts) (every dashboard query) have more queries to borrow from. +## ๐Ÿฉบ Troubleshooting + +**The collector rejects events.** `/api/collect` answers with a JSON `error`: + +| Status | `error` | What to check | +| --- | --- | --- | +| 403 | `origin_not_allowed` | The page's origin isn't in `ANALYTICS_ALLOWED_ORIGINS`. It must match scheme, host, and port exactly (`www.` counts). Browsers report this as a CORS error. Requests with no `Origin` header (curl, a backend) need the server token instead | +| 401 | `invalid_token` | The request has an `Authorization` header, but `ANALYTICS_SERVER_TOKEN` is unset or different | +| 413 | `payload_too_large` | The body is over 1,000,000 bytes. The SDK keeps its batches under this | +| 400 | `invalid_request`, `invalid_json`, `invalid_body` | The batch failed validation or isn't UTF-8 JSON. The `message` says what's wrong | +| 500 | `collector_not_configured` | `RAWTREE_DATABASE`, `RAWTREE_INGEST_KEY`, or `ANALYTICS_ALLOWED_ORIGINS` is missing | +| 503 | `storage_unavailable` | RawTree refused the insert. Check the server log: a wrong key, or a role-bound key whose tables don't exist yet | + +**"All traffic" badge.** No event has stored a user agent yet, so bots can't be told apart. Browser events carry one automatically. + +**Console stays on Sent.** RawTree reads lag writes by a moment. Stored usually appears within seconds, and the console gives up after 30 s. + +**Role-bound keys.** Keys bound to database roles can't create tables. Create `events` and `recordings` first, or inserts fail with `503 storage_unavailable`. + ## ๐Ÿ”’ Privacy notes - Session recording masks all text and inputs by default. `maskAllText: false` shows page text while inputs stay masked. Elements with `rr-block` are never recorded. -- Page URLs keep only allowlisted query parameters (`utm_*` by default) and drop credentials and fragments. +- Page URLs keep only allowlisted query parameters (by default `utm_source`, `utm_medium`, `utm_campaign`, `utm_term`, `utm_content`) and drop credentials and fragments. - RawTree can't delete individual rows yet, so plan retention before collecting real users. ## ๐Ÿ› ๏ธ Development ```sh -npm test # builds the SDK, runs SDK and collector/dashboard tests +npm ci npm run typecheck +npm test # offline: builds the SDK, runs the SDK and app tests with RawTree mocked npm run build # SDK + Next.js ``` -Contributions are welcome: see [CONTRIBUTING.md](CONTRIBUTING.md). CI runs typecheck, tests, and build on every pull request, and Vercel deploys a preview for each pull request and production from `main`. - -Automated tests never touch your real tables: they use temporary tables named with `RAWTREE_TABLE_PREFIX`. Node runs the `.ts` sources directly, so use erasable syntax only and `.ts` extensions in relative imports. +Contributions are welcome. [CONTRIBUTING.md](CONTRIBUTING.md) has the rules and the checks CI runs on every pull request. Vercel deploys a preview for each pull request and production from `main`. | Path | What | | --- | --- | | `packages/analytics/` | The SDK. `src/protocol.ts` is the event contract for both the SDK and the collector | | `app/`, `components/` | Dashboard, recordings, console, and the collector routes | -| `lib/` | Collector, RawTree client, dashboard SQL, crawler classifier, recording reassembly | +| `lib/` | Collector, RawTree client, sign-in access, dashboard SQL, crawler classifier, recording reassembly | +| `lib/site.ts` | Name, tagline, and links shown in the app | | `examples/recipes/` | Tracking recipes (typechecked against the SDK) | -| `test/` | Collector and dashboard tests | -| `AGENTS.md` | Guide for coding agents working in this repo | -| `.github/` | CI workflow, pull request template, and issue forms | +| `test/` | Collector, access, dashboard query, and recording reassembly tests | +| `AGENTS.md` | Notes for coding agents working in this repo | +| `.github/` | CI and release workflows, pull request template, and issue forms | ## ๐Ÿ“„ License diff --git a/app/(dashboard)/layout.tsx b/app/(dashboard)/layout.tsx index 6d6de4f..126772e 100644 --- a/app/(dashboard)/layout.tsx +++ b/app/(dashboard)/layout.tsx @@ -1,47 +1,53 @@ -// App shell ported from Treewatcher (components/app-shell.tsx): a floating card sidebar -// with the RawTree logo and navigation. Each page renders its own sticky toolbar, +// App shell: a floating card sidebar with the logo and navigation. Each page renders its own sticky toolbar, // because only pages receive search params (the date range). /sign-in sits outside this group. import Image from "next/image"; import rawtreeLogo from "../../components/rawtree-logo.svg"; -import { IconExternalLink, IconLogout } from "@tabler/icons-react"; +import { IconLogout } from "@tabler/icons-react"; +import { Button } from "../../components/button.tsx"; import { SidebarNav } from "../../components/sidebar-nav.tsx"; +import { ExternalLink, cardClass, cn } from "../../components/ui.tsx"; import { requireAccess } from "../../lib/access.ts"; import { signOut } from "../sign-in/actions.ts"; +import { SITE } from "../../lib/site.ts"; export default async function DashboardLayout({ children }: { children: React.ReactNode }) { const access = await requireAccess(); return (
-