diff --git a/apps/server/scripts/analytics-backfill.ts b/apps/server/scripts/analytics-backfill.ts new file mode 100644 index 0000000000000..6a096d73e49fa --- /dev/null +++ b/apps/server/scripts/analytics-backfill.ts @@ -0,0 +1,433 @@ +#!/usr/bin/env node +// @effect-diagnostics nodeBuiltinImport:off globalDate:off globalFetch:off globalConsole:off globalTimers:off preferSchemaOverJson:off - a one-off operator script over a SQLite copy, outside any Effect runtime. +/** + * T3-CUSTOM(expbkt3): replay the orchestration V2 event history into PostHog + * session metrics (TEC-1631). + * + * Feeds the same tracker the live server uses + * (`src/telemetry/sessionActivity/SessionActivityReducer.ts`) with the events + * of a SQLite COPY of the server state, and writes the resulting PostHog + * events. It never opens a live state file: make a copy first, for example + * with `VACUUM INTO` (see AGENTS.md, "Test data"). + * + * node apps/server/scripts/analytics-backfill.ts [options] + * + * --dry-run (default) write JSONL and print counts; send nothing + * --out JSONL output path (default analytics-backfill.jsonl) + * --send POST to ${T3CODE_POSTHOG_HOST}/batch/; the key comes + * only from env T3CODE_POSTHOG_KEY + * --until ignore events after this time; required with --send. + * Set it to the moment the live reactor started, so + * the two never overlap. + * + * Events with no user whose email is known are skipped (see toBatchItem). + * --report-days print per-state hours for the last n days (default 7) + * + * Events are processed one thread at a time, sorted by `occurredAt`: imported + * history was appended to the log out of time order, so sequence order alone + * is not time order. Spans still open at the end are not emitted; the live + * server owns them. Every event carries `backfill: true` and a stable + * `$insert_id` and uuid, so a re-run does not duplicate events. + */ +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; + +import { + makeSessionActivityTracker, + type SessionActivityEmission, + type SessionActivityEvent, + toEpochMs, +} from "../src/telemetry/sessionActivity/SessionActivityReducer.ts"; + +const BATCH_SIZE = 500; +const MAX_SEND_ATTEMPTS = 6; + +/** Event types the tracker reads; everything else is skipped in SQL. */ +const RELEVANT_EVENT_TYPES = [ + "run.created", + "run.updated", + "run-attempt.created", + "runtime-request.updated", + "subagent.updated", + "turn-item.updated", + "provider-thread.updated", + "plan.updated", + "message.updated", +] as const; + +interface Options { + readonly dbPath: string; + readonly send: boolean; + readonly outPath: string; + readonly untilMs: number | undefined; + readonly reportDays: number; +} + +function fail(message: string): never { + console.error(`analytics-backfill: ${message}`); + process.exit(1); +} + +function parseArgs(argv: ReadonlyArray): Options { + let dbPath: string | undefined; + let send = false; + let outPath = "analytics-backfill.jsonl"; + let untilMs: number | undefined; + let reportDays = 7; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = () => argv[++index] ?? fail(`${arg} needs a value`); + switch (arg) { + case "--dry-run": + send = false; + break; + case "--send": + send = true; + break; + case "--out": + outPath = next(); + break; + case "--until": + untilMs = toEpochMs(next()) ?? fail("--until must be an ISO time"); + break; + case "--report-days": + reportDays = Number(next()); + break; + default: + if (arg === undefined || arg.startsWith("--")) fail(`unknown option ${arg}`); + dbPath = arg; + } + } + if (dbPath === undefined) fail("pass the path of a SQLite COPY of the state database"); + return { dbPath, send, outPath, untilMs, reportDays }; +} + +/** Refuses any state file under a T3 home `userdata` directory: those are live. */ +function assertNotLiveState(dbPath: string): string { + const resolved = NodeFS.realpathSync(dbPath); + const t3Home = NodePath.join(NodeOS.homedir(), ".t3"); + const relative = NodePath.relative(t3Home, resolved); + const insideT3Home = !relative.startsWith("..") && !NodePath.isAbsolute(relative); + if (insideT3Home && relative.split(NodePath.sep).includes("userdata")) { + fail(`${resolved} is a live T3 state file. Copy it with VACUUM INTO and pass the copy.`); + } + return resolved; +} + +type Rec = Record; + +/** Free-text payload fields dropped as soon as a row is read. */ +const TEXT_KEYS = [ + "prompt", + "result", + "progress", + "title", + "text", + "attachments", + "context", + "markdown", + "steps", + "explanation", + "answers", +] as const; + +function asRec(value: unknown): Rec | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Rec) + : undefined; +} + +/** + * Keeps only the payload fields the tracker reads, so a thread's history fits + * in memory and no prompt, output or message text is held longer than a row. + */ +function slimPayload(type: string, payload: Rec): Rec { + switch (type) { + case "turn-item.updated": + return { + id: payload.id, + type: payload.type, + status: payload.status, + runId: payload.runId ?? null, + ...(asRec(payload.input)?.persistent === true ? { input: { persistent: true } } : {}), + ...(asRec(payload.failure) === undefined + ? {} + : { failure: { class: asRec(payload.failure)?.class } }), + }; + case "subagent.updated": + case "message.updated": + case "plan.updated": { + const rest: Rec = { ...payload }; + for (const key of TEXT_KEYS) delete rest[key]; + return rest; + } + default: + if (type.startsWith("thread.")) { + // The slim analysis copy replaced these with booleans; restore a marker. + const marked: Rec = { ...payload }; + if (payload.hasMattermostThread === true) marked.mattermostThreadUrl = "present"; + if (payload.hasLinearLink === true) marked.linearIssueUrl = "present"; + delete marked.title; + delete marked.branch; + delete marked.worktreePath; + return marked; + } + return payload; + } +} + +interface EventRow { + readonly sequence: number; + readonly stream_id: string; + readonly event_type: string; + readonly occurred_at: string; + readonly payload_json: string; + readonly metadata_json: string; +} + +interface QueuedEvent { + readonly sequence: number; + readonly atMs: number; + readonly event: SessionActivityEvent; +} + +function toEvent(row: EventRow): QueuedEvent | undefined { + const atMs = toEpochMs(row.occurred_at); + if (atMs === undefined) return undefined; + const payload = asRec(JSON.parse(row.payload_json)); + const metadata = asRec(JSON.parse(row.metadata_json)) ?? {}; + if (payload === undefined) return undefined; + const text = (key: string) => + typeof metadata[key] === "string" ? (metadata[key] as string) : undefined; + const runId = text("runId"); + const actorUserId = text("actorUserId"); + const driver = text("driver"); + return { + sequence: row.sequence, + atMs, + event: { + type: row.event_type, + threadId: row.stream_id, + occurredAt: row.occurred_at, + payload: slimPayload(row.event_type, payload), + ...(runId === undefined ? {} : { runId }), + ...(actorUserId === undefined ? {} : { actorUserId }), + ...(driver === undefined ? {} : { driver }), + }, + }; +} + +/** Same derivation as `analyticsIdentity.uuidFromSha256Hex`, so live and backfill ids agree. */ +function uuidFromInsertId(insertId: string): string { + const hex = NodeCrypto.createHash("sha256").update(insertId).digest("hex").padEnd(32, "0"); + const variant = ((Number.parseInt(hex[16] ?? "0", 16) & 0x3) | 0x8).toString(16); + return [ + hex.slice(0, 8), + hex.slice(8, 12), + `8${hex.slice(13, 16)}`, + `${variant}${hex.slice(17, 20)}`, + hex.slice(20, 32), + ].join("-"); +} + +function loadUserEmails(db: NodeSqlite.DatabaseSync): Map { + const emails = new Map(); + const table = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'environment_users'") + .get(); + if (table === undefined) return emails; + for (const row of db + .prepare("SELECT user_id, primary_email FROM environment_users") + .all() as Array<{ user_id: string; primary_email: string | null }>) { + // Same normalisation as analyticsIdentity.ts, so live and backfill name one person. + const email = row.primary_email?.trim().toLowerCase() ?? ""; + if (email !== "") emails.set(row.user_id, email); + } + return emails; +} + +interface BatchItem { + readonly uuid: string; + readonly event: string; + readonly distinct_id: string; + readonly timestamp: string; + readonly properties: Rec; +} + +/** + * The PostHog item for an emission, or undefined when no person is known. + * + * Live sends an unattributed event under the server install's anonymous id, + * which comes from that machine's T3 home, not from the database, so a copy + * cannot reproduce it. Inventing another id would create a fake person, so + * the backfill skips these events instead. Nearly every session has an owner, + * so very little is lost. `--until` keeps the backfill and live from covering + * the same time, so this cannot break live/backfill de-duplication; re-runs of + * the backfill stay de-duplicated by the stable uuid. + */ +function toBatchItem( + emission: SessionActivityEmission, + emails: ReadonlyMap, + instance: string, +): BatchItem | undefined { + const email = emission.userId === undefined ? undefined : emails.get(emission.userId); + if (email === undefined) return undefined; + return { + uuid: uuidFromInsertId(emission.insertId), + event: emission.event, + distinct_id: email, + timestamp: emission.timestamp, + properties: { + ...emission.properties, + backfill: true, + $insert_id: emission.insertId, + $process_person_profile: true, + $groups: { instance }, + instance, + }, + }; +} + +async function sendBatch(host: string, apiKey: string, batch: ReadonlyArray) { + for (let attempt = 1; ; attempt += 1) { + try { + const response = await fetch(`${host}/batch/`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ api_key: apiKey, historical_migration: true, batch }), + }); + if (response.ok) return; + if (response.status < 500 && response.status !== 429) { + fail(`PostHog rejected a batch with HTTP ${response.status}`); + } + throw new Error(`HTTP ${response.status}`); + } catch (cause) { + if (attempt >= MAX_SEND_ATTEMPTS) + fail(`batch failed after ${attempt} attempts: ${String(cause)}`); + await new Promise((resolve) => setTimeout(resolve, 1_000 * 2 ** attempt)); + } + } +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const dbPath = assertNotLiveState(options.dbPath); + const host = process.env.T3CODE_POSTHOG_HOST ?? "https://us.i.posthog.com"; + const apiKey = process.env.T3CODE_POSTHOG_KEY ?? ""; + const instance = process.env.T3CODE_ANALYTICS_INSTANCE ?? "unknown"; + if (options.send && apiKey.trim() === "") fail("--send needs env T3CODE_POSTHOG_KEY"); + if (options.send && options.untilMs === undefined) { + fail( + "--send needs --until : the time the live reactor started. Without it the backfill " + + "and the live server can both report the same time.", + ); + } + + const db = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true }); + const emails = loadUserEmails(db); + const tracker = makeSessionActivityTracker({ backfill: true }); + + const out = options.send ? undefined : NodeFS.openSync(options.outPath, "w"); + const counts = new Map(); + const stateHours = new Map(); + const newest = db + .prepare("SELECT occurred_at FROM orchestration_events ORDER BY sequence DESC LIMIT 1") + .get() as { occurred_at: string } | undefined; + const reportEndMs = options.untilMs ?? toEpochMs(newest?.occurred_at) ?? 0; + const reportFromMs = reportEndMs - options.reportDays * 86_400_000; + let latestMs = 0; + let pending: Array = []; + let sent = 0; + let unattributed = 0; + + const flush = async () => { + if (pending.length === 0) return; + const batch = pending; + pending = []; + if (options.send) { + await sendBatch(host, apiKey, batch); + sent += batch.length; + } else if (out !== undefined) { + NodeFS.writeSync(out, batch.map((item) => JSON.stringify(item)).join("\n") + "\n"); + } + }; + + const collect = async (emissions: ReadonlyArray) => { + for (const emission of emissions) { + counts.set(emission.event, (counts.get(emission.event) ?? 0) + 1); + if (emission.event === "session.state.span") { + const endedMs = toEpochMs(emission.timestamp) ?? 0; + if (endedMs >= reportFromMs) { + const state = String(emission.properties.state); + const hours = Number(emission.properties.durationMs) / 3_600_000; + stateHours.set(state, (stateHours.get(state) ?? 0) + hours); + } + } + const item = toBatchItem(emission, emails, instance); + if (item === undefined) { + unattributed += 1; + continue; + } + pending.push(item); + if (pending.length >= BATCH_SIZE) await flush(); + } + }; + + const placeholders = RELEVANT_EVENT_TYPES.map(() => "?").join(", "); + // Uses idx_orch_events_stream_sequence (aggregate_kind, stream_id, sequence). + const rows = db + .prepare( + `SELECT sequence, stream_id, event_type, occurred_at, payload_json, metadata_json + FROM orchestration_events + WHERE aggregate_kind = 'thread' + AND application_event_version = 2 + AND (event_type LIKE 'thread.%' OR event_type IN (${placeholders})) + ORDER BY aggregate_kind, stream_id, sequence`, + ) + .iterate(...RELEVANT_EVENT_TYPES) as Iterable; + + let threadId: string | undefined; + let threadEvents: Array = []; + let threads = 0; + const replayThread = async () => { + threadEvents.sort((a, b) => a.atMs - b.atMs || a.sequence - b.sequence); + for (const queued of threadEvents) { + if (options.untilMs !== undefined && queued.atMs > options.untilMs) break; + latestMs = Math.max(latestMs, queued.atMs); + await collect(tracker.apply(queued.event)); + } + threadEvents = []; + threads += 1; + }; + + for (const row of rows) { + if (row.stream_id !== threadId) { + if (threadId !== undefined) await replayThread(); + threadId = row.stream_id; + } + const queued = toEvent(row); + if (queued !== undefined) threadEvents.push(queued); + } + if (threadId !== undefined) await replayThread(); + await flush(); + if (out !== undefined) NodeFS.closeSync(out); + db.close(); + + console.log(`threads replayed: ${threads}`); + console.log(`latest event: ${latestMs > 0 ? new Date(latestMs).toISOString() : "none"}`); + console.log(options.send ? `sent: ${sent} events` : `wrote: ${options.outPath}`); + console.log(`skipped, no known user email: ${unattributed}`); + console.log("events by type:"); + for (const [event, count] of [...counts].sort((a, b) => b[1] - a[1])) { + console.log(` ${event}: ${count}`); + } + console.log(`state hours, last ${options.reportDays} days (by span end):`); + for (const [state, hours] of [...stateHours].sort((a, b) => b[1] - a[1])) { + console.log(` ${state}: ${hours.toFixed(1)}`); + } +} + +await main(); diff --git a/apps/server/src/auth/McpOAuth.test.ts b/apps/server/src/auth/McpOAuth.test.ts index 1273b350281a3..e4de0b0187486 100644 --- a/apps/server/src/auth/McpOAuth.test.ts +++ b/apps/server/src/auth/McpOAuth.test.ts @@ -595,3 +595,32 @@ it("matches loopback redirects on everything but the port, and https redirects e expect(matches("https://bot.example/cb", "https://bot.example/cb?x=1")).toBe(false); expect(matches("https://bot.example/cb", "https://evil.example/cb")).toBe(false); }); + +// T3-CUSTOM(expbkt3): BEGIN — hosted connectors such as Google's register many redirect URIs. +it.live("registers a hosted client with up to 20 https redirect URIs", () => + withRoutes((handler) => + Effect.gen(function* () { + const registerMany = (count: number) => + handler( + at("/oauth/mcp/register", { + method: "POST", + headers: { "content-type": "application/json" }, + body: encodeJson({ + client_name: "Google", + redirect_uris: Array.from( + { length: count }, + (_, index) => `https://connector${index}.example/oauth-redirect`, + ), + }), + }), + ); + + expect((yield* registerMany(8)).status).toBe(201); + expect((yield* registerMany(20)).status).toBe(201); + const tooMany = yield* registerMany(21); + expect(tooMany.status).toBe(400); + expect(yield* json(tooMany)).toMatchObject({ error: "invalid_redirect_uri" }); + }), + ), +); +// T3-CUSTOM(expbkt3): END diff --git a/apps/server/src/auth/McpOAuth.ts b/apps/server/src/auth/McpOAuth.ts index 073e0856de314..c48ba746953fc 100644 --- a/apps/server/src/auth/McpOAuth.ts +++ b/apps/server/src/auth/McpOAuth.ts @@ -63,7 +63,8 @@ import { const SIGNING_SECRET_NAME = "mcp-oauth-signing-key"; const AUTHORIZATION_CODE_TTL_MS = 60_000; const MAX_CLIENT_NAME_LENGTH = 100; -const MAX_REDIRECT_URIS = 5; +// T3-CUSTOM(expbkt3): Google's MCP connector registers more than upstream's 5 redirect URIs. +const MAX_REDIRECT_URIS = 20; const MAX_REDIRECT_URI_LENGTH = 512; const DEFAULT_CLIENT_NAME = "MCP client"; const LOOPBACK_HOSTNAMES = new Set(["localhost", "127.0.0.1", "[::1]"]); diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index fe4dc61231c48..c212b834550b5 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -22,6 +22,8 @@ import { resolveServiceLauncherMode } from "../cloud/serviceLauncherClient.ts"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import { resolveServerEnvironmentLabel } from "./ServerEnvironmentLabel.ts"; +// T3-CUSTOM(expbkt3): product analytics config for clients (TEC-1631). +import { loadClientAnalyticsConfig } from "./clientAnalyticsConfig.ts"; import { detectServerEnvironmentMachineKind } from "./ServerEnvironmentMachine.ts"; export class ServerEnvironmentIdPersistenceError extends Schema.TaggedError()( @@ -204,6 +206,8 @@ export const make = Effect.gen(function* () { // the fd and correctly do not advertise. const desktopAppUpdate = serverSelfUpdate === "desktop-managed" && serverConfig.desktopTelemetryControlFd !== undefined; + // T3-CUSTOM(expbkt3): read once at startup, like the rest of the descriptor. + const analytics = yield* loadClientAnalyticsConfig; const descriptor: ExecutionEnvironmentDescriptor = { environmentId, @@ -282,6 +286,8 @@ export const make = Effect.gen(function* () { ...(desktopAppUpdate ? { desktopAppUpdate: true } : {}), serverBrowser: true, }, + // T3-CUSTOM(expbkt3): absent unless an explicit company PostHog key is set. + ...(analytics === null ? {} : { analytics }), }; return ServerEnvironment.of({ diff --git a/apps/server/src/environment/clientAnalyticsConfig.test.ts b/apps/server/src/environment/clientAnalyticsConfig.test.ts new file mode 100644 index 0000000000000..5baf91f76b186 --- /dev/null +++ b/apps/server/src/environment/clientAnalyticsConfig.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as Option from "effect/Option"; + +import { resolveClientAnalyticsConfig } from "./clientAnalyticsConfig.ts"; + +const base = { + posthogKey: Option.some("phc_company"), + posthogHost: Option.none(), + enabled: true, + instance: Option.some("bkt3"), +}; + +describe("resolveClientAnalyticsConfig", () => { + it("advertises an explicit key with the default host and the instance", () => { + expect(resolveClientAnalyticsConfig(base)).toEqual({ + enabled: true, + posthogKey: "phc_company", + posthogHost: "https://us.i.posthog.com", + instance: "bkt3", + }); + }); + + it("stays absent without an explicit key, so clients never use upstream's project", () => { + expect(resolveClientAnalyticsConfig({ ...base, posthogKey: Option.none() })).toBeNull(); + expect(resolveClientAnalyticsConfig({ ...base, posthogKey: Option.some(" ") })).toBeNull(); + }); + + it("stays absent when telemetry is off", () => { + expect(resolveClientAnalyticsConfig({ ...base, enabled: false })).toBeNull(); + }); + + it("omits a blank instance and honours a custom host", () => { + expect( + resolveClientAnalyticsConfig({ + ...base, + instance: Option.some(""), + posthogHost: Option.some("https://eu.i.posthog.com"), + }), + ).toEqual({ + enabled: true, + posthogKey: "phc_company", + posthogHost: "https://eu.i.posthog.com", + }); + }); +}); diff --git a/apps/server/src/environment/clientAnalyticsConfig.ts b/apps/server/src/environment/clientAnalyticsConfig.ts new file mode 100644 index 0000000000000..e4f1a2e8fd264 --- /dev/null +++ b/apps/server/src/environment/clientAnalyticsConfig.ts @@ -0,0 +1,57 @@ +/** + * T3-CUSTOM(expbkt3): product analytics config handed to clients (TEC-1631). + * + * The web and desktop clients send their own PostHog events, to the same + * company project the server reports to. They learn the key from the + * environment descriptor at runtime, so no key is baked into the bundle and + * each environment decides for itself. + * + * Only an explicitly set `T3CODE_POSTHOG_KEY` is advertised. The server's own + * telemetry falls back to upstream's project key; clients must never send + * Beknown usage there, so with no explicit key the field is absent and the + * client stays off. + * + * @module clientAnalyticsConfig + */ +import type { ExecutionEnvironmentAnalytics } from "@t3tools/contracts"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com"; + +const ClientAnalyticsEnvConfig = Config.all({ + posthogKey: Config.String("T3CODE_POSTHOG_KEY").pipe(Config.option), + posthogHost: Config.String("T3CODE_POSTHOG_HOST").pipe(Config.option), + enabled: Config.Boolean("T3CODE_TELEMETRY_ENABLED").pipe(Config.withDefault(true)), + instance: Config.String("T3CODE_ANALYTICS_INSTANCE").pipe(Config.option), +}); + +function nonEmpty(value: Option.Option): string | null { + const trimmed = Option.getOrNull(value)?.trim() ?? ""; + return trimmed.length > 0 ? trimmed : null; +} + +/** Pure mapping from env values to the descriptor field; `null` means omit it. */ +export function resolveClientAnalyticsConfig(input: { + readonly posthogKey: Option.Option; + readonly posthogHost: Option.Option; + readonly enabled: boolean; + readonly instance: Option.Option; +}): ExecutionEnvironmentAnalytics | null { + const posthogKey = nonEmpty(input.posthogKey); + if (!input.enabled || posthogKey === null) return null; + const instance = nonEmpty(input.instance); + return { + enabled: true, + posthogKey, + posthogHost: nonEmpty(input.posthogHost) ?? DEFAULT_POSTHOG_HOST, + ...(instance === null ? {} : { instance }), + }; +} + +/** Reads the env once; a bad value disables client analytics, never the server. */ +export const loadClientAnalyticsConfig: Effect.Effect = + Effect.gen(function* () { + return resolveClientAnalyticsConfig(yield* ClientAnalyticsEnvConfig); + }).pipe(Effect.orElseSucceed(() => null)); diff --git a/apps/server/src/mcp/McpToolAccess.ts b/apps/server/src/mcp/McpToolAccess.ts index 6e882629bbba1..f7fddadc2c107 100644 --- a/apps/server/src/mcp/McpToolAccess.ts +++ b/apps/server/src/mcp/McpToolAccess.ts @@ -16,6 +16,7 @@ import { type DispatchModeRefusal, } from "../orchestration-v2/DispatchModeLimit.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; +import { withMcpToolCallAnalytics } from "./mcpToolAnalytics.expbkt3.ts"; // T3-CUSTOM(expbkt3) import { resolveInteractionMode, resolveRuntimeMode } from "./OrchestratorMcpService.ts"; import { assertFullAccess, @@ -300,7 +301,8 @@ export class HandlersLayer, EX = never, R } const checkedHandlers = (declarations: Declarations): Handlers => - Struct.map(declarations, checkedHandler); + // T3-CUSTOM(expbkt3): count every tool call by name (TEC-1631). + withMcpToolCallAnalytics(Struct.map(declarations, checkedHandler)); /** `Toolkit.toLayer` for handlers that all declare their access. */ export const toLayer = , EX = never, RX = never>( diff --git a/apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts b/apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts new file mode 100644 index 0000000000000..7d34948b314ed --- /dev/null +++ b/apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts @@ -0,0 +1,40 @@ +/** + * T3-CUSTOM(expbkt3): counts every T3 MCP tool call by name (TEC-1631). + * + * `McpToolAccess.toLayer` is the one place every `/mcp` toolkit's handlers are + * built, keyed by tool name, so wrapping them there covers every tool. Only + * the name, caller kind, outcome and duration are recorded, never arguments. + */ +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import { type McpToolCaller, withMcpToolAnalytics } from "../telemetry/serverEvents.ts"; +import * as McpInvocationContext from "./McpInvocationContext.ts"; + +const currentCaller: Effect.Effect = Effect.serviceOption( + McpInvocationContext.McpInvocationContext, +).pipe( + Effect.map((scope) => + Option.isNone(scope) + ? undefined + : { principal: scope.value.principal, actorUserId: scope.value.actorUserId }, + ), +); + +/** `handlers` with each Effect-returning handler timed and recorded under its key. */ +export function withMcpToolCallAnalytics(handlers: Handlers): Handlers { + const wrapped: Record = {}; + for (const [toolName, handler] of Object.entries(handlers as Record)) { + wrapped[toolName] = + typeof handler === "function" + ? (...args: ReadonlyArray) => { + const result = (handler as (...params: ReadonlyArray) => unknown)(...args); + return Effect.isEffect(result) + ? // @effect-diagnostics-next-line anyUnknownInErrorContext:off -- handlers are heterogeneous; types pass through unchanged. + withMcpToolAnalytics(toolName, currentCaller, result) + : result; + } + : handler; + } + return wrapped as Handlers; +} diff --git a/apps/server/src/mcp/toolkits/control/handlers.ts b/apps/server/src/mcp/toolkits/control/handlers.ts index a13364048a13d..6efa32390bb00 100644 --- a/apps/server/src/mcp/toolkits/control/handlers.ts +++ b/apps/server/src/mcp/toolkits/control/handlers.ts @@ -78,6 +78,8 @@ import { hasUserWideScope, resolveMcpSessionTarget } from "../../mcpSessionTarge import { boundSessionDetail } from "./sessionDetail.ts"; import { T3ControlToolkit, T3ControlToolError, type T3CreateSessionTool } from "./tools.ts"; import type { Tool } from "effect/ai"; +import { recordLinearLinked } from "../../../telemetry/serverEvents.ts"; +import { CurrentOrchestrationActorUserId } from "../../../orchestration-v2/forkActor.expbkt3.ts"; const nowIso = Effect.map(DateTime.now, DateTime.formatIso); const decodeOrchestrationCommand = Schema.decodeUnknownEffect(OrchestrationCommandSchema); @@ -1662,10 +1664,17 @@ const handlers = { const after = yield* readLinearLinks(operation, sessionId); const urls = links.map((link) => link.url); const has = (list: typeof before, url: string) => list.some((entry) => entry.url === url); + const added = urls.filter((url) => !has(before, url) && has(after, url)); + // Product analytics: how many items were linked, never which (TEC-1631). + yield* recordLinearLinked({ + threadId: sessionId, + count: added.length, + userId: yield* CurrentOrchestrationActorUserId, + }); return { sessionId, linearLinks: after, - added: urls.filter((url) => !has(before, url) && has(after, url)), + added, alreadyLinked: urls.filter((url) => has(before, url)), // Left out because the session already holds the maximum number of tags. notAdded: urls.filter((url) => !has(before, url) && !has(after, url)), diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 427c79c8c0544..005e2e0e18b53 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -27,6 +27,8 @@ import { settledBackgroundWakeRequest, settledBackgroundWakeRun, } from "./restartBackgroundResume.expbkt3.ts"; +// T3-CUSTOM(expbkt3): session.recovery.* product analytics (TEC-1631). +import { withSessionRecoveryAnalytics } from "../telemetry/serverEvents.ts"; import { cancelledRosterTaskWork, cancelledTurnItemWork, @@ -834,9 +836,12 @@ export const make = Effect.gen(function* () { Effect.mapError((cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause })), ); - const recover = Effect.gen(function* () { - return (yield* reconcile("startup")) satisfies ProviderRuntimeRecoverySummary; - }); + // T3-CUSTOM(expbkt3): session.recovery.* product analytics (TEC-1631). + const recover = withSessionRecoveryAnalytics( + Effect.gen(function* () { + return (yield* reconcile("startup")) satisfies ProviderRuntimeRecoverySummary; + }), + ); return ProviderRuntimeRecoveryService.of({ reconcile, prepareForShutdown, recover }); }); diff --git a/apps/server/src/provider/ProviderInstanceEnvironment.ts b/apps/server/src/provider/ProviderInstanceEnvironment.ts index 77c0c6c2dc888..ccbcf344851e3 100644 --- a/apps/server/src/provider/ProviderInstanceEnvironment.ts +++ b/apps/server/src/provider/ProviderInstanceEnvironment.ts @@ -1,13 +1,15 @@ import type { ProviderInstanceEnvironment } from "@t3tools/contracts"; import { expandHomePath } from "../pathExpansion.ts"; +// T3-CUSTOM(expbkt3): agents never inherit the server's analytics settings (TEC-1631). +import { withoutAnalyticsEnvironment } from "../telemetry/analyticsEnvironment.ts"; export function mergeProviderInstanceEnvironment( environment: ProviderInstanceEnvironment | undefined, baseEnv: NodeJS.ProcessEnv = process.env, ): NodeJS.ProcessEnv { if (!environment || environment.length === 0) { - return baseEnv; + return withoutAnalyticsEnvironment(baseEnv); // T3-CUSTOM(expbkt3): TEC-1631. } const next: NodeJS.ProcessEnv = { ...baseEnv }; @@ -18,5 +20,5 @@ export function mergeProviderInstanceEnvironment( ? expandHomePath(variable.value) : variable.value; } - return next; + return withoutAnalyticsEnvironment(next); // T3-CUSTOM(expbkt3): TEC-1631. } diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 407d5fac55115..4d61c04e81128 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -53,6 +53,7 @@ import { type WebhookRequest, } from "./webhookTemplate.ts"; import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts"; +import { recordScheduledTaskRan } from "../telemetry/serverEvents.ts"; // T3-CUSTOM(expbkt3) // T3-CUSTOM(expbkt3): BK sidebar custom group of the threads a task launches (XFN-59). import { deleteScheduledTaskCustomGroup, @@ -904,6 +905,16 @@ export const layer = Layer.effect( lastRunError, runCount: scheduleSource.runCount + 1, }; + // T3-CUSTOM(expbkt3): product analytics for each finished run (TEC-1631). + yield* recordScheduledTaskRan({ + taskId: task.id, + scheduleType: scheduleSource.schedule.type, + trigger, + creationSource: active.creationSource, + status: lastRunStatus, + durationMs: DateTime.toEpochMillis(completedAt) - DateTime.toEpochMillis(startedAt), + ownerUserId: forkOwner, + }); if (current !== null) { // startedAtIso in the guard ensures this writes only to the row this // run marked as running — a task deleted mid-run and recreated with diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index 9fcf8344f50d7..4e01bb4d051cd 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -13,6 +13,7 @@ import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import * as Metrics from "../observability/Metrics.ts"; import * as RelayDeliveryProof from "./RelayDeliveryProof.ts"; import * as ScheduledTaskService from "./ScheduledTaskService.ts"; +import { recordWebhookDelivered } from "../telemetry/serverEvents.ts"; // T3-CUSTOM(expbkt3) /** Largest request body a webhook accepts. The relay enforces the same cap. */ export const WEBHOOK_MAX_BODY_BYTES = 1024 * 1024; @@ -111,6 +112,8 @@ export const layer = HttpApiBuilder.group( Effect.as({ _tag: "error" as const }), ), ), + // T3-CUSTOM(expbkt3): product analytics, outcome tag only (TEC-1631). + Effect.tap((delivery) => recordWebhookDelivered("scheduled_task", delivery._tag)), ); switch (result._tag) { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index c586afe25aa30..b1383df1e47c0 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -89,6 +89,7 @@ import * as SqlitePersistence from "./persistence/Sqlite.ts"; import * as PullRequestFilesViewed from "./persistence/PullRequestFilesViewed.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; +import * as SessionActivityReactor from "./telemetry/sessionActivity/SessionActivityReactor.ts"; // T3-CUSTOM(expbkt3): TEC-1631. import * as ProviderEventIngestor from "./orchestration-v2/ProviderEventIngestor.ts"; import * as ModelManifest from "./provider/ModelManifest.ts"; import * as ResetCreditCoordinator from "./provider/resetCreditCoordinator.ts"; @@ -656,6 +657,7 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( ProviderUsageLimitsIngestion.layer, layerProviderInstallationRefresh, ReplayMarkers.layer, + SessionActivityReactor.layer, // T3-CUSTOM(expbkt3): session time-in-state metrics (TEC-1631). ).pipe( // Core Services Layer.provideMerge(layerOrchestrationApplication), diff --git a/apps/server/src/session-webhooks/SessionWebhookService.test.ts b/apps/server/src/session-webhooks/SessionWebhookService.test.ts index 5d25910ce3367..e18fbe813f838 100644 --- a/apps/server/src/session-webhooks/SessionWebhookService.test.ts +++ b/apps/server/src/session-webhooks/SessionWebhookService.test.ts @@ -67,6 +67,7 @@ const harness = (mode: "push" | "pull" = "push") => const historyEntered = yield* Deferred.make(); const twoHistoriesEntered = yield* Deferred.make(); const fourHistoriesEntered = yield* Deferred.make(); + const sql = yield* SqlClient.SqlClient; const secrets = new Map(); const receipts = new Map(); const state = { @@ -119,7 +120,17 @@ const harness = (mode: "push" | "pull" = "push") => dispatches: 0, registrations: 0, notifications: [] as string[], + // Production checks the owner with `Effect.runPromise(users.get(...))`: + // a fresh fiber that reads the same SQLite client outside any transaction. + ownerCheckReadsDatabase: false, }; + // oxlint-disable-next-line t3code/no-manual-effect-runtime-in-tests -- Replicates the production owner check's own root fiber; a forked fiber would inherit the transaction connection and hide the deadlock. + const readOnFreshFiber = () => Effect.runPromise(sql`SELECT 1`); + const ownerCheckRead = Effect.suspend(() => + state.ownerCheckReadsDatabase + ? Effect.promise(readOnFreshFiber).pipe(Effect.asVoid) + : Effect.void, + ); const shell = (id = threadId) => ({ id, @@ -198,7 +209,7 @@ const harness = (mode: "push" | "pull" = "push") => : null, ), callbackBinding: () => - Effect.sync(() => + Effect.map(ownerCheckRead, () => state.integrationEnabled ? { instanceId: state.instanceId, @@ -215,7 +226,7 @@ const harness = (mode: "push" | "pull" = "push") => ), assertConnectionOwner: () => state.userEnabled - ? Effect.void + ? ownerCheckRead : Effect.fail( new PersonalMcpSettingsError({ operation: "test", message: "owner-disabled" }), ), @@ -1357,6 +1368,30 @@ describe("durable session webhooks", () => { ).pipe(Effect.provide(NodeHttpServer.layerTest)), ), ); + it.live("acknowledges a push callback when the owner check reads the database", () => + scenario((h) => + Effect.gen(function* () { + h.state.ownerCheckReadsDatabase = true; + // Before the fix the admission transaction called the owner check, which + // waited for the connection that transaction held, until the sender gave up. + // One payload for both deliveries: the live clock would give a second + // `h.event()` a new timestamp, which is a content conflict, not a retry. + const payload = h.event(); + const first = yield* h.receive(payload).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(first)); + assert.strictEqual(Option.getOrThrow(first).duplicate, false); + const repeat = yield* h.receive(payload).pipe(Effect.timeoutOption("2 seconds")); + assert.strictEqual(Option.getOrThrow(repeat).duplicate, true); + assert.strictEqual(h.state.dispatches, 0); + assert.strictEqual( + (yield* h.service.inspect(owner, h.webhook.id)).deliveries[0]?.state, + "queued", + ); + yield* h.service.drain(); + assert.strictEqual(h.state.dispatches, 1); + }), + ), + ); it.effect("returns an existing destination without revealing secrets", () => scenario((h) => Effect.gen(function* () { diff --git a/apps/server/src/session-webhooks/SessionWebhookService.ts b/apps/server/src/session-webhooks/SessionWebhookService.ts index 91e1b787e1c6d..49d6e83cb724c 100644 --- a/apps/server/src/session-webhooks/SessionWebhookService.ts +++ b/apps/server/src/session-webhooks/SessionWebhookService.ts @@ -420,6 +420,14 @@ export const make = Effect.gen(function* () { catch: () => fail(400, "invalid-callback-payload"), }); const fingerprint = callbackFingerprint(body); + // Only SQL runs inside the admission transaction. The SQLite client has one + // connection behind a one-permit semaphore, and the transaction holds that + // permit until it commits. The integration checks the owner with + // `Effect.runPromise(users.get(...))`, a fresh fiber outside this + // transaction, so calling it here waited for the permit this fiber held: + // every push callback hung until the sender's 20 s timeout and rolled back. + // The binding was verified above; the drain loop re-checks it before dispatch + // and `retireReplacedReceiver` ends queued events of a replaced receiver. return yield* sql.withTransaction( Effect.gen(function* () { const existing = @@ -431,14 +439,7 @@ export const make = Effect.gen(function* () { } const current = (yield* sql`SELECT * FROM session_webhooks WHERE id=${id}`)[0]; if (current?.status !== "active") return yield* fail(410, "webhook-disabled-or-removed"); - const currentBinding = yield* integration.callbackBinding( - UserId.make(current.owner_user_id), - ); - if ( - !currentBinding || - !currentBinding.enabled || - receiverTrustBinding(currentBinding) !== current.trust_binding - ) + if (current.trust_binding !== row.trust_binding) return yield* fail(410, "integration-disabled-or-replaced"); const cutoff = DateTime.formatIso(DateTime.makeUnsafe(millis() - 60_000)); const count = diff --git a/apps/server/src/session-webhooks/http.ts b/apps/server/src/session-webhooks/http.ts index 9a10790d2dd40..c0a20b4163beb 100644 --- a/apps/server/src/session-webhooks/http.ts +++ b/apps/server/src/session-webhooks/http.ts @@ -12,6 +12,7 @@ import { import { SessionWebhookError } from "@t3tools/contracts"; import { SessionWebhookService } from "./SessionWebhookService.ts"; import { MAX_CALLBACK_BYTES } from "./protocol.ts"; +import { recordWebhookDelivered, webhookStatusFromHttp } from "../telemetry/serverEvents.ts"; const isSessionWebhookError = Schema.is(SessionWebhookError); export const sessionWebhookRouteLayer = Layer.unwrap( Effect.gen(function* () { @@ -53,6 +54,10 @@ export const sessionWebhookRouteLayer = Layer.unwrap( ), ), ), + // Product analytics: the answer's status class only (TEC-1631). + Effect.tap((response) => + recordWebhookDelivered("session_callback", webhookStatusFromHttp(response.status)), + ), ); return HttpRouter.add("POST", "/api/session-webhooks/:webhookId", receive); }), diff --git a/apps/server/src/telemetry/AnalyticsService.identity.expbkt3.test.ts b/apps/server/src/telemetry/AnalyticsService.identity.expbkt3.test.ts new file mode 100644 index 0000000000000..dff32f3bcc9cf --- /dev/null +++ b/apps/server/src/telemetry/AnalyticsService.identity.expbkt3.test.ts @@ -0,0 +1,244 @@ +// T3-CUSTOM(expbkt3): bkt3 analytics identity, allowlist and key gating (TEC-1631). +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { EnvironmentUserId } from "@t3tools/contracts"; +import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as HttpServer from "effect/http/HttpServer"; +import * as HttpServerRequest from "effect/http/HttpServerRequest"; +import * as HttpServerResponse from "effect/http/HttpServerResponse"; + +import * as ServerConfig from "../config.ts"; +import * as EnvironmentUsers from "../persistence/EnvironmentUsers.ts"; +import { getTelemetryIdentifier } from "./Identify.ts"; +import * as AnalyticsService from "./AnalyticsService.ts"; + +interface SentEvent { + readonly uuid: string; + readonly event: string; + readonly distinct_id: string; + readonly timestamp: string; + readonly properties: Readonly>; +} + +const ALICE = EnvironmentUserId.make("user_clerk_alice"); +const BOB_WITHOUT_EMAIL = EnvironmentUserId.make("user_clerk_bob"); + +const users: ReadonlyArray = [ + { + userId: ALICE, + displayName: "Alice Example", + primaryEmail: " Alice@Example.com ", + avatarUrl: null, + role: "admin", + status: "active", + firstSeenAt: DateTime.makeUnsafe(0), + lastSeenAt: DateTime.makeUnsafe(0), + }, + { + userId: BOB_WITHOUT_EMAIL, + displayName: "Bob", + primaryEmail: null, + avatarUrl: null, + role: "member", + status: "active", + firstSeenAt: DateTime.makeUnsafe(0), + lastSeenAt: DateTime.makeUnsafe(0), + }, +]; + +const fakeUsers = EnvironmentUsers.EnvironmentUserRepository.of({ + upsert: () => Effect.void, + get: () => Effect.succeed(Option.none()), + update: () => Effect.succeed(false), + list: Effect.succeed(users), +}); + +/** Runs `body` against a local batch endpoint and returns every event it received. */ +const captureEvents = ( + env: Record, + body: (analytics: AnalyticsService.AnalyticsService["Service"]) => Effect.Effect, +) => + Effect.gen(function* () { + const sent: Array = []; + let requests = 0; + let installId: string | null = null; + const layerBatchServer = HttpServer.serve( + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + requests += 1; + const payload = yield* request.json.pipe(Effect.orElseSucceed(() => null)); + const batch = (payload as { readonly batch?: ReadonlyArray } | null)?.batch; + if (Array.isArray(batch)) sent.push(...batch); + return HttpServerResponse.jsonUnsafe({}); + }), + ); + const layerRuntime = AnalyticsService.layer.pipe( + Layer.provideMerge( + ServerConfig.ServerConfig.layerTest(process.cwd(), { prefix: "t3-telemetry-identity-" }), + ), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ T3CODE_POSTHOG_HOST: "http://localhost", ...env }), + ), + ), + Layer.provide( + Layer.mergeAll( + Layer.succeed(HostProcessPlatform, "linux"), + Layer.succeed(HostProcessArchitecture, "arm64"), + ), + ), + Layer.provideMerge(NodeHttpServer.layerTest), + ); + yield* Effect.gen(function* () { + yield* Layer.launch(layerBatchServer).pipe(Effect.forkScoped); + const analytics = yield* AnalyticsService.AnalyticsService; + installId = yield* getTelemetryIdentifier; + yield* body(analytics); + yield* analytics.flush; + }).pipe(Effect.provide(layerRuntime)); + return { sent, requests: () => requests, installId: () => installId }; + }); + +it.layer(NodeServices.layer)("AnalyticsService bkt3 identity", (it) => { + it.effect("sends nothing when no PostHog key is set, even with telemetry enabled", () => + Effect.gen(function* () { + const { sent, requests } = yield* captureEvents( + { T3CODE_TELEMETRY_ENABLED: true }, + (analytics) => analytics.record("test.no_key", { count: 1 }), + ); + assert.deepEqual(sent, []); + assert.equal(requests(), 0); + }), + ); + + it.effect("keeps the two-argument record call and drops keys outside the allowlist", () => + Effect.gen(function* () { + const { sent, installId } = yield* captureEvents( + { T3CODE_TELEMETRY_ENABLED: true, T3CODE_POSTHOG_KEY: "phc_test_key" }, + (analytics) => + analytics.record("test.allowlist", { + count: 2, + threadId: "thread-1", + title: "secret session title", + prompt: "secret prompt", + cwd: "/home/someone/repo", + }), + ); + const event = sent.find((entry) => entry.event === "test.allowlist"); + assert.isDefined(event); + assert.equal(event?.properties.count, 2); + assert.equal(event?.properties.threadId, "thread-1"); + assert.isUndefined(event?.properties.title); + assert.isUndefined(event?.properties.prompt); + assert.isUndefined(event?.properties.cwd); + // No user: the install id, and no person profile. + assert.equal(event?.distinct_id, installId()); + assert.equal(event?.properties.$process_person_profile, false); + assert.deepEqual(event?.properties.$groups, { instance: "unknown" }); + assert.equal(event?.properties.instance, "unknown"); + }), + ); + + it.effect("sends a known user's events as their email and identifies them once", () => + Effect.gen(function* () { + const { sent } = yield* captureEvents( + { T3CODE_TELEMETRY_ENABLED: true, T3CODE_POSTHOG_KEY: "phc_test_key" }, + (analytics) => + Effect.gen(function* () { + yield* analytics.record("test.user", { count: 1 }, { userId: ALICE }); + yield* analytics.record("test.user", { count: 2 }, { userId: ALICE }); + yield* analytics.record("test.user", { count: 3 }, { userId: BOB_WITHOUT_EMAIL }); + yield* analytics.record("test.user", { count: 4 }, { userId: "user_unknown" }); + }).pipe(Effect.provideService(EnvironmentUsers.EnvironmentUserRepository, fakeUsers)), + ); + + const identifies = sent.filter((entry) => entry.event === "$identify"); + assert.equal(identifies.length, 1, "one $identify per user per process"); + assert.equal(identifies[0]?.distinct_id, "alice@example.com"); + assert.deepEqual(identifies[0]?.properties.$set, { + email: "alice@example.com", + role: "admin", + name: "Alice Example", + }); + + const byCount = new Map( + sent + .filter((entry) => entry.event === "test.user") + .map((entry) => [entry.properties.count, entry] as const), + ); + for (const count of [1, 2]) { + assert.equal(byCount.get(count)?.distinct_id, "alice@example.com"); + assert.equal(byCount.get(count)?.properties.$process_person_profile, true); + } + // A user without an email, and an unknown user, stay anonymous. + for (const count of [3, 4]) { + assert.notEqual(byCount.get(count)?.distinct_id, "alice@example.com"); + assert.equal(byCount.get(count)?.properties.$process_person_profile, false); + } + }), + ); + + it.effect("stays anonymous when the caller has no user repository", () => + Effect.gen(function* () { + const { sent } = yield* captureEvents( + { T3CODE_TELEMETRY_ENABLED: true, T3CODE_POSTHOG_KEY: "phc_test_key" }, + (analytics) => analytics.record("test.no_repo", { count: 1 }, { userId: ALICE }), + ); + assert.equal(sent.filter((entry) => entry.event === "$identify").length, 0); + const event = sent.find((entry) => entry.event === "test.no_repo"); + assert.equal(event?.properties.$process_person_profile, false); + }), + ); + + it.effect("maps an insert id to a stable uuid and keeps the caller's timestamp", () => + Effect.gen(function* () { + const timestamp = "2026-10-01T10:00:00.000Z"; + const { sent } = yield* captureEvents( + { T3CODE_TELEMETRY_ENABLED: true, T3CODE_POSTHOG_KEY: "phc_test_key" }, + (analytics) => + Effect.gen(function* () { + const options = { insertId: "run-1", timestamp }; + yield* analytics.record("test.backfill", { count: 1 }, options); + yield* analytics.record("test.backfill", { count: 2 }, options); + yield* analytics.record("test.backfill", { count: 3 }, { insertId: "run-2" }); + }), + ); + const events = sent.filter((entry) => entry.event === "test.backfill"); + assert.equal(events.length, 3); + assert.equal(events[0]?.uuid, events[1]?.uuid); + assert.notEqual(events[0]?.uuid, events[2]?.uuid); + assert.match( + events[0]?.uuid ?? "", + /^[0-9a-f]{8}-[0-9a-f]{4}-8[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); + assert.equal(events[0]?.timestamp, timestamp); + assert.equal(events[0]?.properties.$insert_id, "run-1"); + }), + ); + + it.effect("names the instance group and identifies it once per boot", () => + Effect.gen(function* () { + const { sent } = yield* captureEvents( + { + T3CODE_TELEMETRY_ENABLED: true, + T3CODE_POSTHOG_KEY: "phc_test_key", + T3CODE_ANALYTICS_INSTANCE: "bkt3", + }, + (analytics) => analytics.record("test.instance", { count: 1 }), + ); + const groups = sent.filter((entry) => entry.event === "$groupidentify"); + assert.equal(groups.length, 1); + assert.equal(groups[0]?.properties.$group_type, "instance"); + assert.equal(groups[0]?.properties.$group_key, "bkt3"); + const event = sent.find((entry) => entry.event === "test.instance"); + assert.deepEqual(event?.properties.$groups, { instance: "bkt3" }); + assert.equal(event?.properties.instance, "bkt3"); + }), + ); +}); diff --git a/apps/server/src/telemetry/AnalyticsService.test.ts b/apps/server/src/telemetry/AnalyticsService.test.ts index e0a4786ece6ff..96329bad31e25 100644 --- a/apps/server/src/telemetry/AnalyticsService.test.ts +++ b/apps/server/src/telemetry/AnalyticsService.test.ts @@ -23,7 +23,7 @@ interface RecordedBatchRequest { readonly batch?: ReadonlyArray<{ readonly event?: string; readonly properties?: { - readonly index?: number; + readonly count?: number; readonly clientType?: string; readonly serverOs?: string; readonly serverArch?: string; @@ -39,7 +39,7 @@ interface RecordedBatchBody { readonly batch: ReadonlyArray<{ readonly event?: string; readonly properties?: { - readonly index?: number; + readonly count?: number; readonly clientType?: string; readonly serverOs?: string; readonly serverArch?: string; @@ -115,7 +115,7 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { yield* Effect.gen(function* () { const analytics = yield* AnalyticsService.AnalyticsService; for (let index = 0; index < 20; index += 1) { - yield* analytics.record("test.retry", { index }); + yield* analytics.record("test.retry", { count: index }); } // Before the fix this loop sent the batch about once a second. for (let second = 0; second < 600; second += 1) { @@ -181,7 +181,7 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { const analytics = yield* AnalyticsService.AnalyticsService; for (let index = 0; index < 45; index += 1) { - yield* analytics.record("test.flush.drain", { index }); + yield* analytics.record("test.flush.drain", { count: index }); } yield* analytics.flush; @@ -201,7 +201,7 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { const deliveredIndexes = batchRequests.flatMap((request) => request.body.batch .filter((event) => event.event === "test.flush.drain") - .map((event) => event.properties?.index) + .map((event) => event.properties?.count) .filter((index): index is number => typeof index === "number"), ); @@ -267,7 +267,7 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { yield* Effect.gen(function* () { yield* Layer.launch(layerBatchServer).pipe(Effect.forkScoped); const analytics = yield* AnalyticsService.AnalyticsService; - yield* analytics.record("test.disabled", { index: 1 }); + yield* analytics.record("test.disabled", { count: 1 }); yield* analytics.flush; }).pipe(Effect.provide(layerRuntime)); diff --git a/apps/server/src/telemetry/AnalyticsService.ts b/apps/server/src/telemetry/AnalyticsService.ts index 3d10e93a26d6a..c93f083e346d6 100644 --- a/apps/server/src/telemetry/AnalyticsService.ts +++ b/apps/server/src/telemetry/AnalyticsService.ts @@ -17,6 +17,7 @@ import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Hex from "effect/encoding/Hex"; // T3-CUSTOM(expbkt3): insert-id hashing. import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Random from "effect/Random"; @@ -30,12 +31,22 @@ import * as HttpClientResponse from "effect/http/HttpClientResponse"; import packageJson from "../../package.json" with { type: "json" }; import * as ServerConfig from "../config.ts"; import { getTelemetryIdentifier } from "./Identify.ts"; +// T3-CUSTOM(expbkt3): BEGIN — property allowlist and per-user identity (TEC-1631). +import { filterAnalyticsProperties } from "./analyticsProperties.ts"; +import { + type AnalyticsPerson, + makeAnalyticsUserDirectory, + uuidFromSha256Hex, +} from "./analyticsIdentity.ts"; +// T3-CUSTOM(expbkt3): END interface BufferedAnalyticsEvent { readonly uuid: string; readonly event: string; readonly properties?: Readonly>; readonly capturedAt: string; + /** T3-CUSTOM(expbkt3): an identified person's email; unset events use the install id. */ + readonly distinctId?: string; } interface DeliveryState { @@ -67,9 +78,11 @@ export function retryDelayMs(failures: number, random: number): number { } const TelemetryEnvConfig = Config.all({ - posthogKey: Config.String("T3CODE_POSTHOG_KEY").pipe( - Config.withDefault("phc_XOWci4oZP4VvLiEyrFqkFjP4CZn55mjYYBMREK5Wd6m"), - ), + // T3-CUSTOM(expbkt3): no default key. The fork must never send to the upstream + // maintainers' PostHog project, so telemetry stays off until a key is set. + posthogKey: Config.String("T3CODE_POSTHOG_KEY").pipe(Config.withDefault("")), + // T3-CUSTOM(expbkt3): the PostHog `instance` group every event belongs to. + instance: Config.String("T3CODE_ANALYTICS_INSTANCE").pipe(Config.withDefault("unknown")), posthogHost: Config.String("T3CODE_POSTHOG_HOST").pipe( Config.withDefault("https://us.i.posthog.com"), ), @@ -81,13 +94,25 @@ const TelemetryEnvConfig = Config.all({ wslDistroName: Config.String("WSL_DISTRO_NAME").pipe(Config.option), }); +// T3-CUSTOM(expbkt3): BEGIN — record options (TEC-1631). +export interface AnalyticsRecordOptions { + /** Environment user who acted; sent as that person's email when known. */ + readonly userId?: string; + /** ISO time the event happened, for events recorded after the fact. */ + readonly timestamp?: string; + /** Stable id for de-duplication; the same id always maps to the same event uuid. */ + readonly insertId?: string; +} +// T3-CUSTOM(expbkt3): END + export class AnalyticsService extends Context.Service< AnalyticsService, { - /** Record an anonymous event for best-effort buffered delivery. */ + /** Record an event for best-effort buffered delivery. T3-CUSTOM(expbkt3): may identify a user. */ readonly record: ( event: string, properties?: Readonly>, + options?: AnalyticsRecordOptions, // T3-CUSTOM(expbkt3): identity and backfill options. ) => Effect.Effect; /** Flush all currently queued telemetry events. */ @@ -138,21 +163,24 @@ export const make = Effect.gen(function* () { const clientType = serverConfig.mode === "desktop" ? "desktop-app" : "cli-web-client"; const hostPlatform = yield* HostProcessPlatform; const hostArchitecture = yield* HostProcessArchitecture; + // T3-CUSTOM(expbkt3): BEGIN — key-gated telemetry, identity and instance group. + const telemetryEnabled = telemetryConfig.enabled && telemetryConfig.posthogKey.trim() !== ""; + const userDirectory = yield* makeAnalyticsUserDirectory; + const identifiedUsers = yield* Ref.make>(new Set()); + const droppedKeysLogged = yield* Ref.make>(new Set()); + // T3-CUSTOM(expbkt3): END const enqueueBufferedEvent = ( - uuid: string, - event: string, - properties?: Readonly>, + entry: Omit, + timestamp?: string, // T3-CUSTOM(expbkt3): caller-supplied event time. ) => Effect.flatMap(DateTime.now, (now) => Ref.modify(bufferRef, (current) => { const appended = [ ...current, { - uuid, - event, - ...(properties ? { properties } : {}), - capturedAt: DateTime.formatIso(now), + ...entry, // T3-CUSTOM(expbkt3): one entry object. + capturedAt: timestamp ?? DateTime.formatIso(now), // T3-CUSTOM(expbkt3) } satisfies BufferedAnalyticsEvent, ]; @@ -174,17 +202,20 @@ export const make = Effect.gen(function* () { const sendBatch = Effect.fn("AnalyticsService.sendBatch")(function* ( events: ReadonlyArray, ) { - if (!telemetryConfig.enabled || !identifier) return; + if (!telemetryEnabled || !identifier) return; // T3-CUSTOM(expbkt3): key-gated. const payload = { api_key: telemetryConfig.posthogKey, batch: events.map((event) => ({ uuid: event.uuid, event: event.event, - distinct_id: identifier, + distinct_id: event.distinctId ?? identifier, // T3-CUSTOM(expbkt3): identified person. properties: { ...event.properties, - $process_person_profile: false, + // T3-CUSTOM(expbkt3): only identified events build a person profile. + $process_person_profile: event.distinctId !== undefined, + $groups: { instance: telemetryConfig.instance }, + instance: telemetryConfig.instance, platform: hostPlatform, wsl: Option.getOrUndefined(telemetryConfig.wslDistroName), arch: hostArchitecture, @@ -257,16 +288,92 @@ export const make = Effect.gen(function* () { } }); + // T3-CUSTOM(expbkt3): BEGIN — allowlist, identity, insert ids (TEC-1631). + const logDroppedKeys = (event: string, dropped: ReadonlyArray) => + Effect.gen(function* () { + if (dropped.length === 0) return; + const fresh = yield* Ref.modify( + droppedKeysLogged, + (logged): readonly [ReadonlyArray, ReadonlySet] => { + const unseen = dropped.filter((key) => !logged.has(key)); + return [unseen, unseen.length === 0 ? logged : new Set([...logged, ...unseen])]; + }, + ); + for (const key of fresh) { + yield* Effect.logDebug("analytics property key not allowlisted; dropping it", { + event, + key, + }); + } + }); + + const eventUuid = (insertId: string | undefined) => + Effect.gen(function* () { + if (insertId !== undefined) { + const digest = yield* Effect.exit( + crypto.digest("SHA-256", new TextEncoder().encode(insertId)), + ); + if (Exit.isSuccess(digest)) return uuidFromSha256Hex(Hex.encode(digest.value)); + } + const uuid = yield* Effect.exit(crypto.randomUUIDv7); + return Exit.isSuccess(uuid) ? uuid.value : undefined; + }); + + /** Sends the person's profile once per user per process start. */ + const identifyOnce = (userId: string, person: AnalyticsPerson) => + Effect.gen(function* () { + const first = yield* Ref.modify( + identifiedUsers, + (seen): readonly [boolean, ReadonlySet] => + seen.has(userId) ? [false, seen] : [true, new Set([...seen, userId])], + ); + if (!first) return; + const uuid = yield* eventUuid(undefined); + if (uuid === undefined) return; + yield* enqueueBufferedEvent({ + uuid, + event: "$identify", + distinctId: person.email, + properties: { + $set: { + email: person.email, + role: person.role, + ...(person.name === undefined ? {} : { name: person.name }), + }, + }, + }); + }); + const record: AnalyticsService["Service"]["record"] = Effect.fn("AnalyticsService.record")( - function* (event, properties) { - if (!telemetryConfig.enabled || !identifier) return; + function* (event, properties, options) { + if (!telemetryEnabled || !identifier) return; + + const { kept, dropped } = filterAnalyticsProperties(properties); + yield* logDroppedKeys(event, dropped); + + const person = + options?.userId === undefined ? undefined : yield* userDirectory.resolve(options.userId); + if (person !== undefined && options?.userId !== undefined) { + yield* identifyOnce(options.userId, person); + } // Telemetry is best effort: an event without a uuid is not sent. The // Node implementation throws (a defect) rather than failing, so catch both. - const uuid = yield* Effect.exit(crypto.randomUUIDv7); - if (Exit.isFailure(uuid)) return; + const uuid = yield* eventUuid(options?.insertId); + if (uuid === undefined) return; - const enqueueResult = yield* enqueueBufferedEvent(uuid.value, event, properties); + const enqueueResult = yield* enqueueBufferedEvent( + { + uuid, + event, + properties: { + ...kept, + ...(options?.insertId === undefined ? {} : { $insert_id: options.insertId }), + }, + ...(person === undefined ? {} : { distinctId: person.email }), + }, + options?.timestamp, + ); if (enqueueResult.dropped) { yield* Effect.logDebug("analytics buffer full; dropping oldest event", { size: enqueueResult.size, @@ -276,6 +383,24 @@ export const make = Effect.gen(function* () { }, ); + // One group profile per boot, named after the instance. An unnamed instance + // has nothing worth a profile. + if (telemetryEnabled && identifier && telemetryConfig.instance !== "unknown") { + const uuid = yield* eventUuid(undefined); + if (uuid !== undefined) { + yield* enqueueBufferedEvent({ + uuid, + event: "$groupidentify", + properties: { + $group_type: "instance", + $group_key: telemetryConfig.instance, + $group_set: { name: telemetryConfig.instance, t3CodeVersion: packageJson.version }, + }, + }); + } + } + // T3-CUSTOM(expbkt3): END + yield* Effect.forever(Effect.sleep(FLUSH_INTERVAL_MS).pipe(Effect.flatMap(() => flushWhenDue)), { disableYield: true, }).pipe(Effect.forkScoped); diff --git a/apps/server/src/telemetry/analyticsEnvironment.test.ts b/apps/server/src/telemetry/analyticsEnvironment.test.ts new file mode 100644 index 0000000000000..0326ccaf0170f --- /dev/null +++ b/apps/server/src/telemetry/analyticsEnvironment.test.ts @@ -0,0 +1,61 @@ +// T3-CUSTOM(expbkt3): agent processes never see the server's analytics settings (TEC-1631). +import { assert, it } from "@effect/vitest"; + +import { mergeProviderInstanceEnvironment } from "../provider/ProviderInstanceEnvironment.ts"; +import { SERVER_ANALYTICS_ENV_KEYS, withoutAnalyticsEnvironment } from "./analyticsEnvironment.ts"; + +const serverEnv = { + PATH: "/usr/bin", + T3CODE_POSTHOG_KEY: "phc_secret", + T3CODE_POSTHOG_HOST: "https://eu.i.posthog.com", + T3CODE_ANALYTICS_INSTANCE: "bkt3", + T3CODE_TELEMETRY_ENABLED: "true", +}; + +/** What a child sees when the spawner spreads the server env under `env` (`extendEnv: true`). */ +const childSees = (env: NodeJS.ProcessEnv) => + Object.fromEntries( + Object.entries({ ...serverEnv, ...env }).filter(([, value]) => value !== undefined), + ); + +it("masks every analytics key from an env built on the server's env", () => { + const masked = withoutAnalyticsEnvironment({ ...serverEnv, CODEX_HOME: "/codex" }, serverEnv); + for (const key of SERVER_ANALYTICS_ENV_KEYS) { + assert.isTrue(key in masked, key); + assert.isUndefined(masked[key], key); + } + assert.equal(masked.PATH, "/usr/bin"); + assert.equal(masked.CODEX_HOME, "/codex"); +}); + +it("hides inherited keys even when the spawner extends the server env", () => { + // The env passed to the child does not carry the keys, but the server's does. + const masked = withoutAnalyticsEnvironment({ CODEX_HOME: "/codex" }, serverEnv); + assert.deepEqual(childSees(masked), { PATH: "/usr/bin", CODEX_HOME: "/codex" }); +}); + +it("returns the env unchanged when no analytics key is set anywhere", () => { + const env = { PATH: "/usr/bin" }; + assert.strictEqual(withoutAnalyticsEnvironment(env, { HOME: "/home/u" }), env); +}); + +it("does not mutate the env it was given", () => { + const env = { ...serverEnv }; + withoutAnalyticsEnvironment(env, {}); + assert.equal(env.T3CODE_POSTHOG_KEY, "phc_secret"); +}); + +it("strips the keys from every provider environment, with or without instance overrides", () => { + for (const environment of [ + undefined, + [], + [{ name: "CUSTOM_VALUE", value: "x", sensitive: false }], + [{ name: "T3CODE_POSTHOG_KEY", value: "phc_override", sensitive: true }], + ]) { + const merged = mergeProviderInstanceEnvironment(environment, serverEnv); + for (const key of SERVER_ANALYTICS_ENV_KEYS) { + assert.isUndefined(merged[key], key); + } + assert.equal(merged.PATH, "/usr/bin"); + } +}); diff --git a/apps/server/src/telemetry/analyticsEnvironment.ts b/apps/server/src/telemetry/analyticsEnvironment.ts new file mode 100644 index 0000000000000..9a09dad5911e1 --- /dev/null +++ b/apps/server/src/telemetry/analyticsEnvironment.ts @@ -0,0 +1,38 @@ +/** + * T3-CUSTOM(expbkt3): the server's analytics settings stay in the server. + * + * Agent processes (provider CLIs, and every tool or dev server they start) + * must not inherit the PostHog key or instance name: an agent could read the + * key, and a dev server it starts would report into the production project + * as the production instance. + * + * The keys are set to `undefined` rather than deleted. Some spawns use + * `extendEnv: true`, which spreads `process.env` under the given env, and only + * an explicit `undefined` hides the inherited value there. Node drops + * `undefined` entries when it builds the child's environment. + */ + +export const SERVER_ANALYTICS_ENV_KEYS = [ + "T3CODE_POSTHOG_KEY", + "T3CODE_POSTHOG_HOST", + "T3CODE_ANALYTICS_INSTANCE", + "T3CODE_TELEMETRY_ENABLED", +] as const; + +/** + * `env` with every analytics key the child could see (from `env` itself or + * from `inherited`, the server's own environment) masked. Returns `env` + * unchanged when no such key is set anywhere. + */ +export function withoutAnalyticsEnvironment( + env: NodeJS.ProcessEnv, + inherited: Readonly> = process.env, +): NodeJS.ProcessEnv { + const present = SERVER_ANALYTICS_ENV_KEYS.filter( + (key) => env[key] !== undefined || inherited[key] !== undefined, + ); + if (present.length === 0) return env; + const next: NodeJS.ProcessEnv = { ...env }; + for (const key of present) next[key] = undefined; + return next; +} diff --git a/apps/server/src/telemetry/analyticsIdentity.ts b/apps/server/src/telemetry/analyticsIdentity.ts new file mode 100644 index 0000000000000..f4df3fb5e9dae --- /dev/null +++ b/apps/server/src/telemetry/analyticsIdentity.ts @@ -0,0 +1,102 @@ +/** + * T3-CUSTOM(expbkt3): who a bkt3 analytics event belongs to. + * + * An event recorded with a `userId` is sent as that person: their + * `environment_users.primary_email` becomes the PostHog `distinct_id`. The + * user table is read at most once per `USER_DIRECTORY_REFRESH_MS`, never once + * per event. A user without an email, or a caller with no user repository in + * its context, falls back to the install-level anonymous id. + * + * The repository is read from the caller's context rather than the analytics + * layer's, because `AnalyticsService` sits below persistence in the server's + * layer stack. Every runtime service (websocket handlers, MCP tools, reactors) + * has `EnvironmentUserRepository` in its context. + */ +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as Result from "effect/Result"; + +import * as EnvironmentUsers from "../persistence/EnvironmentUsers.ts"; + +export const USER_DIRECTORY_REFRESH_MS = 5 * 60_000; + +/** The person-profile fields PostHog stores for an identified user. */ +export interface AnalyticsPerson { + readonly email: string; + readonly name: string | undefined; + readonly role: string; +} + +interface UserDirectoryState { + /** Epoch ms of the last read attempt, or null before the first. */ + readonly loadedAt: number | null; + readonly people: ReadonlyMap; +} + +/** Users with an email, keyed by user id. Emails are trimmed and lower-cased. */ +export function analyticsPeopleFromUsers( + users: ReadonlyArray, +): ReadonlyMap { + const people = new Map(); + for (const user of users) { + const email = user.primaryEmail?.trim().toLowerCase() ?? ""; + if (email === "") continue; + const name = user.displayName?.trim() ?? ""; + people.set(user.userId, { + email, + name: name === "" ? undefined : name, + role: user.role, + }); + } + return people; +} + +/** Caches `environment_users` for identity lookups. */ +export const makeAnalyticsUserDirectory = Effect.gen(function* () { + const stateRef = yield* Ref.make({ loadedAt: null, people: new Map() }); + + const refresh = Effect.gen(function* () { + const repository = yield* Effect.serviceOption(EnvironmentUsers.EnvironmentUserRepository); + // No repository here: try again on the next lookup, which costs no DB read. + if (Option.isNone(repository)) return; + const now = yield* Clock.currentTimeMillis; + const users = yield* Effect.result(repository.value.list); + yield* Ref.update(stateRef, (state) => ({ + loadedAt: now, + // A failed read keeps the last good map until the next refresh window. + people: Result.isSuccess(users) ? analyticsPeopleFromUsers(users.success) : state.people, + })); + }); + + /** The person for `userId`, or undefined when unknown or without an email. */ + const resolve = (userId: string) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const state = yield* Ref.get(stateRef); + if (state.loadedAt === null || now - state.loadedAt >= USER_DIRECTORY_REFRESH_MS) { + yield* refresh; + } + return (yield* Ref.get(stateRef)).people.get(userId); + }); + + return { resolve } as const; +}); + +/** + * A stable UUID for a caller-chosen insert id, so a re-sent or backfilled + * event lands on the same PostHog `uuid` and is de-duplicated. `sha256Hex` is + * the hex SHA-256 of the insert id. The result has UUID version 8 (custom). + */ +export function uuidFromSha256Hex(sha256Hex: string): string { + const hex = sha256Hex.toLowerCase().padEnd(32, "0"); + const variant = ((Number.parseInt(hex[16] ?? "0", 16) & 0x3) | 0x8).toString(16); + return [ + hex.slice(0, 8), + hex.slice(8, 12), + `8${hex.slice(13, 16)}`, + `${variant}${hex.slice(17, 20)}`, + hex.slice(20, 32), + ].join("-"); +} diff --git a/apps/server/src/telemetry/analyticsProperties.ts b/apps/server/src/telemetry/analyticsProperties.ts new file mode 100644 index 0000000000000..aab8aac216bf4 --- /dev/null +++ b/apps/server/src/telemetry/analyticsProperties.ts @@ -0,0 +1,182 @@ +/** + * T3-CUSTOM(expbkt3): the only event property keys bkt3 sends to PostHog. + * + * Events now carry a person's email as `distinct_id`, so a property must never + * hold free text a person or an agent wrote: titles, prompts, messages, paths, + * branches, repository names, answers, questions or error messages. Add a key + * here only when its values are ids, enums, numbers or booleans. Error detail + * goes in as a class or tag name (`errorClass`, `failureClass`), never a message. + * + * `AnalyticsService.record` drops every other key and logs each dropped key + * once per process. + */ + +const UPSTREAM_KEYS = [ + // server.boot.heartbeat + "threadCount", + "projectCount", + // client.connected, client.thread.started, client.turn.requested + "surface", + "appVersion", + "clientAppVersion", + "clientOs", + "os", + "clientDeviceType", + "webDeployment", + "clientBrowser", + "osMajorVersion", + "clientOsMajorVersion", + "deviceModel", + "clientDeviceModel", + "connectionMethod", + // provider.turn.completed + "terminalStatus", + "usageStatus", + "usageScope", + "hasSubagents", + "inputTokens", + "cachedInputTokens", + "cacheCreationTokens", + "outputTokens", + "reasoningTokens", + "mixedModels", + // chatgpt.auth.*, chatgpt.transfer.* + "flow", + "intent", + "callbackMode", + "outcome", + "failureStage", + "accountCountScope", + "connectedAccountCount", + "connectedConnectionCount", + "savedConnectionCount", + "unidentifiedConnectedConnectionCount", +] as const; + +/** Session activity and run metrics (TEC-1631 worker 2). */ +const SESSION_ACTIVITY_KEYS = [ + "threadId", + "projectId", + "parentThreadId", + "isChildSession", + "state", + "waitKind", + "durationMs", + "startedAt", + "endedAt", + "endReason", + "runId", + "runOrdinal", + "provider", + "providerInstanceId", + "model", + "effort", + "interactionMode", + "runtimeMode", + "activeWorkers", + "maxActiveWorkers", + "ownerUserId", + "senderUserId", + "sessionSource", + "resumedAfterRestart", + "backfill", + "ageMs", + "status", + "queueWaitMs", + "runDurationMs", + "workDurationMs", + "attempts", + "failureClass", + "requestKind", + "waitMs", + "resolution", + "answeredByUserId", + "subagentId", + "driver", + "childThreadPresent", + "decision", + "count", + "runningRuns", + "queuedRuns", + "waitingHuman", + "waitingWorkers", + "activeSubagents", + "snoozed", + "spanStartedAt", + "segmentIndex", + "runTrigger", + "backgroundTasks", + "createdBy", + "origin", + "scope", +] as const; + +/** Server events outside the orchestration stream (TEC-1631 worker 1). */ +const SERVER_EVENT_KEYS = [ + "instance", + "actorKind", + "connectionMode", + // mcp.tool.called + "toolName", + "callerKind", + "ok", + "errorClass", + // scheduled_task.ran + "taskId", + "scheduleType", + "trigger", + "creationSource", + // webhook.delivered + "kind", + // session.recovery.* + "terminalizedRuns", + "stoppedSessions", + "closedRequests", + "retiredEffects", + "requeuedEffects", +] as const; + +export const ALLOWED_ANALYTICS_PROPERTY_KEYS: ReadonlySet = new Set([ + ...UPSTREAM_KEYS, + ...SESSION_ACTIVITY_KEYS, + ...SERVER_EVENT_KEYS, +]); + +/** + * Keys that must never be allowed, whatever someone adds above. Checked by the + * tests so a free-text key cannot slip in by accident. + */ +export const FORBIDDEN_ANALYTICS_PROPERTY_KEYS: ReadonlyArray = [ + "title", + "prompt", + "message", + "text", + "path", + "cwd", + "branch", + "repo", + "answer", + "question", + "error", + "errorMessage", +]; + +/** Splits `properties` into the allowed ones and the names of the dropped keys. */ +export function filterAnalyticsProperties( + properties: Readonly> | undefined, +): { + readonly kept: Readonly> | undefined; + readonly dropped: ReadonlyArray; +} { + if (properties === undefined) return { kept: undefined, dropped: [] }; + const kept: Record = {}; + const dropped: Array = []; + for (const [key, value] of Object.entries(properties)) { + if (ALLOWED_ANALYTICS_PROPERTY_KEYS.has(key)) { + kept[key] = value; + } else { + dropped.push(key); + } + } + return { kept, dropped }; +} diff --git a/apps/server/src/telemetry/serverEvents.test.ts b/apps/server/src/telemetry/serverEvents.test.ts new file mode 100644 index 0000000000000..efcd6674bfd2f --- /dev/null +++ b/apps/server/src/telemetry/serverEvents.test.ts @@ -0,0 +1,142 @@ +// T3-CUSTOM(expbkt3): bkt3 server analytics helpers (TEC-1631). +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import { + ALLOWED_ANALYTICS_PROPERTY_KEYS, + FORBIDDEN_ANALYTICS_PROPERTY_KEYS, + filterAnalyticsProperties, +} from "./analyticsProperties.ts"; +import { uuidFromSha256Hex } from "./analyticsIdentity.ts"; +import * as AnalyticsService from "./AnalyticsService.ts"; +import { + clientConnectionMode, + recordLinearLinked, + webhookStatusFromHttp, + withMcpToolAnalytics, +} from "./serverEvents.ts"; + +interface Recorded { + readonly event: string; + readonly properties: Readonly> | undefined; + readonly options: AnalyticsService.AnalyticsRecordOptions | undefined; +} + +const recordingAnalytics = (recorded: Array) => + AnalyticsService.AnalyticsService.of({ + record: (event, properties, options) => + Effect.sync(() => { + recorded.push({ event, properties, options }); + }), + flush: Effect.void, + }); + +class ToolBroke extends Schema.TaggedError()("ToolBroke", { + detail: Schema.String, +}) {} + +it("never allows a free-text key", () => { + for (const key of FORBIDDEN_ANALYTICS_PROPERTY_KEYS) { + assert.isFalse(ALLOWED_ANALYTICS_PROPERTY_KEYS.has(key), key); + } +}); + +it("allows the session-activity keys other recorders depend on", () => { + for (const key of ["threadId", "waitKind", "durationMs", "snoozed", "ownerUserId"]) { + assert.isTrue(ALLOWED_ANALYTICS_PROPERTY_KEYS.has(key), key); + } +}); + +it("splits properties into kept values and dropped key names", () => { + const { kept, dropped } = filterAnalyticsProperties({ + threadId: "t-1", + title: "a title", + message: "hello", + }); + assert.deepEqual(kept, { threadId: "t-1" }); + assert.deepEqual([...dropped].toSorted(), ["message", "title"]); +}); + +it("derives a version 8 uuid from a sha-256 hex digest", () => { + const hex = "ab".repeat(32); + assert.equal(uuidFromSha256Hex(hex), "abababab-abab-8bab-abab-abababababab"); + assert.equal(uuidFromSha256Hex(hex), uuidFromSha256Hex(hex.toUpperCase())); +}); + +it("classifies how a client reached the server", () => { + assert.equal(clientConnectionMode({ connectionMethod: "relay", host: "localhost" }), "relay"); + assert.equal(clientConnectionMode({ host: "localhost:3773" }), "local"); + assert.equal(clientConnectionMode({ host: "127.0.0.1:3773" }), "local"); + assert.equal(clientConnectionMode({ host: "[::1]:3773" }), "local"); + assert.equal(clientConnectionMode({ host: "dev-server-1.tailab6257.ts.net" }), "tailnet"); + assert.equal(clientConnectionMode({ host: "100.101.102.103:18087" }), "tailnet"); + assert.equal(clientConnectionMode({ host: "bkt3.dev.beknown.live" }), "remote"); + assert.equal(clientConnectionMode({}), "remote"); +}); + +it("names a session-webhook answer by its status", () => { + assert.equal(webhookStatusFromHttp(202), "accepted"); + assert.equal(webhookStatusFromHttp(413), "http_413"); +}); + +it.effect("records an MCP tool call by name and outcome, never its arguments", () => + Effect.gen(function* () { + const recorded: Array = []; + const caller = Effect.succeed({ + principal: "provider-session" as const, + actorUserId: "user_clerk_alice", + }); + yield* withMcpToolAnalytics("t3_send_prompt", caller, Effect.succeed("ok")); + const failed = yield* withMcpToolAnalytics( + "t3_send_prompt", + caller, + Effect.fail(new ToolBroke({ detail: "secret detail" })), + ).pipe(Effect.flip); + assert.equal(failed._tag, "ToolBroke"); + + assert.equal(recorded.length, 0, "nothing is recorded without analytics in context"); + yield* withMcpToolAnalytics("t3_inbox", caller, Effect.void).pipe( + Effect.provideService(AnalyticsService.AnalyticsService, recordingAnalytics(recorded)), + ); + yield* withMcpToolAnalytics( + "t3_inbox", + Effect.succeed(undefined), + Effect.fail(new ToolBroke({ detail: "secret detail" })), + ).pipe( + Effect.ignore, + Effect.provideService(AnalyticsService.AnalyticsService, recordingAnalytics(recorded)), + ); + + assert.equal(recorded.length, 2); + assert.equal(recorded[0]?.event, "mcp.tool.called"); + assert.equal(recorded[0]?.properties?.toolName, "t3_inbox"); + assert.equal(recorded[0]?.properties?.callerKind, "agent"); + assert.equal(recorded[0]?.properties?.ok, true); + assert.deepEqual(recorded[0]?.options, { userId: "user_clerk_alice" }); + assert.equal(recorded[1]?.properties?.ok, false); + assert.equal(recorded[1]?.properties?.callerKind, "unknown"); + assert.equal(recorded[1]?.properties?.errorClass, "ToolBroke"); + assert.isUndefined(recorded[1]?.options); + assert.notInclude(JSON.stringify(recorded), "secret detail"); + }), +); + +it.effect("records linked Linear items as a count only", () => + Effect.gen(function* () { + const recorded: Array = []; + const provide = Effect.provideService( + AnalyticsService.AnalyticsService, + recordingAnalytics(recorded), + ); + yield* recordLinearLinked({ threadId: "thread-1", count: 0, userId: null }).pipe(provide); + yield* recordLinearLinked({ threadId: "thread-1", count: 2, userId: "user-1" }).pipe(provide); + assert.deepEqual(recorded, [ + { + event: "linear.linked", + properties: { threadId: "thread-1", count: 2 }, + options: { userId: "user-1" }, + }, + ]); + }), +); diff --git a/apps/server/src/telemetry/serverEvents.ts b/apps/server/src/telemetry/serverEvents.ts new file mode 100644 index 0000000000000..1c29a9f39e419 --- /dev/null +++ b/apps/server/src/telemetry/serverEvents.ts @@ -0,0 +1,257 @@ +/** + * T3-CUSTOM(expbkt3): bkt3 product events that do not come from the + * orchestration event stream (TEC-1631). + * + * Each helper reads `AnalyticsService` from the caller's context, so a hook + * point is one line and adds no service requirement. Every helper is best + * effort: it never fails or delays the work it reports on. Properties are ids, + * enums, numbers and booleans only; see `analyticsProperties.ts`. + */ +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Option from "effect/Option"; +import * as Result from "effect/Result"; + +import * as AnalyticsService from "./AnalyticsService.ts"; + +/** Records `event` when analytics is in context; otherwise does nothing. */ +export const recordServerEvent = ( + event: string, + properties: Readonly>, + options?: AnalyticsService.AnalyticsRecordOptions, +): Effect.Effect => + Effect.serviceOption(AnalyticsService.AnalyticsService).pipe( + Effect.flatMap((analytics) => + Option.isNone(analytics) + ? Effect.void + : analytics.value.record(event, properties, options).pipe(Effect.ignoreCause), + ), + ); + +const userOption = (userId: string | null | undefined) => + userId === null || userId === undefined || userId === "" ? undefined : { userId }; + +/** + * The tag or class name of a failure, never its message. Tagged errors give + * their `_tag`; other errors their constructor name; anything else "unknown". + */ +export function errorClassOf(cause: Cause.Cause): string { + if (Cause.hasInterruptsOnly(cause)) return "Interrupted"; + const error = Option.getOrUndefined(Cause.findErrorOption(cause)); + const defect = Cause.findDefect(cause); + const failure = error ?? (Result.isSuccess(defect) ? defect.success : undefined); + if (typeof failure === "object" && failure !== null) { + const tag = (failure as { readonly _tag?: unknown })._tag; + if (typeof tag === "string" && /^[A-Za-z0-9_.-]{1,80}$/.test(tag)) return tag; + const name = failure.constructor?.name; + if (typeof name === "string" && /^[A-Za-z0-9_]{1,80}$/.test(name)) return name; + } + return error === undefined ? "Defect" : "unknown"; +} + +// --------------------------------------------------------------------------- +// mcp.tool.called +// --------------------------------------------------------------------------- + +/** Who called a T3 MCP tool. */ +export type McpCallerKind = "agent" | "user" | "operator"; + +/** Maps the MCP invocation principal to the analytics caller kind. */ +export function mcpCallerKind( + principal: "provider-session" | "external-user" | "external-operator", +): McpCallerKind { + switch (principal) { + case "provider-session": + return "agent"; + case "external-user": + return "user"; + case "external-operator": + return "operator"; + } +} + +export interface McpToolCaller { + readonly principal: "provider-session" | "external-user" | "external-operator"; + readonly actorUserId: string | null; +} + +/** + * Times one MCP tool call and records `mcp.tool.called` when it ends. Only + * the tool name is sent, never its arguments or result. `caller` reads the + * invocation scope, or returns undefined when there is none. + */ +export const withMcpToolAnalytics = ( + toolName: string, + caller: Effect.Effect, + effect: Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const startedAt = yield* Clock.currentTimeMillis; + return yield* effect.pipe( + Effect.onExit((exit) => + Effect.gen(function* () { + const endedAt = yield* Clock.currentTimeMillis; + const scope = yield* caller; + yield* recordServerEvent( + "mcp.tool.called", + { + toolName, + callerKind: scope === undefined ? "unknown" : mcpCallerKind(scope.principal), + ok: Exit.isSuccess(exit), + durationMs: Math.max(0, endedAt - startedAt), + ...(Exit.isFailure(exit) ? { errorClass: errorClassOf(exit.cause) } : {}), + }, + userOption(scope?.actorUserId), + ); + }), + ), + ); + }); + +// --------------------------------------------------------------------------- +// scheduled_task.ran +// --------------------------------------------------------------------------- + +export const recordScheduledTaskRan = (input: { + readonly taskId: string; + readonly scheduleType: string; + readonly trigger: "scheduled" | "manual" | "webhook"; + readonly creationSource: string; + readonly status: "succeeded" | "failed"; + readonly durationMs: number; + readonly ownerUserId?: string | null | undefined; +}): Effect.Effect => + recordServerEvent( + "scheduled_task.ran", + { + taskId: input.taskId, + scheduleType: input.scheduleType, + trigger: input.trigger, + creationSource: input.creationSource, + status: input.status, + durationMs: Math.max(0, input.durationMs), + }, + userOption(input.ownerUserId), + ); + +// --------------------------------------------------------------------------- +// webhook.delivered +// --------------------------------------------------------------------------- + +/** + * An inbound webhook delivery and how the server answered it. `kind` is + * "scheduled_task" for `/api/hooks/...` and "session_callback" for + * `/api/session-webhooks/...`. `status` is an outcome tag, never a body. + */ +export const recordWebhookDelivered = ( + kind: "scheduled_task" | "session_callback", + status: string, +): Effect.Effect => recordServerEvent("webhook.delivered", { kind, status }); + +/** The outcome tag for an HTTP status the session-webhook route answered with. */ +export function webhookStatusFromHttp(status: number): string { + return status === 202 ? "accepted" : `http_${status}`; +} + +// --------------------------------------------------------------------------- +// session.recovery.* +// --------------------------------------------------------------------------- + +/** + * Wraps the startup provider-runtime recovery: `session.recovery.attempted` + * before it runs and `session.recovery.finished` with its outcome and counts + * after. Recovery runs once per boot, so `attempts` is 1. + */ +export const withSessionRecoveryAnalytics = ( + recover: Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + yield* recordServerEvent("session.recovery.attempted", { trigger: "startup", attempts: 1 }); + const startedAt = yield* Clock.currentTimeMillis; + return yield* recover.pipe( + Effect.onExit((exit) => + Effect.gen(function* () { + const endedAt = yield* Clock.currentTimeMillis; + yield* recordServerEvent("session.recovery.finished", { + trigger: "startup", + attempts: 1, + outcome: Exit.isSuccess(exit) ? "succeeded" : "failed", + durationMs: Math.max(0, endedAt - startedAt), + ...(Exit.isSuccess(exit) ? recoveryCounts(exit.value) : {}), + ...(Exit.isFailure(exit) ? { errorClass: errorClassOf(exit.cause) } : {}), + }); + }), + ), + ); + }); + +const RECOVERY_COUNT_KEYS = [ + "terminalizedRuns", + "stoppedSessions", + "closedRequests", + "retiredEffects", + "requeuedEffects", +] as const; + +function recoveryCounts(summary: object): Record { + const counts: Record = {}; + for (const key of RECOVERY_COUNT_KEYS) { + const value = (summary as Record)[key]; + if (typeof value === "number") counts[key] = value; + } + return counts; +} + +// --------------------------------------------------------------------------- +// linear.linked +// --------------------------------------------------------------------------- + +/** Linear items newly linked to a session through the T3 MCP tool. */ +export const recordLinearLinked = (input: { + readonly threadId: string; + readonly count: number; + readonly userId?: string | null | undefined; +}): Effect.Effect => + input.count <= 0 + ? Effect.void + : recordServerEvent( + "linear.linked", + { threadId: input.threadId, count: input.count }, + userOption(input.userId), + ); + +// --------------------------------------------------------------------------- +// client.connected +// --------------------------------------------------------------------------- + +export type ClientConnectionMode = "relay" | "local" | "tailnet" | "remote"; + +const TAILNET_IPV4 = /^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\.\d{1,3}\.\d{1,3}$/; + +/** + * How a client reached the server: through the relay, from this machine, over + * the tailnet (a `*.ts.net` name or a 100.64.0.0/10 address), or otherwise. + * `host` is the request's Host header; only its class is sent, never the host. + */ +export function clientConnectionMode(input: { + readonly connectionMethod?: unknown; + readonly host?: string | undefined; +}): ClientConnectionMode { + if (input.connectionMethod === "relay") return "relay"; + const raw = (input.host ?? "").trim().toLowerCase(); + const hostname = raw.startsWith("[") ? raw.slice(1, raw.indexOf("]")) : raw.split(":")[0]; + if ( + hostname === "localhost" || + hostname === "::1" || + hostname?.startsWith("127.") === true || + hostname?.endsWith(".localhost") === true + ) { + return "local"; + } + if (hostname?.endsWith(".ts.net") === true || TAILNET_IPV4.test(hostname ?? "")) { + return "tailnet"; + } + return "remote"; +} diff --git a/apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts b/apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts new file mode 100644 index 0000000000000..c8fcb8d28a6c3 --- /dev/null +++ b/apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts @@ -0,0 +1,98 @@ +/** + * T3-CUSTOM(expbkt3): live session time-in-state metrics (TEC-1631). + * + * Feeds the live orchestration V2 event stream to the pure session tracker + * (`SessionActivityReducer.ts`) and records what it emits through + * `AnalyticsService`. On boot it seeds open sessions from one shell snapshot + * read, so their first span carries `resumedAfterRestart: true`. On shutdown it + * closes the spans of active sessions (not idle or archived ones) with + * `endReason: "server_stopped"`. Every five minutes it records a + * `session.state.snapshot` for concurrency and peak load. + * + * It does nothing, not even subscribe, when telemetry is off or no PostHog key + * is set. + */ +import * as Clock from "effect/Clock"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schedule from "effect/Schedule"; +import * as Stream from "effect/Stream"; + +import { EventSinkV2 } from "../../orchestration-v2/EventSink.ts"; +import { ProjectionStoreV2 } from "../../orchestration-v2/ProjectionStore.ts"; +import { forkParked } from "../../serverActivation.ts"; +import { AnalyticsService } from "../AnalyticsService.ts"; +import { + makeSessionActivityTracker, + type SessionActivityEmission, +} from "./SessionActivityReducer.ts"; + +const SNAPSHOT_INTERVAL = "5 minutes"; + +const SessionActivityConfig = Config.all({ + enabled: Config.Boolean("T3CODE_TELEMETRY_ENABLED").pipe(Config.withDefault(true)), + posthogKey: Config.String("T3CODE_POSTHOG_KEY").pipe(Config.withDefault("")), +}); + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const config = yield* SessionActivityConfig; + if (!config.enabled || config.posthogKey.trim() === "") return; + + const analytics = yield* AnalyticsService; + const eventSink = yield* EventSinkV2; + const projections = yield* ProjectionStoreV2; + const tracker = makeSessionActivityTracker({ backfill: false }); + + const record = (emissions: ReadonlyArray) => + Effect.forEach( + emissions, + (emission) => + analytics.record(emission.event, emission.properties, { + ...(emission.userId === undefined ? {} : { userId: emission.userId }), + timestamp: emission.timestamp, + insertId: emission.insertId, + }), + { discard: true }, + ); + + // Subscribe from the current head, then seed: an event that lands between + // the two reads is applied on top of the seed, never lost. + const afterSequence = yield* eventSink.latestSequence().pipe(Effect.orDie); + const shells = yield* projections.getShellSnapshot({ location: "active" }).pipe( + Effect.map((snapshot) => snapshot.threads), + Effect.catchCause((cause) => + Effect.logWarning("session activity: could not seed open sessions", { cause }).pipe( + Effect.as([]), + ), + ), + ); + const bootMs = yield* Clock.currentTimeMillis; + for (const shell of shells) tracker.seed(shell, bootMs); + + yield* forkParked( + eventSink.stream({ afterSequence }).pipe( + Stream.runForEach((stored) => record(tracker.apply(stored.event))), + Effect.catchCause((cause) => + Effect.logWarning("session activity: event stream failed", { cause }), + ), + ), + ); + + yield* forkParked( + Clock.currentTimeMillis.pipe( + Effect.flatMap((nowMs) => record(tracker.snapshot(nowMs))), + Effect.repeat(Schedule.spaced(SNAPSHOT_INTERVAL)), + Effect.asVoid, + ), + ); + + // Runs before AnalyticsService's own finalizer, which flushes the buffer. + yield* Effect.addFinalizer(() => + Clock.currentTimeMillis.pipe( + Effect.flatMap((nowMs) => record(tracker.closeActive(nowMs, "server_stopped"))), + ), + ); + }), +); diff --git a/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts new file mode 100644 index 0000000000000..c7250bd5b77b0 --- /dev/null +++ b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts @@ -0,0 +1,585 @@ +// T3-CUSTOM(expbkt3): session time-in-state metrics (TEC-1631). +// Fixtures follow real `orchestration_events` rows from bkt3 (ids shortened; +// every text field replaced by a SECRET_* sentinel the privacy test looks for). +import { assert, describe, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; + +import { + makeSessionActivityTracker, + SESSION_ACTIVITY_PROPERTY_KEYS, + type SessionActivityEmission, + type SessionActivityEvent, +} from "./SessionActivityReducer.ts"; + +const THREAD = "d8658052-d2d8-4141-bc61-d91bfc5c570b"; +const OWNER = "user_owner"; +const SENDER = "user_sender"; +const RUN = `run:thread:${THREAD}:ordinal:73`; + +const at = (time: string) => `2026-10-09T${time}Z`; +const epoch = (iso: string) => DateTime.toEpochMillis(DateTime.makeUnsafe(iso)); + +function threadPayload(overrides: Record = {}) { + return { + ownerUserId: OWNER, + memberUserIds: [OWNER], + parentThreadId: null, + createdBy: "user", + creationSource: "web", + id: THREAD, + projectId: "project-1", + title: "SECRET_TITLE", + providerInstanceId: "claudeAgent", + modelSelection: { + instanceId: "claudeAgent", + model: "claude-opus-5-5", + options: [{ id: "effort", value: "high" }], + }, + runtimeMode: "full-access", + interactionMode: "default", + branch: "SECRET_BRANCH", + worktreePath: "/SECRET_PATH", + activeProviderThreadId: null, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: THREAD }, + forkedFrom: null, + createdAt: at("09:00:00.000"), + updatedAt: at("09:00:00.000"), + archivedAt: null, + snoozedUntil: null, + deletedAt: null, + ...overrides, + }; +} + +function event( + type: string, + occurredAt: string, + payload: unknown, + extra: Partial = {}, +): SessionActivityEvent { + return { type, threadId: THREAD, occurredAt, payload, ...extra }; +} + +function runPayload(status: string, fields: Record = {}) { + return { + id: RUN, + threadId: THREAD, + ordinal: 73, + providerInstanceId: "claudeAgent", + modelSelection: { + instanceId: "claudeAgent", + model: "claude-opus-5-5", + options: [ + { id: "effort", value: "high" }, + { id: "fastMode", value: false }, + ], + }, + userMessageId: "message-1", + status, + requestedAt: at("09:06:19.278"), + startedAt: null, + completedAt: null, + ...fields, + }; +} + +function userMessage(fields: Record = {}) { + return { + sentByUserId: SENDER, + createdBy: "user", + creationSource: "web", + id: "message-1", + threadId: THREAD, + runId: null, + role: "user", + text: "SECRET_PROMPT", + attachments: [], + streaming: false, + createdAt: at("09:06:19.000"), + updatedAt: at("09:06:19.000"), + ...fields, + }; +} + +function questionPayload(status: string, resolvedAt: string | null) { + return { + id: "runtime-request:native-request:toolu_01", + nodeId: "node:runtime-request:toolu_01", + kind: "user_input", + status, + responseCapability: { type: "live", providerSessionId: "provider-session:1" }, + createdAt: at("09:07:25.279"), + resolvedAt, + ...(status === "resolved" ? { answers: { q1: "SECRET_ANSWER" } } : {}), + }; +} + +function subagentPayload(status: string, fields: Record = {}) { + return { + id: "node:subagent:task-1", + threadId: THREAD, + runId: RUN, + parentNodeId: "node:root", + origin: "provider_native", + createdBy: "agent", + driver: "claudeAgent", + providerInstanceId: "claudeAgent", + childThreadId: "thread:provider:claudeAgent:native-thread:child", + prompt: "SECRET_SUBAGENT_PROMPT", + title: "SECRET_SUBAGENT_TITLE", + result: status === "completed" ? "SECRET_RESULT" : null, + status, + startedAt: at("09:08:00.000"), + completedAt: status === "completed" ? at("09:20:00.000") : null, + updatedAt: at("09:08:00.000"), + ...fields, + }; +} + +function feed(events: ReadonlyArray, backfill = false) { + const tracker = makeSessionActivityTracker({ backfill }); + const out: Array = []; + for (const item of events) out.push(...tracker.apply(item)); + return { tracker, out }; +} + +const spans = (out: ReadonlyArray) => + out.filter((emission) => emission.event === "session.state.span"); + +/** [state, waitKind, durationMs, endReason] per span. */ +const spanSummary = (out: ReadonlyArray) => + spans(out).map((span) => [ + span.properties.state, + span.properties.waitKind, + span.properties.durationMs, + span.properties.endReason, + ]); + +const startedRun = (time: string) => runPayload("running", { startedAt: at(time) }); +const archivedThread = (time: string) => threadPayload({ archivedAt: at(time) }); + +const created = event("thread.created", at("09:00:00.000"), threadPayload()); + +describe("SessionActivityReducer", () => { + it("counts a question asked mid-run as human wait, not agent time", () => { + const { out } = feed([ + created, + event("message.updated", at("09:06:19.000"), userMessage()), + event("run.created", at("09:06:19.278"), runPayload("starting"), { actorUserId: SENDER }), + event("run-attempt.created", at("09:06:19.278"), { id: "attempt-1", runId: RUN }), + event( + "run.updated", + at("09:06:19.534"), + runPayload("running", { startedAt: at("09:06:19.534") }), + { driver: "claudeAgent" }, + ), + event("runtime-request.updated", at("09:07:25.320"), questionPayload("pending", null), { + runId: RUN, + }), + event( + "runtime-request.updated", + at("09:09:23.778"), + questionPayload("resolved", at("09:09:23.778")), + { runId: RUN, actorUserId: SENDER }, + ), + event( + "run.updated", + at("09:10:09.686"), + runPayload("waiting", { startedAt: at("09:06:19.534") }), + ), + event( + "run.updated", + at("09:10:09.747"), + runPayload("completed", { + startedAt: at("09:06:19.534"), + completedAt: at("09:10:09.747"), + }), + ), + event("thread.visited", at("09:30:00.000"), threadPayload()), + ]); + + assert.deepStrictEqual(spanSummary(out), [ + ["idle", undefined, 379_278, "agent_running"], + ["agent_running", undefined, 66_042, "waiting_human"], + ["waiting_human", "question", 118_458, "agent_running"], + ["agent_running", undefined, 45_908, "idle"], + ]); + const waitSpan = spans(out)[2]; + assert.strictEqual(waitSpan?.properties.runId, RUN); + assert.strictEqual(waitSpan?.properties.senderUserId, SENDER); + assert.strictEqual(waitSpan?.userId, OWNER); + + const resolved = out.find((emission) => emission.event === "human_request.resolved"); + assert.strictEqual(resolved?.properties.waitMs, 118_499); + assert.strictEqual(resolved?.properties.resolution, "answered"); + assert.strictEqual(resolved?.properties.answeredByUserId, SENDER); + + const finished = out.find((emission) => emission.event === "run.finished"); + assert.deepStrictEqual( + { + status: finished?.properties.status, + queueWaitMs: finished?.properties.queueWaitMs, + runDurationMs: finished?.properties.runDurationMs, + attempts: finished?.properties.attempts, + effort: finished?.properties.effort, + runTrigger: finished?.properties.runTrigger, + senderUserId: finished?.properties.senderUserId, + }, + { + status: "completed", + queueWaitMs: 256, + runDurationMs: 230_213, + attempts: 1, + effort: "high", + runTrigger: "human", + senderUserId: SENDER, + }, + ); + }); + + it("waits on workers after the run ends while a subagent still runs", () => { + const { out } = feed([ + created, + event("run.created", at("09:06:19.278"), startedRun("09:06:19.278")), + event("subagent.updated", at("09:08:00.000"), subagentPayload("running")), + event( + "run.updated", + at("09:10:00.000"), + runPayload("completed", { + startedAt: at("09:06:19.278"), + completedAt: at("09:10:00.000"), + }), + ), + event("subagent.updated", at("09:20:00.000"), subagentPayload("completed")), + event("thread.visited", at("09:25:00.000"), threadPayload()), + ]); + + const summary = spanSummary(out); + assert.deepStrictEqual(summary.slice(1), [ + ["agent_running", undefined, 220_722, "waiting_workers"], + ["waiting_workers", undefined, 600_000, "idle"], + ]); + const workerSpan = spans(out)[2]; + assert.strictEqual(workerSpan?.properties.activeWorkers, 0); + assert.strictEqual(workerSpan?.properties.maxActiveWorkers, 1); + + const finished = out.find((emission) => emission.event === "subagent.finished"); + assert.strictEqual(finished?.properties.durationMs, 720_000); + assert.strictEqual(finished?.properties.childThreadPresent, true); + assert.strictEqual(finished?.properties.origin, "provider_native"); + }); + + it("measures queued, running and idle time in order", () => { + const { out } = feed([ + created, + event( + "run.created", + at("09:01:00.000"), + runPayload("queued", { requestedAt: at("09:01:00.000") }), + ), + event( + "run.updated", + at("09:03:00.000"), + runPayload("running", { requestedAt: at("09:01:00.000"), startedAt: at("09:03:00.000") }), + ), + event( + "run.updated", + at("09:08:00.000"), + runPayload("failed", { + requestedAt: at("09:01:00.000"), + startedAt: at("09:03:00.000"), + completedAt: at("09:08:00.000"), + }), + ), + event("thread.visited", at("09:10:00.000"), threadPayload()), + ]); + assert.deepStrictEqual( + spanSummary(out).map(([state, , durationMs]) => [state, durationMs]), + [ + ["idle", 60_000], + ["queued", 120_000], + ["agent_running", 300_000], + ], + ); + const finished = out.find((emission) => emission.event === "run.finished"); + assert.strictEqual(finished?.properties.queueWaitMs, 120_000); + assert.strictEqual(finished?.properties.failureClass, "unknown"); + }); + + it("tracks archive and unarchive as a terminal state with lifecycle events", () => { + const { out } = feed([ + created, + event("thread.archived", at("10:00:00.000"), archivedThread("10:00:00.000")), + event("thread.unarchived", at("12:00:00.000"), threadPayload({ archivedAt: null })), + event("thread.visited", at("12:30:00.000"), threadPayload()), + ]); + assert.deepStrictEqual( + spanSummary(out).map(([state, , durationMs, endReason]) => [state, durationMs, endReason]), + [ + ["idle", 3_600_000, "archived"], + ["archived", 7_200_000, "idle"], + ], + ); + const archived = out.find((emission) => emission.event === "session.archived"); + assert.strictEqual(archived?.properties.ageMs, 3_600_000); + assert.ok(out.some((emission) => emission.event === "session.unarchived")); + }); + + it("waits on a proposed plan until the next run decides it", () => { + const { out } = feed([ + created, + event("run.created", at("09:01:00.000"), startedRun("09:01:00.000")), + event("plan.updated", at("09:04:00.000"), { + id: "plan-1", + threadId: THREAD, + runId: RUN, + nodeId: "node:plan", + status: "active", + kind: "proposed_plan", + markdown: "SECRET_PLAN", + }), + event( + "run.updated", + at("09:05:00.000"), + runPayload("completed", { startedAt: at("09:01:00.000"), completedAt: at("09:05:00.000") }), + ), + event( + "run.created", + at("09:15:00.000"), + runPayload("running", { + id: "run-2", + ordinal: 74, + startedAt: at("09:15:00.000"), + sourcePlanRef: { threadId: THREAD, planId: "plan-1" }, + }), + ), + ]); + const planSpan = spans(out).find((span) => span.properties.waitKind === "plan"); + assert.strictEqual(planSpan?.properties.durationMs, 600_000); + const decided = out.find((emission) => emission.event === "plan.decided"); + assert.strictEqual(decided?.properties.decision, "implemented"); + assert.strictEqual(decided?.properties.waitMs, 600_000); + }); + + it("keeps both spans when one run ends and the next starts in the same millisecond", () => { + const runB = (status: string, fields: Record = {}) => + runPayload(status, { id: "run-B", ordinal: 74, requestedAt: at("09:03:00.000"), ...fields }); + const { out } = feed([ + created, + event("run.created", at("09:01:00.000"), startedRun("09:01:00.000")), + event("run.created", at("09:03:00.000"), runB("queued")), + event( + "run.updated", + at("09:05:00.000"), + runPayload("completed", { startedAt: at("09:01:00.000"), completedAt: at("09:05:00.000") }), + ), + event("run.updated", at("09:05:00.000"), runB("running", { startedAt: at("09:05:00.000") })), + event( + "run.updated", + at("09:07:00.000"), + runB("completed", { startedAt: at("09:05:00.000"), completedAt: at("09:07:00.000") }), + ), + ]); + const emitted = spans(out); + assert.deepStrictEqual( + emitted.map((span) => [ + span.properties.state, + span.properties.runId, + span.properties.durationMs, + ]), + [ + ["idle", undefined, 60_000], + ["agent_running", RUN, 240_000], + ["agent_running", "run-B", 120_000], + ], + ); + assert.ok(emitted.every((span) => Number(span.properties.durationMs) > 0)); + assert.strictEqual(new Set(emitted.map((span) => span.insertId)).size, emitted.length); + }); + + it("ignores later updates of a run it already reported", () => { + const doneA = runPayload("completed", { + startedAt: at("09:01:00.000"), + completedAt: at("09:05:00.000"), + }); + const { out } = feed([ + created, + event("run.created", at("09:01:00.000"), startedRun("09:01:00.000")), + event("run.updated", at("09:05:00.000"), doneA), + event( + "run.created", + at("09:10:00.000"), + runPayload("completed", { + id: "run-B", + ordinal: 74, + startedAt: at("09:10:00.000"), + completedAt: at("09:10:00.000"), + }), + ), + // A delivery acknowledgement re-writes run A long after it finished. + event("run.updated", at("09:30:00.000"), doneA), + event("thread.visited", at("09:40:00.000"), threadPayload()), + ]); + const finishedA = out.filter( + (emission) => emission.event === "run.finished" && emission.properties.runId === RUN, + ); + assert.strictEqual(finishedA.length, 1); + const runningSpans = spans(out).filter((span) => span.properties.state === "agent_running"); + assert.deepStrictEqual( + runningSpans.map((span) => span.properties.runId), + [RUN], + ); + }); + + it("splits a span at UTC midnight", () => { + const tracker = makeSessionActivityTracker({ backfill: false }); + tracker.apply(event("thread.created", "2026-10-09T22:00:00.000Z", threadPayload())); + const out = tracker.apply( + event("thread.archived", "2026-10-10T03:00:00.000Z", threadPayload()), + ); + assert.deepStrictEqual( + spans(out).map((span) => [ + span.properties.durationMs, + span.properties.segmentIndex, + span.properties.endReason, + ]), + [ + [7_200_000, 0, "idle"], + [10_800_000, 1, "archived"], + ], + ); + assert.notStrictEqual(spans(out)[0]?.insertId, spans(out)[1]?.insertId); + }); + + it("ignores provider-native subagent transcript threads", () => { + const tracker = makeSessionActivityTracker({ backfill: false }); + const child = "thread:provider:claudeAgent:native-thread:child"; + const out = tracker.apply({ + type: "thread.created", + threadId: child, + occurredAt: at("09:00:00.000"), + payload: threadPayload({ + id: child, + createdBy: "agent", + creationSource: "provider", + lineage: { parentThreadId: THREAD, relationshipToParent: "subagent", rootThreadId: THREAD }, + }), + }); + assert.deepStrictEqual(out, []); + assert.deepStrictEqual(tracker.snapshot(epoch(at("10:00:00.000"))).length, 1); + }); + + it("replays an imported, already finished run from its recorded start", () => { + const { out } = feed([ + created, + event( + "run.created", + at("09:05:00.000"), + runPayload("completed", { + requestedAt: at("09:02:00.000"), + startedAt: at("09:02:00.000"), + completedAt: at("09:05:00.000"), + }), + ), + event("thread.visited", at("09:06:00.000"), threadPayload()), + ]); + assert.deepStrictEqual( + spanSummary(out).map(([state, , durationMs]) => [state, durationMs]), + [ + ["idle", 120_000], + ["agent_running", 180_000], + ], + ); + }); + + it("marks seeded spans as resumed and closes them on shutdown", () => { + const tracker = makeSessionActivityTracker({ backfill: false }); + const bootMs = epoch(at("09:00:00.000")); + tracker.seed( + { + ...threadPayload(), + activeRunId: RUN, + activityRunStatus: "running", + latestRunId: RUN, + status: "running", + pendingRuntimeRequest: null, + activeSubagentCount: 2, + pendingBackgroundTasks: [], + }, + bootMs, + ); + const snapshot = tracker.snapshot(bootMs + 1_000); + const total = snapshot.find((emission) => emission.properties.scope === "total"); + assert.strictEqual(total?.properties.runningRuns, 1); + assert.strictEqual(total?.properties.activeSubagents, 2); + const perUser = snapshot.find((emission) => emission.properties.scope === "user"); + assert.strictEqual(perUser?.userId, OWNER); + assert.strictEqual(perUser?.properties.count, 1); + + const closed = tracker.closeActive(bootMs + 60_000, "server_stopped"); + assert.strictEqual(closed.length, 1); + assert.strictEqual(closed[0]?.properties.state, "agent_running"); + assert.strictEqual(closed[0]?.properties.resumedAfterRestart, true); + assert.strictEqual(closed[0]?.properties.endReason, "server_stopped"); + assert.strictEqual(closed[0]?.properties.durationMs, 60_000); + }); + + it("never emits text, paths or unlisted keys", () => { + const { tracker, out } = feed( + [ + created, + event("message.updated", at("09:06:19.000"), userMessage()), + event("run.created", at("09:06:19.278"), startedRun("09:06:19.278")), + event("runtime-request.updated", at("09:07:25.320"), questionPayload("pending", null), { + runId: RUN, + }), + event("subagent.updated", at("09:08:00.000"), subagentPayload("running")), + event( + "runtime-request.updated", + at("09:09:23.778"), + questionPayload("resolved", at("09:09:23.778")), + { runId: RUN }, + ), + event("subagent.updated", at("09:20:00.000"), subagentPayload("completed")), + event("turn-item.updated", at("09:09:30.000"), { + id: "turn-item:error", + type: "error", + status: "completed", + runId: RUN, + title: "SECRET_ERROR_TITLE", + failure: { + class: "provider_error", + message: "SECRET_ERROR", + code: null, + retryable: null, + }, + }), + event( + "run.updated", + at("09:21:00.000"), + runPayload("failed", { startedAt: at("09:06:19.278"), completedAt: at("09:21:00.000") }), + ), + event("thread.archived", at("10:00:00.000"), archivedThread("10:00:00.000")), + ], + true, + ); + const all = [ + ...out, + ...tracker.snapshot(epoch(at("11:00:00.000"))), + ...tracker.closeActive(epoch(at("12:00:00.000")), "server_stopped"), + ]; + const allowed = new Set(SESSION_ACTIVITY_PROPERTY_KEYS); + for (const emission of all) { + for (const [key, value] of Object.entries(emission.properties)) { + assert.ok(allowed.has(key), `unlisted key ${key} on ${emission.event}`); + assert.ok( + !(typeof value === "string" && (value.includes("SECRET") || value.includes("/"))), + `${emission.event}.${key} leaks text`, + ); + } + assert.strictEqual(emission.properties.backfill, true); + } + const finished = all.find((emission) => emission.event === "run.finished"); + assert.strictEqual(finished?.properties.failureClass, "provider_error"); + }); +}); diff --git a/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts new file mode 100644 index 0000000000000..5cce5b2c955ec --- /dev/null +++ b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts @@ -0,0 +1,1367 @@ +/** + * T3-CUSTOM(expbkt3): session time-in-state metrics (TEC-1631). + * + * A pure tracker over the orchestration V2 event log. It keeps one mutually + * exclusive STATE per session (thread) and emits a `session.state.span` each + * time that state changes, plus transition events (`session.created`, + * `run.finished`, `human_request.resolved`, `subagent.finished`, + * `plan.decided`, archive and snooze events). The live reactor + * (`SessionActivityReactor.ts`) and the history backfill + * (`apps/server/scripts/analytics-backfill.ts`) feed it the same events, so + * live and historical numbers mean the same thing. Time comes only from event + * `occurredAt`; the tracker never reads a clock or a database. + * + * State precedence, highest first: + * 1. `archived` the thread is archived (terminal until unarchived). + * 2. `waiting_human` a runtime request waits on a person: an approval + * (`waitKind: "approval"`) or a question / MCP + * elicitation (`"question"`). This wins over a running + * run: a question asked mid-run is human wait time. + * 3. `agent_running` a run is preparing, starting or running. + * 4. `waiting_human` (`waitKind: "plan"`) a proposed plan from a settled + * run waits for the next message. Decided by the next + * run, by archiving, or by the plan's own completion. + * 5. `waiting_workers` no run is running, but background work the agent + * waits on is still active: running subagents (provider + * native and delegated `delegate_task` children, from + * `subagent.updated`), active non-persistent + * `dynamic_tool` items, and provider roster tasks that + * hold completion (`backgroundWorkHoldsCompletion`). + * Plain background shell commands (dev servers) and + * pull request watches do not count: they can run for + * days while the agent is done. + * 6. `queued` a run is queued and nothing above applies. + * 7. `idle` nothing is pending; the session waits for the next + * message. `snoozed: true` marks idle time while snoozed. + * + * A run in `waiting` status counts as settled: it is the short window between + * a successful turn and its checkpoint. Provider-native subagent threads + * (Claude Agent tool transcripts) are not sessions; their time is the parent's + * `agent_running` or `waiting_workers` time. + * + * Spans are split at UTC midnight, so per-day sums are exact. A state that + * lasts no time (several events share one millisecond) emits no span. At + * shutdown, spans of active states close with `endReason: "server_stopped"`; + * idle and archived spans are dropped. The next server seeds every open + * session and opens new spans with `resumedAfterRestart: true`. + * + * Properties hold only ids, enums, counts, durations and timestamps. Never add + * prompt, message, title, question or answer text, paths, branches or + * repository names. + */ +import * as DateTime from "effect/DateTime"; +import * as Option from "effect/Option"; + +export type SessionState = + | "waiting_human" + | "agent_running" + | "waiting_workers" + | "queued" + | "idle" + | "archived"; + +export type SessionWaitKind = "approval" | "question" | "plan"; + +export type SessionSource = + | "ui" + | "mcp" + | "linear_runner" + | "schedule" + | "delegate" + | "fork" + | "mattermost" + | "unknown"; + +/** What started a run, from its user message. */ +export type RunTrigger = + | "human" + | "agent" + | "wake" + | "delegated_result" + | "schedule" + | "async_answer" + | "restart" + | "unknown"; + +export type SessionActivityPropertyValue = string | number | boolean; + +export interface SessionActivityEmission { + readonly event: string; + readonly properties: Readonly>; + /** Environment user the event is attributed to: the owner, else the sender. */ + readonly userId?: string; + /** ISO time the event happened. */ + readonly timestamp: string; + /** Stable identity of this event, so a re-run does not count it twice. */ + readonly insertId: string; +} + +type TimeInput = string | number | DateTime.DateTime; + +/** + * The fields the tracker reads from an orchestration V2 domain event. The live + * `OrchestrationV2DomainEvent` and a row decoded from `orchestration_events` + * both fit. + */ +export interface SessionActivityEvent { + readonly type: string; + readonly threadId: string; + readonly occurredAt: TimeInput; + readonly runId?: string | undefined; + readonly actorUserId?: string | undefined; + readonly driver?: string | undefined; + readonly payload: unknown; +} + +export interface SessionActivityTrackerOptions { + /** Set on every event; true for the history backfill. */ + readonly backfill: boolean; +} + +/** Every property key the tracker can emit, for the analytics allowlist. */ +export const SESSION_ACTIVITY_PROPERTY_KEYS = [ + "threadId", + "projectId", + "parentThreadId", + "isChildSession", + "state", + "waitKind", + "durationMs", + "startedAt", + "endedAt", + "endReason", + "spanStartedAt", + "segmentIndex", + "runId", + "runOrdinal", + "runTrigger", + "provider", + "providerInstanceId", + "model", + "effort", + "interactionMode", + "runtimeMode", + "activeWorkers", + "maxActiveWorkers", + "backgroundTasks", + "ownerUserId", + "senderUserId", + "sessionSource", + "creationSource", + "createdBy", + "resumedAfterRestart", + "backfill", + "snoozed", + "ageMs", + "status", + "queueWaitMs", + "runDurationMs", + "workDurationMs", + "attempts", + "failureClass", + "requestKind", + "waitMs", + "resolution", + "answeredByUserId", + "subagentId", + "driver", + "origin", + "childThreadPresent", + "decision", + "scope", + "count", + "runningRuns", + "queuedRuns", + "waitingHuman", + "waitingWorkers", + "activeSubagents", +] as const; + +// --------------------------------------------------------------------------- +// Payload readers. Payloads arrive decoded (live) or as parsed JSON (backfill), +// so every read is defensive and typed `unknown` until checked. + +type Rec = Readonly>; + +function asRec(value: unknown): Rec | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Rec) + : undefined; +} + +function str(record: Rec | undefined, key: string): string | undefined { + const value = record?.[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +/** Epoch milliseconds of an ISO string, epoch number or Effect DateTime. */ +export function toEpochMs(value: unknown): number | undefined { + if (typeof value === "number") return Number.isFinite(value) ? value : undefined; + if (typeof value === "string" || DateTime.isDateTime(value)) { + return Option.match(DateTime.make(value), { + onNone: () => undefined, + onSome: DateTime.toEpochMillis, + }); + } + return undefined; +} + +function isoOf(ms: number): string { + return DateTime.formatIso(DateTime.makeUnsafe(ms)); +} + +const DAY_MS = 86_400_000; + +const ACTIVE_RUN_STATUSES = new Set(["preparing", "starting", "running"]); +const FINAL_RUN_STATUSES = new Set([ + "completed", + "failed", + "interrupted", + "cancelled", + "rolled_back", +]); +const ACTIVE_WORK_STATUSES = new Set(["pending", "running", "waiting"]); +const FINAL_SUBAGENT_STATUSES = new Set(["completed", "failed", "cancelled", "interrupted"]); +/** Request kinds a client tool answers, not a person. */ +const NON_HUMAN_REQUEST_KINDS = new Set(["dynamic_tool_call"]); + +function modelInfo(selection: unknown): { + readonly instanceId?: string; + readonly model?: string; + readonly effort?: string; +} { + const record = asRec(selection); + if (record === undefined) return {}; + let effort: string | undefined; + const options = record.options; + if (Array.isArray(options)) { + for (const option of options) { + const entry = asRec(option); + if (entry?.id === "effort" && typeof entry.value === "string") effort = entry.value; + } + } else { + const value = asRec(options)?.effort; + if (typeof value === "string") effort = value; + } + const instanceId = str(record, "instanceId"); + const model = str(record, "model"); + return { + ...(instanceId === undefined ? {} : { instanceId }), + ...(model === undefined ? {} : { model }), + ...(effort === undefined ? {} : { effort }), + }; +} + +function isProviderNativeSubagentThread(payload: Rec | undefined): boolean { + return ( + asRec(payload?.lineage)?.relationshipToParent === "subagent" && + payload?.creationSource === "provider" + ); +} + +function hasLinearLink(payload: Rec): boolean { + if (str(payload, "linearIssueUrl") !== undefined) return true; + return Array.isArray(payload.linearLinks) && payload.linearLinks.length > 0; +} + +/** + * Best-effort origin of a session, from its first thread payload: + * fork lineage, delegate_task child, Mattermost-bound, Linear-tagged and + * created by a server or MCP client (the Linear runner), web or mobile UI, + * MCP, or a server-created session with a parent (another agent's + * t3_create_session). A scheduled first message upgrades it to `schedule`. + */ +function sessionSourceOf(payload: Rec): SessionSource { + const relationship = asRec(payload.lineage)?.relationshipToParent; + if (relationship === "fork") return "fork"; + if (relationship === "subagent") return "delegate"; + if (payload.forkedFrom !== null && payload.forkedFrom !== undefined) return "fork"; + if (str(payload, "mattermostThreadUrl") !== undefined) return "mattermost"; + const creationSource = str(payload, "creationSource"); + if (hasLinearLink(payload) && (creationSource === "server" || creationSource === "mcp")) { + return "linear_runner"; + } + if (creationSource === "web" || creationSource === "mobile") return "ui"; + if (creationSource === "mcp") return "mcp"; + if (creationSource === "server" && str(payload, "parentThreadId") !== undefined) return "mcp"; + return "unknown"; +} + +function runTriggerOf(message: Rec): RunTrigger { + if (message.delegatedCompletion !== undefined && message.delegatedCompletion !== null) { + return "delegated_result"; + } + if (message.notification !== undefined && message.notification !== null) return "wake"; + if (str(message, "scheduledTaskId") !== undefined) return "schedule"; + if (str(message, "id")?.startsWith("async-answer:") === true) return "async_answer"; + if (str(message, "senderThreadId") !== undefined) return "agent"; + const createdBy = str(message, "createdBy"); + if (createdBy === "agent") return "agent"; + if (createdBy === "user") return "human"; + return "unknown"; +} + +function waitKindOfRequest(kind: string): SessionWaitKind { + return kind === "user_input" || kind === "mcp-elicitation" ? "question" : "approval"; +} + +function resolutionOf(status: string, kind: string, decision: string | undefined): string { + if (status === "expired") return "expired"; + if (status === "cancelled") return "cancelled"; + if (kind === "user_input" || kind === "mcp-elicitation") return "answered"; + switch (decision) { + case "accept": + case "acceptForSession": + case "acceptAlways": + return "approved"; + case "decline": + return "declined"; + case "cancel": + return "cancelled"; + default: + return "resolved"; + } +} + +/** Provider roster tasks the agent waits on; plain commands do not hold it. */ +function holdingRosterTaskCount(tasks: unknown): number { + if (!Array.isArray(tasks)) return 0; + let count = 0; + for (const task of tasks) { + const entry = asRec(task); + if (entry === undefined) continue; + if (entry.kind !== "command" || entry.wakesAgent === true) count += 1; + } + return count; +} + +// --------------------------------------------------------------------------- +// Tracker state. + +interface RunInfo { + readonly id: string; + ordinal?: number; + status: string; + requestedAtMs?: number; + startedAtMs?: number; + completedAtMs?: number; + workStartedAtMs?: number; + userMessageId?: string; + providerInstanceId?: string; + model?: string; + effort?: string; + attempts: number; + failureClass?: string; + senderUserId?: string; + trigger?: RunTrigger; + restartContinuation: boolean; + finished: boolean; + /** An imported run finished later than the event that created it: finish it then. */ + deferredFinal?: { readonly status: string; readonly atMs: number }; +} + +interface RequestInfo { + readonly kind: string; + readonly waitKind: SessionWaitKind; + readonly createdAtMs: number; + readonly runId?: string; + readonly live: boolean; +} + +interface PlanInfo { + readonly runId?: string; + readonly atMs: number; + waitingSinceMs?: number; +} + +interface SubagentInfo { + readonly startedAtMs: number; +} + +interface MessageInfo { + readonly senderUserId?: string; + readonly trigger: RunTrigger; +} + +interface OpenSpan { + readonly key: string; + readonly state: SessionState; + readonly waitKind?: SessionWaitKind; + readonly snoozed: boolean; + readonly run?: RunInfo; + readonly startMs: number; + readonly resumed: boolean; + maxWorkers: number; +} + +interface ThreadTrack { + readonly threadId: string; + known: boolean; + projectId?: string; + parentThreadId?: string; + ownerUserId?: string; + createdAtMs?: number; + creationSource?: string; + createdBy?: string; + sessionSource: SessionSource; + providerInstanceId?: string; + provider?: string; + model?: string; + effort?: string; + interactionMode?: string; + runtimeMode?: string; + activeProviderThreadId?: string; + archived: boolean; + snoozed: boolean; + readonly runs: Map; + lastRun?: RunInfo; + readonly requests: Map; + readonly plans: Map; + readonly decidedPlans: Set; + readonly subagents: Map; + readonly finishedSubagents: Set; + /** Runs already reported; later metadata-only updates of them are ignored. */ + readonly finishedRuns: Set; + readonly backgroundTools: Set; + readonly rosterTasks: Map; + /** Workers a restart seed counted without ids; cleared by the next worker event. */ + seededWorkers: number; + readonly messages: Map; + lastSenderUserId?: string; + span?: OpenSpan; + resumeNextSpan: boolean; +} + +const MAX_TRACKED_MESSAGES = 64; +const MAX_FINISHED_IDS = 512; + +function newThread(threadId: string): ThreadTrack { + return { + threadId, + known: false, + sessionSource: "unknown", + archived: false, + snoozed: false, + runs: new Map(), + requests: new Map(), + plans: new Map(), + decidedPlans: new Set(), + subagents: new Map(), + finishedSubagents: new Set(), + finishedRuns: new Set(), + backgroundTools: new Set(), + rosterTasks: new Map(), + seededWorkers: 0, + messages: new Map(), + resumeNextSpan: false, + }; +} + +function rememberBounded(set: Set, id: string): void { + set.add(id); + if (set.size > MAX_FINISHED_IDS) { + const oldest = set.values().next(); + if (!oldest.done) set.delete(oldest.value); + } +} + +/** Event types that may introduce a session the tracker has not seen yet. */ +function introducesSession(type: string): boolean { + return ( + type.startsWith("thread.") || + type === "run.created" || + type === "run.updated" || + type === "runtime-request.updated" || + type === "message.updated" || + type === "plan.updated" + ); +} + +/** The current-state picture used by the reactor's periodic snapshot. */ +interface StateCounts { + readonly byOwnerState: Map; + runningRuns: number; + queuedRuns: number; + waitingHuman: number; + waitingWorkers: number; + activeSubagents: number; +} + +export interface SessionActivityTracker { + /** Feed one event, in order. Returns the analytics events it produced. */ + readonly apply: (event: SessionActivityEvent) => ReadonlyArray; + /** + * Seed a session from a projection shell after a restart. Emits nothing; the + * span it opens at `nowMs` carries `resumedAfterRestart: true`. + */ + readonly seed: (shell: unknown, nowMs: number) => void; + /** + * Close the open spans of active states (everything but `idle` and + * `archived`), for example with `server_stopped` on shutdown. Idle and + * archived time is cheap to lose and is resumed by the next boot's seed, so + * shutdown records only the few spans that matter within its time budget. + */ + readonly closeActive: ( + nowMs: number, + endReason: string, + ) => ReadonlyArray; + /** Point-in-time counts: one event per (owner, state), plus one totals event. */ + readonly snapshot: (nowMs: number) => ReadonlyArray; +} + +export function makeSessionActivityTracker( + options: SessionActivityTrackerOptions, +): SessionActivityTracker { + const threads = new Map(); + /** Threads that are not sessions (provider-native subagent transcripts, deleted). */ + const ignored = new Set(); + + const emission = ( + t: ThreadTrack | undefined, + event: string, + atMs: number, + insertId: string, + properties: Record, + userId?: string, + ): SessionActivityEmission => { + const clean: Record = {}; + for (const [key, value] of Object.entries(properties)) { + if (value !== undefined) clean[key] = value; + } + clean.backfill = options.backfill; + const attributed = userId ?? t?.ownerUserId ?? t?.lastSenderUserId; + return { + event, + properties: clean, + ...(attributed === undefined ? {} : { userId: attributed }), + timestamp: isoOf(atMs), + insertId, + }; + }; + + const updateThreadFields = (t: ThreadTrack, payload: Rec, atMs: number) => { + const firstPayload = !t.known; + t.known = true; + const projectId = str(payload, "projectId"); + if (projectId !== undefined) t.projectId = projectId; + const parentThreadId = + str(payload, "parentThreadId") ?? str(asRec(payload.lineage), "parentThreadId"); + if (parentThreadId !== undefined) t.parentThreadId = parentThreadId; + const ownerUserId = str(payload, "ownerUserId"); + if (ownerUserId !== undefined) t.ownerUserId = ownerUserId; + t.createdAtMs ??= toEpochMs(payload.createdAt) ?? atMs; + const creationSource = str(payload, "creationSource"); + if (creationSource !== undefined) t.creationSource = creationSource; + const createdBy = str(payload, "createdBy"); + if (createdBy !== undefined) t.createdBy = createdBy; + if (firstPayload) t.sessionSource = sessionSourceOf(payload); + const providerInstanceId = str(payload, "providerInstanceId"); + if (providerInstanceId !== undefined) t.providerInstanceId = providerInstanceId; + const selection = modelInfo(payload.modelSelection); + if (selection.model !== undefined) t.model = selection.model; + if (payload.modelSelection !== undefined) { + if (selection.effort === undefined) delete t.effort; + else t.effort = selection.effort; + } + const interactionMode = str(payload, "interactionMode"); + if (interactionMode !== undefined) t.interactionMode = interactionMode; + const runtimeMode = str(payload, "runtimeMode"); + if (runtimeMode !== undefined) t.runtimeMode = runtimeMode; + if ("activeProviderThreadId" in payload) { + const active = str(payload, "activeProviderThreadId"); + if (active === undefined) delete t.activeProviderThreadId; + else t.activeProviderThreadId = active; + } + if ("archivedAt" in payload) t.archived = toEpochMs(payload.archivedAt) !== undefined; + if ("snoozedUntil" in payload) t.snoozed = toEpochMs(payload.snoozedUntil) !== undefined; + }; + + const baseProps = (t: ThreadTrack) => ({ + threadId: t.threadId, + projectId: t.projectId, + parentThreadId: t.parentThreadId, + isChildSession: t.parentThreadId !== undefined, + ownerUserId: t.ownerUserId, + sessionSource: t.sessionSource, + }); + + const activeWorkers = (t: ThreadTrack) => t.subagents.size + t.seededWorkers; + + const rosterCount = (t: ThreadTrack) => { + if (t.activeProviderThreadId !== undefined) { + return t.rosterTasks.get(t.activeProviderThreadId) ?? 0; + } + let total = 0; + for (const count of t.rosterTasks.values()) total += count; + return total; + }; + + const runById = (t: ThreadTrack, runId: string | undefined): RunInfo | undefined => { + if (runId === undefined) return undefined; + return t.runs.get(runId) ?? (t.lastRun?.id === runId ? t.lastRun : undefined); + }; + + const derive = ( + t: ThreadTrack, + ): { state: SessionState; waitKind?: SessionWaitKind; run?: RunInfo } => { + if (t.archived) return { state: "archived" }; + if (t.requests.size > 0) { + let latest: RequestInfo | undefined; + for (const request of t.requests.values()) { + if (latest === undefined || request.createdAtMs >= latest.createdAtMs) latest = request; + } + const run = runById(t, latest?.runId); + return { + state: "waiting_human", + waitKind: latest?.waitKind ?? "approval", + ...(run === undefined ? {} : { run }), + }; + } + let active: RunInfo | undefined; + let queued: RunInfo | undefined; + for (const run of t.runs.values()) { + if (ACTIVE_RUN_STATUSES.has(run.status)) { + if (active === undefined || (run.ordinal ?? 0) > (active.ordinal ?? 0)) active = run; + } else if (run.status === "queued") { + if (queued === undefined || (run.ordinal ?? 0) < (queued.ordinal ?? 0)) queued = run; + } + } + if (active !== undefined) return { state: "agent_running", run: active }; + if (t.plans.size > 0) { + let latestPlan: PlanInfo | undefined; + for (const plan of t.plans.values()) { + if (latestPlan === undefined || plan.atMs >= latestPlan.atMs) latestPlan = plan; + } + const run = runById(t, latestPlan?.runId); + return { state: "waiting_human", waitKind: "plan", ...(run === undefined ? {} : { run }) }; + } + if (activeWorkers(t) > 0 || t.backgroundTools.size > 0 || rosterCount(t) > 0) { + return { state: "waiting_workers", ...(t.lastRun === undefined ? {} : { run: t.lastRun }) }; + } + if (queued !== undefined) return { state: "queued", run: queued }; + return { state: "idle" }; + }; + + const spanEmissions = ( + t: ThreadTrack, + span: OpenSpan, + endMs: number, + endReason: string, + ): ReadonlyArray => { + // Several events can share one millisecond. A state that lasted no time is + // not a span, and emitting it would also reuse the next span's insert id. + if (endMs <= span.startMs) return []; + const end = endMs; + const run = span.run; + const common = { + ...baseProps(t), + state: span.state, + waitKind: span.waitKind, + endReason, + spanStartedAt: isoOf(span.startMs), + runId: run?.id, + runOrdinal: run?.ordinal, + runTrigger: run?.trigger, + provider: t.provider, + providerInstanceId: run?.providerInstanceId ?? t.providerInstanceId, + model: run?.model ?? t.model, + effort: run?.effort ?? t.effort, + interactionMode: t.interactionMode, + runtimeMode: t.runtimeMode, + activeWorkers: activeWorkers(t), + maxActiveWorkers: Math.max(span.maxWorkers, activeWorkers(t)), + backgroundTasks: t.backgroundTools.size + rosterCount(t), + senderUserId: run?.senderUserId ?? t.lastSenderUserId, + creationSource: t.creationSource, + resumedAfterRestart: span.resumed, + snoozed: span.state === "idle" ? span.snoozed : undefined, + }; + const out: Array = []; + let segmentStart = span.startMs; + let segmentIndex = 0; + do { + const nextMidnight = (Math.floor(segmentStart / DAY_MS) + 1) * DAY_MS; + const segmentEnd = Math.min(end, nextMidnight); + const last = segmentEnd >= end; + out.push( + emission( + t, + "session.state.span", + segmentEnd, + [ + "session.state.span", + t.threadId, + span.state, + span.waitKind ?? "", + run?.id ?? "", + segmentStart, + ].join("|"), + { + ...common, + durationMs: segmentEnd - segmentStart, + startedAt: isoOf(segmentStart), + endedAt: isoOf(segmentEnd), + segmentIndex, + // A segment cut at midnight continues in the same state. + endReason: last ? endReason : span.state, + }, + ), + ); + segmentStart = segmentEnd; + segmentIndex += 1; + } while (segmentStart < end); + return out; + }; + + /** Recompute the state; close the open span and open a new one when it changed. */ + const reconcile = (t: ThreadTrack, atMs: number, out: Array) => { + const next = derive(t); + if (next.state === "waiting_human" && next.waitKind === "plan") { + for (const plan of t.plans.values()) plan.waitingSinceMs ??= atMs; + } + const snoozed = next.state === "idle" && t.snoozed; + const key = `${next.state}|${next.waitKind ?? ""}|${snoozed}|${next.run?.id ?? ""}`; + const workers = activeWorkers(t); + if (t.span !== undefined && t.span.key === key) { + t.span.maxWorkers = Math.max(t.span.maxWorkers, workers); + return; + } + const previous = t.span; + if (previous !== undefined) out.push(...spanEmissions(t, previous, atMs, next.state)); + // A replaced zero-length span hands on its restart flag. + const carriesResume = previous !== undefined && previous.startMs >= atMs && previous.resumed; + t.span = { + key, + state: next.state, + ...(next.waitKind === undefined ? {} : { waitKind: next.waitKind }), + snoozed, + ...(next.run === undefined ? {} : { run: next.run }), + startMs: atMs, + resumed: t.resumeNextSpan || carriesResume, + maxWorkers: workers, + }; + t.resumeNextSpan = false; + }; + + const decidePlan = ( + t: ThreadTrack, + planId: string, + plan: PlanInfo, + atMs: number, + decision: string, + out: Array, + ) => { + t.plans.delete(planId); + rememberBounded(t.decidedPlans, planId); + out.push( + emission(t, "plan.decided", atMs, `plan.decided|${t.threadId}|${planId}`, { + ...baseProps(t), + runId: plan.runId, + decision, + // Zero when the plan was decided before its run settled. + waitMs: plan.waitingSinceMs === undefined ? 0 : Math.max(0, atMs - plan.waitingSinceMs), + }), + ); + }; + + const decidePlans = ( + t: ThreadTrack, + atMs: number, + decisionFor: (planId: string) => string, + out: Array, + ) => { + for (const [planId, plan] of [...t.plans]) { + decidePlan(t, planId, plan, atMs, decisionFor(planId), out); + } + }; + + const closeRequest = ( + t: ThreadTrack, + requestId: string, + request: RequestInfo, + atMs: number, + resolution: string, + resolvedAtMs: number, + answeredByUserId: string | undefined, + out: Array, + ) => { + t.requests.delete(requestId); + out.push( + emission( + t, + "human_request.resolved", + atMs, + `human_request.resolved|${t.threadId}|${requestId}`, + { + ...baseProps(t), + runId: request.runId, + requestKind: request.kind, + waitKind: request.waitKind, + waitMs: Math.max(0, resolvedAtMs - request.createdAtMs), + resolution, + answeredByUserId, + }, + ), + ); + }; + + const finishRun = ( + t: ThreadTrack, + run: RunInfo, + atMs: number, + out: Array, + ) => { + run.finished = true; + rememberBounded(t.finishedRuns, run.id); + t.runs.delete(run.id); + t.lastRun = run; + const completedAtMs = run.completedAtMs ?? atMs; + out.push( + emission(t, "run.finished", completedAtMs, `run.finished|${t.threadId}|${run.id}`, { + ...baseProps(t), + runId: run.id, + runOrdinal: run.ordinal, + runTrigger: run.trigger, + status: run.status, + queueWaitMs: + run.startedAtMs !== undefined && run.requestedAtMs !== undefined + ? Math.max(0, run.startedAtMs - run.requestedAtMs) + : undefined, + runDurationMs: + run.startedAtMs !== undefined ? Math.max(0, completedAtMs - run.startedAtMs) : undefined, + workDurationMs: + run.workStartedAtMs !== undefined + ? Math.max(0, completedAtMs - run.workStartedAtMs) + : undefined, + attempts: run.attempts, + provider: t.provider, + providerInstanceId: run.providerInstanceId ?? t.providerInstanceId, + model: run.model ?? t.model, + effort: run.effort ?? t.effort, + interactionMode: t.interactionMode, + runtimeMode: t.runtimeMode, + senderUserId: run.senderUserId, + failureClass: run.status === "failed" ? (run.failureClass ?? "unknown") : undefined, + }), + ); + // A live request cannot be answered once its run has ended. + for (const [requestId, request] of t.requests) { + if (request.live && request.runId === run.id) { + closeRequest(t, requestId, request, atMs, "abandoned", atMs, undefined, out); + } + } + }; + + const applyRun = ( + t: ThreadTrack, + event: SessionActivityEvent, + payload: Rec, + atMs: number, + out: Array, + ) => { + const id = str(payload, "id"); + if (id === undefined) return; + // A finished run keeps receiving updates (delivery acknowledgements and + // the like). Re-reading one would report it twice, and as an imported run. + if (t.finishedRuns.has(id)) return; + let run = t.runs.get(id); + const firstSeen = run === undefined; + if (run === undefined) { + run = { id, status: "queued", attempts: 0, restartContinuation: false, finished: false }; + t.runs.set(id, run); + } + const status = str(payload, "status"); + if (status !== undefined) run.status = status; + if (typeof payload.ordinal === "number") run.ordinal = payload.ordinal; + const requestedAtMs = toEpochMs(payload.requestedAt); + if (requestedAtMs !== undefined) run.requestedAtMs = requestedAtMs; + const startedAtMs = toEpochMs(payload.startedAt); + if (startedAtMs !== undefined) run.startedAtMs = startedAtMs; + const completedAtMs = toEpochMs(payload.completedAt); + if (completedAtMs !== undefined) run.completedAtMs = completedAtMs; + const workStartedAtMs = toEpochMs(payload.workStartedAt); + if (workStartedAtMs !== undefined) run.workStartedAtMs = workStartedAtMs; + const providerInstanceId = str(payload, "providerInstanceId"); + if (providerInstanceId !== undefined) run.providerInstanceId = providerInstanceId; + const selection = modelInfo(payload.modelSelection); + if (selection.model !== undefined) run.model = selection.model; + if (selection.effort !== undefined) run.effort = selection.effort; + if (str(payload, "restartContinuationOfRunId") !== undefined) run.restartContinuation = true; + const userMessageId = str(payload, "userMessageId"); + if (userMessageId !== undefined && run.userMessageId === undefined) { + run.userMessageId = userMessageId; + const message = t.messages.get(userMessageId); + if (message !== undefined) { + t.messages.delete(userMessageId); + if (message.senderUserId !== undefined) run.senderUserId = message.senderUserId; + run.trigger = message.trigger; + } + } + if (run.restartContinuation) run.trigger = "restart"; + if (event.driver !== undefined) t.provider = event.driver; + + if (event.type === "run.created") { + t.seededWorkers = 0; + if (t.plans.size > 0) { + const sourcePlanId = str(asRec(payload.sourcePlanRef), "planId"); + decidePlans( + t, + atMs, + (planId) => (planId === sourcePlanId ? "implemented" : "follow_up"), + out, + ); + } + // A newer run means an older run's live requests are orphaned. + for (const [requestId, request] of t.requests) { + if (request.live && request.runId !== undefined && request.runId !== id) { + const owner = runById(t, request.runId); + if (owner === undefined || owner.finished) { + closeRequest(t, requestId, request, atMs, "abandoned", atMs, undefined, out); + } + } + } + } + // Imported history writes a run once, already finished, at its completion + // time. Replay its running time from the recorded start instead. + if (firstSeen && FINAL_RUN_STATUSES.has(run.status) && run.startedAtMs !== undefined) { + const finalStatus = run.status; + run.status = "running"; + const runningFromMs = Math.max(run.startedAtMs, t.span?.startMs ?? run.startedAtMs); + if (runningFromMs < atMs) reconcile(t, runningFromMs, out); + if (run.completedAtMs !== undefined && run.completedAtMs > atMs) { + run.deferredFinal = { status: finalStatus, atMs: run.completedAtMs }; + return; + } + run.status = finalStatus; + } + if (FINAL_RUN_STATUSES.has(run.status)) { + delete run.deferredFinal; + finishRun(t, run, atMs, out); + } + }; + + /** Finishes imported runs whose recorded completion time has passed. */ + const settleDeferredRuns = ( + t: ThreadTrack, + atMs: number, + out: Array, + ) => { + const due = [...t.runs.values()] + .filter((run) => run.deferredFinal !== undefined && run.deferredFinal.atMs <= atMs) + .sort((a, b) => (a.deferredFinal?.atMs ?? 0) - (b.deferredFinal?.atMs ?? 0)); + for (const run of due) { + const final = run.deferredFinal; + if (final === undefined) continue; + delete run.deferredFinal; + run.status = final.status; + finishRun(t, run, final.atMs, out); + reconcile(t, final.atMs, out); + } + }; + + const applyRequest = ( + t: ThreadTrack, + event: SessionActivityEvent, + payload: Rec, + atMs: number, + out: Array, + ) => { + const id = str(payload, "id"); + const kind = str(payload, "kind"); + const status = str(payload, "status"); + if (id === undefined || kind === undefined || status === undefined) return; + if (NON_HUMAN_REQUEST_KINDS.has(kind)) return; + if (status === "pending") { + if (!t.requests.has(id)) { + t.requests.set(id, { + kind, + waitKind: waitKindOfRequest(kind), + createdAtMs: toEpochMs(payload.createdAt) ?? atMs, + ...(event.runId === undefined ? {} : { runId: event.runId }), + live: asRec(payload.responseCapability)?.type === "live", + }); + } + return; + } + const request = t.requests.get(id); + if (request === undefined) return; + closeRequest( + t, + id, + request, + atMs, + resolutionOf(status, kind, str(payload, "decision")), + toEpochMs(payload.resolvedAt) ?? atMs, + status === "resolved" ? event.actorUserId : undefined, + out, + ); + }; + + const applySubagent = ( + t: ThreadTrack, + event: SessionActivityEvent, + payload: Rec, + atMs: number, + out: Array, + ) => { + const id = str(payload, "id"); + const status = str(payload, "status"); + if (id === undefined || status === undefined) return; + t.seededWorkers = 0; + if (ACTIVE_WORK_STATUSES.has(status)) { + if (!t.subagents.has(id) && !t.finishedSubagents.has(id)) { + t.subagents.set(id, { startedAtMs: toEpochMs(payload.startedAt) ?? atMs }); + } + return; + } + const info = t.subagents.get(id); + t.subagents.delete(id); + if (!FINAL_SUBAGENT_STATUSES.has(status) || t.finishedSubagents.has(id)) return; + rememberBounded(t.finishedSubagents, id); + const startedAtMs = toEpochMs(payload.startedAt) ?? info?.startedAtMs; + const completedAtMs = toEpochMs(payload.completedAt) ?? atMs; + out.push( + emission(t, "subagent.finished", completedAtMs, `subagent.finished|${t.threadId}|${id}`, { + ...baseProps(t), + runId: str(payload, "runId") ?? event.runId, + subagentId: id, + status, + origin: str(payload, "origin"), + driver: str(payload, "driver"), + durationMs: + startedAtMs === undefined ? undefined : Math.max(0, completedAtMs - startedAtMs), + childThreadPresent: str(payload, "childThreadId") !== undefined, + }), + ); + }; + + const applyTurnItem = (t: ThreadTrack, event: SessionActivityEvent, payload: Rec) => { + const type = payload.type; + if (type === "dynamic_tool") { + const id = str(payload, "id"); + if (id === undefined) return; + const persistent = asRec(payload.input)?.persistent === true; + const status = str(payload, "status"); + if (!persistent && status !== undefined && ACTIVE_WORK_STATUSES.has(status)) { + t.backgroundTools.add(id); + } else { + t.backgroundTools.delete(id); + } + return; + } + if (type === "error") { + const failureClass = str(asRec(payload.failure), "class"); + const run = runById(t, str(payload, "runId") ?? event.runId); + if (failureClass !== undefined && run !== undefined) run.failureClass = failureClass; + } + }; + + const applyPlan = ( + t: ThreadTrack, + payload: Rec, + atMs: number, + out: Array, + ) => { + if (payload.kind !== "proposed_plan") return; + const id = str(payload, "id"); + const status = str(payload, "status"); + if (id === undefined || status === undefined) return; + if (status === "draft" || status === "active") { + if (!t.decidedPlans.has(id) && !t.plans.has(id)) { + const runId = str(payload, "runId"); + t.plans.set(id, { atMs, ...(runId === undefined ? {} : { runId }) }); + } + return; + } + // completed or superseded + const plan = t.plans.get(id); + if (plan !== undefined) decidePlan(t, id, plan, atMs, status, out); + }; + + const applyMessage = (t: ThreadTrack, payload: Rec) => { + if (payload.role !== "user") return; + const id = str(payload, "id"); + if (id === undefined) return; + const senderUserId = str(payload, "sentByUserId"); + const trigger = runTriggerOf(payload); + if (trigger === "schedule" && t.sessionSource === "unknown") t.sessionSource = "schedule"; + if (senderUserId !== undefined && trigger === "human") t.lastSenderUserId = senderUserId; + // The run may have been created before its message reached the log. + for (const run of t.runs.values()) { + if (run.userMessageId === id && run.trigger === undefined) { + if (senderUserId !== undefined) run.senderUserId = senderUserId; + run.trigger = run.restartContinuation ? "restart" : trigger; + return; + } + } + t.messages.set(id, { trigger, ...(senderUserId === undefined ? {} : { senderUserId }) }); + if (t.messages.size > MAX_TRACKED_MESSAGES) { + const oldest = t.messages.keys().next(); + if (!oldest.done) t.messages.delete(oldest.value); + } + }; + + const applyThreadEvent = ( + t: ThreadTrack, + event: SessionActivityEvent, + payload: Rec | undefined, + atMs: number, + out: Array, + ) => { + const wasKnown = t.known; + const wasArchived = t.archived; + const wasSnoozed = t.snoozed; + if (payload !== undefined) updateThreadFields(t, payload, atMs); + if (event.driver !== undefined) t.provider = event.driver; + const ageMs = t.createdAtMs === undefined ? undefined : Math.max(0, atMs - t.createdAtMs); + const lifecycle = (name: string) => + out.push( + emission(t, name, atMs, `${name}|${t.threadId}|${atMs}`, { ...baseProps(t), ageMs }), + ); + switch (event.type) { + case "thread.created": + if (!wasKnown) { + out.push( + emission(t, "session.created", atMs, `session.created|${t.threadId}`, { + ...baseProps(t), + provider: t.provider, + providerInstanceId: t.providerInstanceId, + model: t.model, + effort: t.effort, + interactionMode: t.interactionMode, + runtimeMode: t.runtimeMode, + creationSource: t.creationSource, + createdBy: t.createdBy, + }), + ); + } + return; + case "thread.archived": + t.archived = true; + if (!wasArchived) { + lifecycle("session.archived"); + if (t.plans.size > 0) decidePlans(t, atMs, () => "archived", out); + } + return; + case "thread.unarchived": + t.archived = false; + if (wasArchived) lifecycle("session.unarchived"); + return; + case "thread.snoozed": + t.snoozed = true; + if (!wasSnoozed) lifecycle("session.snoozed"); + return; + case "thread.unsnoozed": + t.snoozed = false; + if (wasSnoozed) lifecycle("session.unsnoozed"); + return; + default: + return; + } + }; + + const apply: SessionActivityTracker["apply"] = (event) => { + if (ignored.has(event.threadId)) return []; + const atMs = toEpochMs(event.occurredAt); + if (atMs === undefined) return []; + const payload = asRec(event.payload); + let t = threads.get(event.threadId); + if (t === undefined) { + if (!introducesSession(event.type)) return []; + if (event.type.startsWith("thread.") && isProviderNativeSubagentThread(payload)) { + ignored.add(event.threadId); + return []; + } + t = newThread(event.threadId); + threads.set(event.threadId, t); + } + const out: Array = []; + settleDeferredRuns(t, atMs, out); + + if (event.type === "thread.deleted") { + if (t.span !== undefined) out.push(...spanEmissions(t, t.span, atMs, "deleted")); + threads.delete(event.threadId); + ignored.add(event.threadId); + return out; + } + + if (event.type.startsWith("thread.")) { + applyThreadEvent(t, event, payload, atMs, out); + } else if (payload !== undefined) { + switch (event.type) { + case "run.created": + case "run.updated": + applyRun(t, event, payload, atMs, out); + break; + case "run-attempt.created": { + const run = runById(t, str(payload, "runId")); + if (run !== undefined) run.attempts += 1; + if (event.driver !== undefined) t.provider = event.driver; + break; + } + case "runtime-request.updated": + applyRequest(t, event, payload, atMs, out); + break; + case "subagent.updated": + applySubagent(t, event, payload, atMs, out); + break; + case "turn-item.updated": + applyTurnItem(t, event, payload); + break; + case "provider-thread.updated": { + const id = str(payload, "id"); + if (id === undefined) break; + t.seededWorkers = 0; + const count = holdingRosterTaskCount(payload.pendingBackgroundTasks); + if (count > 0) t.rosterTasks.set(id, count); + else t.rosterTasks.delete(id); + break; + } + case "plan.updated": + applyPlan(t, payload, atMs, out); + break; + case "message.updated": + applyMessage(t, payload); + break; + default: + break; + } + } + reconcile(t, atMs, out); + return out; + }; + + const seed: SessionActivityTracker["seed"] = (shell, nowMs) => { + const payload = asRec(shell); + const threadId = str(payload, "id"); + if (payload === undefined || threadId === undefined) return; + if (isProviderNativeSubagentThread(payload) || toEpochMs(payload.deletedAt) !== undefined) { + ignored.add(threadId); + return; + } + const t = newThread(threadId); + threads.set(threadId, t); + updateThreadFields(t, payload, nowMs); + const activeRunId = str(payload, "activeRunId"); + const activityStatus = str(payload, "activityRunStatus"); + if ( + activeRunId !== undefined && + activityStatus !== undefined && + ACTIVE_RUN_STATUSES.has(activityStatus) + ) { + const startedAtMs = toEpochMs(payload.activityRunStartedAt); + t.runs.set(activeRunId, { + id: activeRunId, + status: activityStatus, + attempts: 0, + restartContinuation: false, + finished: false, + ...(startedAtMs === undefined ? {} : { startedAtMs }), + }); + } + const latestRunId = str(payload, "latestRunId"); + if (latestRunId !== undefined && payload.status === "queued" && !t.runs.has(latestRunId)) { + t.runs.set(latestRunId, { + id: latestRunId, + status: "queued", + attempts: 0, + restartContinuation: false, + finished: false, + }); + } + const pending = asRec(payload.pendingRuntimeRequest); + const requestId = str(pending, "id"); + const requestKind = str(pending, "kind"); + if ( + requestId !== undefined && + requestKind !== undefined && + !NON_HUMAN_REQUEST_KINDS.has(requestKind) + ) { + t.requests.set(requestId, { + kind: requestKind, + waitKind: waitKindOfRequest(requestKind), + createdAtMs: toEpochMs(pending?.createdAt) ?? nowMs, + // Live requests did not survive the restart; message-answered ones did. + live: false, + }); + } + const subagentCount = + typeof payload.activeSubagentCount === "number" ? payload.activeSubagentCount : 0; + t.seededWorkers = subagentCount + holdingRosterTaskCount(payload.pendingBackgroundTasks); + t.resumeNextSpan = true; + reconcile(t, nowMs, []); + }; + + const closeActive: SessionActivityTracker["closeActive"] = (nowMs, endReason) => { + const out: Array = []; + for (const t of threads.values()) { + if (t.span === undefined || t.span.state === "idle" || t.span.state === "archived") { + continue; + } + out.push(...spanEmissions(t, t.span, nowMs, endReason)); + delete t.span; + } + return out; + }; + + const snapshot: SessionActivityTracker["snapshot"] = (nowMs) => { + const counts: StateCounts = { + byOwnerState: new Map(), + runningRuns: 0, + queuedRuns: 0, + waitingHuman: 0, + waitingWorkers: 0, + activeSubagents: 0, + }; + for (const t of threads.values()) { + const span = t.span; + if (span === undefined || span.state === "archived") continue; + const key = `${t.ownerUserId ?? ""}|${span.state}`; + const entry = counts.byOwnerState.get(key); + if (entry === undefined) { + counts.byOwnerState.set(key, { + ...(t.ownerUserId === undefined ? {} : { ownerUserId: t.ownerUserId }), + state: span.state, + count: 1, + }); + } else { + entry.count += 1; + } + if (span.state === "agent_running") counts.runningRuns += 1; + if (span.state === "waiting_human") counts.waitingHuman += 1; + if (span.state === "waiting_workers") counts.waitingWorkers += 1; + for (const run of t.runs.values()) if (run.status === "queued") counts.queuedRuns += 1; + counts.activeSubagents += activeWorkers(t); + } + const out: Array = []; + for (const entry of counts.byOwnerState.values()) { + out.push( + emission( + undefined, + "session.state.snapshot", + nowMs, + `session.state.snapshot|${entry.ownerUserId ?? ""}|${entry.state}|${nowMs}`, + { scope: "user", ownerUserId: entry.ownerUserId, state: entry.state, count: entry.count }, + entry.ownerUserId, + ), + ); + } + out.push({ + event: "session.state.snapshot", + properties: { + scope: "total", + runningRuns: counts.runningRuns, + queuedRuns: counts.queuedRuns, + waitingHuman: counts.waitingHuman, + waitingWorkers: counts.waitingWorkers, + activeSubagents: counts.activeSubagents, + backfill: options.backfill, + }, + timestamp: isoOf(nowMs), + insertId: `session.state.snapshot|total|${nowMs}`, + }); + return out; + }; + + return { apply, seed, closeActive, snapshot }; +} diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index e793d4ddf5fb5..decc4ed912d76 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -262,6 +262,7 @@ import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts"; import * as HostResources from "./resourceTelemetry/HostResources.ts"; import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; +import { clientConnectionMode } from "./telemetry/serverEvents.ts"; // T3-CUSTOM(expbkt3) import * as UsageService from "./usage/UsageService.ts"; import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; import * as PullRequestService from "./pullRequest/PullRequestService.ts"; @@ -1262,10 +1263,15 @@ const layerWsRpc = ( // client starts is credited to its surface + app version. Best-effort: // attribution must never fail the user's command. const originProps = clientAnalyticsProps; + // T3-CUSTOM(expbkt3): client events carry the signed-in user (TEC-1631). + const analyticsUser = + currentSession.userId === null ? undefined : { userId: currentSession.userId }; const recordClientCommandAnalytics = (command: OrchestrationV2Command) => { switch (command.type) { case "message.dispatch": - return analytics.record("client.turn.requested", originProps).pipe(Effect.ignore); + return analytics + .record("client.turn.requested", originProps, analyticsUser) + .pipe(Effect.ignore); default: return Effect.void; } @@ -2492,13 +2498,14 @@ const layerWsRpc = ( ) .pipe( Effect.tap(() => + // T3-CUSTOM(expbkt3): client events carry the signed-in user (TEC-1631). analytics - .record("client.thread.started", originProps) + .record("client.thread.started", originProps, analyticsUser) .pipe( Effect.andThen( input.initialMessage === undefined ? Effect.void - : analytics.record("client.turn.requested", originProps), + : analytics.record("client.turn.requested", originProps, analyticsUser), ), Effect.ignore, ), @@ -3899,7 +3906,18 @@ export const layer = Layer.unwrap( const clientOrigin = readClientConnectionOrigin(request); const clientAnalyticsProps = readClientAnalyticsProps(request); yield* sessions.recordClientConnection(session.sessionId, clientOrigin); - yield* analytics.record("client.connected", clientAnalyticsProps); + // T3-CUSTOM(expbkt3): the signed-in user and how the client reached us (TEC-1631). + yield* analytics.record( + "client.connected", + { + ...clientAnalyticsProps, + connectionMode: clientConnectionMode({ + ...clientAnalyticsProps, + host: request.headers.host, + }), + }, + session.userId === null ? undefined : { userId: session.userId }, + ); const rpcWebSocketHttpEffect = yield* Effect.gen(function* () { const { protocol, httpEffect } = yield* RpcServer.makeProtocolWithHttpEffectWebsocket; yield* RpcServer.make(ServerWsRpcGroup, WS_RPC_SERVER_OPTIONS).pipe( diff --git a/apps/web/package.json b/apps/web/package.json index 2df68088f3dc8..a209cf55d1e5e 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -68,6 +68,7 @@ "mermaid": "^11.17.2", "morphicons": "^1.7.1", "platejs": "53.3.3", + "posthog-js": "^1.438.9", "react": "19.2.6", "react-dom": "19.2.6", "react-markdown": "^10.1.0", diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 98c9f9dad4a99..945a7c19c3ebd 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -632,6 +632,8 @@ const EMPTY_PROVIDERS: ServerProvider[] = []; const EMPTY_PROVIDER_MODELS: ServerProvider["models"] = []; const EMPTY_USAGE_LIMIT_SOURCES: UsageLimitSourceSnapshots = []; import type { CodexArtifactTemplate } from "@t3tools/client-runtime/codex-artifact-templates"; +// T3-CUSTOM(expbkt3): product analytics (TEC-1631). +import { lastInputWasKeyboard, trackProductEvent } from "../lib/analytics/productAnalytics"; const TIMELINE_SCROLL_CANCEL_SENTINEL = Object.freeze({}); const EMPTY_FEEDBACK_SUBMISSIONS: ReadonlyArray = []; @@ -9229,6 +9231,13 @@ export default function ChatView(props: ChatViewProps) { promptRef.current = ""; clearComposerDraftContent(composerDraftTarget); composerRef.current?.resetCursorState(); + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics + trackProductEvent("plan.action", { + threadId: activeThread.id, + action: followUp.interactionMode === "plan" ? "revise" : "approve", + source: "composer", + }); + // T3-CUSTOM(expbkt3): END const followUpSent = await onSubmitPlanFollowUp({ text: followUp.text, context: buildMessageContext({ @@ -9467,6 +9476,14 @@ export default function ChatView(props: ChatViewProps) { setThreadError(threadIdForSend, attachmentCapabilitiesBeforeUpload.fileBlockReason); return; } + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics + trackProductEvent("composer.sent", { + threadId: threadIdForSend, + interactionMode: sendInteractionMode, + hasAttachments: composerAttachmentsSnapshot.length > 0, + viaKeyboard: lastInputWasKeyboard(), + }); + // T3-CUSTOM(expbkt3): END const turnUsesAttachmentUploads = composerFilesSnapshot.length > 0 ? attachmentCapabilitiesBeforeUpload.supportsAttachmentUploads @@ -10238,11 +10255,15 @@ export default function ChatView(props: ChatViewProps) { async (requestId: RuntimeRequestId, decision: ProviderApprovalDecision) => { if (!activeThreadId || !readEnvironmentScope(environmentId, AuthOrchestrationOperateScope)) return; - if ( - pendingApprovals.find((approval) => approval.requestId === requestId) - ?.responseCapability !== "live" - ) - return; + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics + const pendingApproval = pendingApprovals.find((approval) => approval.requestId === requestId); + if (pendingApproval?.responseCapability !== "live") return; + trackProductEvent("approval.answered_in_ui", { + threadId: activeThreadId, + kind: pendingApproval.requestKind, + decision, + }); + // T3-CUSTOM(expbkt3): END setRespondingRequestIds((existing) => existing.includes(requestId) ? existing : [...existing, requestId], @@ -10305,6 +10326,12 @@ export default function ChatView(props: ChatViewProps) { ); } userInputResponsesInFlight.current.add(responseKey); + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics + trackProductEvent("question.answered_in_ui", { + threadId: activeThreadId, + kind: "user_input", + }); + // T3-CUSTOM(expbkt3): END setRespondingUserInputRequestIds((existing) => existing.includes(requestId) ? existing : [...existing, requestId], @@ -10682,6 +10709,13 @@ export default function ChatView(props: ChatViewProps) { const nextThreadTitle = truncate(buildPlanImplementationThreadTitle(planMarkdown)); const nextThreadModelSelection: ModelSelection = ctxSelectedModelSelection; + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics + trackProductEvent("plan.action", { + threadId: activeThread.id, + action: "approve", + source: "new_thread", + }); + // T3-CUSTOM(expbkt3): END sendInFlightRef.current = true; beginLocalDispatch({ preparingWorktree: false }); const finish = () => { diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index 749b8b933d82b..b7fb3dbaa2906 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -224,6 +224,9 @@ import { import type { Project } from "../types"; import { PullRequestGlyph } from "~/components/pullRequest/pullRequestIcons"; import { readPullRequestListPreferences } from "~/components/pullRequest/pullRequestListPreferences"; +// T3-CUSTOM(expbkt3): product analytics (TEC-1631). +import { trackProductEvent } from "~/lib/analytics/productAnalytics"; +import { commandIdForPaletteItem } from "~/lib/analytics/productAnalytics.logic"; const EMPTY_BROWSE_ENTRIES: FilesystemBrowseResult["entries"] = []; @@ -3200,6 +3203,11 @@ function OpenCommandPaletteDialog(props: { setOpen(false); } + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics + trackProductEvent("command_palette.used", { + commandId: commandIdForPaletteItem(item), + }); + // T3-CUSTOM(expbkt3): END void item.run().catch((error: unknown) => { toastManager.add( stackedThreadToast({ diff --git a/apps/web/src/components/planreview/PlanReviewPanel.tsx b/apps/web/src/components/planreview/PlanReviewPanel.tsx index 299118f7531f0..bb31e24a3580d 100644 --- a/apps/web/src/components/planreview/PlanReviewPanel.tsx +++ b/apps/web/src/components/planreview/PlanReviewPanel.tsx @@ -54,6 +54,7 @@ import { usePlanReviewScrollSurfaceRef, } from "../../fork/planReviewComposerDock"; import { canHidePlanGlobalNote, shouldShowPlanGlobalNote } from "../../fork/planReviewGlobalNote"; +import { trackProductEvent } from "../../lib/analytics/productAnalytics"; interface PlanReviewPanelProps { readonly environmentId: EnvironmentId; @@ -354,6 +355,11 @@ export default function PlanReviewPanel({ const handleSubmit = useCallback( (decision: "approved" | "changes-requested" | "discarded") => { + trackProductEvent("plan.action", { + action: + decision === "approved" ? "approve" : decision === "discarded" ? "discard" : "revise", + source: "plan_review", + }); void submit({ environmentId, input: { diff --git a/apps/web/src/hooks/useSettings.ts b/apps/web/src/hooks/useSettings.ts index 04fab1c1d1c0b..64f209840ccae 100644 --- a/apps/web/src/hooks/useSettings.ts +++ b/apps/web/src/hooks/useSettings.ts @@ -53,6 +53,8 @@ import { useAtomCommand } from "~/state/use-atom-command"; import { useTheme } from "./useTheme"; import { environmentSession, readEnvironmentScope, useEnvironmentScope } from "~/state/session"; import { appAtomRegistry } from "~/rpc/atomRegistry"; +// T3-CUSTOM(expbkt3): product analytics (TEC-1631). +import { trackSettingsChanged } from "~/lib/analytics/productAnalytics"; const CLIENT_SETTINGS_PERSISTENCE_ERROR_SCOPE = "[CLIENT_SETTINGS]"; @@ -477,6 +479,8 @@ function useUpdateSettingsTarget(environmentId: EnvironmentId | null) { const sharedSettingsSyncTargetIds = useSharedSettingsSyncTargetIds(true); const updateSettings = useCallback( (patch: UnifiedSettingsPatch) => { + // T3-CUSTOM(expbkt3): TEC-1631 product analytics + trackSettingsChanged(Object.keys(patch)); const { serverPatch, clientPatch } = splitPatch(patch); const canWriteServerPatch = @@ -577,6 +581,8 @@ export function useUpdatePrimarySettings() { export function useUpdateClientSettings() { return useCallback((patch: ClientSettingsPatch) => { + // T3-CUSTOM(expbkt3): TEC-1631 product analytics + trackSettingsChanged(Object.keys(patch)); return persistClientSettingsPatch(patch); }, []); } diff --git a/apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx b/apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx new file mode 100644 index 0000000000000..f55f354e4d6e7 --- /dev/null +++ b/apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx @@ -0,0 +1,127 @@ +/** + * T3-CUSTOM(expbkt3): wires client product analytics to app state (TEC-1631). + * + * Mounted once under the router. It hands the primary environment's analytics + * config to the service and, only while that is on, mounts the trackers for + * identity, views, opened threads and app visibility. With analytics off it + * renders nothing and subscribes to nothing beyond the primary environment. + * + * @module ProductAnalyticsCoordinator + */ +import { useParams, useRouterState } from "@tanstack/react-router"; +import { useEffect, useMemo, useRef } from "react"; + +import { APP_VERSION } from "../../branding"; +import { isElectron } from "../../env"; +import { usePrimaryEnvironment } from "../../state/environments"; +import { useCurrentUserId } from "../../state/identity"; +import { useOrgMembers } from "../../state/orgMembers"; +import { productAnalytics, trackProductEvent } from "./productAnalytics"; +import { + createVisibilityReporter, + resolveConnectionMode, + viewNameFromRouteId, +} from "./productAnalytics.logic"; + +/** A visibility change must hold this long before it is reported. */ +const VISIBILITY_SETTLE_MS = 5_000; + +let appOpenedSent = false; + +export function ProductAnalyticsCoordinator() { + const primary = usePrimaryEnvironment(); + const analytics = primary?.serverConfig?.environment.analytics ?? null; + const target = primary?.entry.target ?? null; + const key = analytics !== null && analytics.enabled ? analytics.posthogKey : null; + const host = analytics !== null && analytics.enabled ? analytics.posthogHost : null; + const instance = analytics?.instance ?? null; + const connectionMode = useMemo(() => resolveConnectionMode(target), [target]); + + useEffect(() => { + productAnalytics.configure({ + config: + key === null || host === null + ? null + : { + enabled: true, + posthogKey: key, + posthogHost: host, + ...(instance === null ? {} : { instance }), + }, + surface: isElectron ? "desktop" : "web", + appVersion: APP_VERSION, + connectionMode, + }); + if (key !== null && !appOpenedSent) { + appOpenedSent = true; + trackProductEvent("app.opened"); + } + }, [key, host, instance, connectionMode]); + + if (key === null) return null; + return ( + <> + + + + + ); +} + +/** + * Identifies by work email only, as the server does. Without an email the + * events stay anonymous, and posthog-js merges them once the email arrives. + */ +function ProductAnalyticsIdentity() { + const userId = useCurrentUserId(); + const { resolveUser } = useOrgMembers(); + const email = userId === null ? null : resolveUser(userId).email; + useEffect(() => { + productAnalytics.identify(email); + }, [email]); + return null; +} + +function ProductAnalyticsRouteTracker() { + const routeId = useRouterState({ select: (state) => state.matches.at(-1)?.routeId ?? null }); + const { threadId: routeThreadId } = useParams({ strict: false }); + const threadId = typeof routeThreadId === "string" ? routeThreadId : null; + const view = viewNameFromRouteId(routeId); + const lastThreadId = useRef(null); + + useEffect(() => { + if (view !== null) trackProductEvent("view.changed", { view }); + }, [view]); + + useEffect(() => { + if (threadId === null || threadId === lastThreadId.current) return; + lastThreadId.current = threadId; + trackProductEvent("session.opened", { threadId }); + }, [threadId]); + return null; +} + +/** App visibility (settled, so window peeks send nothing) and last input kind. */ +function ProductAnalyticsActivityTracker() { + useEffect(() => { + const reporter = createVisibilityReporter({ + delayMs: VISIBILITY_SETTLE_MS, + emit: (state) => + trackProductEvent(state === "foreground" ? "app.foregrounded" : "app.backgrounded"), + }); + const onVisibility = () => + reporter.update(document.visibilityState === "hidden" ? "background" : "foreground"); + const onKeyDown = () => productAnalytics.noteInput("keyboard"); + const onPointerDown = () => productAnalytics.noteInput("pointer"); + document.addEventListener("visibilitychange", onVisibility); + window.addEventListener("keydown", onKeyDown, { capture: true, passive: true }); + window.addEventListener("pointerdown", onPointerDown, { capture: true, passive: true }); + return () => { + reporter.dispose(); + document.removeEventListener("visibilitychange", onVisibility); + window.removeEventListener("keydown", onKeyDown, { capture: true }); + window.removeEventListener("pointerdown", onPointerDown, { capture: true }); + }; + }, []); + return null; +} diff --git a/apps/web/src/lib/analytics/productAnalytics.logic.test.ts b/apps/web/src/lib/analytics/productAnalytics.logic.test.ts new file mode 100644 index 0000000000000..024b68bf22592 --- /dev/null +++ b/apps/web/src/lib/analytics/productAnalytics.logic.test.ts @@ -0,0 +1,205 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + commandIdForPaletteItem, + commandIdFromPaletteValue, + createKeyThrottle, + createVisibilityReporter, + isAnalyticsSettingKey, + resolveConnectionMode, + sanitizeAnalyticsProperties, + stripUrlProperties, + viewNameFromRouteId, +} from "./productAnalytics.logic"; + +describe("sanitizeAnalyticsProperties", () => { + it("keeps only the event's allowlisted keys", () => { + expect( + sanitizeAnalyticsProperties("composer.sent", { + threadId: "thread-1", + interactionMode: "plan", + hasAttachments: true, + viaKeyboard: false, + prompt: "secret text", + title: "Fix the login page", + }), + ).toEqual({ + threadId: "thread-1", + interactionMode: "plan", + hasAttachments: true, + viaKeyboard: false, + }); + }); + + it("drops objects, long strings and non-finite numbers", () => { + expect( + sanitizeAnalyticsProperties("settings.changed", { settingKey: { nested: "value" } }), + ).toEqual({}); + expect(sanitizeAnalyticsProperties("view.changed", { view: "x".repeat(500) })).toEqual({}); + expect(sanitizeAnalyticsProperties("command_palette.used", { commandId: Number.NaN })).toEqual( + {}, + ); + }); + + it("keeps nothing for unknown events or missing properties", () => { + expect(sanitizeAnalyticsProperties("custom.event", { threadId: "t" })).toEqual({}); + expect(sanitizeAnalyticsProperties("app.opened", { threadId: "t" })).toEqual({}); + expect(sanitizeAnalyticsProperties("composer.sent", undefined)).toEqual({}); + }); +}); + +describe("viewNameFromRouteId", () => { + it("names a view by its route template, never by ids", () => { + expect(viewNameFromRouteId("/_chat/$environmentId/$threadId")).toBe( + "/chat/$environmentId/$threadId", + ); + expect(viewNameFromRouteId("/_chat/draft/$draftId")).toBe("/chat/draft/$draftId"); + expect(viewNameFromRouteId("/_chat/")).toBe("/chat"); + expect(viewNameFromRouteId("/settings/general")).toBe("/settings/general"); + expect(viewNameFromRouteId("__root__")).toBe("/"); + }); + + it("drops segments that look like concrete values", () => { + expect(viewNameFromRouteId("/projects/3f2a9c1e-uuid/thread title")).toBe("/projects"); + expect(viewNameFromRouteId(null)).toBeNull(); + }); +}); + +describe("palette command ids", () => { + it("keeps the command kind and drops environment ids, paths and text", () => { + expect(commandIdFromPaletteValue("action:add-project:env-123:local")).toBe( + "action:add-project", + ); + expect(commandIdFromPaletteValue("appearance:dark")).toBe("appearance:dark"); + expect(commandIdFromPaletteValue("thread:01HZX9")).toBe("thread"); + expect(commandIdFromPaletteValue("/home/user/repo")).toBeNull(); + }); + + it("prefers the keybinding command and hides project script names", () => { + expect(commandIdForPaletteItem({ shortcutCommand: "terminal.toggle", value: "x" })).toBe( + "terminal.toggle", + ); + expect(commandIdForPaletteItem({ shortcutCommand: "script.deploy-prod.run", value: "x" })).toBe( + "script.run", + ); + expect(commandIdForPaletteItem({ value: "action:new-thread" })).toBe("action:new-thread"); + }); +}); + +describe("isAnalyticsSettingKey", () => { + it("accepts identifiers only", () => { + expect(isAnalyticsSettingKey("followUpBehavior")).toBe(true); + expect(isAnalyticsSettingKey("some value with spaces")).toBe(false); + expect(isAnalyticsSettingKey("/path")).toBe(false); + }); +}); + +describe("resolveConnectionMode", () => { + const primary = (httpBaseUrl: string) => + ({ _tag: "PrimaryConnectionTarget", httpBaseUrl }) as const; + + it("tells local, tailnet and remote primaries apart", () => { + expect(resolveConnectionMode(primary("http://localhost:3773"))).toBe("local"); + expect(resolveConnectionMode(primary("http://127.0.0.1:3773"))).toBe("local"); + expect(resolveConnectionMode(primary("https://dev-server-1.tailab6257.ts.net"))).toBe( + "tailnet", + ); + expect(resolveConnectionMode(primary("http://100.101.5.7:3773"))).toBe("tailnet"); + expect(resolveConnectionMode(primary("https://bkt3.example.com"))).toBe("remote"); + }); + + it("maps other targets by kind", () => { + expect(resolveConnectionMode({ _tag: "RelayConnectionTarget" })).toBe("relay"); + expect(resolveConnectionMode({ _tag: "SshConnectionTarget" })).toBe("ssh"); + expect(resolveConnectionMode({ _tag: "BearerConnectionTarget" })).toBe("remote"); + expect(resolveConnectionMode(null)).toBeNull(); + }); +}); + +describe("stripUrlProperties", () => { + it("removes the URL and title properties PostHog adds", () => { + expect( + stripUrlProperties({ + $current_url: "https://bkt3/chat/env/thread", + $pathname: "/chat/env/thread", + $title: "Fix login", + $browser: "Chrome", + view: "/chat", + }), + ).toEqual({ $browser: "Chrome", view: "/chat" }); + }); + + it("removes every session-entry, initial and previous-pageview property by prefix", () => { + expect( + stripUrlProperties({ + $session_entry_url: "https://bkt3/pull-requests?repository=owner/repo", + $session_entry_pathname: "/pull-requests", + $session_entry_referrer: "https://github.com/owner/repo", + $session_entry_host: "bkt3", + $session_entry_some_future_key: "x", + $initial_person_info: { u: "https://bkt3/chat/env/thread", r: "$direct" }, + $initial_referrer_info: { $referrer: "https://github.com" }, + $prev_pageview_pathname: "/chat/env/thread", + $host: "dev-server-1.tailab6257.ts.net", + utm_source: "newsletter", + gclid: "abc", + $session_id: "s1", + instance: "bkt3", + }), + ).toEqual({ $session_id: "s1", instance: "bkt3" }); + }); +}); + +describe("createVisibilityReporter", () => { + function harness() { + const timers = new Map void>(); + let nextId = 0; + const emitted: string[] = []; + const reporter = createVisibilityReporter({ + delayMs: 1_000, + emit: (state) => emitted.push(state), + setTimer: (callback) => { + nextId += 1; + timers.set(nextId, callback); + return nextId; + }, + clearTimer: (handle) => timers.delete(handle as number), + }); + const flush = () => { + for (const [id, callback] of [...timers]) { + timers.delete(id); + callback(); + } + }; + return { reporter, emitted, flush }; + } + + it("sends a settled change once", () => { + const { reporter, emitted, flush } = harness(); + reporter.update("background"); + flush(); + reporter.update("background"); + flush(); + expect(emitted).toEqual(["background"]); + }); + + it("sends nothing for a quick hide and show", () => { + const { reporter, emitted, flush } = harness(); + reporter.update("background"); + reporter.update("foreground"); + flush(); + expect(emitted).toEqual([]); + }); +}); + +describe("createKeyThrottle", () => { + it("allows each key once per window", () => { + let now = 0; + const allow = createKeyThrottle(1_000, () => now); + expect(allow("a")).toBe(true); + expect(allow("a")).toBe(false); + expect(allow("b")).toBe(true); + now = 1_000; + expect(allow("a")).toBe(true); + }); +}); diff --git a/apps/web/src/lib/analytics/productAnalytics.logic.ts b/apps/web/src/lib/analytics/productAnalytics.logic.ts new file mode 100644 index 0000000000000..399d12f19dd4d --- /dev/null +++ b/apps/web/src/lib/analytics/productAnalytics.logic.ts @@ -0,0 +1,278 @@ +/** + * T3-CUSTOM(expbkt3): pure rules for client product analytics (TEC-1631). + * + * Everything that decides *what* leaves the client lives here, so it can be + * tested without PostHog: the named events, the property allowlist per event, + * route → view names, palette command ids, and the URL properties PostHog adds + * on its own that we strip before send. Nothing here sends text, titles, + * paths, branch or repository names, or setting values. + * + * @module productAnalytics.logic + */ + +export type ProductAnalyticsSurface = "web" | "desktop"; +export type ProductAnalyticsConnectionMode = "local" | "tailnet" | "remote" | "relay" | "ssh"; + +/** Properties each event may carry. Anything else is dropped before capture. */ +export const PRODUCT_ANALYTICS_EVENTS = { + "app.opened": [], + "app.foregrounded": [], + "app.backgrounded": [], + "view.changed": ["view"], + "session.opened": ["threadId"], + "composer.sent": ["threadId", "interactionMode", "hasAttachments", "viaKeyboard"], + "command_palette.used": ["commandId"], + "settings.changed": ["settingKey"], + "approval.answered_in_ui": ["threadId", "kind", "decision"], + "question.answered_in_ui": ["threadId", "kind"], + "plan.action": ["threadId", "action", "source"], +} as const satisfies Record>; + +export type ProductAnalyticsEvent = keyof typeof PRODUCT_ANALYTICS_EVENTS; +export type ProductAnalyticsProperties = Readonly>; + +/** Longest string value kept. Ids and enum tokens fit; free text does not. */ +const MAX_STRING_PROPERTY_LENGTH = 96; + +/** + * Keeps only the allowlisted keys of `event` whose values are short strings, + * finite numbers or booleans. Unknown events keep nothing. + */ +export function sanitizeAnalyticsProperties( + event: string, + properties: Readonly> | undefined, +): ProductAnalyticsProperties { + const allowed: ReadonlyArray | undefined = + PRODUCT_ANALYTICS_EVENTS[event as ProductAnalyticsEvent]; + if (allowed === undefined || properties === undefined) return {}; + const result: Record = {}; + for (const key of allowed) { + const value = properties[key]; + if (typeof value === "boolean") result[key] = value; + else if (typeof value === "number" && Number.isFinite(value)) result[key] = value; + else if ( + typeof value === "string" && + value.length > 0 && + value.length <= MAX_STRING_PROPERTY_LENGTH + ) { + result[key] = value; + } + } + return result; +} + +/** + * A view name from a TanStack route id, which is the route *template* + * (`/_chat/$environmentId/$threadId`), never the visited URL. Pathless layout + * segments (`_chat`) lose their underscore; params keep only their name. + */ +export function viewNameFromRouteId(routeId: string | null | undefined): string | null { + if (!routeId) return null; + if (routeId === "__root__") return "/"; + const segments = routeId + .split("/") + .filter((segment) => segment.length > 0) + .map((segment) => (segment.startsWith("_") ? segment.slice(1) : segment)) + .filter((segment) => /^\$?[A-Za-z][\w-]*$/.test(segment)); + return `/${segments.join("/")}`; +} + +const COMMAND_SEGMENT = /^[a-z][a-z-]*$/; + +/** + * A stable command id from a palette item `value` such as + * `action:add-project::local`. Keeps at most the first two + * segments, and stops at the first one that is not a plain lowercase word, so + * ids, paths and search text never leave. + */ +export function commandIdFromPaletteValue(value: string): string | null { + const kept: string[] = []; + for (const segment of value.split(":").slice(0, 2)) { + if (!COMMAND_SEGMENT.test(segment)) break; + kept.push(segment); + } + return kept.length === 0 ? null : kept.join(":"); +} + +/** + * The command id for a palette item: its keybinding command when it has one, + * else the cleaned `value`. Project scripts (`script..run`) collapse to + * `script.run`, since the name is the user's own. + */ +export function commandIdForPaletteItem(item: { + readonly shortcutCommand?: string | undefined; + readonly value: string; +}): string | null { + if (item.shortcutCommand !== undefined) { + return item.shortcutCommand.startsWith("script.") ? "script.run" : item.shortcutCommand; + } + return commandIdFromPaletteValue(item.value); +} + +/** Settings keys are camelCase identifiers; anything else is not a key. */ +export function isAnalyticsSettingKey(key: string): boolean { + return /^[A-Za-z][A-Za-z0-9]{0,63}$/.test(key); +} + +const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]", "::1"]); + +/** How this client reaches its primary environment, from the primary target. */ +export function resolveConnectionMode( + target: + | { readonly _tag: "PrimaryConnectionTarget"; readonly httpBaseUrl: string } + | { readonly _tag: "RelayConnectionTarget" } + | { readonly _tag: "BearerConnectionTarget" } + | { readonly _tag: "SshConnectionTarget" } + | null, +): ProductAnalyticsConnectionMode | null { + if (target === null) return null; + switch (target._tag) { + case "RelayConnectionTarget": + return "relay"; + case "SshConnectionTarget": + return "ssh"; + case "BearerConnectionTarget": + return "remote"; + case "PrimaryConnectionTarget": { + let hostname: string; + try { + hostname = new URL(target.httpBaseUrl).hostname.toLowerCase(); + } catch { + return null; + } + if (LOOPBACK_HOSTS.has(hostname)) return "local"; + if (hostname.endsWith(".ts.net") || isTailnetIpv4(hostname)) return "tailnet"; + return "remote"; + } + } +} + +function isTailnetIpv4(hostname: string): boolean { + const match = /^100\.(\d{1,3})\.\d{1,3}\.\d{1,3}$/.exec(hostname); + if (!match) return false; + const second = Number(match[1]); + return second >= 64 && second <= 127; +} + +/** + * Properties PostHog adds on its own that carry the visited URL (thread ids, + * `?repository=owner/repo`), the host, the referrer or the window title + * (thread titles). They are removed in `before_send`, from event properties + * and from `$set` / `$set_once`. Checked against posthog-js 1.438. + */ +export const STRIPPED_POSTHOG_PROPERTIES = [ + "$current_url", + "$pathname", + "$host", + "$title", + "$referrer", + "$referring_domain", + "$external_click_url", + "$el_text", + "$elements", + "$elements_chain", + // Campaign and click ids are read from the URL query string. + "utm_source", + "utm_medium", + "utm_campaign", + "utm_content", + "utm_term", + "gad_source", + "mc_cid", + "gclid", + "gclsrc", + "dclid", + "gbraid", + "wbraid", + "fbclid", + "msclkid", + "twclid", + "li_fat_id", + "igshid", + "ttclid", + "rdt_cid", + "epik", + "qclid", + "sccid", + "oppref", + "irclid", + "_kx", +] as const; + +/** + * Prefixes of PostHog property families built from the URL or referrer: + * `$initial_*` (first-touch person info), `$session_entry_*` (the session's + * entry URL, path, host and referrer), `$prev_pageview_*` (the previous path). + */ +export const STRIPPED_POSTHOG_PROPERTY_PREFIXES = [ + "$initial_", + "$session_entry_", + "$prev_pageview_", +] as const; + +const STRIPPED_PROPERTY_SET: ReadonlySet = new Set(STRIPPED_POSTHOG_PROPERTIES); + +function isStrippedProperty(key: string): boolean { + return ( + STRIPPED_PROPERTY_SET.has(key) || + STRIPPED_POSTHOG_PROPERTY_PREFIXES.some((prefix) => key.startsWith(prefix)) + ); +} + +export function stripUrlProperties>(properties: T): T { + const next: Record = {}; + for (const [key, value] of Object.entries(properties)) { + if (!isStrippedProperty(key)) next[key] = value; + } + return next as T; +} + +/** + * Reports app visibility once it has been stable for `delayMs`. A quick + * hide-then-show (window switch, tab peek) sends nothing; a real change sends + * one event, and the same state is never sent twice in a row. + */ +export function createVisibilityReporter(input: { + readonly delayMs: number; + readonly emit: (state: "foreground" | "background") => void; + readonly setTimer?: (callback: () => void, ms: number) => unknown; + readonly clearTimer?: (handle: unknown) => void; +}) { + const setTimer = + input.setTimer ?? ((callback: () => void, ms: number) => globalThis.setTimeout(callback, ms)); + const clearTimer = + input.clearTimer ?? + ((handle: unknown) => globalThis.clearTimeout(handle as ReturnType)); + let lastEmitted: "foreground" | "background" = "foreground"; + let timer: unknown = null; + return { + update(state: "foreground" | "background"): void { + if (timer !== null) { + clearTimer(timer); + timer = null; + } + if (state === lastEmitted) return; + timer = setTimer(() => { + timer = null; + lastEmitted = state; + input.emit(state); + }, input.delayMs); + }, + dispose(): void { + if (timer !== null) clearTimer(timer); + timer = null; + }, + }; +} + +/** True at most once per `windowMs` for each key. */ +export function createKeyThrottle(windowMs: number, now: () => number = Date.now) { + const lastByKey = new Map(); + return (key: string): boolean => { + const at = now(); + const last = lastByKey.get(key); + if (last !== undefined && at - last < windowMs) return false; + lastByKey.set(key, at); + return true; + }; +} diff --git a/apps/web/src/lib/analytics/productAnalytics.test.ts b/apps/web/src/lib/analytics/productAnalytics.test.ts new file mode 100644 index 0000000000000..b71151b9db272 --- /dev/null +++ b/apps/web/src/lib/analytics/productAnalytics.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + createProductAnalytics, + normalizeAnalyticsEmail, + type ProductAnalyticsClient, + type ProductAnalyticsContext, + type ProductAnalyticsLoader, +} from "./productAnalytics"; + +interface FakeClient { + readonly key: string; + readonly captured: Array<{ event: string; properties: unknown }>; + readonly identified: string[]; + readonly registered: Array>; + readonly optIns: unknown[]; + optedOut: boolean; +} + +function fakeLoader() { + const clients: FakeClient[] = []; + const load: ProductAnalyticsLoader = async (key) => { + const fake: FakeClient = { + key, + captured: [], + identified: [], + registered: [], + optIns: [], + optedOut: false, + }; + clients.push(fake); + // opt_out_capturing is present so a regression that calls it is visible. + const client = { + capture: (event: string, properties?: unknown) => { + fake.captured.push({ event, properties }); + return undefined; + }, + identify: (id?: string) => { + if (id !== undefined) fake.identified.push(id); + }, + register: (properties: Record) => { + fake.registered.push(properties); + }, + opt_in_capturing: (options?: unknown) => { + fake.optIns.push(options); + }, + opt_out_capturing: () => { + fake.optedOut = true; + }, + } as unknown as ProductAnalyticsClient; + return client; + }; + return { load, clients }; +} + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + +function context(config: ProductAnalyticsContext["config"]): ProductAnalyticsContext { + return { config, surface: "web", appVersion: "1.2.3", connectionMode: "tailnet" }; +} + +const enabledConfig = { + enabled: true, + posthogKey: "phc_company", + posthogHost: "https://us.i.posthog.com", + instance: "bkt3", +}; + +describe("createProductAnalytics", () => { + it("is a no-op and never loads PostHog when the config is missing", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.track("app.opened"); + analytics.configure(context(null)); + analytics.track("composer.sent", { threadId: "t1" }); + await flush(); + expect(loader.clients).toEqual([]); + expect(analytics.isActive()).toBe(false); + }); + + it("is a no-op when the server marks analytics disabled", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.configure(context({ ...enabledConfig, enabled: false })); + analytics.track("app.opened"); + await flush(); + expect(loader.clients).toEqual([]); + }); + + it("queues events until loaded, then sends sanitized properties", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.configure(context(enabledConfig)); + analytics.identify(" Person@Beknown.Work "); + analytics.track("composer.sent", { threadId: "t1", prompt: "do not send me" }); + await flush(); + const [client] = loader.clients; + expect(client?.captured).toEqual([{ event: "composer.sent", properties: { threadId: "t1" } }]); + expect(client?.identified).toEqual(["person@beknown.work"]); + expect(client?.optIns).toEqual([{ captureEventName: null }]); + expect(client?.registered.at(-1)).toEqual({ + surface: "web", + appVersion: "1.2.3", + instance: "bkt3", + connectionMode: "tailnet", + }); + }); + + it("never opts out: a config flap stops forwarding and reuses the same client", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.configure(context(enabledConfig)); + await flush(); + analytics.configure(context(null)); + analytics.track("app.opened"); + analytics.configure(context(enabledConfig)); + await flush(); + analytics.track("app.foregrounded"); + expect(loader.clients).toHaveLength(1); + expect(loader.clients[0]?.optedOut).toBe(false); + expect(loader.clients[0]?.captured.map((entry) => entry.event)).toEqual(["app.foregrounded"]); + }); + + it("sends to the new key only after a key change, without opting the old one out", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.configure(context(enabledConfig)); + await flush(); + analytics.configure(context({ ...enabledConfig, posthogKey: "phc_other" })); + await flush(); + analytics.track("app.opened"); + expect(loader.clients.map((client) => client.optedOut)).toEqual([false, false]); + expect(loader.clients[0]?.captured).toEqual([]); + expect(loader.clients[1]?.captured.map((entry) => entry.event)).toEqual(["app.opened"]); + }); + + it("stays anonymous until an email is known, so one person is never split", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.configure(context(enabledConfig)); + analytics.identify(null); + analytics.identify("user_2abc"); + analytics.track("app.opened"); + await flush(); + const [client] = loader.clients; + expect(client?.identified).toEqual([]); + expect(client?.captured.map((entry) => entry.event)).toEqual(["app.opened"]); + analytics.identify("person@beknown.work"); + analytics.identify("PERSON@beknown.work"); + expect(client?.identified).toEqual(["person@beknown.work"]); + }); +}); + +describe("normalizeAnalyticsEmail", () => { + it("trims and lower-cases emails and rejects anything else", () => { + expect(normalizeAnalyticsEmail(" A@B.co ")).toBe("a@b.co"); + expect(normalizeAnalyticsEmail("user_2abc")).toBeNull(); + expect(normalizeAnalyticsEmail(null)).toBeNull(); + }); +}); diff --git a/apps/web/src/lib/analytics/productAnalytics.ts b/apps/web/src/lib/analytics/productAnalytics.ts new file mode 100644 index 0000000000000..7fa36d37edefe --- /dev/null +++ b/apps/web/src/lib/analytics/productAnalytics.ts @@ -0,0 +1,226 @@ +/** + * T3-CUSTOM(expbkt3): client product analytics (TEC-1631). + * + * Call sites use `trackProductEvent(name, props)`: one line, no awaits, and a + * no-op until the primary environment advertises an analytics config. Only + * then is `posthog-js` loaded, by dynamic import, so the main bundle and + * startup cost do not change while analytics is off. + * + * Only the primary environment's config is used. A client connected to + * several servers never mixes keys: a new key retires the old client first. + * + * @module productAnalytics + */ +import type { ExecutionEnvironmentAnalytics } from "@t3tools/contracts"; +import type { PostHog, PostHogConfig } from "posthog-js"; + +import { + createKeyThrottle, + isAnalyticsSettingKey, + type ProductAnalyticsConnectionMode, + type ProductAnalyticsEvent, + type ProductAnalyticsSurface, + sanitizeAnalyticsProperties, + stripUrlProperties, +} from "./productAnalytics.logic"; + +export interface ProductAnalyticsContext { + readonly config: ExecutionEnvironmentAnalytics | null; + readonly surface: ProductAnalyticsSurface; + readonly appVersion: string; + readonly connectionMode: ProductAnalyticsConnectionMode | null; +} + +/** The part of the PostHog client this module calls. */ +export type ProductAnalyticsClient = Pick< + PostHog, + "capture" | "identify" | "register" | "opt_in_capturing" +>; + +export type ProductAnalyticsLoader = ( + key: string, + options: Partial, + name: string, +) => Promise; + +/** Events captured before `posthog-js` finishes loading. Older ones are dropped. */ +const MAX_PENDING_EVENTS = 50; + +interface ActiveSession { + readonly key: string; + posthog: ProductAnalyticsClient | null; + readonly pending: Array<{ event: string; properties: Record }>; +} + +const loadPostHog: ProductAnalyticsLoader = async (key, options, name) => { + const { default: posthog } = await import("posthog-js"); + return posthog.init(key, options, name); +}; + +function postHogOptions(host: string): Partial { + return { + api_host: host, + autocapture: false, + capture_pageview: false, + capture_pageleave: false, + disable_session_recording: true, + persistence: "localStorage", + person_profiles: "identified_only", + // Named events only: no surveys, flags, heatmaps, vitals or remote scripts. + disable_surveys: true, + advanced_disable_flags: true, + disable_web_experiments: true, + disable_external_dependency_loading: true, + enable_heatmaps: false, + capture_performance: false, + capture_exceptions: false, + capture_dead_clicks: false, + rageclick: false, + before_send: (event) => ({ + ...event, + properties: stripUrlProperties(event.properties), + ...(event.$set === undefined ? {} : { $set: stripUrlProperties(event.$set) }), + ...(event.$set_once === undefined ? {} : { $set_once: stripUrlProperties(event.$set_once) }), + }), + }; +} + +/** Matches the server's distinct id (`analyticsIdentity.ts`): trimmed, lower case. */ +export function normalizeAnalyticsEmail(email: string | null | undefined): string | null { + const normalized = email?.trim().toLowerCase() ?? ""; + return normalized.includes("@") ? normalized : null; +} + +export function createProductAnalytics(load: ProductAnalyticsLoader = loadPostHog) { + // One PostHog instance per key for the page's life. Turning analytics off only + // stops forwarding: `opt_out_capturing` would persist for a year per token, + // so a brief config flap must never reach it. + const clients = new Map>(); + const identifiedAs = new WeakMap(); + let session: ActiveSession | null = null; + let superProperties: Record = {}; + let email: string | null = null; + let lastInputWasKeyboard = false; + + const applyIdentity = (client: ProductAnalyticsClient) => { + client.register(superProperties); + // Until the email is known, events stay anonymous; identify(email) later + // merges them into the person. Identifying by a user id first would split it. + if (email !== null && identifiedAs.get(client) !== email) { + client.identify(email); + identifiedAs.set(client, email); + } + }; + + const clientFor = (key: string, host: string): Promise => { + const existing = clients.get(key); + if (existing !== undefined) return existing; + const created = load(key, postHogOptions(host), `bkt3_${clients.size + 1}`).then((client) => { + // Clears an opt-out a build that still called opt_out_capturing persisted. + client.opt_in_capturing({ captureEventName: null }); + return client; + }); + clients.set(key, created); + // A failed load may be retried by the next configure. + created.catch(() => clients.delete(key)); + return created; + }; + + return { + /** Apply the primary environment's config. `null` or disabled turns it off. */ + configure(context: ProductAnalyticsContext): void { + const config = context.config?.enabled === true ? context.config : null; + superProperties = { + surface: context.surface, + appVersion: context.appVersion, + ...(config?.instance === undefined ? {} : { instance: config.instance }), + ...(context.connectionMode === null ? {} : { connectionMode: context.connectionMode }), + }; + if (config === null) { + session = null; + return; + } + if (session !== null && session.key === config.posthogKey) { + if (session.posthog !== null) applyIdentity(session.posthog); + return; + } + const next: ActiveSession = { key: config.posthogKey, posthog: null, pending: [] }; + session = next; + clientFor(config.posthogKey, config.posthogHost) + .then((client) => { + if (session !== next) return; + next.posthog = client; + applyIdentity(client); + for (const { event, properties } of next.pending.splice(0)) { + client.capture(event, properties); + } + }) + .catch(() => { + // Analytics must never break the app; a failed load stays off. + if (session === next) session = null; + }); + }, + + /** The user's work email. Anything that is not an email is ignored. */ + identify(rawEmail: string | null): void { + const normalized = normalizeAnalyticsEmail(rawEmail); + if (normalized === null) return; + email = normalized; + if (session?.posthog) applyIdentity(session.posthog); + }, + + track(event: ProductAnalyticsEvent, properties?: Readonly>): void { + if (session === null) return; + const clean = sanitizeAnalyticsProperties(event, properties); + if (session.posthog !== null) { + session.posthog.capture(event, clean); + return; + } + if (session.pending.length >= MAX_PENDING_EVENTS) session.pending.shift(); + session.pending.push({ event, properties: clean }); + }, + + isActive(): boolean { + return session !== null; + }, + + noteInput(kind: "keyboard" | "pointer"): void { + lastInputWasKeyboard = kind === "keyboard"; + }, + + /** Whether the last user input before now was a key press. */ + lastInputWasKeyboard(): boolean { + return lastInputWasKeyboard; + }, + }; +} + +export type ProductAnalytics = ReturnType; + +export const productAnalytics: ProductAnalytics = createProductAnalytics(); + +/** One-line hook for call sites. A no-op while analytics is off. */ +export function trackProductEvent( + event: ProductAnalyticsEvent, + properties?: Readonly>, +): void { + productAnalytics.track(event, properties); +} + +export function lastInputWasKeyboard(): boolean { + return productAnalytics.lastInputWasKeyboard(); +} + +/** Drags and repeated writes of one setting count once per window. */ +const SETTINGS_CHANGED_WINDOW_MS = 10_000; +const settingsChangedThrottle = createKeyThrottle(SETTINGS_CHANGED_WINDOW_MS); + +/** `settings.changed` per key in a patch. Keys only, never values. */ +export function trackSettingsChanged(keys: ReadonlyArray): void { + if (!productAnalytics.isActive()) return; + for (const settingKey of keys) { + if (isAnalyticsSettingKey(settingKey) && settingsChangedThrottle(settingKey)) { + productAnalytics.track("settings.changed", { settingKey }); + } + } +} diff --git a/apps/web/src/routes/__root.tsx b/apps/web/src/routes/__root.tsx index 5c8be802c5e96..b175eb9a5e836 100644 --- a/apps/web/src/routes/__root.tsx +++ b/apps/web/src/routes/__root.tsx @@ -40,6 +40,8 @@ import { ChatGptWelcomeCoordinator } from "../components/settings/ChatGptWelcome import { PlanAgentSelectionHeal } from "../fork/PlanAgentSelectionHeal"; // T3-CUSTOM(expbkt3): delivery runs independently of the selected thread or draft. import { DurableOutboxCoordinator } from "../fork/DurableOutboxCoordinator"; +// T3-CUSTOM(expbkt3): product analytics, off unless the primary environment enables it. +import { ProductAnalyticsCoordinator } from "../lib/analytics/ProductAnalyticsCoordinator"; import { ProviderAuthCallbackCoordinator } from "../components/settings/ProviderAuthCallbackCoordinator"; import { ThemeEditorHost } from "../components/settings/ThemeEditorHost"; import { useCopyToClipboard } from "../hooks/useCopyToClipboard"; @@ -228,6 +230,7 @@ function RootRouteView() { {/* T3-CUSTOM(expbkt3): heal saved plan-agent options after authenticated settings load. */} {primaryEnvironmentAuthenticated ? : null} + {primaryEnvironmentAuthenticated ? : null} turn on the instance. Trigger a manual bkt3 deploy from the `t3.dev` instance > or from a human shell. +## Product analytics + +bkt3 sends product events to the company PostHog project. The +server reads these variables: + +| Variable | Meaning | Default | +| --------------------------- | --------------------------------------------------------------------------------- | -------------------------- | +| `T3CODE_TELEMETRY_ENABLED` | Set `false` to send nothing. | `true` | +| `T3CODE_POSTHOG_KEY` | PostHog project API key. Without a key, telemetry is off. | none | +| `T3CODE_POSTHOG_HOST` | PostHog ingestion host. | `https://us.i.posthog.com` | +| `T3CODE_ANALYTICS_INSTANCE` | Name of this instance, sent as the PostHog `instance` group (for example `bkt3`). | `unknown` | + +The fork has no built-in key, so it never sends to the upstream maintainers' +project. Never commit the key. Production keeps the values in +`/home/ubuntu/.t3/nextbkt3-dev/posthog.env` (mode `600`, owner `ubuntu`), which +sets `T3CODE_POSTHOG_KEY`, `T3CODE_POSTHOG_HOST` and +`T3CODE_ANALYTICS_INSTANCE=bkt3`. + +The existing drop-in `/etc/systemd/system/t3-nextbkt3.service.d/clerk.conf` sets +`Environment=T3CODE_TELEMETRY_ENABLED=false`. To turn analytics on, add a second +drop-in named `zz-analytics.conf`. systemd reads drop-ins in name order, so this +one comes after `clerk.conf` and its value replaces the `false`: + +```ini +# /etc/systemd/system/t3-nextbkt3.service.d/zz-analytics.conf +[Service] +EnvironmentFile=-/home/ubuntu/.t3/nextbkt3-dev/posthog.env +Environment=T3CODE_TELEMETRY_ENABLED=true +``` + +```bash +sudo systemctl daemon-reload +sudo systemctl show t3-nextbkt3 -p Environment # expect T3CODE_TELEMETRY_ENABLED=true +``` + +The new environment takes effect at the next restart of the service. To turn +analytics off again, remove `zz-analytics.conf`, reload systemd and restart. +Provider and agent processes never inherit these variables: the server masks +them when it builds a provider's environment. + +Events carry ids, enums, numbers and booleans only. Free text (titles, prompts, +messages, paths, branches, errors) is dropped by the allowlist in +`apps/server/src/telemetry/analyticsProperties.ts`. An event with a known user +is sent as that user's `environment_users.primary_email`. + ## Temporary files `start.sh` exports `TMPDIR=/tmp` so the service and every agent session diff --git a/deploy/bkt3/posthog-dashboards.py b/deploy/bkt3/posthog-dashboards.py new file mode 100644 index 0000000000000..f905be391d8c0 --- /dev/null +++ b/deploy/bkt3/posthog-dashboards.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +"""Create the bkt3 product-metrics dashboards and alerts in PostHog (TEC-1631). + +Reads POSTHOG_PERSONAL_API_KEY and POSTHOG_HOST from the environment +(source ~/.config/posthog/dev-server-1.env) and POSTHOG_PROJECT_ID +(source ~/.t3/nextbkt3-dev/posthog.env). Idempotent: a dashboard that +already exists is left alone unless --replace is given. + + python3 deploy/bkt3/posthog-dashboards.py --check # run every query once + python3 deploy/bkt3/posthog-dashboards.py # create what is missing + python3 deploy/bkt3/posthog-dashboards.py --replace # delete and recreate + +Every query filters on the `instance` property, so other apps that share the +PostHog project do not leak into these numbers. +""" + +import json +import os +import sys +import urllib.error +import urllib.request + +INSTANCE = os.environ.get("T3CODE_ANALYTICS_INSTANCE", "bkt3") +F = f"properties.instance = '{INSTANCE}'" +TZ = "Asia/Kolkata" +DAYS = 90 + +# Time in each state comes from session.state.span. Spans are split at UTC +# midnight, so summing durationMs per day is exact. +SPAN = f"event = 'session.state.span' AND {F} AND timestamp > now() - INTERVAL {DAYS} DAY" + + +def table(sql): + return {"display": "ActionsTable", "sql": sql} + + +def line(sql): + return {"display": "ActionsLineGraph", "sql": sql} + + +def bar(sql): + return {"display": "ActionsStackedBar", "sql": sql} + + +DASHBOARDS = { + "bkt3 · Usage": { + "Daily active people": line(f""" + SELECT toDate(timestamp) AS day, count(DISTINCT distinct_id) AS people + FROM events + WHERE {F} AND timestamp > now() - INTERVAL {DAYS} DAY + AND event IN ('session.created', 'run.finished', 'composer.sent', + 'human_request.resolved', 'app.opened') + AND distinct_id LIKE '%@%' + GROUP BY day ORDER BY day"""), + "Sessions created per day, by person": bar(f""" + SELECT toDate(timestamp) AS day, distinct_id AS person, count() AS sessions + FROM events + WHERE event = 'session.created' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, person ORDER BY day"""), + "Runs per day, by provider": bar(f""" + SELECT toDate(timestamp) AS day, properties.providerInstanceId AS provider, + count() AS runs + FROM events + WHERE event = 'run.finished' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, provider ORDER BY day"""), + "Runs by hour of day (IST) and weekday, last 30 days": table(f""" + SELECT toDayOfWeek(toTimeZone(timestamp, '{TZ}')) AS weekday, + toHour(toTimeZone(timestamp, '{TZ}')) AS hour_ist, count() AS runs + FROM events + WHERE event = 'run.finished' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY weekday, hour_ist ORDER BY weekday, hour_ist"""), + "Sessions by source per day": bar(f""" + SELECT toDate(timestamp) AS day, properties.sessionSource AS source, + count() AS sessions + FROM events + WHERE event = 'session.created' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, source ORDER BY day"""), + "Client surfaces opened per day": bar(f""" + SELECT toDate(timestamp) AS day, properties.surface AS surface, count() AS opens + FROM events + WHERE event = 'app.opened' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, surface ORDER BY day"""), + }, + "bkt3 · Session time": { + "Hours per state per day": bar(f""" + SELECT toDate(timestamp) AS day, + if(properties.state = 'waiting_human', + concat('waiting_human:', toString(properties.waitKind)), + toString(properties.state)) AS state, + round(sum(toFloat(properties.durationMs)) / 3600000, 1) AS hours + FROM events + WHERE {SPAN} AND properties.state NOT IN ('idle', 'archived') + GROUP BY day, state ORDER BY day"""), + "Agent running hours per person per day": bar(f""" + SELECT toDate(timestamp) AS day, distinct_id AS person, + round(sum(toFloat(properties.durationMs)) / 3600000, 1) AS hours + FROM events + WHERE {SPAN} AND properties.state = 'agent_running' + GROUP BY day, person ORDER BY day"""), + "Time waiting on a person, by person and kind (last 30 days)": table(f""" + SELECT distinct_id AS person, properties.waitKind AS kind, + round(sum(toFloat(properties.durationMs)) / 3600000, 1) AS hours, + count() AS waits + FROM events + WHERE {SPAN} AND properties.state = 'waiting_human' + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY person, kind ORDER BY hours DESC"""), + "Time waiting on background workers per day": line(f""" + SELECT toDate(timestamp) AS day, + round(sum(toFloat(properties.durationMs)) / 3600000, 1) AS hours + FROM events + WHERE {SPAN} AND properties.state = 'waiting_workers' + GROUP BY day ORDER BY day"""), + "Person answer time (minutes), last 30 days": table(f""" + SELECT coalesce(properties.answeredByUserId, distinct_id) AS person, + properties.requestKind AS kind, count() AS answered, + round(quantile(0.5)(toFloat(properties.waitMs)) / 60000, 1) AS p50_min, + round(quantile(0.9)(toFloat(properties.waitMs)) / 60000, 1) AS p90_min + FROM events + WHERE event = 'human_request.resolved' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY person, kind ORDER BY answered DESC"""), + "Share of active time by state, per person (last 30 days)": table(f""" + SELECT distinct_id AS person, + round(sumIf(toFloat(properties.durationMs), properties.state = 'agent_running') / 3600000, 1) AS running_h, + round(sumIf(toFloat(properties.durationMs), properties.state = 'waiting_human') / 3600000, 1) AS waiting_human_h, + round(sumIf(toFloat(properties.durationMs), properties.state = 'waiting_workers') / 3600000, 1) AS waiting_workers_h, + round(sumIf(toFloat(properties.durationMs), properties.state = 'queued') / 3600000, 2) AS queued_h + FROM events + WHERE {SPAN} AND timestamp > now() - INTERVAL 30 DAY + GROUP BY person ORDER BY running_h DESC"""), + }, + "bkt3 · Agent quality": { + "Run outcome per day": bar(f""" + SELECT toDate(timestamp) AS day, properties.status AS status, count() AS runs + FROM events + WHERE event = 'run.finished' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, status ORDER BY day"""), + "Run time and failure rate by model (last 30 days)": table(f""" + SELECT properties.model AS model, count() AS runs, + round(quantile(0.5)(toFloat(properties.runDurationMs)) / 60000, 1) AS p50_min, + round(quantile(0.95)(toFloat(properties.runDurationMs)) / 60000, 1) AS p95_min, + round(100 * countIf(properties.status = 'failed') / count(), 1) AS failed_pct, + round(100 * countIf(properties.status = 'interrupted') / count(), 1) AS interrupted_pct + FROM events + WHERE event = 'run.finished' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY model ORDER BY runs DESC"""), + "Failure classes (last 30 days)": table(f""" + SELECT properties.failureClass AS failure, properties.providerInstanceId AS provider, + count() AS runs + FROM events + WHERE event = 'run.finished' AND properties.status = 'failed' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY failure, provider ORDER BY runs DESC"""), + "Queue wait p95 (seconds) per day": line(f""" + SELECT toDate(timestamp) AS day, + round(quantile(0.95)(toFloat(properties.queueWaitMs)) / 1000, 1) AS p95_s + FROM events + WHERE event = 'run.finished' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day ORDER BY day"""), + "Session recoveries per day": bar(f""" + SELECT toDate(timestamp) AS day, properties.outcome AS outcome, count() AS recoveries + FROM events + WHERE event = 'session.recovery.finished' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, outcome ORDER BY day"""), + "Subagents per day, by status": bar(f""" + SELECT toDate(timestamp) AS day, properties.status AS status, count() AS subagents + FROM events + WHERE event = 'subagent.finished' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, status ORDER BY day"""), + }, + "bkt3 · Load": { + "Peak concurrent sessions per hour, by state": line(f""" + SELECT toStartOfHour(timestamp) AS hour, + max(toFloat(properties.runningRuns)) AS running, + max(toFloat(properties.waitingHuman)) AS waiting_human, + max(toFloat(properties.waitingWorkers)) AS waiting_workers, + max(toFloat(properties.queuedRuns)) AS queued, + max(toFloat(properties.activeSubagents)) AS subagents + FROM events + WHERE event = 'session.state.snapshot' AND properties.scope = 'total' AND {F} + AND timestamp > now() - INTERVAL 14 DAY + GROUP BY hour ORDER BY hour"""), + "Peak running sessions per person per day": table(f""" + SELECT toDate(timestamp) AS day, distinct_id AS person, + max(toFloat(properties.count)) AS peak_running + FROM events + WHERE event = 'session.state.snapshot' AND properties.scope = 'user' + AND properties.state = 'agent_running' AND {F} + AND timestamp > now() - INTERVAL 14 DAY + GROUP BY day, person ORDER BY day DESC, peak_running DESC"""), + "Connected clients per day, by connection mode": bar(f""" + SELECT toDate(timestamp) AS day, properties.connectionMode AS mode, count() AS connects + FROM events + WHERE event = 'client.connected' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, mode ORDER BY day"""), + }, + "bkt3 · Automation": { + "MCP tool calls (last 30 days)": table(f""" + SELECT properties.toolName AS tool, properties.callerKind AS caller, count() AS calls, + round(100 * countIf(properties.ok = false) / count(), 1) AS error_pct, + round(quantile(0.95)(toFloat(properties.durationMs))) AS p95_ms + FROM events + WHERE event = 'mcp.tool.called' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY tool, caller ORDER BY calls DESC LIMIT 50"""), + "Scheduled task runs per day, by status": bar(f""" + SELECT toDate(timestamp) AS day, properties.status AS status, count() AS runs + FROM events + WHERE event = 'scheduled_task.ran' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, status ORDER BY day"""), + "Webhook deliveries per day": bar(f""" + SELECT toDate(timestamp) AS day, properties.kind AS kind, properties.status AS status, + count() AS deliveries + FROM events + WHERE event = 'webhook.delivered' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, kind, status ORDER BY day"""), + "Plans: decisions and wait (last 30 days)": table(f""" + SELECT properties.decision AS decision, count() AS plans, + round(quantile(0.5)(toFloat(properties.waitMs)) / 60000, 1) AS p50_wait_min + FROM events + WHERE event = 'plan.decided' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY decision ORDER BY plans DESC"""), + "Top client views (last 30 days)": table(f""" + SELECT properties.view AS view, count() AS views, count(DISTINCT distinct_id) AS people + FROM events + WHERE event = 'view.changed' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY view ORDER BY views DESC LIMIT 30"""), + }, +} + +# Alerts need a trends insight. Thresholds come from 30 days of history: +# a normal day has under 40 failed runs; the server sends a snapshot every 5 min. +ALERTS = [ + { + "insight": "Alert · server snapshots per hour", + "event": "session.state.snapshot", + "interval": "hour", + "properties": [{"key": "scope", "value": ["total"], "operator": "exact", "type": "event"}], + "alert": "bkt3 server stopped sending metrics", + "bounds": {"lower": 1}, + "calculation_interval": "hourly", + }, + { + "insight": "Alert · failed runs per day", + "event": "run.finished", + "interval": "day", + "properties": [{"key": "status", "value": ["failed"], "operator": "exact", "type": "event"}], + "alert": "bkt3 failed runs above 40 in a day", + "bounds": {"upper": 40}, + "calculation_interval": "daily", + }, +] + + +def api(method, path, body=None): + host = os.environ["POSTHOG_HOST"].rstrip("/") + req = urllib.request.Request( + f"{host}{path}", + method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={ + "Authorization": f"Bearer {os.environ['POSTHOG_PERSONAL_API_KEY']}", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(req, timeout=60) as res: + raw = res.read() + return json.loads(raw) if raw else None + except urllib.error.HTTPError as err: + raise SystemExit(f"{method} {path} -> {err.code}: {err.read().decode()[:500]}") + + +def hogql(sql): + return { + "kind": "DataVisualizationNode", + "source": {"kind": "HogQLQuery", "query": " ".join(sql.split())}, + } + + +def main(): + pid = os.environ["POSTHOG_PROJECT_ID"] + base = f"/api/projects/{pid}" + args = set(sys.argv[1:]) + + if "--check" in args: + for dash, insights in DASHBOARDS.items(): + for name, spec in insights.items(): + res = api("POST", f"{base}/query/", {"query": hogql(spec["sql"])["source"]}) + print(f"ok {dash} / {name}: {len(res.get('results') or [])} rows") + return + + existing = {d["name"]: d for d in api("GET", f"{base}/dashboards/?limit=200")["results"]} + for dash, insights in DASHBOARDS.items(): + if dash in existing: + if "--replace" not in args: + print(f"skip {dash} (exists, id {existing[dash]['id']})") + continue + api("PATCH", f"{base}/dashboards/{existing[dash]['id']}/", {"deleted": True}) + d = api("POST", f"{base}/dashboards/", {"name": dash, "description": "TEC-1631 bkt3 product metrics"}) + for name, spec in insights.items(): + query = hogql(spec["sql"]) + query["display"] = spec["display"] + api("POST", f"{base}/insights/", {"name": name, "query": query, "dashboards": [d["id"]]}) + print(f"made {dash} -> {os.environ['POSTHOG_HOST'].rstrip('/')}/project/{pid}/dashboard/{d['id']}") + + me = api("GET", "/api/users/@me/") + alerts = {a["name"] for a in (api("GET", f"{base}/alerts/") or {}).get("results", [])} + for spec in ALERTS: + if spec["alert"] in alerts: + print(f"skip alert {spec['alert']} (exists)") + continue + insight = api("POST", f"{base}/insights/", { + "name": spec["insight"], + "query": { + "kind": "InsightVizNode", + "source": { + "kind": "TrendsQuery", + "series": [{"kind": "EventsNode", "event": spec["event"], "math": "total"}], + "interval": spec["interval"], + "dateRange": {"date_from": "-7d"}, + "properties": [{"key": "instance", "value": [INSTANCE], "operator": "exact", "type": "event"}] + + spec["properties"], + }, + }, + }) + api("POST", f"{base}/alerts/", { + "name": spec["alert"], + "insight": insight["id"], + "subscribed_users": [me["id"]], + "threshold": {"configuration": {"type": "absolute", "bounds": spec["bounds"]}}, + "condition": {"type": "absolute_value"}, + "config": {"type": "TrendsAlertConfig", "series_index": 0}, + "calculation_interval": spec["calculation_interval"], + "enabled": True, + }) + print(f"made alert {spec['alert']}") + + +if __name__ == "__main__": + main() diff --git a/knip.jsonc b/knip.jsonc index 8e1b9a45161e2..607baf19f5d4f 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -37,6 +37,8 @@ "entry": [ "src/bin.ts!", "src/claude-history-worker.ts!", + // T3-CUSTOM(expbkt3): TEC-1631 product analytics + "scripts/analytics-backfill.ts", "scripts/cli.ts", "scripts/evaluate-thread-titles.ts", "scripts/measure-pr-preview.ts", diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 0059cddf08d8d..50ed7f11000a9 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -250,6 +250,23 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ }); export type ExecutionEnvironmentCapabilities = typeof ExecutionEnvironmentCapabilities.Type; +// T3-CUSTOM(expbkt3): BEGIN — product analytics config for the client (TEC-1631). +/** + * Where a client sends its product analytics for this environment. Present + * only when the server has an explicit PostHog project key and telemetry is + * on, so the client bundle never carries a key of its own. `posthogKey` is a + * public write-only `phc_` key. + */ +export const ExecutionEnvironmentAnalytics = Schema.Struct({ + enabled: Schema.Boolean, + posthogKey: TrimmedNonEmptyString, + posthogHost: TrimmedNonEmptyString, + /** Which deployment the events come from, such as `bkt3` or `expbkt3`. */ + instance: Schema.optionalKey(TrimmedNonEmptyString), +}); +export type ExecutionEnvironmentAnalytics = typeof ExecutionEnvironmentAnalytics.Type; +// T3-CUSTOM(expbkt3): END + export const ExecutionEnvironmentDescriptor = Schema.Struct({ environmentId: EnvironmentId, label: TrimmedNonEmptyString, @@ -258,6 +275,8 @@ export const ExecutionEnvironmentDescriptor = Schema.Struct({ /** Absent on hosts from before explicit orchestration protocol negotiation. */ orchestrationProtocolVersion: Schema.optionalKey(Schema.Int), capabilities: ExecutionEnvironmentCapabilities, + // T3-CUSTOM(expbkt3): absent on servers without product analytics (TEC-1631). + analytics: Schema.optionalKey(ExecutionEnvironmentAnalytics), }); export type ExecutionEnvironmentDescriptor = typeof ExecutionEnvironmentDescriptor.Type; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0b4b37d7b91be..aa73fdf2186fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -505,7 +505,7 @@ importers: version: 4.0.1(patch_hash=359f6fb2f7b3ec145bb72208edb9034f02489791aa2491a55cdbd69bd56ee0d2)(@types/node@24.12.4)(@vitest/ui@5.0.1)(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) '@pierre/trees': specifier: 1.0.0-beta.6 - version: 1.0.0-beta.6(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) + version: 1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) '@types/react': specifier: ~19.3.0 version: 19.3.0 @@ -680,7 +680,7 @@ importers: version: 1.5.2(patch_hash=b33a575eb1382270ed0df956307e4b6ffe4b8e4349cefeb43e14f648c0fdc3b0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) '@pierre/trees': specifier: 1.0.0-beta.6 - version: 1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0) + version: 1.0.0-beta.6(@pierre/theme@2.0.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0) '@platejs/basic-nodes': specifier: 53.0.0 version: 53.0.0(platejs@53.3.3(@types/react@19.3.0)(immer@10.2.0)(react-dom@19.2.6(react@19.2.6))(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.2.6)(utf-8-validate@6.0.6))(react@19.2.6)(scheduler@0.28.0)(use-sync-external-store@1.6.0(react@19.2.6)))(react-dom@19.2.6(react@19.2.6))(react@19.2.6) @@ -804,6 +804,9 @@ importers: platejs: specifier: 53.3.3 version: 53.3.3(@types/react@19.3.0)(immer@10.2.0)(react-dom@19.2.6(react@19.2.6))(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.2.6)(utf-8-validate@6.0.6))(react@19.2.6)(scheduler@0.28.0)(use-sync-external-store@1.6.0(react@19.2.6)) + posthog-js: + specifier: ^1.438.9 + version: 1.438.9(@types/react@19.3.0)(preact-render-to-string@6.6.5(preact@11.0.0-beta.0))(react@19.2.6) react: specifier: 19.2.6 version: 19.2.6 @@ -4676,6 +4679,15 @@ packages: '@polka/url@1.0.0-next.29': resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} + '@posthog/browser-common@0.9.4': + resolution: {integrity: sha512-QlG0WZFG9iTp9cmXTypB4cgnxBAD3GImrj9AwmA84uh8uy5vIVgVLjoTTdOAW29XL+kh/eLn/5ZtBQt2JQfR+w==} + + '@posthog/core@1.58.1': + resolution: {integrity: sha512-z0ISzMUoF1izukfLIUSjGZBdO5H2BPdqN8MX27OKYGQ1sDj/r+WW9Nq7m5WsQY56NOxVsFtHClx9iHj6MYM3Tg==} + + '@posthog/types@1.415.3': + resolution: {integrity: sha512-y90ARRBbyXcZDwuXXXvAnbjiuxQ+4FmUI4sasI19AHOPgdWPyd2Kpo0c0B2pKIR+fq+BcRokhF35eAdA54Mfbw==} + '@puppeteer/browsers@3.2.2': resolution: {integrity: sha512-q2BU4YfO9h/Wt7IcWPcggpOOqLk2Tbs1hDwolvKZrweRjy751OJBKMN9zO5bfD0pzU7X/tvKw/exQds4pM/LOg==} engines: {node: '>=22.12.0'} @@ -7161,6 +7173,9 @@ packages: core-js@3.47.0: resolution: {integrity: sha512-c3Q2VVkGAUyupsjRnaNX6u8Dq2vAdzm9iuPj5FW0fRxzlxgq9Q39MDq10IvmQSpLgHQNyQzQmOo6bgGHmH3NNg==} + core-js@3.50.0: + resolution: {integrity: sha512-BRWgOLKkFeCgRudR6zrs8p9XJZcE14grzKMMssoYrk6krtuEZ7MTKPIY5RzOnqsEKIR9kst7wNzphttraT+Yqw==} + core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} @@ -8331,6 +8346,9 @@ packages: ffi-rs@1.3.2: resolution: {integrity: sha512-4s8dX9VbBw/jd5NOuE3EJRqXaIVdjMyiumeeDzrOhtjQRwp6Bz2za7iksWXTnvTQKV/tTdm1s1w7mObe92zPjQ==} + fflate@0.4.9: + resolution: {integrity: sha512-zdxgIEddhfsyCaWpJ2SdXEP8ZMrKJ6+5jl4OupODcywU0IhRk6gdXuVGcPICyfx2H97hVK7xmJtRLPjkxAX8Vw==} + fflate@0.8.3: resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} @@ -10115,6 +10133,17 @@ packages: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} + posthog-js@1.438.9: + resolution: {integrity: sha512-/KTZQF3sPlQbgLAlSncxv3z8dz4yex+JmIbCmGKIYkIVupdqESiqXx3SadPkWKgqgHtnQGbPSqRVe4vgpXnrGw==} + peerDependencies: + '@types/react': '>=16.8.0' + react: '>=16.8.0' + peerDependenciesMeta: + '@types/react': + optional: true + react: + optional: true + postject@1.0.0-alpha.6: resolution: {integrity: sha512-b9Eb8h2eVqNE8edvKdwqkrY6O7kAwmI8kcnBv1NScolYJbo59XUF0noFq+lxbC1yN20bmC0WBEbDC5H/7ASb0A==} engines: {node: '>=14.0.0'} @@ -10125,6 +10154,14 @@ packages: peerDependencies: preact: '>=10 || >= 11.0.0-0' + preact@10.29.8: + resolution: {integrity: sha512-ej2aVZ+vZ8WO7tvlQWRM9N63A0KzF9q4mWJfDUHgYaIofWY9hu74QdnQrjoPMmZi2/nZ5gN0bJCQF49xQqx09Q==} + peerDependencies: + preact-render-to-string: '>=5' + peerDependenciesMeta: + preact-render-to-string: + optional: true + preact@11.0.0-beta.0: resolution: {integrity: sha512-IcODoASASYwJ9kxz7+MJeiJhvLriwSb4y4mHIyxdgaRZp6kPUud7xytrk/6GZw8U3y6EFJaRb5wi9SrEK+8+lg==} @@ -10252,6 +10289,9 @@ packages: resolution: {integrity: sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==} engines: {node: '>=0.6'} + query-selector-shadow-dom@1.0.1: + resolution: {integrity: sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==} + query-string@7.1.3: resolution: {integrity: sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==} engines: {node: '>=6'} @@ -11741,6 +11781,12 @@ packages: web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + web-vitals@6.2.1: + resolution: {integrity: sha512-rLcLXA2sx6+9dE88NHFubwTtGxpK4yYBLj6qHPdFoCaLr0cXGb4efOqtKLlm4loGA4OEKHIQKMKzZkKyOh5ctw==} + + web-vitals@6.2.3: + resolution: {integrity: sha512-Dmy+vdO1UPMqXiAtZSNayhl4f794Bxfd94W4HIj2HlYtczEQfEOon969memlcpcHrTuOKLa5baFjX7WbgmOq+A==} + webcrypto-core@1.9.2: resolution: {integrity: sha512-gsXecm82UQNlTBURJGuqOWy1Ww08S3kZUcr3aOJS02Pk0xLtkfeUAVC0u0xhgdonFme80edSJUIJyuvL/7250Q==} @@ -15358,20 +15404,20 @@ snapshots: '@pierre/theme@2.0.0': {} - '@pierre/theming@1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0)': + '@pierre/theming@1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': optionalDependencies: '@pierre/theme': 1.1.0 - '@shikijs/themes': 4.3.0 - react: 19.2.6 - react-dom: 19.2.6(react@19.2.6) + '@shikijs/themes': 4.2.0 + react: 19.3.0 + react-dom: 19.3.0(react@19.3.0) shiki: 4.2.0 - '@pierre/theming@1.0.0(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': + '@pierre/theming@1.0.0(@pierre/theme@2.0.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0)': optionalDependencies: '@pierre/theme': 2.0.0 - '@shikijs/themes': 4.2.0 - react: 19.3.0 - react-dom: 19.3.0(react@19.3.0) + '@shikijs/themes': 4.3.0 + react: 19.2.6 + react-dom: 19.2.6(react@19.2.6) shiki: 4.2.0 '@pierre/theming@1.0.1(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': @@ -15390,25 +15436,25 @@ snapshots: react-dom: 19.2.6(react@19.2.6) shiki: 4.2.0 - '@pierre/trees@1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0)': + '@pierre/trees@1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': dependencies: - '@pierre/theming': 1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0) + '@pierre/theming': 1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) preact: 11.0.0-beta.0 preact-render-to-string: 6.6.5(preact@11.0.0-beta.0) - react: 19.2.6 - react-dom: 19.2.6(react@19.2.6) + react: 19.3.0 + react-dom: 19.3.0(react@19.3.0) transitivePeerDependencies: - '@pierre/theme' - '@shikijs/themes' - shiki - '@pierre/trees@1.0.0-beta.6(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': + '@pierre/trees@1.0.0-beta.6(@pierre/theme@2.0.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0)': dependencies: - '@pierre/theming': 1.0.0(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) + '@pierre/theming': 1.0.0(@pierre/theme@2.0.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0) preact: 11.0.0-beta.0 preact-render-to-string: 6.6.5(preact@11.0.0-beta.0) - react: 19.3.0 - react-dom: 19.3.0(react@19.3.0) + react: 19.2.6 + react-dom: 19.2.6(react@19.2.6) transitivePeerDependencies: - '@pierre/theme' - '@shikijs/themes' @@ -15585,6 +15631,17 @@ snapshots: '@polka/url@1.0.0-next.29': {} + '@posthog/browser-common@0.9.4': + dependencies: + '@posthog/core': 1.58.1 + '@posthog/types': 1.415.3 + + '@posthog/core@1.58.1': + dependencies: + '@posthog/types': 1.415.3 + + '@posthog/types@1.415.3': {} + '@puppeteer/browsers@3.2.2(yauzl@3.4.0)': dependencies: modern-tar: 0.8.5 @@ -16016,7 +16073,7 @@ snapshots: '@shikijs/types@4.2.0': dependencies: '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@shikijs/types@4.3.0': dependencies: @@ -18097,6 +18154,8 @@ snapshots: core-js@3.47.0: {} + core-js@3.50.0: {} + core-util-is@1.0.2: {} cors@2.8.6: @@ -19309,10 +19368,6 @@ snapshots: dependencies: walk-up-path: 4.0.0 - fdir@6.5.0(picomatch@4.0.4): - optionalDependencies: - picomatch: 4.0.4 - fdir@6.5.0(picomatch@4.0.7): optionalDependencies: picomatch: 4.0.7 @@ -19333,6 +19388,8 @@ snapshots: '@yuuang/ffi-rs-win32-ia32-msvc': 1.3.2 '@yuuang/ffi-rs-win32-x64-msvc': 1.3.2 + fflate@0.4.9: {} + fflate@0.8.3: {} filelist@1.0.6: @@ -21668,6 +21725,24 @@ snapshots: xtend: 4.0.2 optional: true + posthog-js@1.438.9(@types/react@19.3.0)(preact-render-to-string@6.6.5(preact@11.0.0-beta.0))(react@19.2.6): + dependencies: + '@posthog/browser-common': 0.9.4 + '@posthog/core': 1.58.1 + '@posthog/types': 1.415.3 + core-js: 3.50.0 + dompurify: 3.4.16 + fflate: 0.4.9 + preact: 10.29.8(preact-render-to-string@6.6.5(preact@11.0.0-beta.0)) + query-selector-shadow-dom: 1.0.1 + web-vitals: 6.2.3 + web-vitals-soft-navs: web-vitals@6.2.1 + optionalDependencies: + '@types/react': 19.3.0 + react: 19.2.6 + transitivePeerDependencies: + - preact-render-to-string + postject@1.0.0-alpha.6: dependencies: commander: 9.5.0 @@ -21677,6 +21752,10 @@ snapshots: dependencies: preact: 11.0.0-beta.0 + preact@10.29.8(preact-render-to-string@6.6.5(preact@11.0.0-beta.0)): + optionalDependencies: + preact-render-to-string: 6.6.5(preact@11.0.0-beta.0) + preact@11.0.0-beta.0: {} prettier@3.8.3: {} @@ -21832,6 +21911,8 @@ snapshots: es-define-property: 1.0.1 side-channel: 1.1.1 + query-selector-shadow-dom@1.0.1: {} + query-string@7.1.3: dependencies: decode-uri-component: 0.2.2 @@ -22587,7 +22668,7 @@ snapshots: '@shikijs/themes': 4.2.0 '@shikijs/types': 4.2.0 '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 should-equal@2.0.0: dependencies: @@ -22971,8 +23052,8 @@ snapshots: tinyglobby@0.2.17: dependencies: - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 tinypool@2.1.2: {} @@ -23573,6 +23654,10 @@ snapshots: web-namespaces@2.0.1: {} + web-vitals@6.2.1: {} + + web-vitals@6.2.3: {} + webcrypto-core@1.9.2: dependencies: '@peculiar/asn1-schema': 2.8.0