From dd34521a45aacaaf488b58f67b922534c73842d1 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Sat, 10 Oct 2026 16:27:06 +0000 Subject: [PATCH 01/20] fix(server): accept up to 20 MCP OAuth redirect URIs so Google can register Google's MCP connector registers more than 5 redirect URIs, and the register endpoint refused it with invalid_redirect_uri. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/auth/McpOAuth.test.ts | 29 +++++++++++++++++++++++++++ apps/server/src/auth/McpOAuth.ts | 3 ++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/apps/server/src/auth/McpOAuth.test.ts b/apps/server/src/auth/McpOAuth.test.ts index 1273b350281a3..e4de0b0187486 100644 --- a/apps/server/src/auth/McpOAuth.test.ts +++ b/apps/server/src/auth/McpOAuth.test.ts @@ -595,3 +595,32 @@ it("matches loopback redirects on everything but the port, and https redirects e expect(matches("https://bot.example/cb", "https://bot.example/cb?x=1")).toBe(false); expect(matches("https://bot.example/cb", "https://evil.example/cb")).toBe(false); }); + +// T3-CUSTOM(expbkt3): BEGIN — hosted connectors such as Google's register many redirect URIs. +it.live("registers a hosted client with up to 20 https redirect URIs", () => + withRoutes((handler) => + Effect.gen(function* () { + const registerMany = (count: number) => + handler( + at("/oauth/mcp/register", { + method: "POST", + headers: { "content-type": "application/json" }, + body: encodeJson({ + client_name: "Google", + redirect_uris: Array.from( + { length: count }, + (_, index) => `https://connector${index}.example/oauth-redirect`, + ), + }), + }), + ); + + expect((yield* registerMany(8)).status).toBe(201); + expect((yield* registerMany(20)).status).toBe(201); + const tooMany = yield* registerMany(21); + expect(tooMany.status).toBe(400); + expect(yield* json(tooMany)).toMatchObject({ error: "invalid_redirect_uri" }); + }), + ), +); +// T3-CUSTOM(expbkt3): END diff --git a/apps/server/src/auth/McpOAuth.ts b/apps/server/src/auth/McpOAuth.ts index 073e0856de314..c48ba746953fc 100644 --- a/apps/server/src/auth/McpOAuth.ts +++ b/apps/server/src/auth/McpOAuth.ts @@ -63,7 +63,8 @@ import { const SIGNING_SECRET_NAME = "mcp-oauth-signing-key"; const AUTHORIZATION_CODE_TTL_MS = 60_000; const MAX_CLIENT_NAME_LENGTH = 100; -const MAX_REDIRECT_URIS = 5; +// T3-CUSTOM(expbkt3): Google's MCP connector registers more than upstream's 5 redirect URIs. +const MAX_REDIRECT_URIS = 20; const MAX_REDIRECT_URI_LENGTH = 512; const DEFAULT_CLIENT_NAME = "MCP client"; const LOOPBACK_HOSTNAMES = new Set(["localhost", "127.0.0.1", "[::1]"]); From c06d3c4d8341b2db4a9b5c0d8b774e53bca3f409 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Mon, 12 Oct 2026 07:24:45 +0000 Subject: [PATCH 02/20] fix(mcp): t3_answer sees Codex async questions and says which questions block A Codex async question is a runtime request answered by message. The shell leaves those out of pendingRuntimeRequest and only flags them, so the agent toolkit's loader read nothing: t3_inbox showed the question without options and t3_answer said nothing waits. The loader now reads pending message-mode requests whenever the shell flags one. Every question shows whether it blocks the agent; t3_answer takes a short requestId, reports how a non-blocking answer reached the agent, and dismisses a non-blocking question with dismiss: true. t3_send_message no longer refuses for a non-blocking question. Co-Authored-By: Claude Opus 5.5 --- .../server/src/mcp/toolkits/agent/handlers.ts | 214 ++++++++++++++---- .../src/mcp/toolkits/agent/instructions.ts | 2 +- .../src/mcp/toolkits/agent/pending.test.ts | 169 ++++++++++++++ apps/server/src/mcp/toolkits/agent/pending.ts | 59 ++++- .../src/mcp/toolkits/agent/render.test.ts | 88 +++++++ apps/server/src/mcp/toolkits/agent/render.ts | 52 ++++- apps/server/src/mcp/toolkits/agent/tools.ts | 16 +- apps/server/src/mcp/toolkits/agent/view.ts | 4 +- .../src/mcp/toolkits/agent/wait.test.ts | 41 +++- apps/server/src/mcp/toolkits/agent/wait.ts | 8 +- .../scheduledTaskWatch.expbkt3.test.ts | 9 +- .../scheduledTaskWatch.expbkt3.ts | 6 +- docs/user/t3-mcp-control.md | 11 +- 13 files changed, 605 insertions(+), 74 deletions(-) create mode 100644 apps/server/src/mcp/toolkits/agent/pending.test.ts create mode 100644 apps/server/src/mcp/toolkits/agent/render.test.ts diff --git a/apps/server/src/mcp/toolkits/agent/handlers.ts b/apps/server/src/mcp/toolkits/agent/handlers.ts index 71ac84a200b6e..2118c276245fa 100644 --- a/apps/server/src/mcp/toolkits/agent/handlers.ts +++ b/apps/server/src/mcp/toolkits/agent/handlers.ts @@ -50,7 +50,9 @@ import { modelSelectionFor, resolveModel } from "./models.ts"; import { EMPTY_PENDING, loadPendingDetail, + matchesRequest, planIsCurrent, + requestRef, type PendingApproval, type PendingDetail, type PendingQuestion, @@ -196,9 +198,11 @@ const sessionCursor = (caller: Caller, threadId: ThreadId) => const waitHint = (ref: string, cursor: string | undefined) => callHint("t3_wait", { sessions: [ref], ...(cursor === undefined ? {} : { cursor }) }); -/** A question as `format: "json"` returns it: long texts cut like the Markdown. */ -const boundQuestion = (pending: PendingQuestion): PendingQuestion => ({ +/** A question as `format: "json"` returns it: long texts cut like the Markdown, and whether it blocks the agent. */ +const boundQuestion = (pending: PendingQuestion) => ({ ...pending, + requestRef: requestRef(pending.requestId), + blocksAgent: !pending.async, questions: pending.questions.map((question) => ({ ...question, question: cutText(question.question, 1_000), @@ -371,17 +375,20 @@ const inbox = Effect.fn("agentToolkit.inbox")(function* (input: Tool.Parameters< // Each last-reply read scans that session's timeline index; a few are enough. const readReply = candidate.kind === "unread" && unreadReads < 8; if (readReply) unreadReads += 1; + const question = + candidate.kind === "question" + ? detail.question + : candidate.kind === "async-question" + ? (detail.asyncQuestions[0] ?? null) + : null; + // The shell has no time for a non-blocking question; the question has. + const askedMs = question?.async === true ? toMillisOrNull(question.askedAt) : null; return { kind: candidate.kind, shell: candidate.shell, context: context(candidate.shell), - waitingSinceMs: candidate.waitingSinceMs, - question: - candidate.kind === "question" - ? detail.question - : candidate.kind === "async-question" - ? (detail.asyncQuestions[0] ?? null) - : null, + waitingSinceMs: askedMs ?? candidate.waitingSinceMs, + question, approval: candidate.kind === "approval" ? detail.approval : null, plan: candidate.kind === "plan" ? detail.plan : null, lastReply: readReply ? yield* lastReply(caller, candidate.shell) : null, @@ -438,6 +445,71 @@ const inbox = Effect.fn("agentToolkit.inbox")(function* (input: Tool.Parameters< // -- t3_answer ----------------------------------------------------------------- +/** An ISO time as epoch milliseconds, or null. */ +const toMillisOrNull = (iso: string | null) => { + if (iso === null) return null; + const ms = Date.parse(iso); + return Number.isNaN(ms) ? null : ms; +}; + +/** The open questions, one line each, for a refusal that names the right requestId. */ +const describeOpenQuestions = (open: ReadonlyArray) => + open + .map( + (question, index) => + `${index + 1}. requestId "${requestRef(question.requestId)}" (${question.async ? "non-blocking" : "blocking"}): ${oneLine(question.questions[0]?.question ?? "a question", 100)}`, + ) + .join("; "); + +const DELIVERY_TEXT = { + steered: "It reached the agent as a message in its running turn.", + started: "It reached the agent as a message and started a new turn.", + queued: "It waits as a message and runs after the current turn.", +} as const; + +type AnswerDelivery = keyof typeof DELIVERY_TEXT; + +/** + * How an answer to a non-blocking question reached the agent. The answer is + * sent as a message with a fixed id, and its user turn item records whether + * it joined the running turn, started one or waits behind it (as + * ThreadManagementService.sendToThread reads it). Null when that cannot be read. + */ +const answerDelivery = ( + caller: Caller, + threadId: ThreadId, + requestId: string, +): Effect.Effect => + Effect.gen(function* () { + const messageId = MessageId.make(`async-answer:${requestId}`); + const { messages, runs } = yield* caller.threads.getThreadRecords( + threadId, + ["messages", "runs"], + { messageIds: [messageId] }, + ); + const runId = messages.find((message) => message.id === messageId)?.runId ?? null; + if (runId === null) return null; + const { turnItems } = yield* caller.threads.getThreadRecords(threadId, ["turnItems"], { + turnItemTypes: ["user_message"], + turnItemRunIds: [runId], + }); + const item = turnItems.find( + (candidate) => candidate.type === "user_message" && candidate.messageId === messageId, + ); + // A queued message gets its turn item only when its turn starts. + const delivery: AnswerDelivery | null = + item?.type !== "user_message" + ? runs.find((run) => run.id === runId)?.status === "queued" + ? "queued" + : null + : item.inputIntent === "queued_turn" + ? "queued" + : item.inputIntent === "turn_start" + ? "started" + : "steered"; + return delivery; + }).pipe(Effect.orElseSucceed(() => null)); + const DECISIONS: Record = { approve: "accept", approve_for_session: "acceptForSession", @@ -453,11 +525,42 @@ const describeWaiting = (shell: SessionShell, detail: PendingDetail) => { return `It waits on an approval: pass decision (${callHint("t3_answer", { session: ref, decision: "approve" })}).`; if (detail.question !== null || detail.asyncQuestions.length > 0) return `It waits on a question: pass choice, text or answers (see ${callHint("t3_session_brief", { session: ref })}).`; - if (shell.hasActionableProposedPlan && attentionOf(shell) === "plan") + const attention = attentionOf(shell); + if (shell.hasActionableProposedPlan && attention === "plan") return `It waits on a plan review: pass plan (${callHint("t3_answer", { session: ref, plan: "approve" })}).`; + if (attention === "question" || attention === "async-question" || attention === "approval") + return `It reports an open ${attention === "approval" ? "approval" : "question"}, but T3 Code could not read it just now. Read it with ${callHint("t3_session_brief", { session: ref })}, then try again.`; return `Nothing waits in it. To tell the agent something, use ${callHint("t3_send_message", { session: ref, message: "…" })}. If you sent this answer before, it was delivered.`; }; +/** The reply to t3_answer: what it did, how a message answer reached the agent, and where the session stands now. */ +const answerResult = Effect.fn("agentToolkit.answerResult")(function* (input: { + readonly caller: Caller; + readonly shell: SessionShell; + readonly cursor: string | undefined; + readonly nowMs: number; + readonly origin: string | undefined; + readonly answered: string; + readonly delivery: AnswerDelivery | null; +}) { + const { caller, shell, cursor, nowMs, answered, delivery } = input; + const after = yield* caller.threads + .getThreadShell(shell.id) + .pipe(Effect.orElseSucceed(() => null)); + const now = after ?? shell; + const projects = yield* loadProjects(); + const context = contextFor(caller, projects, nowMs, input.origin)(now); + return result( + [ + `## ${answered}`, + ...(delivery === null ? [] : [DELIVERY_TEXT[delivery]]), + `${sessionHeading(now, context)} — ${sessionActivity(now, nowMs)}`, + `Next: ${waitHint(shortRef(shell.id), cursor)} to hear when it needs you again.`, + ].join("\n"), + { answered, delivery, session: sessionData(now, context), cursor: cursor ?? null }, + ); +}); + const answer = Effect.fn("agentToolkit.answer")(function* ( input: Tool.Parameters, ) { @@ -481,12 +584,13 @@ const answer = Effect.fn("agentToolkit.answer")(function* ( input.choice !== undefined || input.text !== undefined || input.answers !== undefined ? "question" : null, + input.dismiss === true ? "dismiss" : null, ].filter((part) => part !== null); if (asked.length !== 1) { return yield* fail( "answer_required", asked.length === 0 - ? "Pass one answer: choice, text or answers for a question; decision for an approval; or plan for a plan review." + ? "Pass one answer: choice, text or answers for a question (or dismiss: true for a non-blocking one); decision for an approval; or plan for a plan review." : `Pass one kind of answer per call; you passed ${asked.join(" and ")}.`, ); } @@ -566,7 +670,7 @@ const answer = Effect.fn("agentToolkit.answer")(function* ( const approval = detail.approval; if ( approval === null || - (input.requestId !== undefined && input.requestId !== approval.requestId) + (input.requestId !== undefined && !matchesRequest(approval.requestId, input.requestId)) ) { return yield* fail( "nothing_waiting", @@ -599,17 +703,56 @@ const answer = Effect.fn("agentToolkit.answer")(function* ( .pipe(Effect.mapError(dispatchError), withModeLimit(caller)); answered = `Sent ${input.decision} for the ${approval.requestKind} request${approval.prompt === null ? "" : ` \`${oneLine(approval.prompt, 120)}\``}.`; } else { + // The blocking question first, then the non-blocking ones, oldest first. const open = [...(detail.question === null ? [] : [detail.question]), ...detail.asyncQuestions]; + const dismiss = input.dismiss === true; + const wanted = input.requestId; const target = - input.requestId === undefined - ? open[0] - : open.find((candidate) => candidate.requestId === input.requestId); + wanted === undefined + ? (dismiss ? open.filter((candidate) => candidate.async) : open)[0] + : open.find((candidate) => matchesRequest(candidate.requestId, wanted)); if (target === undefined) { return yield* fail( "nothing_waiting", - `No such question waits in ${ref}. ${describeWaiting(shell, detail)}`, + open.length === 0 + ? `No question waits in ${ref}. ${describeWaiting(shell, detail)}` + : wanted !== undefined + ? `No open question in ${ref} has requestId "${wanted}"; it was answered or dismissed already, or the id is wrong. Open questions: ${describeOpenQuestions(open)}.` + : `${ref} has no non-blocking question to dismiss. Open questions: ${describeOpenQuestions(open)}.`, ); } + // Codex heads every question "Question", so name it by its text. + const header = oneLine(target.questions[0]?.question ?? "the question", 80); + if (dismiss) { + if (!target.async) { + return yield* fail( + "not_dismissible", + `This question blocks the agent, so it cannot be dismissed. Answer it, or stop the turn with ${callHint("t3_send_message", { session: ref, message: "…", ifBusy: "interrupt", force: true })}.`, + ); + } + yield* caller.threads + .dispatch({ + type: "thread.user-input.dismiss", + commandId: yield* commandIdFor( + scope, + "dismiss", + `${shell.id}\u0000${target.requestId}`, + input.clientRequestId, + ), + threadId: shell.id, + requestId: RuntimeRequestId.make(target.requestId), + }) + .pipe(Effect.mapError(dispatchError), withModeLimit(caller)); + return yield* answerResult({ + caller, + shell, + cursor, + nowMs, + origin, + answered: `Dismissed the non-blocking question "${header}". The agent is not told.`, + delivery: null, + }); + } const answers = buildAnswers(target.questions, input, target.textOnly); if (Result.isFailure(answers)) return yield* fail("invalid_answer", answers.failure); yield* caller.threads @@ -626,27 +769,17 @@ const answer = Effect.fn("agentToolkit.answer")(function* ( answers: answers.success, }) .pipe(Effect.mapError(dispatchError), withModeLimit(caller)); - answered = `Answered ${target.questions.length === 1 ? `"${oneLine(target.questions[0]?.header ?? "the question", 80)}"` : `${target.questions.length} questions`}: ${Object.values( + answered = `Answered ${target.questions.length === 1 ? `"${header}"` : `${target.questions.length} questions`}: ${Object.values( answers.success, ) .map((value) => (typeof value === "string" ? oneLine(value, 80) : value.join(", "))) .join(" · ")}.`; + if (target.async) { + const delivery = yield* answerDelivery(caller, shell.id, target.requestId); + return yield* answerResult({ caller, shell, cursor, nowMs, origin, answered, delivery }); + } } - - const after = yield* caller.threads - .getThreadShell(shell.id) - .pipe(Effect.orElseSucceed(() => null)); - const now = after ?? shell; - const projects = yield* loadProjects(); - const context = contextFor(caller, projects, nowMs, origin)(now); - return result( - [ - `## ${answered}`, - `${sessionHeading(now, context)} — ${sessionActivity(now, nowMs)}`, - `Next: ${waitHint(ref, cursor)} to hear when it needs you again.`, - ].join("\n"), - { answered, session: sessionData(now, context), cursor: cursor ?? null }, - ); + return yield* answerResult({ caller, shell, cursor, nowMs, origin, answered, delivery: null }); }); // -- t3_find_sessions ------------------------------------------------------------ @@ -864,9 +997,9 @@ const brief = Effect.fn("agentToolkit.brief")(function* ( if (detail.approval !== null) lines.push("", "### Waits on an approval", ...renderApproval(detail.approval, ref)); if (detail.question !== null) - lines.push("", "### Waits on a question", ...renderQuestion(detail.question, ref)); + lines.push("", "### Blocking question", ...renderQuestion(detail.question, ref)); for (const question of detail.asyncQuestions) { - lines.push("", "### Asked while it keeps working", ...renderQuestion(question, ref)); + lines.push("", "### Non-blocking question", ...renderQuestion(question, ref)); } if ((attention === "plan" && planIsCurrent(shell, detail.plan)) || include.has("plan")) { lines.push("", "### Plan", ...renderPlan(detail.plan, ref, 0)); @@ -1154,13 +1287,11 @@ const sendMessage = Effect.fn("agentToolkit.sendMessage")(function* ( const text = input.message.trim(); if (text.length === 0) return yield* fail("message_required", "Pass message: the text to send."); const attention = attentionOf(shell); - if ( - input.force !== true && - (attention === "question" || attention === "approval" || attention === "async-question") - ) { + // A non-blocking question did not stop the agent, so a message does not drop it. + if (input.force !== true && (attention === "question" || attention === "approval")) { return yield* fail( "question_open", - `${ref} waits on ${attention === "approval" ? "an approval" : "a question"}. Answer it with t3_answer (see ${callHint("t3_session_brief", { session: ref })}), or pass force: true to send anyway; the agent may then drop the open ${attention === "approval" ? "approval" : "question"}.`, + `${ref} waits on ${attention === "approval" ? "an approval" : "a blocking question"}. Answer it with t3_answer (see ${callHint("t3_session_brief", { session: ref })}), or pass force: true to send anyway; the agent may then drop the open ${attention === "approval" ? "approval" : "question"}.`, ); } yield* assertTargetWithinLimits(caller.limits, shell); @@ -1201,6 +1332,11 @@ const sendMessage = Effect.fn("agentToolkit.sendMessage")(function* ( [ `## Message ${delivery}`, `\`${ref}\` ${shell.title} · run ${sent.run.id} (${sent.run.status})`, + ...(attention === "async-question" + ? [ + `A non-blocking question stays open in ${ref}. Answer or dismiss it with t3_answer (see ${callHint("t3_session_brief", { session: ref })}).`, + ] + : []), `Next: ${waitHint(ref, cursor)} to hear when it answers, asks or fails.`, ].join("\n"), { diff --git a/apps/server/src/mcp/toolkits/agent/instructions.ts b/apps/server/src/mcp/toolkits/agent/instructions.ts index 1de7bef2cf899..0a230fa7e1c32 100644 --- a/apps/server/src/mcp/toolkits/agent/instructions.ts +++ b/apps/server/src/mcp/toolkits/agent/instructions.ts @@ -6,7 +6,7 @@ export const MCP_SERVER_INSTRUCTIONS = [ "T3 Code runs coding-agent sessions for people. Start with t3_whoami.", "- t3_inbox: everything that waits on a person (questions with options, approvals, plans, failures), each with a ready t3_answer call.", - "- t3_answer: answer a question (choice, text, answers), an approval (decision) or a plan (approve, revise with feedback, discard).", + "- t3_answer: answer a question (choice, text, answers), an approval (decision) or a plan (approve, revise with feedback, discard). A blocking question pauses the agent; a non-blocking one (a Codex async question) does not, so its answer arrives as a new message, or dismiss it.", "- t3_find_sessions, then t3_session_brief: find a session and read where it stands.", "- t3_start_session and t3_send_message: start work, or send to a session.", "- t3_wait: wait for the next question, approval, finish or failure with a cursor. Do not poll t3_inbox.", diff --git a/apps/server/src/mcp/toolkits/agent/pending.test.ts b/apps/server/src/mcp/toolkits/agent/pending.test.ts new file mode 100644 index 0000000000000..9b6e65e0b4bc3 --- /dev/null +++ b/apps/server/src/mcp/toolkits/agent/pending.test.ts @@ -0,0 +1,169 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; + +import type { Caller } from "../../threadAccess.ts"; +import { loadPendingDetail, matchesRequest, requestRef } from "./pending.ts"; +import type { SessionShell } from "./view.ts"; + +// The ids and texts of a Codex async question on bkt3 (2026-10-12), which +// t3_answer could not see: the shell only flags it. +const ASYNC_ID = + "runtime-request:provider:codex:provider-turn:provider-turn%3Aprovider%3Acodex%3Anative-turn%3A01a12ec9-fa33-7680-9a4e-a5e0af57da87:native-request:async%3Acall_04f18b02c891414083d01ab154ff366e:9e9c4c3e-50f1-4f9e-a521-b41c3a04c7af"; +const BLOCKING_ID = + "runtime-request:provider:codex:provider-turn:provider-turn%3Aprovider%3Acodex%3Anative-turn%3A01a12ec9-fa33-7680-9a4e-a5e0af57da87:native-request:7:2b982706-8007-48e8-8e46-f0308942084c"; +const ASKED = "2026-10-12T06:34:48.759Z"; +const OPTION = "Prime-service starts dbt four times daily (recommended)."; + +const runtimeRequest = (fields: Record) => ({ + nodeId: "node-1", + providerTurnId: "provider-turn-1", + nativeRequestRef: null, + kind: "user_input", + status: "pending", + createdAt: DateTime.makeUnsafe(ASKED), + resolvedAt: null, + ...fields, +}); + +const questionItem = (requestId: string, question: string, responseMode?: "message") => ({ + id: `turn-item:${requestId}`, + type: "user_input_request", + status: "waiting", + requestId, + runId: "run-1", + updatedAt: DateTime.makeUnsafe(ASKED), + questions: [ + { + id: "0", + header: "Question", + question, + options: [{ label: OPTION, description: OPTION }], + }, + ], + ...(responseMode === undefined ? {} : { responseMode }), +}); + +// Only the fields the loader reads. +const shell = (fields: Partial>): SessionShell => + ({ + id: "86157111-97e0-449f-9a7e-f65b82072544", + pendingRuntimeRequest: null, + hasPendingAsyncUserInput: false, + hasActionableProposedPlan: false, + ...fields, + }) as unknown as SessionShell; + +const callerWith = (records: { + readonly runtimeRequests: ReadonlyArray; + readonly turnItems: ReadonlyArray; +}) => { + const reads: Array = []; + const caller = { + threads: { + getThreadRecords: (_threadId: unknown, _fields: unknown, filter: unknown) => { + reads.push(filter); + return Effect.succeed(records); + }, + }, + } as unknown as Caller; + return { caller, reads }; +}; + +describe("loadPendingDetail", () => { + it.effect("reads a Codex async question that the shell only flags", () => + Effect.gen(function* () { + const { caller, reads } = callerWith({ + runtimeRequests: [ + runtimeRequest({ id: ASYNC_ID, responseCapability: { type: "message" } }), + // Answered already: not open. + runtimeRequest({ + id: "runtime-request:old:11111111-2222-3333-4444-555555555555", + status: "resolved", + responseCapability: { type: "message" }, + }), + ], + turnItems: [ + questionItem( + ASYNC_ID, + "TEC-1582: Which DAILY refresh trigger must I implement?", + "message", + ), + questionItem( + "runtime-request:old:11111111-2222-3333-4444-555555555555", + "Old", + "message", + ), + ], + }); + const detail = yield* loadPendingDetail(caller, shell({ hasPendingAsyncUserInput: true }), { + includePlan: false, + }); + expect(reads).toEqual([{ turnItemTypes: ["user_input_request"] }]); + expect(detail.question).toBeNull(); + expect(detail.asyncQuestions).toEqual([ + { + requestId: ASYNC_ID, + questions: [expect.objectContaining({ question: expect.stringContaining("DAILY") })], + textOnly: true, + async: true, + askedAt: ASKED, + }, + ]); + }), + ); + + it.effect("keeps a blocking question apart from a non-blocking one", () => + Effect.gen(function* () { + const { caller } = callerWith({ + runtimeRequests: [ + runtimeRequest({ + id: BLOCKING_ID, + responseCapability: { type: "live", providerSessionId: "provider-session-1" }, + }), + runtimeRequest({ id: ASYNC_ID, responseCapability: { type: "message" } }), + ], + turnItems: [ + questionItem(BLOCKING_ID, "Blocking?"), + questionItem(ASYNC_ID, "Later?", "message"), + ], + }); + const detail = yield* loadPendingDetail( + caller, + shell({ + hasPendingAsyncUserInput: true, + pendingRuntimeRequest: { + id: BLOCKING_ID, + kind: "user_input", + createdAt: DateTime.makeUnsafe(ASKED), + }, + }), + { includePlan: false }, + ); + expect(detail.question).toMatchObject({ + requestId: BLOCKING_ID, + async: false, + textOnly: false, + }); + expect(detail.asyncQuestions.map((question) => question.requestId)).toEqual([ASYNC_ID]); + }), + ); + + it.effect("reads nothing when nothing waits", () => + Effect.gen(function* () { + const { caller, reads } = callerWith({ runtimeRequests: [], turnItems: [] }); + const detail = yield* loadPendingDetail(caller, shell({}), { includePlan: false }); + expect(reads).toEqual([]); + expect(detail.asyncQuestions).toEqual([]); + }), + ); +}); + +describe("matchesRequest", () => { + it("accepts a request id in full or in the short form t3_inbox shows", () => { + expect(requestRef(ASYNC_ID)).toBe("9e9c4c3e"); + expect(matchesRequest(ASYNC_ID, ASYNC_ID)).toBe(true); + expect(matchesRequest(ASYNC_ID, " 9E9C4C3E ")).toBe(true); + expect(matchesRequest(ASYNC_ID, "2b982706")).toBe(false); + }); +}); diff --git a/apps/server/src/mcp/toolkits/agent/pending.ts b/apps/server/src/mcp/toolkits/agent/pending.ts index 1869151d66160..d827932995e25 100644 --- a/apps/server/src/mcp/toolkits/agent/pending.ts +++ b/apps/server/src/mcp/toolkits/agent/pending.ts @@ -5,6 +5,13 @@ * session's own records filtered to the few turn items that matter: the open * question and its options, the open approval and its decisions, the * questions the agent asked while it kept working, and the newest plan. + * + * A question blocks the agent when its provider holds the turn open for the + * answer. A non-blocking one (a Codex `delivery: "async"` question, or an + * older fork activity question) did not stop the agent: the answer reaches it + * as a new message, and a person may dismiss the question instead. The shell + * leaves non-blocking requests out of `pendingRuntimeRequest`, so they are + * read from the session's runtime requests whenever it flags one. */ import { type OrchestrationV2TurnItem, @@ -19,18 +26,27 @@ import * as Schema from "effect/Schema"; import { findForkAsyncRequest } from "../../../orchestration-v2/forkAsyncResponse.expbkt3.ts"; import type { Caller } from "../../threadAccess.ts"; import type { QuestionView } from "./answers.ts"; -import { toIso, type SessionShell } from "./view.ts"; +import { shortRef, toIso, type SessionShell } from "./view.ts"; export interface PendingQuestion { readonly requestId: string; readonly questions: ReadonlyArray; /** The request resumes the agent with a message, so each answer must be text. */ readonly textOnly: boolean; - /** Asked while the agent kept working. */ + /** Non-blocking: asked while the agent kept working; the answer arrives as a new message. */ readonly async: boolean; readonly askedAt: string | null; } +/** The short form of a request id that t3_answer accepts: the first 8 characters of its last UUID. */ +export const requestRef = (requestId: string): string => shortRef(requestId); + +/** Whether `ref` names this request, by its full id or its short form. */ +export const matchesRequest = (requestId: string, ref: string): boolean => { + const wanted = ref.trim(); + return requestId === wanted || requestRef(requestId) === wanted.toLowerCase(); +}; + export interface ApprovalOptionView { readonly decision: ProviderApprovalDecision; readonly label: string; @@ -134,8 +150,9 @@ export const loadPendingDetail = Effect.fn("agentToolkit.loadPendingDetail")(fun ) { const request = shell.pendingRuntimeRequest; const asyncIds = shell.pendingAsyncUserInputIds ?? []; + const nonBlocking = shell.hasPendingAsyncUserInput === true; const types = new Set(); - if (request?.kind === "user_input") types.add("user_input_request"); + if (request?.kind === "user_input" || nonBlocking) types.add("user_input_request"); else if (request !== null && request.kind !== "auth_refresh") types.add("approval_request"); if (asyncIds.length > 0) types.add("dynamic_tool"); if (options.includePlan && shell.hasActionableProposedPlan) types.add("proposed_plan"); @@ -198,7 +215,34 @@ export const loadPendingDetail = Effect.fn("agentToolkit.loadPendingDetail")(fun } } - const asyncQuestions = asyncIds.flatMap((requestId): ReadonlyArray => { + // Requests that resume the agent with a message: a Codex async question. + const messageQuestions = runtimeRequests + .filter( + (runtime) => + runtime.status === "pending" && + runtime.kind === "user_input" && + runtime.responseCapability.type === "message", + ) + .toSorted((left, right) => left.createdAt.epochMilliseconds - right.createdAt.epochMilliseconds) + .flatMap((runtime): ReadonlyArray => { + const item = turnItems.find( + (candidate) => + candidate.type === "user_input_request" && candidate.requestId === runtime.id, + ); + if (item?.type !== "user_input_request") return []; + return [ + { + requestId: runtime.id, + questions: item.questions, + textOnly: true, + async: true, + askedAt: toIso(runtime.createdAt), + }, + ]; + }); + + // Older fork activity questions, listed on the session itself. + const activityQuestions = asyncIds.flatMap((requestId): ReadonlyArray => { const activity = findForkAsyncRequest(turnItems, requestId); if (activity?.kind !== "user-input.requested") return []; const payload = decodeAsyncPayload(activity.payload); @@ -214,6 +258,13 @@ export const loadPendingDetail = Effect.fn("agentToolkit.loadPendingDetail")(fun ]; }); + const asyncQuestions = [ + ...messageQuestions, + ...activityQuestions.filter( + (candidate) => !messageQuestions.some((other) => other.requestId === candidate.requestId), + ), + ]; + // Ordinals restart in every run, so the newest plan is the newest update. const planItem = turnItems .filter((item) => item.type === "proposed_plan") diff --git a/apps/server/src/mcp/toolkits/agent/render.test.ts b/apps/server/src/mcp/toolkits/agent/render.test.ts new file mode 100644 index 0000000000000..5af232eaea03d --- /dev/null +++ b/apps/server/src/mcp/toolkits/agent/render.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; + +import type { PendingQuestion } from "./pending.ts"; +import { QUESTION_KIND_TEXT, renderQuestion, sessionActivity } from "./render.ts"; +import type { SessionShell } from "./view.ts"; + +const NOW = Date.UTC(2026, 9, 12, 7, 0, 0); +const at = (minutesAgo: number) => DateTime.makeUnsafe(NOW - minutesAgo * 60_000); +const ASYNC_ID = "runtime-request:codex:native-request:async:9e9c4c3e-50f1-4f9e-a521-b41c3a04c7af"; +const OPTION = "Prime-service starts dbt four times daily (recommended)."; + +const question = (fields: Partial): PendingQuestion => ({ + requestId: ASYNC_ID, + questions: [ + { + id: "0", + header: "Question", + question: "Which DAILY refresh trigger must I implement?", + // Codex sends no description; T3 copies the label into it. + options: [{ label: OPTION, description: OPTION }], + }, + ], + textOnly: true, + async: true, + askedAt: null, + ...fields, +}); + +// Only the fields the state reads. +const shell = (fields: Partial>): SessionShell => + ({ + id: "86157111-97e0-449f-9a7e-f65b82072544", + status: "running", + activityRunStatus: null, + activeRunId: "run-1", + activityRunStartedAt: at(3), + pendingRuntimeRequest: null, + hasPendingAsyncUserInput: false, + hasActionableProposedPlan: false, + pendingBackgroundTasks: [], + updatedAt: at(1), + archivedAt: null, + deletedAt: null, + snoozedUntil: null, + ...fields, + }) as unknown as SessionShell; + +describe("renderQuestion", () => { + it("says a non-blocking question did not stop the agent, and how to answer or dismiss it", () => { + const lines = renderQuestion(question({}), "86157111"); + expect(lines[0]?.trim()).toBe(QUESTION_KIND_TEXT.nonBlocking); + // The label once, not again as its own description. + expect(lines).toContain(` 1. ${OPTION}`); + expect(lines.at(-1)).toBe( + ' → `t3_answer {"session":"86157111","requestId":"9e9c4c3e","choice":1}` · or `t3_answer {"session":"86157111","requestId":"9e9c4c3e","dismiss":true}`', + ); + }); + + it("says a blocking question pauses the agent, and needs no request id", () => { + const lines = renderQuestion(question({ async: false, textOnly: false }), "86157111"); + expect(lines[0]?.trim()).toBe(QUESTION_KIND_TEXT.blocking); + expect(lines.at(-1)).toBe(' → `t3_answer {"session":"86157111","choice":1}`'); + }); +}); + +describe("sessionActivity", () => { + it("does not call a working agent with a non-blocking question paused", () => { + expect(sessionActivity(shell({ hasPendingAsyncUserInput: true }), NOW)).toBe( + "running for 3m · asked a non-blocking question", + ); + expect( + sessionActivity( + shell({ hasPendingAsyncUserInput: true, status: "completed", activeRunId: null }), + NOW, + ), + ).toBe("idle · asked a non-blocking question"); + }); + + it("still says a blocking question waits on a person", () => { + expect( + sessionActivity( + shell({ pendingRuntimeRequest: { kind: "user_input", createdAt: at(10) } }), + NOW, + ), + ).toBe("waits on a person for 10m"); + }); +}); diff --git a/apps/server/src/mcp/toolkits/agent/render.ts b/apps/server/src/mcp/toolkits/agent/render.ts index b251b1638c261..8c7b507638d03 100644 --- a/apps/server/src/mcp/toolkits/agent/render.ts +++ b/apps/server/src/mcp/toolkits/agent/render.ts @@ -8,12 +8,19 @@ import type { ProviderApprovalDecision } from "@t3tools/contracts"; import type { QuestionView } from "./answers.ts"; -import type { PendingApproval, PendingPlan, PendingQuestion } from "./pending.ts"; import { + requestRef, + type PendingApproval, + type PendingPlan, + type PendingQuestion, +} from "./pending.ts"; +import { + attentionOf, callHint, cutText, duration, INBOX_KIND_LABEL, + isAgentActive, isUrgent, linearKeysOf, oneLine, @@ -66,10 +73,15 @@ export const sessionActivity = (shell: SessionShell, nowMs: number): string => { const state = sessionStateOf(shell, nowMs); const started = toMillis(shell.activityRunStartedAt ?? shell.latestRunStartedAt ?? null); const updated = toMillis(shell.updatedAt) ?? nowMs; + const running = started === null ? "running" : `running for ${duration(started, nowMs)}`; switch (state) { case "running": - return started === null ? "running" : `running for ${duration(started, nowMs)}`; + return running; case "waiting": { + // A non-blocking question waits on a person, but the agent did not stop for it. + if (attentionOf(shell) === "async-question") { + return `${isAgentActive(shell) ? running : "idle"} · asked a non-blocking question`; + } const since = toMillis(shell.pendingRuntimeRequest?.createdAt ?? null) ?? updated; return `waits on a person for ${duration(since, nowMs)}`; } @@ -93,30 +105,42 @@ export const renderSessionLine = ( }; const renderOptions = (question: QuestionView, indent: string): ReadonlyArray => [ - ...question.options.map( - (option, index) => - `${indent}${index + 1}. ${option.label} — ${oneLine(option.description, 140)}`, - ), + ...question.options.map((option, index) => { + // Codex sends options without descriptions; T3 copies the label into them. + const description = option.description.trim(); + return description.length === 0 || description === option.label.trim() + ? `${indent}${index + 1}. ${option.label}` + : `${indent}${index + 1}. ${option.label} — ${oneLine(description, 140)}`; + }), ...(question.allowCustomAnswer === false ? [] : [`${indent}(or your own answer as text)`]), ]; -/** A question request: each question with its numbered options, and the call that answers it. */ +/** What answering does, for a blocking and a non-blocking question. */ +export const QUESTION_KIND_TEXT = { + blocking: "Blocking: the agent is paused until you answer.", + nonBlocking: + "Non-blocking: the agent asked and did not stop. Your answer reaches it as a new message (into its running turn, or as a new turn). The options are suggestions; text works too. Dismiss it if no answer is needed.", +} as const; + +/** A question request: whether it blocks, each question with its numbered options, and the calls that answer it. */ export const renderQuestion = ( pending: PendingQuestion, sessionRef: string, ): ReadonlyArray => { const several = pending.questions.length > 1; - const lines: Array = []; + const lines: Array = [ + ` ${pending.async ? QUESTION_KIND_TEXT.nonBlocking : QUESTION_KIND_TEXT.blocking}`, + ]; pending.questions.forEach((question, index) => { lines.push( ` ${several ? `Q${index + 1}. ` : ""}**${question.header}**: ${oneLine(question.question, 400)}${question.multiSelect === true ? " (pick one or more)" : ""}`, ); lines.push(...renderOptions(question, several ? " " : " ")); }); - // t3_answer picks the blocking question first, so only a question asked while - // the agent works needs its (long) request id. + // t3_answer picks the blocking question first, so only a non-blocking one + // needs its request id, in the short form. const base = pending.async - ? { session: sessionRef, requestId: pending.requestId } + ? { session: sessionRef, requestId: requestRef(pending.requestId) } : { session: sessionRef }; const first = pending.questions[0]; const example = several @@ -127,7 +151,11 @@ export const renderQuestion = ( : first !== undefined && first.options.length > 0 ? callHint("t3_answer", { ...base, choice: 1 }) : callHint("t3_answer", { ...base, text: "…" }); - lines.push(` → ${example}`); + lines.push( + pending.async + ? ` → ${example} · or ${callHint("t3_answer", { ...base, dismiss: true })}` + : ` → ${example}`, + ); return lines; }; diff --git a/apps/server/src/mcp/toolkits/agent/tools.ts b/apps/server/src/mcp/toolkits/agent/tools.ts index c5b0057543e83..6be5533ce8d00 100644 --- a/apps/server/src/mcp/toolkits/agent/tools.ts +++ b/apps/server/src/mcp/toolkits/agent/tools.ts @@ -114,7 +114,7 @@ export const InboxTool = readOnly( kinds: Schema.optional( described( Schema.Array(Schema.Literals(["approval", "question", "plan", "failed", "unread"])), - "Only these kinds. 'question' includes questions the agent asked while it keeps working.", + "Only these kinds. 'question' includes non-blocking questions, which the agent asked without stopping.", ), ), includeSnoozed: Schema.optional( @@ -141,13 +141,13 @@ const ChoiceValue = Schema.Union([Schema.Int, Schema.String]); export const AnswerTool = destructive( Tool.make("t3_answer", { description: - "Answer what a session waits for, as the person you act for. Pass one of: choice (an option number or label, or a list for a multi-select question), text (your own answer), answers (one entry per question when a request asks several), decision (for an approval), or plan (approve, revise with feedback, or discard). Omit requestId to answer the oldest open item. The tool refuses when nothing of that kind waits, when the request was already answered, and when an agent tries to answer in its own session. Get the options from t3_inbox or t3_session_brief.", + "Answer what a session waits for, as the person you act for. Pass one of: choice (an option number or label, or a list for a multi-select question), text (your own answer), answers (one entry per question when a request asks several), decision (for an approval), or plan (approve, revise with feedback, or discard). A question is blocking (the agent is paused until you answer) or non-blocking (the agent asked and did not stop, as a Codex async question does; your answer reaches it as a new message, and dismiss: true closes the question without an answer). Omit requestId to answer the blocking question first, then the oldest non-blocking one. The tool refuses when nothing of that kind waits, when the request was already answered, and when an agent tries to answer in its own session. Get the options from t3_inbox or t3_session_brief.", parameters: Schema.Struct({ session: sessionRef(SESSION_REF_HELP), requestId: Schema.optional( described( Schema.String, - "The open request to answer. Omit to answer the oldest open item of the kind your answer fits.", + "The open request to answer: its id, or the short form t3_inbox shows. Omit to answer the oldest open item of the kind your answer fits.", ), ), choice: Schema.optional( @@ -174,6 +174,12 @@ export const AnswerTool = destructive( "For a request that asks several questions: one entry per question.", ), ), + dismiss: Schema.optional( + described( + Schema.Boolean, + "true closes a non-blocking question without an answer, as Dismiss does in the app; the agent is not told. A blocking question cannot be dismissed.", + ), + ), decision: Schema.optional( described( Schema.Literals(["approve", "approve_for_session", "approve_always", "deny", "cancel"]), @@ -361,7 +367,7 @@ export const StartSessionTool = changing( export const SendMessageTool = changing( Tool.make("t3_send_message", { description: - "Send a message to a session. When it is busy, ifBusy decides: queue (the default) runs it after the current turn, steer adds it to the current turn, interrupt stops the turn and starts this one. When the session waits on a question or approval, the tool refuses and points to t3_answer, so an open question is never lost; pass force: true to send anyway.", + "Send a message to a session. When it is busy, ifBusy decides: queue (the default) runs it after the current turn, steer adds it to the current turn, interrupt stops the turn and starts this one. When the session waits on a blocking question or an approval, the tool refuses and points to t3_answer, so an open question is never lost; pass force: true to send anyway. A non-blocking question does not stop a message, and it stays open.", parameters: Schema.Struct({ session: sessionRef(SESSION_REF_HELP), message: described(Schema.String, "The message to send."), @@ -374,7 +380,7 @@ export const SendMessageTool = changing( force: Schema.optional( described( Schema.Boolean, - "Send even when a question or approval is open. Defaults to false.", + "Send even when a blocking question or an approval is open. Defaults to false.", ), ), clientRequestId: Schema.optional( diff --git a/apps/server/src/mcp/toolkits/agent/view.ts b/apps/server/src/mcp/toolkits/agent/view.ts index 345338944b080..2b5fb9aed7fa6 100644 --- a/apps/server/src/mcp/toolkits/agent/view.ts +++ b/apps/server/src/mcp/toolkits/agent/view.ts @@ -34,8 +34,8 @@ export const INBOX_KIND_ORDER: ReadonlyArray = [ export const INBOX_KIND_LABEL: Record = { approval: "Approval", - question: "Question", - "async-question": "Question (agent keeps working)", + question: "Question (blocking)", + "async-question": "Question (non-blocking)", plan: "Plan to review", failed: "Failed", unread: "Finished, not read", diff --git a/apps/server/src/mcp/toolkits/agent/wait.test.ts b/apps/server/src/mcp/toolkits/agent/wait.test.ts index daff15e2ee5b0..caf692e4150d5 100644 --- a/apps/server/src/mcp/toolkits/agent/wait.test.ts +++ b/apps/server/src/mcp/toolkits/agent/wait.test.ts @@ -18,10 +18,14 @@ describe("changeOf", () => { changeOf( stored({ type: "runtime-request.updated", - payload: { status: "pending", kind: "user_input" }, + payload: { + status: "pending", + kind: "user_input", + responseCapability: { type: "live", providerSessionId: "session-1" }, + }, }), - )?.change, - ).toBe("needs-human"); + ), + ).toMatchObject({ change: "needs-human", what: "asked a blocking question" }); expect( changeOf( stored({ @@ -69,6 +73,37 @@ describe("changeOf", () => { expect(changeOf(stopped(9_000))).toBeNull(); }); + it("says a Codex async question does not block the agent", () => { + expect( + changeOf( + stored({ + type: "runtime-request.updated", + payload: { + status: "pending", + kind: "user_input", + responseCapability: { type: "message" }, + }, + }), + ), + ).toMatchObject({ + change: "needs-human", + what: "asked a non-blocking question and keeps working", + }); + // Answering it resolves the request; that is not a new question. + expect( + changeOf( + stored({ + type: "runtime-request.updated", + payload: { + status: "resolved", + kind: "user_input", + responseCapability: { type: "message" }, + }, + }), + ), + ).toBeNull(); + }); + it("reports a question asked while the agent keeps working, in either input form", () => { expect( changeOf( diff --git a/apps/server/src/mcp/toolkits/agent/wait.ts b/apps/server/src/mcp/toolkits/agent/wait.ts index 43b555c6ab28b..834aa1a1b37c4 100644 --- a/apps/server/src/mcp/toolkits/agent/wait.ts +++ b/apps/server/src/mcp/toolkits/agent/wait.ts @@ -58,7 +58,11 @@ export function changeOf(stored: OrchestrationV2StoredEvent): Omit { it("fires on an open question or approval at once", () => { expect( - watchedChange(shell({ pendingRuntimeRequest: { kind: "user_input" } }), all, SINCE)?.change, - ).toBe("needs-human"); + watchedChange(shell({ pendingRuntimeRequest: { kind: "user_input" } }), all, SINCE), + ).toEqual({ change: "needs-human", what: "asks a blocking question" }); + // A Codex async question: the agent may still work, and it still needs a person. + expect(watchedChange(shell({ hasPendingAsyncUserInput: true }), all, SINCE)).toEqual({ + change: "needs-human", + what: "asks a non-blocking question", + }); expect( watchedChange(shell({ pendingRuntimeRequest: { kind: "command" } }), all, SINCE)?.what, ).toBe("waits for an approval"); diff --git a/apps/server/src/scheduledTasks/scheduledTaskWatch.expbkt3.ts b/apps/server/src/scheduledTasks/scheduledTaskWatch.expbkt3.ts index 2c194df46ed3c..b223a5cfc8f61 100644 --- a/apps/server/src/scheduledTasks/scheduledTaskWatch.expbkt3.ts +++ b/apps/server/src/scheduledTasks/scheduledTaskWatch.expbkt3.ts @@ -78,8 +78,10 @@ export const watchedChange = ( const completedMs = millis(shell.latestRunCompletedAt); const newRun = completedMs !== null && completedMs > sinceMs; if (until.has("needs-human")) { - if (attention === "question" || attention === "async-question") { - return { change: "needs-human", what: "asks a question" }; + if (attention === "question") + return { change: "needs-human", what: "asks a blocking question" }; + if (attention === "async-question") { + return { change: "needs-human", what: "asks a non-blocking question" }; } if (attention === "approval") return { change: "needs-human", what: "waits for an approval" }; if (attention === "plan" && newRun) { diff --git a/docs/user/t3-mcp-control.md b/docs/user/t3-mcp-control.md index 8795cb9d3ec16..3133f9c726e98 100644 --- a/docs/user/t3-mcp-control.md +++ b/docs/user/t3-mcp-control.md @@ -145,7 +145,7 @@ omits it to mean its own. | `t3_find_sessions` | Find sessions by text, ticket, pull request, branch, project or state, one line each. | | `t3_session_brief` | Read where one session stands: state, what it waits on, its goal and its last messages. | | `t3_start_session` | Start a session in a project with a first message, a model alias (`opus`, `sonnet`, `codex`) and a mode. | -| `t3_send_message` | Send to a session. It refuses while a question or approval is open, so the question is not lost. | +| `t3_send_message` | Send to a session. It refuses while a blocking question or an approval is open, so it is not lost. | | `t3_wait` | Wait for the next question, approval, finish or failure, with a cursor so no change is missed. | | `t3_schedule` | Send a prompt later or on a clock, into a session or into a new session each run; also pause, resume, delete. | | `t3_schedules` | List the schedules you can manage, with where they send, their next run and their last result. | @@ -160,7 +160,14 @@ guide in the MCP `instructions` when a client connects. `t3_answer` never answers inside the caller's own session: a person answers an agent's questions, approvals and plans. The answer records the person the -caller acts for. `t3_wait` returns within 50 seconds by default, because most +caller acts for. + +A question is blocking or non-blocking, and the tools say which. A blocking +question pauses the agent until someone answers it. A non-blocking question, +such as a Codex async question, did not stop the agent: the answer reaches it +as a new message, in its running turn or as a new turn, and +`t3_answer` with `dismiss: true` closes the question without an answer. +`t3_send_message` refuses only while a blocking question or an approval is open. `t3_wait` returns within 50 seconds by default, because most MCP clients stop a call after 60 seconds; call it again with the cursor it returns. `t3_start_session`, `t3_send_message` and `t3_answer` take a `clientRequestId`, so a retry with the same id does not act twice. From f9a49411e66c17515c8c4bdc08cfb23f51c985e7 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Mon, 12 Oct 2026 07:35:45 +0000 Subject: [PATCH 03/20] fix(mcp): keep approval details beside async questions; a retry answers only its question Review fixes: - an approval next to a non-blocking question loaded no details, because the approval read sat in an else branch; - t3_answer without requestId now refuses when more than one question is open, and every answer hint carries the short requestId, so a retried answer cannot land on the next question; - a secret request (the shell stands it in as a question) now says that a person enters it in the app, instead of asking to try again; - t3_inbox counts the non-blocking questions it does not show; - the delivery check reads only the answer's own run. Co-Authored-By: Claude Opus 5.5 --- .../server/src/mcp/toolkits/agent/handlers.ts | 40 ++++++--- .../src/mcp/toolkits/agent/pending.test.ts | 83 +++++++++++++++++++ apps/server/src/mcp/toolkits/agent/pending.ts | 24 +++++- .../src/mcp/toolkits/agent/render.test.ts | 49 ++++++++++- apps/server/src/mcp/toolkits/agent/render.ts | 31 +++++-- apps/server/src/mcp/toolkits/agent/tools.ts | 4 +- docs/user/t3-mcp-control.md | 4 +- 7 files changed, 208 insertions(+), 27 deletions(-) diff --git a/apps/server/src/mcp/toolkits/agent/handlers.ts b/apps/server/src/mcp/toolkits/agent/handlers.ts index 2118c276245fa..0c5e211dfbbcc 100644 --- a/apps/server/src/mcp/toolkits/agent/handlers.ts +++ b/apps/server/src/mcp/toolkits/agent/handlers.ts @@ -65,6 +65,7 @@ import { renderInboxEntry, renderPlan, renderQuestion, + renderSecret, renderSessionLine, sessionActivity, sessionHeading, @@ -389,6 +390,8 @@ const inbox = Effect.fn("agentToolkit.inbox")(function* (input: Tool.Parameters< context: context(candidate.shell), waitingSinceMs: askedMs ?? candidate.waitingSinceMs, question, + secret: detail.secret, + moreQuestions: detail.asyncQuestions.filter((other) => other !== question).length, approval: candidate.kind === "approval" ? detail.approval : null, plan: candidate.kind === "plan" ? detail.plan : null, lastReply: readReply ? yield* lastReply(caller, candidate.shell) : null, @@ -433,6 +436,8 @@ const inbox = Effect.fn("agentToolkit.inbox")(function* (input: Tool.Parameters< waitingSince: isoFromMillis(entry.waitingSinceMs), question: entry.question === null ? null : boundQuestion(entry.question), approval: entry.approval === null ? null : boundApproval(entry.approval), + secret: entry.secret, + moreQuestions: entry.moreQuestions, plan: entry.plan === null ? null @@ -482,17 +487,16 @@ const answerDelivery = ( ): Effect.Effect => Effect.gen(function* () { const messageId = MessageId.make(`async-answer:${requestId}`); - const { messages, runs } = yield* caller.threads.getThreadRecords( - threadId, - ["messages", "runs"], - { messageIds: [messageId] }, - ); + const { messages } = yield* caller.threads.getThreadRecords(threadId, ["messages"], { + messageIds: [messageId], + }); const runId = messages.find((message) => message.id === messageId)?.runId ?? null; if (runId === null) return null; - const { turnItems } = yield* caller.threads.getThreadRecords(threadId, ["turnItems"], { - turnItemTypes: ["user_message"], - turnItemRunIds: [runId], - }); + const { runs, turnItems } = yield* caller.threads.getThreadRecords( + threadId, + ["runs", "turnItems"], + { runIds: [runId], turnItemTypes: ["user_message"], turnItemRunIds: [runId] }, + ); const item = turnItems.find( (candidate) => candidate.type === "user_message" && candidate.messageId === messageId, ); @@ -523,6 +527,8 @@ const describeWaiting = (shell: SessionShell, detail: PendingDetail) => { const ref = shortRef(shell.id); if (detail.approval !== null) return `It waits on an approval: pass decision (${callHint("t3_answer", { session: ref, decision: "approve" })}).`; + if (detail.secret !== null) + return `It waits on a secret ("${oneLine(detail.secret.label, 80)}"), which a person must enter in the app; no tool can answer it.`; if (detail.question !== null || detail.asyncQuestions.length > 0) return `It waits on a question: pass choice, text or answers (see ${callHint("t3_session_brief", { session: ref })}).`; const attention = attentionOf(shell); @@ -707,9 +713,17 @@ const answer = Effect.fn("agentToolkit.answer")(function* ( const open = [...(detail.question === null ? [] : [detail.question]), ...detail.asyncQuestions]; const dismiss = input.dismiss === true; const wanted = input.requestId; + const fitting = dismiss ? open.filter((candidate) => candidate.async) : open; + // Without a request id a retry could land on the next open question. + if (wanted === undefined && fitting.length > 1) { + return yield* fail( + "request_id_required", + `${ref} has ${fitting.length} open ${dismiss ? "non-blocking " : ""}questions. Pass requestId to pick one: ${describeOpenQuestions(fitting)}.`, + ); + } const target = wanted === undefined - ? (dismiss ? open.filter((candidate) => candidate.async) : open)[0] + ? fitting[0] : open.find((candidate) => matchesRequest(candidate.requestId, wanted)); if (target === undefined) { return yield* fail( @@ -996,6 +1010,8 @@ const brief = Effect.fn("agentToolkit.brief")(function* ( ]; if (detail.approval !== null) lines.push("", "### Waits on an approval", ...renderApproval(detail.approval, ref)); + if (detail.secret !== null) + lines.push("", "### Waits on a secret", ...renderSecret(detail.secret)); if (detail.question !== null) lines.push("", "### Blocking question", ...renderQuestion(detail.question, ref)); for (const question of detail.asyncQuestions) { @@ -1044,6 +1060,7 @@ const brief = Effect.fn("agentToolkit.brief")(function* ( tagged: members, waitsOn: { approval: detail.approval === null ? null : boundApproval(detail.approval), + secret: detail.secret, question: detail.question === null ? null : boundQuestion(detail.question), asyncQuestions: detail.asyncQuestions.map(boundQuestion), plan: @@ -1087,6 +1104,7 @@ const firstOutcome = Effect.fn("agentToolkit.firstOutcome")(function* ( return [ "### It waits on you", ...(detail.approval !== null ? renderApproval(detail.approval, ref) : []), + ...(detail.secret !== null ? renderSecret(detail.secret) : []), ...(question !== null ? renderQuestion(question, ref) : []), ]; } @@ -1437,6 +1455,7 @@ const wait = Effect.fn("agentToolkit.wait")(function* (input: Tool.Parameters renderQuestion(question, ref)), ...(attention === "plan" ? renderPlan(detail.plan, ref, 600) : []), @@ -1460,6 +1479,7 @@ const wait = Effect.fn("agentToolkit.wait")(function* (input: Tool.Parameters { }), ); + it.effect("keeps an approval's details while a non-blocking question is open", () => + Effect.gen(function* () { + const approvalId = + "runtime-request:codex:native-request:9:aaaaaaaa-1111-2222-3333-444444444444"; + const { caller, reads } = callerWith({ + runtimeRequests: [ + runtimeRequest({ + id: approvalId, + kind: "command", + responseCapability: { type: "live", providerSessionId: "provider-session-1" }, + }), + runtimeRequest({ id: ASYNC_ID, responseCapability: { type: "message" } }), + ], + turnItems: [ + { + id: `turn-item:${approvalId}`, + type: "approval_request", + status: "waiting", + requestId: approvalId, + requestKind: "command", + title: "Run the migration", + prompt: "It changes the schema.", + runId: "run-1", + updatedAt: DateTime.makeUnsafe(ASKED), + options: [{ decision: "accept", label: "Yes" }], + }, + questionItem(ASYNC_ID, "Later?", "message"), + ], + }); + const detail = yield* loadPendingDetail( + caller, + shell({ + hasPendingAsyncUserInput: true, + pendingRuntimeRequest: { + id: approvalId, + kind: "command", + createdAt: DateTime.makeUnsafe(ASKED), + }, + }), + { includePlan: false }, + ); + expect(reads[0]).toEqual({ turnItemTypes: ["user_input_request", "approval_request"] }); + expect(detail.approval).toMatchObject({ + title: "Run the migration", + options: [{ decision: "accept", label: "Yes" }], + }); + expect(detail.asyncQuestions.map((question) => question.requestId)).toEqual([ASYNC_ID]); + }), + ); + + it.effect("names a secret request, which the shell stands in as a question", () => + Effect.gen(function* () { + const secretId = "turn-item:provider:claudeAgent:native-item:secret-1"; + const { caller } = callerWith({ + runtimeRequests: [], + turnItems: [ + { + id: secretId, + type: "secret_request", + status: "waiting", + label: "GitHub token", + reason: "To open the pull request.", + secretStatus: "pending", + updatedAt: DateTime.makeUnsafe(ASKED), + }, + ], + }); + const detail = yield* loadPendingDetail( + caller, + shell({ + pendingRuntimeRequest: { + id: secretId, + kind: "user_input", + createdAt: DateTime.makeUnsafe(ASKED), + }, + }), + { includePlan: false }, + ); + expect(detail.question).toBeNull(); + expect(detail.secret).toEqual({ label: "GitHub token", reason: "To open the pull request." }); + }), + ); + it.effect("reads nothing when nothing waits", () => Effect.gen(function* () { const { caller, reads } = callerWith({ runtimeRequests: [], turnItems: [] }); diff --git a/apps/server/src/mcp/toolkits/agent/pending.ts b/apps/server/src/mcp/toolkits/agent/pending.ts index d827932995e25..935803593be14 100644 --- a/apps/server/src/mcp/toolkits/agent/pending.ts +++ b/apps/server/src/mcp/toolkits/agent/pending.ts @@ -81,10 +81,17 @@ export interface PendingPlan { export const planIsCurrent = (shell: SessionShell, plan: PendingPlan | null): boolean => plan === null || plan.runId === null || plan.runId === shell.latestRunId; +/** A secret the agent asked for; only a person can enter it, in the app. */ +export interface PendingSecret { + readonly label: string; + readonly reason: string; +} + export interface PendingDetail { readonly question: PendingQuestion | null; readonly asyncQuestions: ReadonlyArray; readonly approval: PendingApproval | null; + readonly secret: PendingSecret | null; readonly plan: PendingPlan | null; } @@ -92,6 +99,7 @@ export const EMPTY_PENDING: PendingDetail = { question: null, asyncQuestions: [], approval: null, + secret: null, plan: null, }; @@ -153,7 +161,11 @@ export const loadPendingDetail = Effect.fn("agentToolkit.loadPendingDetail")(fun const nonBlocking = shell.hasPendingAsyncUserInput === true; const types = new Set(); if (request?.kind === "user_input" || nonBlocking) types.add("user_input_request"); - else if (request !== null && request.kind !== "auth_refresh") types.add("approval_request"); + // The shell stands a secret request in as a pending question. + if (request?.kind === "user_input") types.add("secret_request"); + if (request !== null && request.kind !== "user_input" && request.kind !== "auth_refresh") { + types.add("approval_request"); + } if (asyncIds.length > 0) types.add("dynamic_tool"); if (options.includePlan && shell.hasActionableProposedPlan) types.add("proposed_plan"); if (types.size === 0) return EMPTY_PENDING; @@ -168,6 +180,7 @@ export const loadPendingDetail = Effect.fn("agentToolkit.loadPendingDetail")(fun let question: PendingQuestion | null = null; let approval: PendingApproval | null = null; + let secret: PendingSecret | null = null; if (request !== null) { const runtime = runtimeRequests.find((candidate) => candidate.id === request.id); const item = turnItems.find( @@ -175,7 +188,12 @@ export const loadPendingDetail = Effect.fn("agentToolkit.loadPendingDetail")(fun (candidate.type === "user_input_request" || candidate.type === "approval_request") && candidate.requestId === request.id, ); - if (item?.type === "user_input_request") { + const secretItem = turnItems.find( + (candidate) => candidate.type === "secret_request" && candidate.id === request.id, + ); + if (secretItem?.type === "secret_request") { + secret = { label: secretItem.label, reason: secretItem.reason }; + } else if (item?.type === "user_input_request") { question = { requestId: request.id, questions: item.questions, @@ -280,5 +298,5 @@ export const loadPendingDetail = Effect.fn("agentToolkit.loadPendingDetail")(fun } : null; - return { question, asyncQuestions, approval, plan } satisfies PendingDetail; + return { question, asyncQuestions, approval, secret, plan } satisfies PendingDetail; }); diff --git a/apps/server/src/mcp/toolkits/agent/render.test.ts b/apps/server/src/mcp/toolkits/agent/render.test.ts index 5af232eaea03d..21ae10cf5195d 100644 --- a/apps/server/src/mcp/toolkits/agent/render.test.ts +++ b/apps/server/src/mcp/toolkits/agent/render.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "@effect/vitest"; import * as DateTime from "effect/DateTime"; import type { PendingQuestion } from "./pending.ts"; -import { QUESTION_KIND_TEXT, renderQuestion, sessionActivity } from "./render.ts"; +import { QUESTION_KIND_TEXT, renderInboxEntry, renderQuestion, sessionActivity } from "./render.ts"; import type { SessionShell } from "./view.ts"; const NOW = Date.UTC(2026, 9, 12, 7, 0, 0); @@ -43,6 +43,9 @@ const shell = (fields: Partial>): SessionShell => archivedAt: null, deletedAt: null, snoozedUntil: null, + priority: null, + linearLinks: [], + pullRequests: [], ...fields, }) as unknown as SessionShell; @@ -57,10 +60,12 @@ describe("renderQuestion", () => { ); }); - it("says a blocking question pauses the agent, and needs no request id", () => { + it("says a blocking question pauses the agent, and offers no dismiss", () => { const lines = renderQuestion(question({ async: false, textOnly: false }), "86157111"); expect(lines[0]?.trim()).toBe(QUESTION_KIND_TEXT.blocking); - expect(lines.at(-1)).toBe(' → `t3_answer {"session":"86157111","choice":1}`'); + expect(lines.at(-1)).toBe( + ' → `t3_answer {"session":"86157111","requestId":"9e9c4c3e","choice":1}`', + ); }); }); @@ -86,3 +91,41 @@ describe("sessionActivity", () => { ).toBe("waits on a person for 10m"); }); }); + +describe("renderInboxEntry", () => { + it("counts the non-blocking questions it does not show", () => { + const text = renderInboxEntry(1, { + kind: "async-question", + shell: shell({ hasPendingAsyncUserInput: true, title: "Child work" }), + context: { projectTitle: null, link: null, nowMs: NOW }, + waitingSinceMs: NOW - 60_000, + question: question({}), + approval: null, + secret: null, + moreQuestions: 2, + plan: null, + lastReply: null, + }); + expect(text).toContain("**Question (non-blocking)**"); + expect(text).toContain( + ' +2 more non-blocking questions: see `t3_session_brief {"session":"86157111"}`', + ); + }); + + it("says no tool answers a secret request", () => { + const text = renderInboxEntry(1, { + kind: "question", + shell: shell({ title: "Child work" }), + context: { projectTitle: null, link: null, nowMs: NOW }, + waitingSinceMs: NOW - 60_000, + question: null, + approval: null, + secret: { label: "GitHub token", reason: "To open the pull request." }, + moreQuestions: 0, + plan: null, + lastReply: null, + }); + expect(text).toContain("Asks for a secret: **GitHub token** — To open the pull request."); + expect(text).toContain("no tool can answer it"); + }); +}); diff --git a/apps/server/src/mcp/toolkits/agent/render.ts b/apps/server/src/mcp/toolkits/agent/render.ts index 8c7b507638d03..8a25e32bab7f3 100644 --- a/apps/server/src/mcp/toolkits/agent/render.ts +++ b/apps/server/src/mcp/toolkits/agent/render.ts @@ -13,6 +13,7 @@ import { type PendingApproval, type PendingPlan, type PendingQuestion, + type PendingSecret, } from "./pending.ts"; import { attentionOf, @@ -137,11 +138,9 @@ export const renderQuestion = ( ); lines.push(...renderOptions(question, several ? " " : " ")); }); - // t3_answer picks the blocking question first, so only a non-blocking one - // needs its request id, in the short form. - const base = pending.async - ? { session: sessionRef, requestId: requestRef(pending.requestId) } - : { session: sessionRef }; + // The request id, in the short form, keeps a retried answer on this question + // when the session has others open. + const base = { session: sessionRef, requestId: requestRef(pending.requestId) }; const first = pending.questions[0]; const example = several ? callHint("t3_answer", { @@ -159,6 +158,12 @@ export const renderQuestion = ( return lines; }; +/** A secret request: no tool answers it; a person enters it in the app. */ +export const renderSecret = (secret: PendingSecret): ReadonlyArray => [ + ` Asks for a secret: **${oneLine(secret.label, 120)}**${secret.reason.trim().length === 0 ? "" : ` — ${oneLine(secret.reason, 300)}`}`, + " A person must enter it in the app (open the session's link); no tool can answer it.", +]; + export const renderApproval = ( approval: PendingApproval, sessionRef: string, @@ -219,6 +224,9 @@ export interface InboxEntry { readonly waitingSinceMs: number; readonly question: PendingQuestion | null; readonly approval: PendingApproval | null; + readonly secret: PendingSecret | null; + /** Open non-blocking questions this entry does not show. */ + readonly moreQuestions: number; readonly plan: PendingPlan | null; /** The agent's newest reply, read for unread results only. */ readonly lastReply: string | null; @@ -233,10 +241,12 @@ export const renderInboxEntry = (index: number, entry: InboxEntry): string => { switch (entry.kind) { case "question": case "async-question": - if (entry.question === null) { - lines.push(` → read it with ${callHint("t3_session_brief", { session: ref })}`); - } else { + if (entry.question !== null) { lines.push(...renderQuestion(entry.question, ref)); + } else if (entry.secret !== null) { + lines.push(...renderSecret(entry.secret)); + } else { + lines.push(` → read it with ${callHint("t3_session_brief", { session: ref })}`); } break; case "approval": @@ -259,6 +269,11 @@ export const renderInboxEntry = (index: number, entry: InboxEntry): string => { break; } } + if (entry.moreQuestions > 0) { + lines.push( + ` +${entry.moreQuestions} more non-blocking ${entry.moreQuestions === 1 ? "question" : "questions"}: see ${callHint("t3_session_brief", { session: ref })}`, + ); + } if (context.link !== null) lines.push(` Link: ${context.link}`); return lines.join("\n"); }; diff --git a/apps/server/src/mcp/toolkits/agent/tools.ts b/apps/server/src/mcp/toolkits/agent/tools.ts index 6be5533ce8d00..e07628c3c608b 100644 --- a/apps/server/src/mcp/toolkits/agent/tools.ts +++ b/apps/server/src/mcp/toolkits/agent/tools.ts @@ -141,13 +141,13 @@ const ChoiceValue = Schema.Union([Schema.Int, Schema.String]); export const AnswerTool = destructive( Tool.make("t3_answer", { description: - "Answer what a session waits for, as the person you act for. Pass one of: choice (an option number or label, or a list for a multi-select question), text (your own answer), answers (one entry per question when a request asks several), decision (for an approval), or plan (approve, revise with feedback, or discard). A question is blocking (the agent is paused until you answer) or non-blocking (the agent asked and did not stop, as a Codex async question does; your answer reaches it as a new message, and dismiss: true closes the question without an answer). Omit requestId to answer the blocking question first, then the oldest non-blocking one. The tool refuses when nothing of that kind waits, when the request was already answered, and when an agent tries to answer in its own session. Get the options from t3_inbox or t3_session_brief.", + "Answer what a session waits for, as the person you act for. Pass one of: choice (an option number or label, or a list for a multi-select question), text (your own answer), answers (one entry per question when a request asks several), decision (for an approval), or plan (approve, revise with feedback, or discard). A question is blocking (the agent is paused until you answer) or non-blocking (the agent asked and did not stop, as a Codex async question does; your answer reaches it as a new message, and dismiss: true closes the question without an answer). Pass the requestId t3_inbox shows; you may omit it only when one question is open. The tool refuses when nothing of that kind waits, when the request was already answered, and when an agent tries to answer in its own session. Get the options from t3_inbox or t3_session_brief.", parameters: Schema.Struct({ session: sessionRef(SESSION_REF_HELP), requestId: Schema.optional( described( Schema.String, - "The open request to answer: its id, or the short form t3_inbox shows. Omit to answer the oldest open item of the kind your answer fits.", + "The open request to answer: its id, or the short form t3_inbox shows. Required when a session has more than one open question, so a retry cannot answer another one.", ), ), choice: Schema.optional( diff --git a/docs/user/t3-mcp-control.md b/docs/user/t3-mcp-control.md index 3133f9c726e98..b703fdbd6ed1c 100644 --- a/docs/user/t3-mcp-control.md +++ b/docs/user/t3-mcp-control.md @@ -167,7 +167,9 @@ question pauses the agent until someone answers it. A non-blocking question, such as a Codex async question, did not stop the agent: the answer reaches it as a new message, in its running turn or as a new turn, and `t3_answer` with `dismiss: true` closes the question without an answer. -`t3_send_message` refuses only while a blocking question or an approval is open. `t3_wait` returns within 50 seconds by default, because most +`t3_send_message` refuses only while a blocking question or an approval is open. +When a session has more than one open question, pass the `requestId` that +`t3_inbox` shows, so a retry cannot answer a different question. `t3_wait` returns within 50 seconds by default, because most MCP clients stop a call after 60 seconds; call it again with the cursor it returns. `t3_start_session`, `t3_send_message` and `t3_answer` take a `clientRequestId`, so a retry with the same id does not act twice. From 4bb4763adfeb0f82afebf8d5198d6aaf138a5d08 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Mon, 12 Oct 2026 07:45:20 +0000 Subject: [PATCH 04/20] fix(mcp): require requestId with clientRequestId when answering; fix the secret id type - the secret stand-in id is compared as a string (TS2367: TurnItemId vs RuntimeRequestId); - t3_answer refuses a question, dismiss or decision that has a clientRequestId but no requestId, so a retry cannot land on a request asked since; approval hints carry the short requestId; - t3_send_message names a secret request instead of pointing to t3_answer; - secret text in JSON output is cut like other agent-written text. Co-Authored-By: Claude Opus 5.5 --- .../server/src/mcp/toolkits/agent/handlers.ts | 45 ++++++++++++++----- apps/server/src/mcp/toolkits/agent/pending.ts | 4 +- apps/server/src/mcp/toolkits/agent/render.ts | 4 +- apps/server/src/mcp/toolkits/agent/tools.ts | 4 +- docs/user/t3-mcp-control.md | 5 ++- 5 files changed, 45 insertions(+), 17 deletions(-) diff --git a/apps/server/src/mcp/toolkits/agent/handlers.ts b/apps/server/src/mcp/toolkits/agent/handlers.ts index 0c5e211dfbbcc..ffc2d3ff9eb39 100644 --- a/apps/server/src/mcp/toolkits/agent/handlers.ts +++ b/apps/server/src/mcp/toolkits/agent/handlers.ts @@ -56,6 +56,7 @@ import { type PendingApproval, type PendingDetail, type PendingQuestion, + type PendingSecret, } from "./pending.ts"; import { DECISION_NAMES, @@ -214,6 +215,12 @@ const boundQuestion = (pending: PendingQuestion) => ({ })), }); +/** A secret request as `format: "json"` returns it; its reason is the agent's own text. */ +const boundSecret = (secret: PendingSecret | null) => + secret === null + ? null + : { label: cutText(secret.label, 200), reason: cutText(secret.reason, 1_000) }; + /** An approval as `format: "json"` returns it; a heredoc command can be very long. */ const boundApproval = (approval: PendingApproval): PendingApproval => ({ ...approval, @@ -436,7 +443,7 @@ const inbox = Effect.fn("agentToolkit.inbox")(function* (input: Tool.Parameters< waitingSince: isoFromMillis(entry.waitingSinceMs), question: entry.question === null ? null : boundQuestion(entry.question), approval: entry.approval === null ? null : boundApproval(entry.approval), - secret: entry.secret, + secret: boundSecret(entry.secret), moreQuestions: entry.moreQuestions, plan: entry.plan === null @@ -526,7 +533,7 @@ const DECISIONS: Record = { const describeWaiting = (shell: SessionShell, detail: PendingDetail) => { const ref = shortRef(shell.id); if (detail.approval !== null) - return `It waits on an approval: pass decision (${callHint("t3_answer", { session: ref, decision: "approve" })}).`; + return `It waits on an approval: pass decision (${callHint("t3_answer", { session: ref, requestId: requestRef(detail.approval.requestId), decision: "approve" })}).`; if (detail.secret !== null) return `It waits on a secret ("${oneLine(detail.secret.label, 80)}"), which a person must enter in the app; no tool can answer it.`; if (detail.question !== null || detail.asyncQuestions.length > 0) @@ -674,6 +681,13 @@ const answer = Effect.fn("agentToolkit.answer")(function* ( answered = `${input.plan === "approve" ? "Approved" : input.plan === "revise" ? "Asked for changes to" : "Discarded"} the plan "${open.title}"${submitted.turnStarted ? "; the agent is working on it" : ""}.`; } else if (input.decision !== undefined) { const approval = detail.approval; + // A retry keyed to whatever is open could approve a newer, unseen request. + if (approval !== null && input.requestId === undefined && input.clientRequestId !== undefined) { + return yield* fail( + "request_id_required", + `Pass requestId with clientRequestId, so a retry decides the same request: ${callHint("t3_answer", { session: ref, requestId: requestRef(approval.requestId), decision: input.decision })}.`, + ); + } if ( approval === null || (input.requestId !== undefined && !matchesRequest(approval.requestId, input.requestId)) @@ -714,12 +728,14 @@ const answer = Effect.fn("agentToolkit.answer")(function* ( const dismiss = input.dismiss === true; const wanted = input.requestId; const fitting = dismiss ? open.filter((candidate) => candidate.async) : open; - // Without a request id a retry could land on the next open question. - if (wanted === undefined && fitting.length > 1) { - return yield* fail( - "request_id_required", - `${ref} has ${fitting.length} open ${dismiss ? "non-blocking " : ""}questions. Pass requestId to pick one: ${describeOpenQuestions(fitting)}.`, - ); + // Without a request id, a retry could land on a question asked since. + if (wanted === undefined && (fitting.length > 1 || input.clientRequestId !== undefined)) { + if (fitting.length > 0) { + return yield* fail( + "request_id_required", + `${fitting.length > 1 ? `${ref} has ${fitting.length} open ${dismiss ? "non-blocking " : ""}questions. Pass requestId to pick one` : "Pass requestId with clientRequestId, so a retry answers the same question"}: ${describeOpenQuestions(fitting)}.`, + ); + } } const target = wanted === undefined @@ -1060,7 +1076,7 @@ const brief = Effect.fn("agentToolkit.brief")(function* ( tagged: members, waitsOn: { approval: detail.approval === null ? null : boundApproval(detail.approval), - secret: detail.secret, + secret: boundSecret(detail.secret), question: detail.question === null ? null : boundQuestion(detail.question), asyncQuestions: detail.asyncQuestions.map(boundQuestion), plan: @@ -1307,9 +1323,16 @@ const sendMessage = Effect.fn("agentToolkit.sendMessage")(function* ( const attention = attentionOf(shell); // A non-blocking question did not stop the agent, so a message does not drop it. if (input.force !== true && (attention === "question" || attention === "approval")) { + // The shell reports a secret request as a question; no tool answers it. + const secret = + attention === "question" + ? (yield* loadPendingDetail(caller, shell, { includePlan: false })).secret + : null; return yield* fail( "question_open", - `${ref} waits on ${attention === "approval" ? "an approval" : "a blocking question"}. Answer it with t3_answer (see ${callHint("t3_session_brief", { session: ref })}), or pass force: true to send anyway; the agent may then drop the open ${attention === "approval" ? "approval" : "question"}.`, + secret !== null + ? `${ref} waits on a secret ("${oneLine(secret.label, 80)}"), which a person must enter in the app; no tool can answer it. Pass force: true to send anyway.` + : `${ref} waits on ${attention === "approval" ? "an approval" : "a blocking question"}. Answer it with t3_answer (see ${callHint("t3_session_brief", { session: ref })}), or pass force: true to send anyway; the agent may then drop the open ${attention === "approval" ? "approval" : "question"}.`, ); } yield* assertTargetWithinLimits(caller.limits, shell); @@ -1479,7 +1502,7 @@ const wait = Effect.fn("agentToolkit.wait")(function* (input: Tool.Parameters candidate.type === "secret_request" && candidate.id === request.id, + (candidate) => candidate.type === "secret_request" && candidate.id === standInId, ); if (secretItem?.type === "secret_request") { secret = { label: secretItem.label, reason: secretItem.reason }; diff --git a/apps/server/src/mcp/toolkits/agent/render.ts b/apps/server/src/mcp/toolkits/agent/render.ts index 8a25e32bab7f3..22553e030e3a4 100644 --- a/apps/server/src/mcp/toolkits/agent/render.ts +++ b/apps/server/src/mcp/toolkits/agent/render.ts @@ -187,7 +187,9 @@ export const renderApproval = ( if (option.warning !== undefined) lines.push(` Warning on ${DECISION_NAMES[option.decision]}: ${option.warning}`); } - lines.push(` → ${callHint("t3_answer", { session: sessionRef, decision: "approve" })}`); + lines.push( + ` → ${callHint("t3_answer", { session: sessionRef, requestId: requestRef(approval.requestId), decision: "approve" })}`, + ); return lines; }; diff --git a/apps/server/src/mcp/toolkits/agent/tools.ts b/apps/server/src/mcp/toolkits/agent/tools.ts index e07628c3c608b..582c28478d5bd 100644 --- a/apps/server/src/mcp/toolkits/agent/tools.ts +++ b/apps/server/src/mcp/toolkits/agent/tools.ts @@ -141,13 +141,13 @@ const ChoiceValue = Schema.Union([Schema.Int, Schema.String]); export const AnswerTool = destructive( Tool.make("t3_answer", { description: - "Answer what a session waits for, as the person you act for. Pass one of: choice (an option number or label, or a list for a multi-select question), text (your own answer), answers (one entry per question when a request asks several), decision (for an approval), or plan (approve, revise with feedback, or discard). A question is blocking (the agent is paused until you answer) or non-blocking (the agent asked and did not stop, as a Codex async question does; your answer reaches it as a new message, and dismiss: true closes the question without an answer). Pass the requestId t3_inbox shows; you may omit it only when one question is open. The tool refuses when nothing of that kind waits, when the request was already answered, and when an agent tries to answer in its own session. Get the options from t3_inbox or t3_session_brief.", + "Answer what a session waits for, as the person you act for. Pass one of: choice (an option number or label, or a list for a multi-select question), text (your own answer), answers (one entry per question when a request asks several), decision (for an approval), or plan (approve, revise with feedback, or discard). A question is blocking (the agent is paused until you answer) or non-blocking (the agent asked and did not stop, as a Codex async question does; your answer reaches it as a new message, and dismiss: true closes the question without an answer). Pass the requestId t3_inbox shows; omit it only when one item is open and you pass no clientRequestId. The tool refuses when nothing of that kind waits, when the request was already answered, and when an agent tries to answer in its own session. Get the options from t3_inbox or t3_session_brief.", parameters: Schema.Struct({ session: sessionRef(SESSION_REF_HELP), requestId: Schema.optional( described( Schema.String, - "The open request to answer: its id, or the short form t3_inbox shows. Required when a session has more than one open question, so a retry cannot answer another one.", + "The open request to answer: its id, or the short form t3_inbox shows. Required with clientRequestId, and when a session has more than one open question, so a retry cannot answer another request.", ), ), choice: Schema.optional( diff --git a/docs/user/t3-mcp-control.md b/docs/user/t3-mcp-control.md index b703fdbd6ed1c..12e98e06aed3c 100644 --- a/docs/user/t3-mcp-control.md +++ b/docs/user/t3-mcp-control.md @@ -168,8 +168,9 @@ such as a Codex async question, did not stop the agent: the answer reaches it as a new message, in its running turn or as a new turn, and `t3_answer` with `dismiss: true` closes the question without an answer. `t3_send_message` refuses only while a blocking question or an approval is open. -When a session has more than one open question, pass the `requestId` that -`t3_inbox` shows, so a retry cannot answer a different question. `t3_wait` returns within 50 seconds by default, because most +Pass the `requestId` that `t3_inbox` shows when a session has more than one +open question, and with every `clientRequestId`, so a retry cannot answer a +different request. `t3_wait` returns within 50 seconds by default, because most MCP clients stop a call after 60 seconds; call it again with the cursor it returns. `t3_start_session`, `t3_send_message` and `t3_answer` take a `clientRequestId`, so a retry with the same id does not act twice. From c0b8d2f91b59fe0f887719baf585be4e7e116d91 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Wed, 14 Oct 2026 10:23:58 +0000 Subject: [PATCH 05/20] feat(server): bkt3 product metrics in PostHog (TEC-1631) - Session time-in-state tracker: one state per session (agent_running, waiting_human, waiting_workers, queued, idle, archived) emitted as session.state.span events, plus run/request/subagent/plan events and a 5-minute concurrency snapshot. Same reducer drives a history backfill. - Identity: distinct_id is the environment user's work email, instance property and group, property allowlist that drops free text. - No upstream default PostHog key; telemetry stays off without one. - Agent child processes no longer inherit the analytics env. - Server events for MCP tool calls, scheduled tasks, webhooks, session recovery and Linear links. - Web/desktop client analytics via lazily loaded posthog-js, named events only, config delivered by the server at runtime. Co-Authored-By: Claude Opus 5.5 --- apps/server/scripts/analytics-backfill.ts | 403 +++++ .../src/environment/ServerEnvironment.ts | 6 + .../environment/clientAnalyticsConfig.test.ts | 45 + .../src/environment/clientAnalyticsConfig.ts | 57 + apps/server/src/mcp/McpToolAccess.ts | 4 +- .../src/mcp/mcpToolAnalytics.expbkt3.ts | 39 + .../src/mcp/toolkits/control/handlers.ts | 11 +- .../ProviderRuntimeRecoveryService.ts | 11 +- .../provider/ProviderInstanceEnvironment.ts | 6 +- .../scheduledTasks/ScheduledTaskService.ts | 11 + .../server/src/scheduledTasks/webhookRoute.ts | 3 + apps/server/src/server.ts | 2 + apps/server/src/session-webhooks/http.ts | 5 + .../AnalyticsService.identity.expbkt3.test.ts | 244 +++ .../src/telemetry/AnalyticsService.test.ts | 12 +- apps/server/src/telemetry/AnalyticsService.ts | 163 +- .../telemetry/analyticsEnvironment.test.ts | 61 + .../src/telemetry/analyticsEnvironment.ts | 38 + .../server/src/telemetry/analyticsIdentity.ts | 102 ++ .../src/telemetry/analyticsProperties.ts | 182 +++ .../server/src/telemetry/serverEvents.test.ts | 142 ++ apps/server/src/telemetry/serverEvents.ts | 257 ++++ .../sessionActivity/SessionActivityReactor.ts | 97 ++ .../SessionActivityReducer.test.ts | 515 +++++++ .../sessionActivity/SessionActivityReducer.ts | 1336 +++++++++++++++++ apps/server/src/ws.ts | 26 +- apps/web/package.json | 1 + apps/web/src/components/ChatView.tsx | 34 +- apps/web/src/components/CommandPalette.tsx | 6 + .../components/planreview/PlanReviewPanel.tsx | 6 + apps/web/src/hooks/useSettings.ts | 4 + .../analytics/ProductAnalyticsCoordinator.tsx | 125 ++ .../analytics/productAnalytics.logic.test.ts | 185 +++ .../lib/analytics/productAnalytics.logic.ts | 230 +++ .../lib/analytics/productAnalytics.test.ts | 116 ++ .../web/src/lib/analytics/productAnalytics.ts | 218 +++ apps/web/src/routes/__root.tsx | 3 + deploy/bkt3/README.md | 45 + packages/contracts/src/environment.ts | 19 + pnpm-lock.yaml | 137 +- 40 files changed, 4840 insertions(+), 67 deletions(-) create mode 100644 apps/server/scripts/analytics-backfill.ts create mode 100644 apps/server/src/environment/clientAnalyticsConfig.test.ts create mode 100644 apps/server/src/environment/clientAnalyticsConfig.ts create mode 100644 apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts create mode 100644 apps/server/src/telemetry/AnalyticsService.identity.expbkt3.test.ts create mode 100644 apps/server/src/telemetry/analyticsEnvironment.test.ts create mode 100644 apps/server/src/telemetry/analyticsEnvironment.ts create mode 100644 apps/server/src/telemetry/analyticsIdentity.ts create mode 100644 apps/server/src/telemetry/analyticsProperties.ts create mode 100644 apps/server/src/telemetry/serverEvents.test.ts create mode 100644 apps/server/src/telemetry/serverEvents.ts create mode 100644 apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts create mode 100644 apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts create mode 100644 apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts create mode 100644 apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx create mode 100644 apps/web/src/lib/analytics/productAnalytics.logic.test.ts create mode 100644 apps/web/src/lib/analytics/productAnalytics.logic.ts create mode 100644 apps/web/src/lib/analytics/productAnalytics.test.ts create mode 100644 apps/web/src/lib/analytics/productAnalytics.ts diff --git a/apps/server/scripts/analytics-backfill.ts b/apps/server/scripts/analytics-backfill.ts new file mode 100644 index 0000000000000..95404f72c9081 --- /dev/null +++ b/apps/server/scripts/analytics-backfill.ts @@ -0,0 +1,403 @@ +#!/usr/bin/env node +// @effect-diagnostics nodeBuiltinImport:off globalDate:off globalFetch:off globalConsole:off globalTimers:off preferSchemaOverJson:off - a one-off operator script over a SQLite copy, outside any Effect runtime. +/** + * T3-CUSTOM(expbkt3): replay the orchestration V2 event history into PostHog + * session metrics (TEC-1631). + * + * Feeds the same tracker the live server uses + * (`src/telemetry/sessionActivity/SessionActivityReducer.ts`) with the events + * of a SQLite COPY of the server state, and writes the resulting PostHog + * events. It never opens a live state file: make a copy first, for example + * with `VACUUM INTO` (see AGENTS.md, "Test data"). + * + * node apps/server/scripts/analytics-backfill.ts [options] + * + * --dry-run (default) write JSONL and print counts; send nothing + * --out JSONL output path (default analytics-backfill.jsonl) + * --send POST to ${T3CODE_POSTHOG_HOST}/batch/; the key comes + * only from env T3CODE_POSTHOG_KEY + * --until ignore events after this time. Set it to the moment + * the live reactor started, so the two never overlap. + * --report-days print per-state hours for the last n days (default 7) + * + * Events are processed one thread at a time, sorted by `occurredAt`: imported + * history was appended to the log out of time order, so sequence order alone + * is not time order. Spans still open at the end are not emitted; the live + * server owns them. Every event carries `backfill: true` and a stable + * `$insert_id` and uuid, so a re-run does not duplicate events. + */ +import * as NodeCrypto from "node:crypto"; +import * as NodeFs from "node:fs"; +import * as NodeOs from "node:os"; +import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; + +import { + makeSessionActivityTracker, + type SessionActivityEmission, + type SessionActivityEvent, + toEpochMs, +} from "../src/telemetry/sessionActivity/SessionActivityReducer.ts"; + +const BATCH_SIZE = 500; +const MAX_SEND_ATTEMPTS = 6; + +/** Event types the tracker reads; everything else is skipped in SQL. */ +const RELEVANT_EVENT_TYPES = [ + "run.created", + "run.updated", + "run-attempt.created", + "runtime-request.updated", + "subagent.updated", + "turn-item.updated", + "provider-thread.updated", + "plan.updated", + "message.updated", +] as const; + +interface Options { + readonly dbPath: string; + readonly send: boolean; + readonly outPath: string; + readonly untilMs: number | undefined; + readonly reportDays: number; +} + +function fail(message: string): never { + console.error(`analytics-backfill: ${message}`); + process.exit(1); +} + +function parseArgs(argv: ReadonlyArray): Options { + let dbPath: string | undefined; + let send = false; + let outPath = "analytics-backfill.jsonl"; + let untilMs: number | undefined; + let reportDays = 7; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = () => argv[++index] ?? fail(`${arg} needs a value`); + switch (arg) { + case "--dry-run": + send = false; + break; + case "--send": + send = true; + break; + case "--out": + outPath = next(); + break; + case "--until": + untilMs = toEpochMs(next()) ?? fail("--until must be an ISO time"); + break; + case "--report-days": + reportDays = Number(next()); + break; + default: + if (arg === undefined || arg.startsWith("--")) fail(`unknown option ${arg}`); + dbPath = arg; + } + } + if (dbPath === undefined) fail("pass the path of a SQLite COPY of the state database"); + return { dbPath, send, outPath, untilMs, reportDays }; +} + +/** Refuses any state file under a T3 home `userdata` directory: those are live. */ +function assertNotLiveState(dbPath: string): string { + const resolved = NodeFs.realpathSync(dbPath); + const t3Home = NodePath.join(NodeOs.homedir(), ".t3"); + const relative = NodePath.relative(t3Home, resolved); + const insideT3Home = !relative.startsWith("..") && !NodePath.isAbsolute(relative); + if (insideT3Home && relative.split(NodePath.sep).includes("userdata")) { + fail(`${resolved} is a live T3 state file. Copy it with VACUUM INTO and pass the copy.`); + } + return resolved; +} + +type Rec = Record; + +/** Free-text payload fields dropped as soon as a row is read. */ +const TEXT_KEYS = [ + "prompt", + "result", + "progress", + "title", + "text", + "attachments", + "context", + "markdown", + "steps", + "explanation", + "answers", +] as const; + +function asRec(value: unknown): Rec | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Rec) + : undefined; +} + +/** + * Keeps only the payload fields the tracker reads, so a thread's history fits + * in memory and no prompt, output or message text is held longer than a row. + */ +function slimPayload(type: string, payload: Rec): Rec { + switch (type) { + case "turn-item.updated": + return { + id: payload.id, + type: payload.type, + status: payload.status, + runId: payload.runId ?? null, + ...(asRec(payload.input)?.persistent === true ? { input: { persistent: true } } : {}), + ...(asRec(payload.failure) === undefined + ? {} + : { failure: { class: asRec(payload.failure)?.class } }), + }; + case "subagent.updated": + case "message.updated": + case "plan.updated": { + const rest: Rec = { ...payload }; + for (const key of TEXT_KEYS) delete rest[key]; + return rest; + } + default: + if (type.startsWith("thread.")) { + // The slim analysis copy replaced these with booleans; restore a marker. + const marked: Rec = { ...payload }; + if (payload.hasMattermostThread === true) marked.mattermostThreadUrl = "present"; + if (payload.hasLinearLink === true) marked.linearIssueUrl = "present"; + delete marked.title; + delete marked.branch; + delete marked.worktreePath; + return marked; + } + return payload; + } +} + +interface EventRow { + readonly sequence: number; + readonly stream_id: string; + readonly event_type: string; + readonly occurred_at: string; + readonly payload_json: string; + readonly metadata_json: string; +} + +interface QueuedEvent { + readonly sequence: number; + readonly atMs: number; + readonly event: SessionActivityEvent; +} + +function toEvent(row: EventRow): QueuedEvent | undefined { + const atMs = toEpochMs(row.occurred_at); + if (atMs === undefined) return undefined; + const payload = asRec(JSON.parse(row.payload_json)); + const metadata = asRec(JSON.parse(row.metadata_json)) ?? {}; + if (payload === undefined) return undefined; + const text = (key: string) => + typeof metadata[key] === "string" ? (metadata[key] as string) : undefined; + const runId = text("runId"); + const actorUserId = text("actorUserId"); + const driver = text("driver"); + return { + sequence: row.sequence, + atMs, + event: { + type: row.event_type, + threadId: row.stream_id, + occurredAt: row.occurred_at, + payload: slimPayload(row.event_type, payload), + ...(runId === undefined ? {} : { runId }), + ...(actorUserId === undefined ? {} : { actorUserId }), + ...(driver === undefined ? {} : { driver }), + }, + }; +} + +/** Same derivation as `analyticsIdentity.uuidFromSha256Hex`, so live and backfill ids agree. */ +function uuidFromInsertId(insertId: string): string { + const hex = NodeCrypto.createHash("sha256").update(insertId).digest("hex").padEnd(32, "0"); + const variant = ((Number.parseInt(hex[16] ?? "0", 16) & 0x3) | 0x8).toString(16); + return [ + hex.slice(0, 8), + hex.slice(8, 12), + `8${hex.slice(13, 16)}`, + `${variant}${hex.slice(17, 20)}`, + hex.slice(20, 32), + ].join("-"); +} + +function loadUserEmails(db: NodeSqlite.DatabaseSync): Map { + const emails = new Map(); + const table = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'environment_users'") + .get(); + if (table === undefined) return emails; + for (const row of db + .prepare("SELECT user_id, primary_email FROM environment_users") + .all() as Array<{ user_id: string; primary_email: string | null }>) { + if (row.primary_email) emails.set(row.user_id, row.primary_email); + } + return emails; +} + +interface BatchItem { + readonly uuid: string; + readonly event: string; + readonly distinct_id: string; + readonly timestamp: string; + readonly properties: Rec; +} + +function toBatchItem( + emission: SessionActivityEmission, + emails: ReadonlyMap, + instance: string, +): BatchItem { + const email = emission.userId === undefined ? undefined : emails.get(emission.userId); + return { + uuid: uuidFromInsertId(emission.insertId), + event: emission.event, + distinct_id: email ?? `bkt3-instance:${instance}`, + timestamp: emission.timestamp, + properties: { + ...emission.properties, + backfill: true, + $insert_id: emission.insertId, + $process_person_profile: email !== undefined, + $groups: { instance }, + instance, + }, + }; +} + +async function sendBatch(host: string, apiKey: string, batch: ReadonlyArray) { + for (let attempt = 1; ; attempt += 1) { + try { + const response = await fetch(`${host}/batch/`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ api_key: apiKey, historical_migration: true, batch }), + }); + if (response.ok) return; + if (response.status < 500 && response.status !== 429) { + fail(`PostHog rejected a batch with HTTP ${response.status}`); + } + throw new Error(`HTTP ${response.status}`); + } catch (cause) { + if (attempt >= MAX_SEND_ATTEMPTS) + fail(`batch failed after ${attempt} attempts: ${String(cause)}`); + await new Promise((resolve) => setTimeout(resolve, 1_000 * 2 ** attempt)); + } + } +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const dbPath = assertNotLiveState(options.dbPath); + const host = process.env.T3CODE_POSTHOG_HOST ?? "https://us.i.posthog.com"; + const apiKey = process.env.T3CODE_POSTHOG_KEY ?? ""; + const instance = process.env.T3CODE_ANALYTICS_INSTANCE ?? "unknown"; + if (options.send && apiKey.trim() === "") fail("--send needs env T3CODE_POSTHOG_KEY"); + + const db = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true }); + const emails = loadUserEmails(db); + const tracker = makeSessionActivityTracker({ backfill: true }); + + const out = options.send ? undefined : NodeFs.openSync(options.outPath, "w"); + const counts = new Map(); + const stateHours = new Map(); + const newest = db + .prepare("SELECT occurred_at FROM orchestration_events ORDER BY sequence DESC LIMIT 1") + .get() as { occurred_at: string } | undefined; + const reportEndMs = options.untilMs ?? toEpochMs(newest?.occurred_at) ?? 0; + const reportFromMs = reportEndMs - options.reportDays * 86_400_000; + let latestMs = 0; + let pending: Array = []; + let sent = 0; + + const flush = async () => { + if (pending.length === 0) return; + const batch = pending; + pending = []; + if (options.send) { + await sendBatch(host, apiKey, batch); + sent += batch.length; + } else if (out !== undefined) { + NodeFs.writeSync(out, batch.map((item) => JSON.stringify(item)).join("\n") + "\n"); + } + }; + + const collect = async (emissions: ReadonlyArray) => { + for (const emission of emissions) { + counts.set(emission.event, (counts.get(emission.event) ?? 0) + 1); + if (emission.event === "session.state.span") { + const endedMs = toEpochMs(emission.timestamp) ?? 0; + if (endedMs >= reportFromMs) { + const state = String(emission.properties.state); + const hours = Number(emission.properties.durationMs) / 3_600_000; + stateHours.set(state, (stateHours.get(state) ?? 0) + hours); + } + } + pending.push(toBatchItem(emission, emails, instance)); + if (pending.length >= BATCH_SIZE) await flush(); + } + }; + + const placeholders = RELEVANT_EVENT_TYPES.map(() => "?").join(", "); + // Uses idx_orch_events_stream_sequence (aggregate_kind, stream_id, sequence). + const rows = db + .prepare( + `SELECT sequence, stream_id, event_type, occurred_at, payload_json, metadata_json + FROM orchestration_events + WHERE aggregate_kind = 'thread' + AND application_event_version = 2 + AND (event_type LIKE 'thread.%' OR event_type IN (${placeholders})) + ORDER BY aggregate_kind, stream_id, sequence`, + ) + .iterate(...RELEVANT_EVENT_TYPES) as Iterable; + + let threadId: string | undefined; + let threadEvents: Array = []; + let threads = 0; + const replayThread = async () => { + threadEvents.sort((a, b) => a.atMs - b.atMs || a.sequence - b.sequence); + for (const queued of threadEvents) { + if (options.untilMs !== undefined && queued.atMs > options.untilMs) break; + latestMs = Math.max(latestMs, queued.atMs); + await collect(tracker.apply(queued.event)); + } + threadEvents = []; + threads += 1; + }; + + for (const row of rows) { + if (row.stream_id !== threadId) { + if (threadId !== undefined) await replayThread(); + threadId = row.stream_id; + } + const queued = toEvent(row); + if (queued !== undefined) threadEvents.push(queued); + } + if (threadId !== undefined) await replayThread(); + await flush(); + if (out !== undefined) NodeFs.closeSync(out); + db.close(); + + console.log(`threads replayed: ${threads}`); + console.log(`latest event: ${latestMs > 0 ? new Date(latestMs).toISOString() : "none"}`); + console.log(options.send ? `sent: ${sent} events` : `wrote: ${options.outPath}`); + console.log("events by type:"); + for (const [event, count] of [...counts].sort((a, b) => b[1] - a[1])) { + console.log(` ${event}: ${count}`); + } + console.log(`state hours, last ${options.reportDays} days (by span end):`); + for (const [state, hours] of [...stateHours].sort((a, b) => b[1] - a[1])) { + console.log(` ${state}: ${hours.toFixed(1)}`); + } +} + +await main(); diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index fe4dc61231c48..c212b834550b5 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -22,6 +22,8 @@ import { resolveServiceLauncherMode } from "../cloud/serviceLauncherClient.ts"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import { resolveServerEnvironmentLabel } from "./ServerEnvironmentLabel.ts"; +// T3-CUSTOM(expbkt3): product analytics config for clients (TEC-1631). +import { loadClientAnalyticsConfig } from "./clientAnalyticsConfig.ts"; import { detectServerEnvironmentMachineKind } from "./ServerEnvironmentMachine.ts"; export class ServerEnvironmentIdPersistenceError extends Schema.TaggedError()( @@ -204,6 +206,8 @@ export const make = Effect.gen(function* () { // the fd and correctly do not advertise. const desktopAppUpdate = serverSelfUpdate === "desktop-managed" && serverConfig.desktopTelemetryControlFd !== undefined; + // T3-CUSTOM(expbkt3): read once at startup, like the rest of the descriptor. + const analytics = yield* loadClientAnalyticsConfig; const descriptor: ExecutionEnvironmentDescriptor = { environmentId, @@ -282,6 +286,8 @@ export const make = Effect.gen(function* () { ...(desktopAppUpdate ? { desktopAppUpdate: true } : {}), serverBrowser: true, }, + // T3-CUSTOM(expbkt3): absent unless an explicit company PostHog key is set. + ...(analytics === null ? {} : { analytics }), }; return ServerEnvironment.of({ diff --git a/apps/server/src/environment/clientAnalyticsConfig.test.ts b/apps/server/src/environment/clientAnalyticsConfig.test.ts new file mode 100644 index 0000000000000..5baf91f76b186 --- /dev/null +++ b/apps/server/src/environment/clientAnalyticsConfig.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as Option from "effect/Option"; + +import { resolveClientAnalyticsConfig } from "./clientAnalyticsConfig.ts"; + +const base = { + posthogKey: Option.some("phc_company"), + posthogHost: Option.none(), + enabled: true, + instance: Option.some("bkt3"), +}; + +describe("resolveClientAnalyticsConfig", () => { + it("advertises an explicit key with the default host and the instance", () => { + expect(resolveClientAnalyticsConfig(base)).toEqual({ + enabled: true, + posthogKey: "phc_company", + posthogHost: "https://us.i.posthog.com", + instance: "bkt3", + }); + }); + + it("stays absent without an explicit key, so clients never use upstream's project", () => { + expect(resolveClientAnalyticsConfig({ ...base, posthogKey: Option.none() })).toBeNull(); + expect(resolveClientAnalyticsConfig({ ...base, posthogKey: Option.some(" ") })).toBeNull(); + }); + + it("stays absent when telemetry is off", () => { + expect(resolveClientAnalyticsConfig({ ...base, enabled: false })).toBeNull(); + }); + + it("omits a blank instance and honours a custom host", () => { + expect( + resolveClientAnalyticsConfig({ + ...base, + instance: Option.some(""), + posthogHost: Option.some("https://eu.i.posthog.com"), + }), + ).toEqual({ + enabled: true, + posthogKey: "phc_company", + posthogHost: "https://eu.i.posthog.com", + }); + }); +}); diff --git a/apps/server/src/environment/clientAnalyticsConfig.ts b/apps/server/src/environment/clientAnalyticsConfig.ts new file mode 100644 index 0000000000000..e4f1a2e8fd264 --- /dev/null +++ b/apps/server/src/environment/clientAnalyticsConfig.ts @@ -0,0 +1,57 @@ +/** + * T3-CUSTOM(expbkt3): product analytics config handed to clients (TEC-1631). + * + * The web and desktop clients send their own PostHog events, to the same + * company project the server reports to. They learn the key from the + * environment descriptor at runtime, so no key is baked into the bundle and + * each environment decides for itself. + * + * Only an explicitly set `T3CODE_POSTHOG_KEY` is advertised. The server's own + * telemetry falls back to upstream's project key; clients must never send + * Beknown usage there, so with no explicit key the field is absent and the + * client stays off. + * + * @module clientAnalyticsConfig + */ +import type { ExecutionEnvironmentAnalytics } from "@t3tools/contracts"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com"; + +const ClientAnalyticsEnvConfig = Config.all({ + posthogKey: Config.String("T3CODE_POSTHOG_KEY").pipe(Config.option), + posthogHost: Config.String("T3CODE_POSTHOG_HOST").pipe(Config.option), + enabled: Config.Boolean("T3CODE_TELEMETRY_ENABLED").pipe(Config.withDefault(true)), + instance: Config.String("T3CODE_ANALYTICS_INSTANCE").pipe(Config.option), +}); + +function nonEmpty(value: Option.Option): string | null { + const trimmed = Option.getOrNull(value)?.trim() ?? ""; + return trimmed.length > 0 ? trimmed : null; +} + +/** Pure mapping from env values to the descriptor field; `null` means omit it. */ +export function resolveClientAnalyticsConfig(input: { + readonly posthogKey: Option.Option; + readonly posthogHost: Option.Option; + readonly enabled: boolean; + readonly instance: Option.Option; +}): ExecutionEnvironmentAnalytics | null { + const posthogKey = nonEmpty(input.posthogKey); + if (!input.enabled || posthogKey === null) return null; + const instance = nonEmpty(input.instance); + return { + enabled: true, + posthogKey, + posthogHost: nonEmpty(input.posthogHost) ?? DEFAULT_POSTHOG_HOST, + ...(instance === null ? {} : { instance }), + }; +} + +/** Reads the env once; a bad value disables client analytics, never the server. */ +export const loadClientAnalyticsConfig: Effect.Effect = + Effect.gen(function* () { + return resolveClientAnalyticsConfig(yield* ClientAnalyticsEnvConfig); + }).pipe(Effect.orElseSucceed(() => null)); diff --git a/apps/server/src/mcp/McpToolAccess.ts b/apps/server/src/mcp/McpToolAccess.ts index 6e882629bbba1..f7fddadc2c107 100644 --- a/apps/server/src/mcp/McpToolAccess.ts +++ b/apps/server/src/mcp/McpToolAccess.ts @@ -16,6 +16,7 @@ import { type DispatchModeRefusal, } from "../orchestration-v2/DispatchModeLimit.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; +import { withMcpToolCallAnalytics } from "./mcpToolAnalytics.expbkt3.ts"; // T3-CUSTOM(expbkt3) import { resolveInteractionMode, resolveRuntimeMode } from "./OrchestratorMcpService.ts"; import { assertFullAccess, @@ -300,7 +301,8 @@ export class HandlersLayer, EX = never, R } const checkedHandlers = (declarations: Declarations): Handlers => - Struct.map(declarations, checkedHandler); + // T3-CUSTOM(expbkt3): count every tool call by name (TEC-1631). + withMcpToolCallAnalytics(Struct.map(declarations, checkedHandler)); /** `Toolkit.toLayer` for handlers that all declare their access. */ export const toLayer = , EX = never, RX = never>( diff --git a/apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts b/apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts new file mode 100644 index 0000000000000..8a7555d9c3fa1 --- /dev/null +++ b/apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts @@ -0,0 +1,39 @@ +/** + * T3-CUSTOM(expbkt3): counts every T3 MCP tool call by name (TEC-1631). + * + * `McpToolAccess.toLayer` is the one place every `/mcp` toolkit's handlers are + * built, keyed by tool name, so wrapping them there covers every tool. Only + * the name, caller kind, outcome and duration are recorded, never arguments. + */ +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import { type McpToolCaller, withMcpToolAnalytics } from "../telemetry/serverEvents.ts"; +import * as McpInvocationContext from "./McpInvocationContext.ts"; + +const currentCaller: Effect.Effect = Effect.serviceOption( + McpInvocationContext.McpInvocationContext, +).pipe( + Effect.map((scope) => + Option.isNone(scope) + ? undefined + : { principal: scope.value.principal, actorUserId: scope.value.actorUserId }, + ), +); + +/** `handlers` with each Effect-returning handler timed and recorded under its key. */ +export function withMcpToolCallAnalytics(handlers: Handlers): Handlers { + const wrapped: Record = {}; + for (const [toolName, handler] of Object.entries(handlers as Record)) { + wrapped[toolName] = + typeof handler === "function" + ? (...args: ReadonlyArray) => { + const result = (handler as (...params: ReadonlyArray) => unknown)(...args); + return Effect.isEffect(result) + ? withMcpToolAnalytics(toolName, currentCaller, result) + : result; + } + : handler; + } + return wrapped as Handlers; +} diff --git a/apps/server/src/mcp/toolkits/control/handlers.ts b/apps/server/src/mcp/toolkits/control/handlers.ts index a13364048a13d..6efa32390bb00 100644 --- a/apps/server/src/mcp/toolkits/control/handlers.ts +++ b/apps/server/src/mcp/toolkits/control/handlers.ts @@ -78,6 +78,8 @@ import { hasUserWideScope, resolveMcpSessionTarget } from "../../mcpSessionTarge import { boundSessionDetail } from "./sessionDetail.ts"; import { T3ControlToolkit, T3ControlToolError, type T3CreateSessionTool } from "./tools.ts"; import type { Tool } from "effect/ai"; +import { recordLinearLinked } from "../../../telemetry/serverEvents.ts"; +import { CurrentOrchestrationActorUserId } from "../../../orchestration-v2/forkActor.expbkt3.ts"; const nowIso = Effect.map(DateTime.now, DateTime.formatIso); const decodeOrchestrationCommand = Schema.decodeUnknownEffect(OrchestrationCommandSchema); @@ -1662,10 +1664,17 @@ const handlers = { const after = yield* readLinearLinks(operation, sessionId); const urls = links.map((link) => link.url); const has = (list: typeof before, url: string) => list.some((entry) => entry.url === url); + const added = urls.filter((url) => !has(before, url) && has(after, url)); + // Product analytics: how many items were linked, never which (TEC-1631). + yield* recordLinearLinked({ + threadId: sessionId, + count: added.length, + userId: yield* CurrentOrchestrationActorUserId, + }); return { sessionId, linearLinks: after, - added: urls.filter((url) => !has(before, url) && has(after, url)), + added, alreadyLinked: urls.filter((url) => has(before, url)), // Left out because the session already holds the maximum number of tags. notAdded: urls.filter((url) => !has(before, url) && !has(after, url)), diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 427c79c8c0544..005e2e0e18b53 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -27,6 +27,8 @@ import { settledBackgroundWakeRequest, settledBackgroundWakeRun, } from "./restartBackgroundResume.expbkt3.ts"; +// T3-CUSTOM(expbkt3): session.recovery.* product analytics (TEC-1631). +import { withSessionRecoveryAnalytics } from "../telemetry/serverEvents.ts"; import { cancelledRosterTaskWork, cancelledTurnItemWork, @@ -834,9 +836,12 @@ export const make = Effect.gen(function* () { Effect.mapError((cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause })), ); - const recover = Effect.gen(function* () { - return (yield* reconcile("startup")) satisfies ProviderRuntimeRecoverySummary; - }); + // T3-CUSTOM(expbkt3): session.recovery.* product analytics (TEC-1631). + const recover = withSessionRecoveryAnalytics( + Effect.gen(function* () { + return (yield* reconcile("startup")) satisfies ProviderRuntimeRecoverySummary; + }), + ); return ProviderRuntimeRecoveryService.of({ reconcile, prepareForShutdown, recover }); }); diff --git a/apps/server/src/provider/ProviderInstanceEnvironment.ts b/apps/server/src/provider/ProviderInstanceEnvironment.ts index 77c0c6c2dc888..ccbcf344851e3 100644 --- a/apps/server/src/provider/ProviderInstanceEnvironment.ts +++ b/apps/server/src/provider/ProviderInstanceEnvironment.ts @@ -1,13 +1,15 @@ import type { ProviderInstanceEnvironment } from "@t3tools/contracts"; import { expandHomePath } from "../pathExpansion.ts"; +// T3-CUSTOM(expbkt3): agents never inherit the server's analytics settings (TEC-1631). +import { withoutAnalyticsEnvironment } from "../telemetry/analyticsEnvironment.ts"; export function mergeProviderInstanceEnvironment( environment: ProviderInstanceEnvironment | undefined, baseEnv: NodeJS.ProcessEnv = process.env, ): NodeJS.ProcessEnv { if (!environment || environment.length === 0) { - return baseEnv; + return withoutAnalyticsEnvironment(baseEnv); // T3-CUSTOM(expbkt3): TEC-1631. } const next: NodeJS.ProcessEnv = { ...baseEnv }; @@ -18,5 +20,5 @@ export function mergeProviderInstanceEnvironment( ? expandHomePath(variable.value) : variable.value; } - return next; + return withoutAnalyticsEnvironment(next); // T3-CUSTOM(expbkt3): TEC-1631. } diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 407d5fac55115..4d61c04e81128 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -53,6 +53,7 @@ import { type WebhookRequest, } from "./webhookTemplate.ts"; import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts"; +import { recordScheduledTaskRan } from "../telemetry/serverEvents.ts"; // T3-CUSTOM(expbkt3) // T3-CUSTOM(expbkt3): BK sidebar custom group of the threads a task launches (XFN-59). import { deleteScheduledTaskCustomGroup, @@ -904,6 +905,16 @@ export const layer = Layer.effect( lastRunError, runCount: scheduleSource.runCount + 1, }; + // T3-CUSTOM(expbkt3): product analytics for each finished run (TEC-1631). + yield* recordScheduledTaskRan({ + taskId: task.id, + scheduleType: scheduleSource.schedule.type, + trigger, + creationSource: active.creationSource, + status: lastRunStatus, + durationMs: DateTime.toEpochMillis(completedAt) - DateTime.toEpochMillis(startedAt), + ownerUserId: forkOwner, + }); if (current !== null) { // startedAtIso in the guard ensures this writes only to the row this // run marked as running — a task deleted mid-run and recreated with diff --git a/apps/server/src/scheduledTasks/webhookRoute.ts b/apps/server/src/scheduledTasks/webhookRoute.ts index 9fcf8344f50d7..4e01bb4d051cd 100644 --- a/apps/server/src/scheduledTasks/webhookRoute.ts +++ b/apps/server/src/scheduledTasks/webhookRoute.ts @@ -13,6 +13,7 @@ import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder"; import * as Metrics from "../observability/Metrics.ts"; import * as RelayDeliveryProof from "./RelayDeliveryProof.ts"; import * as ScheduledTaskService from "./ScheduledTaskService.ts"; +import { recordWebhookDelivered } from "../telemetry/serverEvents.ts"; // T3-CUSTOM(expbkt3) /** Largest request body a webhook accepts. The relay enforces the same cap. */ export const WEBHOOK_MAX_BODY_BYTES = 1024 * 1024; @@ -111,6 +112,8 @@ export const layer = HttpApiBuilder.group( Effect.as({ _tag: "error" as const }), ), ), + // T3-CUSTOM(expbkt3): product analytics, outcome tag only (TEC-1631). + Effect.tap((delivery) => recordWebhookDelivered("scheduled_task", delivery._tag)), ); switch (result._tag) { diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index c586afe25aa30..b1383df1e47c0 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -89,6 +89,7 @@ import * as SqlitePersistence from "./persistence/Sqlite.ts"; import * as PullRequestFilesViewed from "./persistence/PullRequestFilesViewed.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; +import * as SessionActivityReactor from "./telemetry/sessionActivity/SessionActivityReactor.ts"; // T3-CUSTOM(expbkt3): TEC-1631. import * as ProviderEventIngestor from "./orchestration-v2/ProviderEventIngestor.ts"; import * as ModelManifest from "./provider/ModelManifest.ts"; import * as ResetCreditCoordinator from "./provider/resetCreditCoordinator.ts"; @@ -656,6 +657,7 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( ProviderUsageLimitsIngestion.layer, layerProviderInstallationRefresh, ReplayMarkers.layer, + SessionActivityReactor.layer, // T3-CUSTOM(expbkt3): session time-in-state metrics (TEC-1631). ).pipe( // Core Services Layer.provideMerge(layerOrchestrationApplication), diff --git a/apps/server/src/session-webhooks/http.ts b/apps/server/src/session-webhooks/http.ts index 9a10790d2dd40..c0a20b4163beb 100644 --- a/apps/server/src/session-webhooks/http.ts +++ b/apps/server/src/session-webhooks/http.ts @@ -12,6 +12,7 @@ import { import { SessionWebhookError } from "@t3tools/contracts"; import { SessionWebhookService } from "./SessionWebhookService.ts"; import { MAX_CALLBACK_BYTES } from "./protocol.ts"; +import { recordWebhookDelivered, webhookStatusFromHttp } from "../telemetry/serverEvents.ts"; const isSessionWebhookError = Schema.is(SessionWebhookError); export const sessionWebhookRouteLayer = Layer.unwrap( Effect.gen(function* () { @@ -53,6 +54,10 @@ export const sessionWebhookRouteLayer = Layer.unwrap( ), ), ), + // Product analytics: the answer's status class only (TEC-1631). + Effect.tap((response) => + recordWebhookDelivered("session_callback", webhookStatusFromHttp(response.status)), + ), ); return HttpRouter.add("POST", "/api/session-webhooks/:webhookId", receive); }), diff --git a/apps/server/src/telemetry/AnalyticsService.identity.expbkt3.test.ts b/apps/server/src/telemetry/AnalyticsService.identity.expbkt3.test.ts new file mode 100644 index 0000000000000..dff32f3bcc9cf --- /dev/null +++ b/apps/server/src/telemetry/AnalyticsService.identity.expbkt3.test.ts @@ -0,0 +1,244 @@ +// T3-CUSTOM(expbkt3): bkt3 analytics identity, allowlist and key gating (TEC-1631). +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { EnvironmentUserId } from "@t3tools/contracts"; +import { HostProcessArchitecture, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as HttpServer from "effect/http/HttpServer"; +import * as HttpServerRequest from "effect/http/HttpServerRequest"; +import * as HttpServerResponse from "effect/http/HttpServerResponse"; + +import * as ServerConfig from "../config.ts"; +import * as EnvironmentUsers from "../persistence/EnvironmentUsers.ts"; +import { getTelemetryIdentifier } from "./Identify.ts"; +import * as AnalyticsService from "./AnalyticsService.ts"; + +interface SentEvent { + readonly uuid: string; + readonly event: string; + readonly distinct_id: string; + readonly timestamp: string; + readonly properties: Readonly>; +} + +const ALICE = EnvironmentUserId.make("user_clerk_alice"); +const BOB_WITHOUT_EMAIL = EnvironmentUserId.make("user_clerk_bob"); + +const users: ReadonlyArray = [ + { + userId: ALICE, + displayName: "Alice Example", + primaryEmail: " Alice@Example.com ", + avatarUrl: null, + role: "admin", + status: "active", + firstSeenAt: DateTime.makeUnsafe(0), + lastSeenAt: DateTime.makeUnsafe(0), + }, + { + userId: BOB_WITHOUT_EMAIL, + displayName: "Bob", + primaryEmail: null, + avatarUrl: null, + role: "member", + status: "active", + firstSeenAt: DateTime.makeUnsafe(0), + lastSeenAt: DateTime.makeUnsafe(0), + }, +]; + +const fakeUsers = EnvironmentUsers.EnvironmentUserRepository.of({ + upsert: () => Effect.void, + get: () => Effect.succeed(Option.none()), + update: () => Effect.succeed(false), + list: Effect.succeed(users), +}); + +/** Runs `body` against a local batch endpoint and returns every event it received. */ +const captureEvents = ( + env: Record, + body: (analytics: AnalyticsService.AnalyticsService["Service"]) => Effect.Effect, +) => + Effect.gen(function* () { + const sent: Array = []; + let requests = 0; + let installId: string | null = null; + const layerBatchServer = HttpServer.serve( + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + requests += 1; + const payload = yield* request.json.pipe(Effect.orElseSucceed(() => null)); + const batch = (payload as { readonly batch?: ReadonlyArray } | null)?.batch; + if (Array.isArray(batch)) sent.push(...batch); + return HttpServerResponse.jsonUnsafe({}); + }), + ); + const layerRuntime = AnalyticsService.layer.pipe( + Layer.provideMerge( + ServerConfig.ServerConfig.layerTest(process.cwd(), { prefix: "t3-telemetry-identity-" }), + ), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ T3CODE_POSTHOG_HOST: "http://localhost", ...env }), + ), + ), + Layer.provide( + Layer.mergeAll( + Layer.succeed(HostProcessPlatform, "linux"), + Layer.succeed(HostProcessArchitecture, "arm64"), + ), + ), + Layer.provideMerge(NodeHttpServer.layerTest), + ); + yield* Effect.gen(function* () { + yield* Layer.launch(layerBatchServer).pipe(Effect.forkScoped); + const analytics = yield* AnalyticsService.AnalyticsService; + installId = yield* getTelemetryIdentifier; + yield* body(analytics); + yield* analytics.flush; + }).pipe(Effect.provide(layerRuntime)); + return { sent, requests: () => requests, installId: () => installId }; + }); + +it.layer(NodeServices.layer)("AnalyticsService bkt3 identity", (it) => { + it.effect("sends nothing when no PostHog key is set, even with telemetry enabled", () => + Effect.gen(function* () { + const { sent, requests } = yield* captureEvents( + { T3CODE_TELEMETRY_ENABLED: true }, + (analytics) => analytics.record("test.no_key", { count: 1 }), + ); + assert.deepEqual(sent, []); + assert.equal(requests(), 0); + }), + ); + + it.effect("keeps the two-argument record call and drops keys outside the allowlist", () => + Effect.gen(function* () { + const { sent, installId } = yield* captureEvents( + { T3CODE_TELEMETRY_ENABLED: true, T3CODE_POSTHOG_KEY: "phc_test_key" }, + (analytics) => + analytics.record("test.allowlist", { + count: 2, + threadId: "thread-1", + title: "secret session title", + prompt: "secret prompt", + cwd: "/home/someone/repo", + }), + ); + const event = sent.find((entry) => entry.event === "test.allowlist"); + assert.isDefined(event); + assert.equal(event?.properties.count, 2); + assert.equal(event?.properties.threadId, "thread-1"); + assert.isUndefined(event?.properties.title); + assert.isUndefined(event?.properties.prompt); + assert.isUndefined(event?.properties.cwd); + // No user: the install id, and no person profile. + assert.equal(event?.distinct_id, installId()); + assert.equal(event?.properties.$process_person_profile, false); + assert.deepEqual(event?.properties.$groups, { instance: "unknown" }); + assert.equal(event?.properties.instance, "unknown"); + }), + ); + + it.effect("sends a known user's events as their email and identifies them once", () => + Effect.gen(function* () { + const { sent } = yield* captureEvents( + { T3CODE_TELEMETRY_ENABLED: true, T3CODE_POSTHOG_KEY: "phc_test_key" }, + (analytics) => + Effect.gen(function* () { + yield* analytics.record("test.user", { count: 1 }, { userId: ALICE }); + yield* analytics.record("test.user", { count: 2 }, { userId: ALICE }); + yield* analytics.record("test.user", { count: 3 }, { userId: BOB_WITHOUT_EMAIL }); + yield* analytics.record("test.user", { count: 4 }, { userId: "user_unknown" }); + }).pipe(Effect.provideService(EnvironmentUsers.EnvironmentUserRepository, fakeUsers)), + ); + + const identifies = sent.filter((entry) => entry.event === "$identify"); + assert.equal(identifies.length, 1, "one $identify per user per process"); + assert.equal(identifies[0]?.distinct_id, "alice@example.com"); + assert.deepEqual(identifies[0]?.properties.$set, { + email: "alice@example.com", + role: "admin", + name: "Alice Example", + }); + + const byCount = new Map( + sent + .filter((entry) => entry.event === "test.user") + .map((entry) => [entry.properties.count, entry] as const), + ); + for (const count of [1, 2]) { + assert.equal(byCount.get(count)?.distinct_id, "alice@example.com"); + assert.equal(byCount.get(count)?.properties.$process_person_profile, true); + } + // A user without an email, and an unknown user, stay anonymous. + for (const count of [3, 4]) { + assert.notEqual(byCount.get(count)?.distinct_id, "alice@example.com"); + assert.equal(byCount.get(count)?.properties.$process_person_profile, false); + } + }), + ); + + it.effect("stays anonymous when the caller has no user repository", () => + Effect.gen(function* () { + const { sent } = yield* captureEvents( + { T3CODE_TELEMETRY_ENABLED: true, T3CODE_POSTHOG_KEY: "phc_test_key" }, + (analytics) => analytics.record("test.no_repo", { count: 1 }, { userId: ALICE }), + ); + assert.equal(sent.filter((entry) => entry.event === "$identify").length, 0); + const event = sent.find((entry) => entry.event === "test.no_repo"); + assert.equal(event?.properties.$process_person_profile, false); + }), + ); + + it.effect("maps an insert id to a stable uuid and keeps the caller's timestamp", () => + Effect.gen(function* () { + const timestamp = "2026-10-01T10:00:00.000Z"; + const { sent } = yield* captureEvents( + { T3CODE_TELEMETRY_ENABLED: true, T3CODE_POSTHOG_KEY: "phc_test_key" }, + (analytics) => + Effect.gen(function* () { + const options = { insertId: "run-1", timestamp }; + yield* analytics.record("test.backfill", { count: 1 }, options); + yield* analytics.record("test.backfill", { count: 2 }, options); + yield* analytics.record("test.backfill", { count: 3 }, { insertId: "run-2" }); + }), + ); + const events = sent.filter((entry) => entry.event === "test.backfill"); + assert.equal(events.length, 3); + assert.equal(events[0]?.uuid, events[1]?.uuid); + assert.notEqual(events[0]?.uuid, events[2]?.uuid); + assert.match( + events[0]?.uuid ?? "", + /^[0-9a-f]{8}-[0-9a-f]{4}-8[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); + assert.equal(events[0]?.timestamp, timestamp); + assert.equal(events[0]?.properties.$insert_id, "run-1"); + }), + ); + + it.effect("names the instance group and identifies it once per boot", () => + Effect.gen(function* () { + const { sent } = yield* captureEvents( + { + T3CODE_TELEMETRY_ENABLED: true, + T3CODE_POSTHOG_KEY: "phc_test_key", + T3CODE_ANALYTICS_INSTANCE: "bkt3", + }, + (analytics) => analytics.record("test.instance", { count: 1 }), + ); + const groups = sent.filter((entry) => entry.event === "$groupidentify"); + assert.equal(groups.length, 1); + assert.equal(groups[0]?.properties.$group_type, "instance"); + assert.equal(groups[0]?.properties.$group_key, "bkt3"); + const event = sent.find((entry) => entry.event === "test.instance"); + assert.deepEqual(event?.properties.$groups, { instance: "bkt3" }); + assert.equal(event?.properties.instance, "bkt3"); + }), + ); +}); diff --git a/apps/server/src/telemetry/AnalyticsService.test.ts b/apps/server/src/telemetry/AnalyticsService.test.ts index e0a4786ece6ff..96329bad31e25 100644 --- a/apps/server/src/telemetry/AnalyticsService.test.ts +++ b/apps/server/src/telemetry/AnalyticsService.test.ts @@ -23,7 +23,7 @@ interface RecordedBatchRequest { readonly batch?: ReadonlyArray<{ readonly event?: string; readonly properties?: { - readonly index?: number; + readonly count?: number; readonly clientType?: string; readonly serverOs?: string; readonly serverArch?: string; @@ -39,7 +39,7 @@ interface RecordedBatchBody { readonly batch: ReadonlyArray<{ readonly event?: string; readonly properties?: { - readonly index?: number; + readonly count?: number; readonly clientType?: string; readonly serverOs?: string; readonly serverArch?: string; @@ -115,7 +115,7 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { yield* Effect.gen(function* () { const analytics = yield* AnalyticsService.AnalyticsService; for (let index = 0; index < 20; index += 1) { - yield* analytics.record("test.retry", { index }); + yield* analytics.record("test.retry", { count: index }); } // Before the fix this loop sent the batch about once a second. for (let second = 0; second < 600; second += 1) { @@ -181,7 +181,7 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { const analytics = yield* AnalyticsService.AnalyticsService; for (let index = 0; index < 45; index += 1) { - yield* analytics.record("test.flush.drain", { index }); + yield* analytics.record("test.flush.drain", { count: index }); } yield* analytics.flush; @@ -201,7 +201,7 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { const deliveredIndexes = batchRequests.flatMap((request) => request.body.batch .filter((event) => event.event === "test.flush.drain") - .map((event) => event.properties?.index) + .map((event) => event.properties?.count) .filter((index): index is number => typeof index === "number"), ); @@ -267,7 +267,7 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => { yield* Effect.gen(function* () { yield* Layer.launch(layerBatchServer).pipe(Effect.forkScoped); const analytics = yield* AnalyticsService.AnalyticsService; - yield* analytics.record("test.disabled", { index: 1 }); + yield* analytics.record("test.disabled", { count: 1 }); yield* analytics.flush; }).pipe(Effect.provide(layerRuntime)); diff --git a/apps/server/src/telemetry/AnalyticsService.ts b/apps/server/src/telemetry/AnalyticsService.ts index 3d10e93a26d6a..c93f083e346d6 100644 --- a/apps/server/src/telemetry/AnalyticsService.ts +++ b/apps/server/src/telemetry/AnalyticsService.ts @@ -17,6 +17,7 @@ import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Hex from "effect/encoding/Hex"; // T3-CUSTOM(expbkt3): insert-id hashing. import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Random from "effect/Random"; @@ -30,12 +31,22 @@ import * as HttpClientResponse from "effect/http/HttpClientResponse"; import packageJson from "../../package.json" with { type: "json" }; import * as ServerConfig from "../config.ts"; import { getTelemetryIdentifier } from "./Identify.ts"; +// T3-CUSTOM(expbkt3): BEGIN — property allowlist and per-user identity (TEC-1631). +import { filterAnalyticsProperties } from "./analyticsProperties.ts"; +import { + type AnalyticsPerson, + makeAnalyticsUserDirectory, + uuidFromSha256Hex, +} from "./analyticsIdentity.ts"; +// T3-CUSTOM(expbkt3): END interface BufferedAnalyticsEvent { readonly uuid: string; readonly event: string; readonly properties?: Readonly>; readonly capturedAt: string; + /** T3-CUSTOM(expbkt3): an identified person's email; unset events use the install id. */ + readonly distinctId?: string; } interface DeliveryState { @@ -67,9 +78,11 @@ export function retryDelayMs(failures: number, random: number): number { } const TelemetryEnvConfig = Config.all({ - posthogKey: Config.String("T3CODE_POSTHOG_KEY").pipe( - Config.withDefault("phc_XOWci4oZP4VvLiEyrFqkFjP4CZn55mjYYBMREK5Wd6m"), - ), + // T3-CUSTOM(expbkt3): no default key. The fork must never send to the upstream + // maintainers' PostHog project, so telemetry stays off until a key is set. + posthogKey: Config.String("T3CODE_POSTHOG_KEY").pipe(Config.withDefault("")), + // T3-CUSTOM(expbkt3): the PostHog `instance` group every event belongs to. + instance: Config.String("T3CODE_ANALYTICS_INSTANCE").pipe(Config.withDefault("unknown")), posthogHost: Config.String("T3CODE_POSTHOG_HOST").pipe( Config.withDefault("https://us.i.posthog.com"), ), @@ -81,13 +94,25 @@ const TelemetryEnvConfig = Config.all({ wslDistroName: Config.String("WSL_DISTRO_NAME").pipe(Config.option), }); +// T3-CUSTOM(expbkt3): BEGIN — record options (TEC-1631). +export interface AnalyticsRecordOptions { + /** Environment user who acted; sent as that person's email when known. */ + readonly userId?: string; + /** ISO time the event happened, for events recorded after the fact. */ + readonly timestamp?: string; + /** Stable id for de-duplication; the same id always maps to the same event uuid. */ + readonly insertId?: string; +} +// T3-CUSTOM(expbkt3): END + export class AnalyticsService extends Context.Service< AnalyticsService, { - /** Record an anonymous event for best-effort buffered delivery. */ + /** Record an event for best-effort buffered delivery. T3-CUSTOM(expbkt3): may identify a user. */ readonly record: ( event: string, properties?: Readonly>, + options?: AnalyticsRecordOptions, // T3-CUSTOM(expbkt3): identity and backfill options. ) => Effect.Effect; /** Flush all currently queued telemetry events. */ @@ -138,21 +163,24 @@ export const make = Effect.gen(function* () { const clientType = serverConfig.mode === "desktop" ? "desktop-app" : "cli-web-client"; const hostPlatform = yield* HostProcessPlatform; const hostArchitecture = yield* HostProcessArchitecture; + // T3-CUSTOM(expbkt3): BEGIN — key-gated telemetry, identity and instance group. + const telemetryEnabled = telemetryConfig.enabled && telemetryConfig.posthogKey.trim() !== ""; + const userDirectory = yield* makeAnalyticsUserDirectory; + const identifiedUsers = yield* Ref.make>(new Set()); + const droppedKeysLogged = yield* Ref.make>(new Set()); + // T3-CUSTOM(expbkt3): END const enqueueBufferedEvent = ( - uuid: string, - event: string, - properties?: Readonly>, + entry: Omit, + timestamp?: string, // T3-CUSTOM(expbkt3): caller-supplied event time. ) => Effect.flatMap(DateTime.now, (now) => Ref.modify(bufferRef, (current) => { const appended = [ ...current, { - uuid, - event, - ...(properties ? { properties } : {}), - capturedAt: DateTime.formatIso(now), + ...entry, // T3-CUSTOM(expbkt3): one entry object. + capturedAt: timestamp ?? DateTime.formatIso(now), // T3-CUSTOM(expbkt3) } satisfies BufferedAnalyticsEvent, ]; @@ -174,17 +202,20 @@ export const make = Effect.gen(function* () { const sendBatch = Effect.fn("AnalyticsService.sendBatch")(function* ( events: ReadonlyArray, ) { - if (!telemetryConfig.enabled || !identifier) return; + if (!telemetryEnabled || !identifier) return; // T3-CUSTOM(expbkt3): key-gated. const payload = { api_key: telemetryConfig.posthogKey, batch: events.map((event) => ({ uuid: event.uuid, event: event.event, - distinct_id: identifier, + distinct_id: event.distinctId ?? identifier, // T3-CUSTOM(expbkt3): identified person. properties: { ...event.properties, - $process_person_profile: false, + // T3-CUSTOM(expbkt3): only identified events build a person profile. + $process_person_profile: event.distinctId !== undefined, + $groups: { instance: telemetryConfig.instance }, + instance: telemetryConfig.instance, platform: hostPlatform, wsl: Option.getOrUndefined(telemetryConfig.wslDistroName), arch: hostArchitecture, @@ -257,16 +288,92 @@ export const make = Effect.gen(function* () { } }); + // T3-CUSTOM(expbkt3): BEGIN — allowlist, identity, insert ids (TEC-1631). + const logDroppedKeys = (event: string, dropped: ReadonlyArray) => + Effect.gen(function* () { + if (dropped.length === 0) return; + const fresh = yield* Ref.modify( + droppedKeysLogged, + (logged): readonly [ReadonlyArray, ReadonlySet] => { + const unseen = dropped.filter((key) => !logged.has(key)); + return [unseen, unseen.length === 0 ? logged : new Set([...logged, ...unseen])]; + }, + ); + for (const key of fresh) { + yield* Effect.logDebug("analytics property key not allowlisted; dropping it", { + event, + key, + }); + } + }); + + const eventUuid = (insertId: string | undefined) => + Effect.gen(function* () { + if (insertId !== undefined) { + const digest = yield* Effect.exit( + crypto.digest("SHA-256", new TextEncoder().encode(insertId)), + ); + if (Exit.isSuccess(digest)) return uuidFromSha256Hex(Hex.encode(digest.value)); + } + const uuid = yield* Effect.exit(crypto.randomUUIDv7); + return Exit.isSuccess(uuid) ? uuid.value : undefined; + }); + + /** Sends the person's profile once per user per process start. */ + const identifyOnce = (userId: string, person: AnalyticsPerson) => + Effect.gen(function* () { + const first = yield* Ref.modify( + identifiedUsers, + (seen): readonly [boolean, ReadonlySet] => + seen.has(userId) ? [false, seen] : [true, new Set([...seen, userId])], + ); + if (!first) return; + const uuid = yield* eventUuid(undefined); + if (uuid === undefined) return; + yield* enqueueBufferedEvent({ + uuid, + event: "$identify", + distinctId: person.email, + properties: { + $set: { + email: person.email, + role: person.role, + ...(person.name === undefined ? {} : { name: person.name }), + }, + }, + }); + }); + const record: AnalyticsService["Service"]["record"] = Effect.fn("AnalyticsService.record")( - function* (event, properties) { - if (!telemetryConfig.enabled || !identifier) return; + function* (event, properties, options) { + if (!telemetryEnabled || !identifier) return; + + const { kept, dropped } = filterAnalyticsProperties(properties); + yield* logDroppedKeys(event, dropped); + + const person = + options?.userId === undefined ? undefined : yield* userDirectory.resolve(options.userId); + if (person !== undefined && options?.userId !== undefined) { + yield* identifyOnce(options.userId, person); + } // Telemetry is best effort: an event without a uuid is not sent. The // Node implementation throws (a defect) rather than failing, so catch both. - const uuid = yield* Effect.exit(crypto.randomUUIDv7); - if (Exit.isFailure(uuid)) return; + const uuid = yield* eventUuid(options?.insertId); + if (uuid === undefined) return; - const enqueueResult = yield* enqueueBufferedEvent(uuid.value, event, properties); + const enqueueResult = yield* enqueueBufferedEvent( + { + uuid, + event, + properties: { + ...kept, + ...(options?.insertId === undefined ? {} : { $insert_id: options.insertId }), + }, + ...(person === undefined ? {} : { distinctId: person.email }), + }, + options?.timestamp, + ); if (enqueueResult.dropped) { yield* Effect.logDebug("analytics buffer full; dropping oldest event", { size: enqueueResult.size, @@ -276,6 +383,24 @@ export const make = Effect.gen(function* () { }, ); + // One group profile per boot, named after the instance. An unnamed instance + // has nothing worth a profile. + if (telemetryEnabled && identifier && telemetryConfig.instance !== "unknown") { + const uuid = yield* eventUuid(undefined); + if (uuid !== undefined) { + yield* enqueueBufferedEvent({ + uuid, + event: "$groupidentify", + properties: { + $group_type: "instance", + $group_key: telemetryConfig.instance, + $group_set: { name: telemetryConfig.instance, t3CodeVersion: packageJson.version }, + }, + }); + } + } + // T3-CUSTOM(expbkt3): END + yield* Effect.forever(Effect.sleep(FLUSH_INTERVAL_MS).pipe(Effect.flatMap(() => flushWhenDue)), { disableYield: true, }).pipe(Effect.forkScoped); diff --git a/apps/server/src/telemetry/analyticsEnvironment.test.ts b/apps/server/src/telemetry/analyticsEnvironment.test.ts new file mode 100644 index 0000000000000..0326ccaf0170f --- /dev/null +++ b/apps/server/src/telemetry/analyticsEnvironment.test.ts @@ -0,0 +1,61 @@ +// T3-CUSTOM(expbkt3): agent processes never see the server's analytics settings (TEC-1631). +import { assert, it } from "@effect/vitest"; + +import { mergeProviderInstanceEnvironment } from "../provider/ProviderInstanceEnvironment.ts"; +import { SERVER_ANALYTICS_ENV_KEYS, withoutAnalyticsEnvironment } from "./analyticsEnvironment.ts"; + +const serverEnv = { + PATH: "/usr/bin", + T3CODE_POSTHOG_KEY: "phc_secret", + T3CODE_POSTHOG_HOST: "https://eu.i.posthog.com", + T3CODE_ANALYTICS_INSTANCE: "bkt3", + T3CODE_TELEMETRY_ENABLED: "true", +}; + +/** What a child sees when the spawner spreads the server env under `env` (`extendEnv: true`). */ +const childSees = (env: NodeJS.ProcessEnv) => + Object.fromEntries( + Object.entries({ ...serverEnv, ...env }).filter(([, value]) => value !== undefined), + ); + +it("masks every analytics key from an env built on the server's env", () => { + const masked = withoutAnalyticsEnvironment({ ...serverEnv, CODEX_HOME: "/codex" }, serverEnv); + for (const key of SERVER_ANALYTICS_ENV_KEYS) { + assert.isTrue(key in masked, key); + assert.isUndefined(masked[key], key); + } + assert.equal(masked.PATH, "/usr/bin"); + assert.equal(masked.CODEX_HOME, "/codex"); +}); + +it("hides inherited keys even when the spawner extends the server env", () => { + // The env passed to the child does not carry the keys, but the server's does. + const masked = withoutAnalyticsEnvironment({ CODEX_HOME: "/codex" }, serverEnv); + assert.deepEqual(childSees(masked), { PATH: "/usr/bin", CODEX_HOME: "/codex" }); +}); + +it("returns the env unchanged when no analytics key is set anywhere", () => { + const env = { PATH: "/usr/bin" }; + assert.strictEqual(withoutAnalyticsEnvironment(env, { HOME: "/home/u" }), env); +}); + +it("does not mutate the env it was given", () => { + const env = { ...serverEnv }; + withoutAnalyticsEnvironment(env, {}); + assert.equal(env.T3CODE_POSTHOG_KEY, "phc_secret"); +}); + +it("strips the keys from every provider environment, with or without instance overrides", () => { + for (const environment of [ + undefined, + [], + [{ name: "CUSTOM_VALUE", value: "x", sensitive: false }], + [{ name: "T3CODE_POSTHOG_KEY", value: "phc_override", sensitive: true }], + ]) { + const merged = mergeProviderInstanceEnvironment(environment, serverEnv); + for (const key of SERVER_ANALYTICS_ENV_KEYS) { + assert.isUndefined(merged[key], key); + } + assert.equal(merged.PATH, "/usr/bin"); + } +}); diff --git a/apps/server/src/telemetry/analyticsEnvironment.ts b/apps/server/src/telemetry/analyticsEnvironment.ts new file mode 100644 index 0000000000000..9a09dad5911e1 --- /dev/null +++ b/apps/server/src/telemetry/analyticsEnvironment.ts @@ -0,0 +1,38 @@ +/** + * T3-CUSTOM(expbkt3): the server's analytics settings stay in the server. + * + * Agent processes (provider CLIs, and every tool or dev server they start) + * must not inherit the PostHog key or instance name: an agent could read the + * key, and a dev server it starts would report into the production project + * as the production instance. + * + * The keys are set to `undefined` rather than deleted. Some spawns use + * `extendEnv: true`, which spreads `process.env` under the given env, and only + * an explicit `undefined` hides the inherited value there. Node drops + * `undefined` entries when it builds the child's environment. + */ + +export const SERVER_ANALYTICS_ENV_KEYS = [ + "T3CODE_POSTHOG_KEY", + "T3CODE_POSTHOG_HOST", + "T3CODE_ANALYTICS_INSTANCE", + "T3CODE_TELEMETRY_ENABLED", +] as const; + +/** + * `env` with every analytics key the child could see (from `env` itself or + * from `inherited`, the server's own environment) masked. Returns `env` + * unchanged when no such key is set anywhere. + */ +export function withoutAnalyticsEnvironment( + env: NodeJS.ProcessEnv, + inherited: Readonly> = process.env, +): NodeJS.ProcessEnv { + const present = SERVER_ANALYTICS_ENV_KEYS.filter( + (key) => env[key] !== undefined || inherited[key] !== undefined, + ); + if (present.length === 0) return env; + const next: NodeJS.ProcessEnv = { ...env }; + for (const key of present) next[key] = undefined; + return next; +} diff --git a/apps/server/src/telemetry/analyticsIdentity.ts b/apps/server/src/telemetry/analyticsIdentity.ts new file mode 100644 index 0000000000000..f4df3fb5e9dae --- /dev/null +++ b/apps/server/src/telemetry/analyticsIdentity.ts @@ -0,0 +1,102 @@ +/** + * T3-CUSTOM(expbkt3): who a bkt3 analytics event belongs to. + * + * An event recorded with a `userId` is sent as that person: their + * `environment_users.primary_email` becomes the PostHog `distinct_id`. The + * user table is read at most once per `USER_DIRECTORY_REFRESH_MS`, never once + * per event. A user without an email, or a caller with no user repository in + * its context, falls back to the install-level anonymous id. + * + * The repository is read from the caller's context rather than the analytics + * layer's, because `AnalyticsService` sits below persistence in the server's + * layer stack. Every runtime service (websocket handlers, MCP tools, reactors) + * has `EnvironmentUserRepository` in its context. + */ +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as Result from "effect/Result"; + +import * as EnvironmentUsers from "../persistence/EnvironmentUsers.ts"; + +export const USER_DIRECTORY_REFRESH_MS = 5 * 60_000; + +/** The person-profile fields PostHog stores for an identified user. */ +export interface AnalyticsPerson { + readonly email: string; + readonly name: string | undefined; + readonly role: string; +} + +interface UserDirectoryState { + /** Epoch ms of the last read attempt, or null before the first. */ + readonly loadedAt: number | null; + readonly people: ReadonlyMap; +} + +/** Users with an email, keyed by user id. Emails are trimmed and lower-cased. */ +export function analyticsPeopleFromUsers( + users: ReadonlyArray, +): ReadonlyMap { + const people = new Map(); + for (const user of users) { + const email = user.primaryEmail?.trim().toLowerCase() ?? ""; + if (email === "") continue; + const name = user.displayName?.trim() ?? ""; + people.set(user.userId, { + email, + name: name === "" ? undefined : name, + role: user.role, + }); + } + return people; +} + +/** Caches `environment_users` for identity lookups. */ +export const makeAnalyticsUserDirectory = Effect.gen(function* () { + const stateRef = yield* Ref.make({ loadedAt: null, people: new Map() }); + + const refresh = Effect.gen(function* () { + const repository = yield* Effect.serviceOption(EnvironmentUsers.EnvironmentUserRepository); + // No repository here: try again on the next lookup, which costs no DB read. + if (Option.isNone(repository)) return; + const now = yield* Clock.currentTimeMillis; + const users = yield* Effect.result(repository.value.list); + yield* Ref.update(stateRef, (state) => ({ + loadedAt: now, + // A failed read keeps the last good map until the next refresh window. + people: Result.isSuccess(users) ? analyticsPeopleFromUsers(users.success) : state.people, + })); + }); + + /** The person for `userId`, or undefined when unknown or without an email. */ + const resolve = (userId: string) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + const state = yield* Ref.get(stateRef); + if (state.loadedAt === null || now - state.loadedAt >= USER_DIRECTORY_REFRESH_MS) { + yield* refresh; + } + return (yield* Ref.get(stateRef)).people.get(userId); + }); + + return { resolve } as const; +}); + +/** + * A stable UUID for a caller-chosen insert id, so a re-sent or backfilled + * event lands on the same PostHog `uuid` and is de-duplicated. `sha256Hex` is + * the hex SHA-256 of the insert id. The result has UUID version 8 (custom). + */ +export function uuidFromSha256Hex(sha256Hex: string): string { + const hex = sha256Hex.toLowerCase().padEnd(32, "0"); + const variant = ((Number.parseInt(hex[16] ?? "0", 16) & 0x3) | 0x8).toString(16); + return [ + hex.slice(0, 8), + hex.slice(8, 12), + `8${hex.slice(13, 16)}`, + `${variant}${hex.slice(17, 20)}`, + hex.slice(20, 32), + ].join("-"); +} diff --git a/apps/server/src/telemetry/analyticsProperties.ts b/apps/server/src/telemetry/analyticsProperties.ts new file mode 100644 index 0000000000000..aab8aac216bf4 --- /dev/null +++ b/apps/server/src/telemetry/analyticsProperties.ts @@ -0,0 +1,182 @@ +/** + * T3-CUSTOM(expbkt3): the only event property keys bkt3 sends to PostHog. + * + * Events now carry a person's email as `distinct_id`, so a property must never + * hold free text a person or an agent wrote: titles, prompts, messages, paths, + * branches, repository names, answers, questions or error messages. Add a key + * here only when its values are ids, enums, numbers or booleans. Error detail + * goes in as a class or tag name (`errorClass`, `failureClass`), never a message. + * + * `AnalyticsService.record` drops every other key and logs each dropped key + * once per process. + */ + +const UPSTREAM_KEYS = [ + // server.boot.heartbeat + "threadCount", + "projectCount", + // client.connected, client.thread.started, client.turn.requested + "surface", + "appVersion", + "clientAppVersion", + "clientOs", + "os", + "clientDeviceType", + "webDeployment", + "clientBrowser", + "osMajorVersion", + "clientOsMajorVersion", + "deviceModel", + "clientDeviceModel", + "connectionMethod", + // provider.turn.completed + "terminalStatus", + "usageStatus", + "usageScope", + "hasSubagents", + "inputTokens", + "cachedInputTokens", + "cacheCreationTokens", + "outputTokens", + "reasoningTokens", + "mixedModels", + // chatgpt.auth.*, chatgpt.transfer.* + "flow", + "intent", + "callbackMode", + "outcome", + "failureStage", + "accountCountScope", + "connectedAccountCount", + "connectedConnectionCount", + "savedConnectionCount", + "unidentifiedConnectedConnectionCount", +] as const; + +/** Session activity and run metrics (TEC-1631 worker 2). */ +const SESSION_ACTIVITY_KEYS = [ + "threadId", + "projectId", + "parentThreadId", + "isChildSession", + "state", + "waitKind", + "durationMs", + "startedAt", + "endedAt", + "endReason", + "runId", + "runOrdinal", + "provider", + "providerInstanceId", + "model", + "effort", + "interactionMode", + "runtimeMode", + "activeWorkers", + "maxActiveWorkers", + "ownerUserId", + "senderUserId", + "sessionSource", + "resumedAfterRestart", + "backfill", + "ageMs", + "status", + "queueWaitMs", + "runDurationMs", + "workDurationMs", + "attempts", + "failureClass", + "requestKind", + "waitMs", + "resolution", + "answeredByUserId", + "subagentId", + "driver", + "childThreadPresent", + "decision", + "count", + "runningRuns", + "queuedRuns", + "waitingHuman", + "waitingWorkers", + "activeSubagents", + "snoozed", + "spanStartedAt", + "segmentIndex", + "runTrigger", + "backgroundTasks", + "createdBy", + "origin", + "scope", +] as const; + +/** Server events outside the orchestration stream (TEC-1631 worker 1). */ +const SERVER_EVENT_KEYS = [ + "instance", + "actorKind", + "connectionMode", + // mcp.tool.called + "toolName", + "callerKind", + "ok", + "errorClass", + // scheduled_task.ran + "taskId", + "scheduleType", + "trigger", + "creationSource", + // webhook.delivered + "kind", + // session.recovery.* + "terminalizedRuns", + "stoppedSessions", + "closedRequests", + "retiredEffects", + "requeuedEffects", +] as const; + +export const ALLOWED_ANALYTICS_PROPERTY_KEYS: ReadonlySet = new Set([ + ...UPSTREAM_KEYS, + ...SESSION_ACTIVITY_KEYS, + ...SERVER_EVENT_KEYS, +]); + +/** + * Keys that must never be allowed, whatever someone adds above. Checked by the + * tests so a free-text key cannot slip in by accident. + */ +export const FORBIDDEN_ANALYTICS_PROPERTY_KEYS: ReadonlyArray = [ + "title", + "prompt", + "message", + "text", + "path", + "cwd", + "branch", + "repo", + "answer", + "question", + "error", + "errorMessage", +]; + +/** Splits `properties` into the allowed ones and the names of the dropped keys. */ +export function filterAnalyticsProperties( + properties: Readonly> | undefined, +): { + readonly kept: Readonly> | undefined; + readonly dropped: ReadonlyArray; +} { + if (properties === undefined) return { kept: undefined, dropped: [] }; + const kept: Record = {}; + const dropped: Array = []; + for (const [key, value] of Object.entries(properties)) { + if (ALLOWED_ANALYTICS_PROPERTY_KEYS.has(key)) { + kept[key] = value; + } else { + dropped.push(key); + } + } + return { kept, dropped }; +} diff --git a/apps/server/src/telemetry/serverEvents.test.ts b/apps/server/src/telemetry/serverEvents.test.ts new file mode 100644 index 0000000000000..efcd6674bfd2f --- /dev/null +++ b/apps/server/src/telemetry/serverEvents.test.ts @@ -0,0 +1,142 @@ +// T3-CUSTOM(expbkt3): bkt3 server analytics helpers (TEC-1631). +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import { + ALLOWED_ANALYTICS_PROPERTY_KEYS, + FORBIDDEN_ANALYTICS_PROPERTY_KEYS, + filterAnalyticsProperties, +} from "./analyticsProperties.ts"; +import { uuidFromSha256Hex } from "./analyticsIdentity.ts"; +import * as AnalyticsService from "./AnalyticsService.ts"; +import { + clientConnectionMode, + recordLinearLinked, + webhookStatusFromHttp, + withMcpToolAnalytics, +} from "./serverEvents.ts"; + +interface Recorded { + readonly event: string; + readonly properties: Readonly> | undefined; + readonly options: AnalyticsService.AnalyticsRecordOptions | undefined; +} + +const recordingAnalytics = (recorded: Array) => + AnalyticsService.AnalyticsService.of({ + record: (event, properties, options) => + Effect.sync(() => { + recorded.push({ event, properties, options }); + }), + flush: Effect.void, + }); + +class ToolBroke extends Schema.TaggedError()("ToolBroke", { + detail: Schema.String, +}) {} + +it("never allows a free-text key", () => { + for (const key of FORBIDDEN_ANALYTICS_PROPERTY_KEYS) { + assert.isFalse(ALLOWED_ANALYTICS_PROPERTY_KEYS.has(key), key); + } +}); + +it("allows the session-activity keys other recorders depend on", () => { + for (const key of ["threadId", "waitKind", "durationMs", "snoozed", "ownerUserId"]) { + assert.isTrue(ALLOWED_ANALYTICS_PROPERTY_KEYS.has(key), key); + } +}); + +it("splits properties into kept values and dropped key names", () => { + const { kept, dropped } = filterAnalyticsProperties({ + threadId: "t-1", + title: "a title", + message: "hello", + }); + assert.deepEqual(kept, { threadId: "t-1" }); + assert.deepEqual([...dropped].toSorted(), ["message", "title"]); +}); + +it("derives a version 8 uuid from a sha-256 hex digest", () => { + const hex = "ab".repeat(32); + assert.equal(uuidFromSha256Hex(hex), "abababab-abab-8bab-abab-abababababab"); + assert.equal(uuidFromSha256Hex(hex), uuidFromSha256Hex(hex.toUpperCase())); +}); + +it("classifies how a client reached the server", () => { + assert.equal(clientConnectionMode({ connectionMethod: "relay", host: "localhost" }), "relay"); + assert.equal(clientConnectionMode({ host: "localhost:3773" }), "local"); + assert.equal(clientConnectionMode({ host: "127.0.0.1:3773" }), "local"); + assert.equal(clientConnectionMode({ host: "[::1]:3773" }), "local"); + assert.equal(clientConnectionMode({ host: "dev-server-1.tailab6257.ts.net" }), "tailnet"); + assert.equal(clientConnectionMode({ host: "100.101.102.103:18087" }), "tailnet"); + assert.equal(clientConnectionMode({ host: "bkt3.dev.beknown.live" }), "remote"); + assert.equal(clientConnectionMode({}), "remote"); +}); + +it("names a session-webhook answer by its status", () => { + assert.equal(webhookStatusFromHttp(202), "accepted"); + assert.equal(webhookStatusFromHttp(413), "http_413"); +}); + +it.effect("records an MCP tool call by name and outcome, never its arguments", () => + Effect.gen(function* () { + const recorded: Array = []; + const caller = Effect.succeed({ + principal: "provider-session" as const, + actorUserId: "user_clerk_alice", + }); + yield* withMcpToolAnalytics("t3_send_prompt", caller, Effect.succeed("ok")); + const failed = yield* withMcpToolAnalytics( + "t3_send_prompt", + caller, + Effect.fail(new ToolBroke({ detail: "secret detail" })), + ).pipe(Effect.flip); + assert.equal(failed._tag, "ToolBroke"); + + assert.equal(recorded.length, 0, "nothing is recorded without analytics in context"); + yield* withMcpToolAnalytics("t3_inbox", caller, Effect.void).pipe( + Effect.provideService(AnalyticsService.AnalyticsService, recordingAnalytics(recorded)), + ); + yield* withMcpToolAnalytics( + "t3_inbox", + Effect.succeed(undefined), + Effect.fail(new ToolBroke({ detail: "secret detail" })), + ).pipe( + Effect.ignore, + Effect.provideService(AnalyticsService.AnalyticsService, recordingAnalytics(recorded)), + ); + + assert.equal(recorded.length, 2); + assert.equal(recorded[0]?.event, "mcp.tool.called"); + assert.equal(recorded[0]?.properties?.toolName, "t3_inbox"); + assert.equal(recorded[0]?.properties?.callerKind, "agent"); + assert.equal(recorded[0]?.properties?.ok, true); + assert.deepEqual(recorded[0]?.options, { userId: "user_clerk_alice" }); + assert.equal(recorded[1]?.properties?.ok, false); + assert.equal(recorded[1]?.properties?.callerKind, "unknown"); + assert.equal(recorded[1]?.properties?.errorClass, "ToolBroke"); + assert.isUndefined(recorded[1]?.options); + assert.notInclude(JSON.stringify(recorded), "secret detail"); + }), +); + +it.effect("records linked Linear items as a count only", () => + Effect.gen(function* () { + const recorded: Array = []; + const provide = Effect.provideService( + AnalyticsService.AnalyticsService, + recordingAnalytics(recorded), + ); + yield* recordLinearLinked({ threadId: "thread-1", count: 0, userId: null }).pipe(provide); + yield* recordLinearLinked({ threadId: "thread-1", count: 2, userId: "user-1" }).pipe(provide); + assert.deepEqual(recorded, [ + { + event: "linear.linked", + properties: { threadId: "thread-1", count: 2 }, + options: { userId: "user-1" }, + }, + ]); + }), +); diff --git a/apps/server/src/telemetry/serverEvents.ts b/apps/server/src/telemetry/serverEvents.ts new file mode 100644 index 0000000000000..1c29a9f39e419 --- /dev/null +++ b/apps/server/src/telemetry/serverEvents.ts @@ -0,0 +1,257 @@ +/** + * T3-CUSTOM(expbkt3): bkt3 product events that do not come from the + * orchestration event stream (TEC-1631). + * + * Each helper reads `AnalyticsService` from the caller's context, so a hook + * point is one line and adds no service requirement. Every helper is best + * effort: it never fails or delays the work it reports on. Properties are ids, + * enums, numbers and booleans only; see `analyticsProperties.ts`. + */ +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Option from "effect/Option"; +import * as Result from "effect/Result"; + +import * as AnalyticsService from "./AnalyticsService.ts"; + +/** Records `event` when analytics is in context; otherwise does nothing. */ +export const recordServerEvent = ( + event: string, + properties: Readonly>, + options?: AnalyticsService.AnalyticsRecordOptions, +): Effect.Effect => + Effect.serviceOption(AnalyticsService.AnalyticsService).pipe( + Effect.flatMap((analytics) => + Option.isNone(analytics) + ? Effect.void + : analytics.value.record(event, properties, options).pipe(Effect.ignoreCause), + ), + ); + +const userOption = (userId: string | null | undefined) => + userId === null || userId === undefined || userId === "" ? undefined : { userId }; + +/** + * The tag or class name of a failure, never its message. Tagged errors give + * their `_tag`; other errors their constructor name; anything else "unknown". + */ +export function errorClassOf(cause: Cause.Cause): string { + if (Cause.hasInterruptsOnly(cause)) return "Interrupted"; + const error = Option.getOrUndefined(Cause.findErrorOption(cause)); + const defect = Cause.findDefect(cause); + const failure = error ?? (Result.isSuccess(defect) ? defect.success : undefined); + if (typeof failure === "object" && failure !== null) { + const tag = (failure as { readonly _tag?: unknown })._tag; + if (typeof tag === "string" && /^[A-Za-z0-9_.-]{1,80}$/.test(tag)) return tag; + const name = failure.constructor?.name; + if (typeof name === "string" && /^[A-Za-z0-9_]{1,80}$/.test(name)) return name; + } + return error === undefined ? "Defect" : "unknown"; +} + +// --------------------------------------------------------------------------- +// mcp.tool.called +// --------------------------------------------------------------------------- + +/** Who called a T3 MCP tool. */ +export type McpCallerKind = "agent" | "user" | "operator"; + +/** Maps the MCP invocation principal to the analytics caller kind. */ +export function mcpCallerKind( + principal: "provider-session" | "external-user" | "external-operator", +): McpCallerKind { + switch (principal) { + case "provider-session": + return "agent"; + case "external-user": + return "user"; + case "external-operator": + return "operator"; + } +} + +export interface McpToolCaller { + readonly principal: "provider-session" | "external-user" | "external-operator"; + readonly actorUserId: string | null; +} + +/** + * Times one MCP tool call and records `mcp.tool.called` when it ends. Only + * the tool name is sent, never its arguments or result. `caller` reads the + * invocation scope, or returns undefined when there is none. + */ +export const withMcpToolAnalytics = ( + toolName: string, + caller: Effect.Effect, + effect: Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const startedAt = yield* Clock.currentTimeMillis; + return yield* effect.pipe( + Effect.onExit((exit) => + Effect.gen(function* () { + const endedAt = yield* Clock.currentTimeMillis; + const scope = yield* caller; + yield* recordServerEvent( + "mcp.tool.called", + { + toolName, + callerKind: scope === undefined ? "unknown" : mcpCallerKind(scope.principal), + ok: Exit.isSuccess(exit), + durationMs: Math.max(0, endedAt - startedAt), + ...(Exit.isFailure(exit) ? { errorClass: errorClassOf(exit.cause) } : {}), + }, + userOption(scope?.actorUserId), + ); + }), + ), + ); + }); + +// --------------------------------------------------------------------------- +// scheduled_task.ran +// --------------------------------------------------------------------------- + +export const recordScheduledTaskRan = (input: { + readonly taskId: string; + readonly scheduleType: string; + readonly trigger: "scheduled" | "manual" | "webhook"; + readonly creationSource: string; + readonly status: "succeeded" | "failed"; + readonly durationMs: number; + readonly ownerUserId?: string | null | undefined; +}): Effect.Effect => + recordServerEvent( + "scheduled_task.ran", + { + taskId: input.taskId, + scheduleType: input.scheduleType, + trigger: input.trigger, + creationSource: input.creationSource, + status: input.status, + durationMs: Math.max(0, input.durationMs), + }, + userOption(input.ownerUserId), + ); + +// --------------------------------------------------------------------------- +// webhook.delivered +// --------------------------------------------------------------------------- + +/** + * An inbound webhook delivery and how the server answered it. `kind` is + * "scheduled_task" for `/api/hooks/...` and "session_callback" for + * `/api/session-webhooks/...`. `status` is an outcome tag, never a body. + */ +export const recordWebhookDelivered = ( + kind: "scheduled_task" | "session_callback", + status: string, +): Effect.Effect => recordServerEvent("webhook.delivered", { kind, status }); + +/** The outcome tag for an HTTP status the session-webhook route answered with. */ +export function webhookStatusFromHttp(status: number): string { + return status === 202 ? "accepted" : `http_${status}`; +} + +// --------------------------------------------------------------------------- +// session.recovery.* +// --------------------------------------------------------------------------- + +/** + * Wraps the startup provider-runtime recovery: `session.recovery.attempted` + * before it runs and `session.recovery.finished` with its outcome and counts + * after. Recovery runs once per boot, so `attempts` is 1. + */ +export const withSessionRecoveryAnalytics = ( + recover: Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + yield* recordServerEvent("session.recovery.attempted", { trigger: "startup", attempts: 1 }); + const startedAt = yield* Clock.currentTimeMillis; + return yield* recover.pipe( + Effect.onExit((exit) => + Effect.gen(function* () { + const endedAt = yield* Clock.currentTimeMillis; + yield* recordServerEvent("session.recovery.finished", { + trigger: "startup", + attempts: 1, + outcome: Exit.isSuccess(exit) ? "succeeded" : "failed", + durationMs: Math.max(0, endedAt - startedAt), + ...(Exit.isSuccess(exit) ? recoveryCounts(exit.value) : {}), + ...(Exit.isFailure(exit) ? { errorClass: errorClassOf(exit.cause) } : {}), + }); + }), + ), + ); + }); + +const RECOVERY_COUNT_KEYS = [ + "terminalizedRuns", + "stoppedSessions", + "closedRequests", + "retiredEffects", + "requeuedEffects", +] as const; + +function recoveryCounts(summary: object): Record { + const counts: Record = {}; + for (const key of RECOVERY_COUNT_KEYS) { + const value = (summary as Record)[key]; + if (typeof value === "number") counts[key] = value; + } + return counts; +} + +// --------------------------------------------------------------------------- +// linear.linked +// --------------------------------------------------------------------------- + +/** Linear items newly linked to a session through the T3 MCP tool. */ +export const recordLinearLinked = (input: { + readonly threadId: string; + readonly count: number; + readonly userId?: string | null | undefined; +}): Effect.Effect => + input.count <= 0 + ? Effect.void + : recordServerEvent( + "linear.linked", + { threadId: input.threadId, count: input.count }, + userOption(input.userId), + ); + +// --------------------------------------------------------------------------- +// client.connected +// --------------------------------------------------------------------------- + +export type ClientConnectionMode = "relay" | "local" | "tailnet" | "remote"; + +const TAILNET_IPV4 = /^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\.\d{1,3}\.\d{1,3}$/; + +/** + * How a client reached the server: through the relay, from this machine, over + * the tailnet (a `*.ts.net` name or a 100.64.0.0/10 address), or otherwise. + * `host` is the request's Host header; only its class is sent, never the host. + */ +export function clientConnectionMode(input: { + readonly connectionMethod?: unknown; + readonly host?: string | undefined; +}): ClientConnectionMode { + if (input.connectionMethod === "relay") return "relay"; + const raw = (input.host ?? "").trim().toLowerCase(); + const hostname = raw.startsWith("[") ? raw.slice(1, raw.indexOf("]")) : raw.split(":")[0]; + if ( + hostname === "localhost" || + hostname === "::1" || + hostname?.startsWith("127.") === true || + hostname?.endsWith(".localhost") === true + ) { + return "local"; + } + if (hostname?.endsWith(".ts.net") === true || TAILNET_IPV4.test(hostname ?? "")) { + return "tailnet"; + } + return "remote"; +} diff --git a/apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts b/apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts new file mode 100644 index 0000000000000..f5460974ce2a0 --- /dev/null +++ b/apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts @@ -0,0 +1,97 @@ +/** + * T3-CUSTOM(expbkt3): live session time-in-state metrics (TEC-1631). + * + * Feeds the live orchestration V2 event stream to the pure session tracker + * (`SessionActivityReducer.ts`) and records what it emits through + * `AnalyticsService`. On boot it seeds open sessions from one shell snapshot + * read, so their first span carries `resumedAfterRestart: true`. On shutdown it + * closes every open span with `endReason: "server_stopped"`. Every five + * minutes it records a `session.state.snapshot` for concurrency and peak load. + * + * It does nothing, not even subscribe, when telemetry is off or no PostHog key + * is set. + */ +import * as Clock from "effect/Clock"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schedule from "effect/Schedule"; +import * as Stream from "effect/Stream"; + +import { EventSinkV2 } from "../../orchestration-v2/EventSink.ts"; +import { ProjectionStoreV2 } from "../../orchestration-v2/ProjectionStore.ts"; +import { forkParked } from "../../serverActivation.ts"; +import { AnalyticsService } from "../AnalyticsService.ts"; +import { + makeSessionActivityTracker, + type SessionActivityEmission, +} from "./SessionActivityReducer.ts"; + +const SNAPSHOT_INTERVAL = "5 minutes"; + +const SessionActivityConfig = Config.all({ + enabled: Config.Boolean("T3CODE_TELEMETRY_ENABLED").pipe(Config.withDefault(true)), + posthogKey: Config.String("T3CODE_POSTHOG_KEY").pipe(Config.withDefault("")), +}); + +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + const config = yield* SessionActivityConfig; + if (!config.enabled || config.posthogKey.trim() === "") return; + + const analytics = yield* AnalyticsService; + const eventSink = yield* EventSinkV2; + const projections = yield* ProjectionStoreV2; + const tracker = makeSessionActivityTracker({ backfill: false }); + + const record = (emissions: ReadonlyArray) => + Effect.forEach( + emissions, + (emission) => + analytics.record(emission.event, emission.properties, { + ...(emission.userId === undefined ? {} : { userId: emission.userId }), + timestamp: emission.timestamp, + insertId: emission.insertId, + }), + { discard: true }, + ); + + // Subscribe from the current head, then seed: an event that lands between + // the two reads is applied on top of the seed, never lost. + const afterSequence = yield* eventSink.latestSequence().pipe(Effect.orDie); + const shells = yield* projections.getShellSnapshot({ location: "active" }).pipe( + Effect.map((snapshot) => snapshot.threads), + Effect.catchCause((cause) => + Effect.logWarning("session activity: could not seed open sessions", { cause }).pipe( + Effect.as([]), + ), + ), + ); + const bootMs = yield* Clock.currentTimeMillis; + for (const shell of shells) tracker.seed(shell, bootMs); + + yield* forkParked( + eventSink.stream({ afterSequence }).pipe( + Stream.runForEach((stored) => record(tracker.apply(stored.event))), + Effect.catchCause((cause) => + Effect.logWarning("session activity: event stream failed", { cause }), + ), + ), + ); + + yield* forkParked( + Clock.currentTimeMillis.pipe( + Effect.flatMap((nowMs) => record(tracker.snapshot(nowMs))), + Effect.repeat(Schedule.spaced(SNAPSHOT_INTERVAL)), + Effect.asVoid, + ), + ); + + // Runs before AnalyticsService's own finalizer, which flushes the buffer. + yield* Effect.addFinalizer(() => + Clock.currentTimeMillis.pipe( + Effect.flatMap((nowMs) => record(tracker.closeAll(nowMs, "server_stopped"))), + ), + ); + }), +); diff --git a/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts new file mode 100644 index 0000000000000..dda19f2f74873 --- /dev/null +++ b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts @@ -0,0 +1,515 @@ +// T3-CUSTOM(expbkt3): session time-in-state metrics (TEC-1631). +// Fixtures follow real `orchestration_events` rows from bkt3 (ids shortened; +// every text field replaced by a SECRET_* sentinel the privacy test looks for). +import { assert, describe, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; + +import { + makeSessionActivityTracker, + SESSION_ACTIVITY_PROPERTY_KEYS, + type SessionActivityEmission, + type SessionActivityEvent, +} from "./SessionActivityReducer.ts"; + +const THREAD = "d8658052-d2d8-4141-bc61-d91bfc5c570b"; +const OWNER = "user_owner"; +const SENDER = "user_sender"; +const RUN = `run:thread:${THREAD}:ordinal:73`; + +const at = (time: string) => `2026-10-09T${time}Z`; +const epoch = (iso: string) => DateTime.toEpochMillis(DateTime.makeUnsafe(iso)); + +function threadPayload(overrides: Record = {}) { + return { + ownerUserId: OWNER, + memberUserIds: [OWNER], + parentThreadId: null, + createdBy: "user", + creationSource: "web", + id: THREAD, + projectId: "project-1", + title: "SECRET_TITLE", + providerInstanceId: "claudeAgent", + modelSelection: { + instanceId: "claudeAgent", + model: "claude-opus-5-5", + options: [{ id: "effort", value: "high" }], + }, + runtimeMode: "full-access", + interactionMode: "default", + branch: "SECRET_BRANCH", + worktreePath: "/SECRET_PATH", + activeProviderThreadId: null, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: THREAD }, + forkedFrom: null, + createdAt: at("09:00:00.000"), + updatedAt: at("09:00:00.000"), + archivedAt: null, + snoozedUntil: null, + deletedAt: null, + ...overrides, + }; +} + +function event( + type: string, + occurredAt: string, + payload: unknown, + extra: Partial = {}, +): SessionActivityEvent { + return { type, threadId: THREAD, occurredAt, payload, ...extra }; +} + +function runPayload(status: string, fields: Record = {}) { + return { + id: RUN, + threadId: THREAD, + ordinal: 73, + providerInstanceId: "claudeAgent", + modelSelection: { + instanceId: "claudeAgent", + model: "claude-opus-5-5", + options: [ + { id: "effort", value: "high" }, + { id: "fastMode", value: false }, + ], + }, + userMessageId: "message-1", + status, + requestedAt: at("09:06:19.278"), + startedAt: null, + completedAt: null, + ...fields, + }; +} + +function userMessage(fields: Record = {}) { + return { + sentByUserId: SENDER, + createdBy: "user", + creationSource: "web", + id: "message-1", + threadId: THREAD, + runId: null, + role: "user", + text: "SECRET_PROMPT", + attachments: [], + streaming: false, + createdAt: at("09:06:19.000"), + updatedAt: at("09:06:19.000"), + ...fields, + }; +} + +function questionPayload(status: string, resolvedAt: string | null) { + return { + id: "runtime-request:native-request:toolu_01", + nodeId: "node:runtime-request:toolu_01", + kind: "user_input", + status, + responseCapability: { type: "live", providerSessionId: "provider-session:1" }, + createdAt: at("09:07:25.279"), + resolvedAt, + ...(status === "resolved" ? { answers: { q1: "SECRET_ANSWER" } } : {}), + }; +} + +function subagentPayload(status: string, fields: Record = {}) { + return { + id: "node:subagent:task-1", + threadId: THREAD, + runId: RUN, + parentNodeId: "node:root", + origin: "provider_native", + createdBy: "agent", + driver: "claudeAgent", + providerInstanceId: "claudeAgent", + childThreadId: "thread:provider:claudeAgent:native-thread:child", + prompt: "SECRET_SUBAGENT_PROMPT", + title: "SECRET_SUBAGENT_TITLE", + result: status === "completed" ? "SECRET_RESULT" : null, + status, + startedAt: at("09:08:00.000"), + completedAt: status === "completed" ? at("09:20:00.000") : null, + updatedAt: at("09:08:00.000"), + ...fields, + }; +} + +function feed(events: ReadonlyArray, backfill = false) { + const tracker = makeSessionActivityTracker({ backfill }); + const out: Array = []; + for (const item of events) out.push(...tracker.apply(item)); + return { tracker, out }; +} + +const spans = (out: ReadonlyArray) => + out.filter((emission) => emission.event === "session.state.span"); + +/** [state, waitKind, durationMs, endReason] per span. */ +const spanSummary = (out: ReadonlyArray) => + spans(out).map((span) => [ + span.properties.state, + span.properties.waitKind, + span.properties.durationMs, + span.properties.endReason, + ]); + +const startedRun = (time: string) => runPayload("running", { startedAt: at(time) }); +const archivedThread = (time: string) => threadPayload({ archivedAt: at(time) }); + +const created = event("thread.created", at("09:00:00.000"), threadPayload()); + +describe("SessionActivityReducer", () => { + it("counts a question asked mid-run as human wait, not agent time", () => { + const { out } = feed([ + created, + event("message.updated", at("09:06:19.000"), userMessage()), + event("run.created", at("09:06:19.278"), runPayload("starting"), { actorUserId: SENDER }), + event("run-attempt.created", at("09:06:19.278"), { id: "attempt-1", runId: RUN }), + event( + "run.updated", + at("09:06:19.534"), + runPayload("running", { startedAt: at("09:06:19.534") }), + { driver: "claudeAgent" }, + ), + event("runtime-request.updated", at("09:07:25.320"), questionPayload("pending", null), { + runId: RUN, + }), + event( + "runtime-request.updated", + at("09:09:23.778"), + questionPayload("resolved", at("09:09:23.778")), + { runId: RUN, actorUserId: SENDER }, + ), + event( + "run.updated", + at("09:10:09.686"), + runPayload("waiting", { startedAt: at("09:06:19.534") }), + ), + event( + "run.updated", + at("09:10:09.747"), + runPayload("completed", { + startedAt: at("09:06:19.534"), + completedAt: at("09:10:09.747"), + }), + ), + event("thread.visited", at("09:30:00.000"), threadPayload()), + ]); + + assert.deepStrictEqual(spanSummary(out), [ + ["idle", undefined, 379_278, "agent_running"], + ["agent_running", undefined, 66_042, "waiting_human"], + ["waiting_human", "question", 118_458, "agent_running"], + ["agent_running", undefined, 45_908, "idle"], + ]); + const waitSpan = spans(out)[2]; + assert.strictEqual(waitSpan?.properties.runId, RUN); + assert.strictEqual(waitSpan?.properties.senderUserId, SENDER); + assert.strictEqual(waitSpan?.userId, OWNER); + + const resolved = out.find((emission) => emission.event === "human_request.resolved"); + assert.strictEqual(resolved?.properties.waitMs, 118_499); + assert.strictEqual(resolved?.properties.resolution, "answered"); + assert.strictEqual(resolved?.properties.answeredByUserId, SENDER); + + const finished = out.find((emission) => emission.event === "run.finished"); + assert.deepStrictEqual( + { + status: finished?.properties.status, + queueWaitMs: finished?.properties.queueWaitMs, + runDurationMs: finished?.properties.runDurationMs, + attempts: finished?.properties.attempts, + effort: finished?.properties.effort, + runTrigger: finished?.properties.runTrigger, + senderUserId: finished?.properties.senderUserId, + }, + { + status: "completed", + queueWaitMs: 256, + runDurationMs: 230_213, + attempts: 1, + effort: "high", + runTrigger: "human", + senderUserId: SENDER, + }, + ); + }); + + it("waits on workers after the run ends while a subagent still runs", () => { + const { out } = feed([ + created, + event("run.created", at("09:06:19.278"), startedRun("09:06:19.278")), + event("subagent.updated", at("09:08:00.000"), subagentPayload("running")), + event( + "run.updated", + at("09:10:00.000"), + runPayload("completed", { + startedAt: at("09:06:19.278"), + completedAt: at("09:10:00.000"), + }), + ), + event("subagent.updated", at("09:20:00.000"), subagentPayload("completed")), + event("thread.visited", at("09:25:00.000"), threadPayload()), + ]); + + const summary = spanSummary(out); + assert.deepStrictEqual(summary.slice(1), [ + ["agent_running", undefined, 220_722, "waiting_workers"], + ["waiting_workers", undefined, 600_000, "idle"], + ]); + const workerSpan = spans(out)[2]; + assert.strictEqual(workerSpan?.properties.activeWorkers, 0); + assert.strictEqual(workerSpan?.properties.maxActiveWorkers, 1); + + const finished = out.find((emission) => emission.event === "subagent.finished"); + assert.strictEqual(finished?.properties.durationMs, 720_000); + assert.strictEqual(finished?.properties.childThreadPresent, true); + assert.strictEqual(finished?.properties.origin, "provider_native"); + }); + + it("measures queued, running and idle time in order", () => { + const { out } = feed([ + created, + event( + "run.created", + at("09:01:00.000"), + runPayload("queued", { requestedAt: at("09:01:00.000") }), + ), + event( + "run.updated", + at("09:03:00.000"), + runPayload("running", { requestedAt: at("09:01:00.000"), startedAt: at("09:03:00.000") }), + ), + event( + "run.updated", + at("09:08:00.000"), + runPayload("failed", { + requestedAt: at("09:01:00.000"), + startedAt: at("09:03:00.000"), + completedAt: at("09:08:00.000"), + }), + ), + event("thread.visited", at("09:10:00.000"), threadPayload()), + ]); + assert.deepStrictEqual( + spanSummary(out).map(([state, , durationMs]) => [state, durationMs]), + [ + ["idle", 60_000], + ["queued", 120_000], + ["agent_running", 300_000], + ], + ); + const finished = out.find((emission) => emission.event === "run.finished"); + assert.strictEqual(finished?.properties.queueWaitMs, 120_000); + assert.strictEqual(finished?.properties.failureClass, "unknown"); + }); + + it("tracks archive and unarchive as a terminal state with lifecycle events", () => { + const { out } = feed([ + created, + event("thread.archived", at("10:00:00.000"), archivedThread("10:00:00.000")), + event("thread.unarchived", at("12:00:00.000"), threadPayload({ archivedAt: null })), + event("thread.visited", at("12:30:00.000"), threadPayload()), + ]); + assert.deepStrictEqual( + spanSummary(out).map(([state, , durationMs, endReason]) => [state, durationMs, endReason]), + [ + ["idle", 3_600_000, "archived"], + ["archived", 7_200_000, "idle"], + ], + ); + const archived = out.find((emission) => emission.event === "session.archived"); + assert.strictEqual(archived?.properties.ageMs, 3_600_000); + assert.ok(out.some((emission) => emission.event === "session.unarchived")); + }); + + it("waits on a proposed plan until the next run decides it", () => { + const { out } = feed([ + created, + event("run.created", at("09:01:00.000"), startedRun("09:01:00.000")), + event("plan.updated", at("09:04:00.000"), { + id: "plan-1", + threadId: THREAD, + runId: RUN, + nodeId: "node:plan", + status: "active", + kind: "proposed_plan", + markdown: "SECRET_PLAN", + }), + event( + "run.updated", + at("09:05:00.000"), + runPayload("completed", { startedAt: at("09:01:00.000"), completedAt: at("09:05:00.000") }), + ), + event( + "run.created", + at("09:15:00.000"), + runPayload("running", { + id: "run-2", + ordinal: 74, + startedAt: at("09:15:00.000"), + sourcePlanRef: { threadId: THREAD, planId: "plan-1" }, + }), + ), + ]); + const planSpan = spans(out).find((span) => span.properties.waitKind === "plan"); + assert.strictEqual(planSpan?.properties.durationMs, 600_000); + const decided = out.find((emission) => emission.event === "plan.decided"); + assert.strictEqual(decided?.properties.decision, "implemented"); + assert.strictEqual(decided?.properties.waitMs, 600_000); + }); + + it("splits a span at UTC midnight", () => { + const tracker = makeSessionActivityTracker({ backfill: false }); + tracker.apply(event("thread.created", "2026-10-09T22:00:00.000Z", threadPayload())); + const out = tracker.apply( + event("thread.archived", "2026-10-10T03:00:00.000Z", threadPayload()), + ); + assert.deepStrictEqual( + spans(out).map((span) => [ + span.properties.durationMs, + span.properties.segmentIndex, + span.properties.endReason, + ]), + [ + [7_200_000, 0, "idle"], + [10_800_000, 1, "archived"], + ], + ); + assert.notStrictEqual(spans(out)[0]?.insertId, spans(out)[1]?.insertId); + }); + + it("ignores provider-native subagent transcript threads", () => { + const tracker = makeSessionActivityTracker({ backfill: false }); + const child = "thread:provider:claudeAgent:native-thread:child"; + const out = tracker.apply({ + type: "thread.created", + threadId: child, + occurredAt: at("09:00:00.000"), + payload: threadPayload({ + id: child, + createdBy: "agent", + creationSource: "provider", + lineage: { parentThreadId: THREAD, relationshipToParent: "subagent", rootThreadId: THREAD }, + }), + }); + assert.deepStrictEqual(out, []); + assert.deepStrictEqual(tracker.snapshot(epoch(at("10:00:00.000"))).length, 1); + }); + + it("replays an imported, already finished run from its recorded start", () => { + const { out } = feed([ + created, + event( + "run.created", + at("09:05:00.000"), + runPayload("completed", { + requestedAt: at("09:02:00.000"), + startedAt: at("09:02:00.000"), + completedAt: at("09:05:00.000"), + }), + ), + event("thread.visited", at("09:06:00.000"), threadPayload()), + ]); + assert.deepStrictEqual( + spanSummary(out).map(([state, , durationMs]) => [state, durationMs]), + [ + ["idle", 120_000], + ["agent_running", 180_000], + ], + ); + }); + + it("marks seeded spans as resumed and closes them on shutdown", () => { + const tracker = makeSessionActivityTracker({ backfill: false }); + const bootMs = epoch(at("09:00:00.000")); + tracker.seed( + { + ...threadPayload(), + activeRunId: RUN, + activityRunStatus: "running", + latestRunId: RUN, + status: "running", + pendingRuntimeRequest: null, + activeSubagentCount: 2, + pendingBackgroundTasks: [], + }, + bootMs, + ); + const snapshot = tracker.snapshot(bootMs + 1_000); + const total = snapshot.find((emission) => emission.properties.scope === "total"); + assert.strictEqual(total?.properties.runningRuns, 1); + assert.strictEqual(total?.properties.activeSubagents, 2); + const perUser = snapshot.find((emission) => emission.properties.scope === "user"); + assert.strictEqual(perUser?.userId, OWNER); + assert.strictEqual(perUser?.properties.count, 1); + + const closed = tracker.closeAll(bootMs + 60_000, "server_stopped"); + assert.strictEqual(closed.length, 1); + assert.strictEqual(closed[0]?.properties.state, "agent_running"); + assert.strictEqual(closed[0]?.properties.resumedAfterRestart, true); + assert.strictEqual(closed[0]?.properties.endReason, "server_stopped"); + assert.strictEqual(closed[0]?.properties.durationMs, 60_000); + }); + + it("never emits text, paths or unlisted keys", () => { + const { tracker, out } = feed( + [ + created, + event("message.updated", at("09:06:19.000"), userMessage()), + event("run.created", at("09:06:19.278"), startedRun("09:06:19.278")), + event("runtime-request.updated", at("09:07:25.320"), questionPayload("pending", null), { + runId: RUN, + }), + event("subagent.updated", at("09:08:00.000"), subagentPayload("running")), + event( + "runtime-request.updated", + at("09:09:23.778"), + questionPayload("resolved", at("09:09:23.778")), + { runId: RUN }, + ), + event("subagent.updated", at("09:20:00.000"), subagentPayload("completed")), + event("turn-item.updated", at("09:09:30.000"), { + id: "turn-item:error", + type: "error", + status: "completed", + runId: RUN, + title: "SECRET_ERROR_TITLE", + failure: { + class: "provider_error", + message: "SECRET_ERROR", + code: null, + retryable: null, + }, + }), + event( + "run.updated", + at("09:21:00.000"), + runPayload("failed", { startedAt: at("09:06:19.278"), completedAt: at("09:21:00.000") }), + ), + event("thread.archived", at("10:00:00.000"), archivedThread("10:00:00.000")), + ], + true, + ); + const all = [ + ...out, + ...tracker.snapshot(epoch(at("11:00:00.000"))), + ...tracker.closeAll(epoch(at("12:00:00.000")), "server_stopped"), + ]; + const allowed = new Set(SESSION_ACTIVITY_PROPERTY_KEYS); + for (const emission of all) { + for (const [key, value] of Object.entries(emission.properties)) { + assert.ok(allowed.has(key), `unlisted key ${key} on ${emission.event}`); + assert.ok( + !(typeof value === "string" && (value.includes("SECRET") || value.includes("/"))), + `${emission.event}.${key} leaks text`, + ); + } + assert.strictEqual(emission.properties.backfill, true); + } + const finished = all.find((emission) => emission.event === "run.finished"); + assert.strictEqual(finished?.properties.failureClass, "provider_error"); + }); +}); diff --git a/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts new file mode 100644 index 0000000000000..db3541fb341c2 --- /dev/null +++ b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts @@ -0,0 +1,1336 @@ +/** + * T3-CUSTOM(expbkt3): session time-in-state metrics (TEC-1631). + * + * A pure tracker over the orchestration V2 event log. It keeps one mutually + * exclusive STATE per session (thread) and emits a `session.state.span` each + * time that state changes, plus transition events (`session.created`, + * `run.finished`, `human_request.resolved`, `subagent.finished`, + * `plan.decided`, archive and snooze events). The live reactor + * (`SessionActivityReactor.ts`) and the history backfill + * (`apps/server/scripts/analytics-backfill.ts`) feed it the same events, so + * live and historical numbers mean the same thing. Time comes only from event + * `occurredAt`; the tracker never reads a clock or a database. + * + * State precedence, highest first: + * 1. `archived` the thread is archived (terminal until unarchived). + * 2. `waiting_human` a runtime request waits on a person: an approval + * (`waitKind: "approval"`) or a question / MCP + * elicitation (`"question"`). This wins over a running + * run: a question asked mid-run is human wait time. + * 3. `agent_running` a run is preparing, starting or running. + * 4. `waiting_human` (`waitKind: "plan"`) a proposed plan from a settled + * run waits for the next message. Decided by the next + * run, by archiving, or by the plan's own completion. + * 5. `waiting_workers` no run is running, but background work the agent + * waits on is still active: running subagents (provider + * native and delegated `delegate_task` children, from + * `subagent.updated`), active non-persistent + * `dynamic_tool` items, and provider roster tasks that + * hold completion (`backgroundWorkHoldsCompletion`). + * Plain background shell commands (dev servers) and + * pull request watches do not count: they can run for + * days while the agent is done. + * 6. `queued` a run is queued and nothing above applies. + * 7. `idle` nothing is pending; the session waits for the next + * message. `snoozed: true` marks idle time while snoozed. + * + * A run in `waiting` status counts as settled: it is the short window between + * a successful turn and its checkpoint. Provider-native subagent threads + * (Claude Agent tool transcripts) are not sessions; their time is the parent's + * `agent_running` or `waiting_workers` time. + * + * Spans are split at UTC midnight, so per-day sums are exact. A span that + * started before a server restart is closed with `endReason: "server_stopped"`; + * the next server opens a new one with `resumedAfterRestart: true`. + * + * Properties hold only ids, enums, counts, durations and timestamps. Never add + * prompt, message, title, question or answer text, paths, branches or + * repository names. + */ +import * as DateTime from "effect/DateTime"; +import * as Option from "effect/Option"; + +export type SessionState = + | "waiting_human" + | "agent_running" + | "waiting_workers" + | "queued" + | "idle" + | "archived"; + +export type SessionWaitKind = "approval" | "question" | "plan"; + +export type SessionSource = + | "ui" + | "mcp" + | "linear_runner" + | "schedule" + | "delegate" + | "fork" + | "mattermost" + | "unknown"; + +/** What started a run, from its user message. */ +export type RunTrigger = + | "human" + | "agent" + | "wake" + | "delegated_result" + | "schedule" + | "async_answer" + | "restart" + | "unknown"; + +export type SessionActivityPropertyValue = string | number | boolean; + +export interface SessionActivityEmission { + readonly event: string; + readonly properties: Readonly>; + /** Environment user the event is attributed to: the owner, else the sender. */ + readonly userId?: string; + /** ISO time the event happened. */ + readonly timestamp: string; + /** Stable identity of this event, so a re-run does not count it twice. */ + readonly insertId: string; +} + +type TimeInput = string | number | DateTime.DateTime; + +/** + * The fields the tracker reads from an orchestration V2 domain event. The live + * `OrchestrationV2DomainEvent` and a row decoded from `orchestration_events` + * both fit. + */ +export interface SessionActivityEvent { + readonly type: string; + readonly threadId: string; + readonly occurredAt: TimeInput; + readonly runId?: string | undefined; + readonly actorUserId?: string | undefined; + readonly driver?: string | undefined; + readonly payload: unknown; +} + +export interface SessionActivityTrackerOptions { + /** Set on every event; true for the history backfill. */ + readonly backfill: boolean; +} + +/** Every property key the tracker can emit, for the analytics allowlist. */ +export const SESSION_ACTIVITY_PROPERTY_KEYS = [ + "threadId", + "projectId", + "parentThreadId", + "isChildSession", + "state", + "waitKind", + "durationMs", + "startedAt", + "endedAt", + "endReason", + "spanStartedAt", + "segmentIndex", + "runId", + "runOrdinal", + "runTrigger", + "provider", + "providerInstanceId", + "model", + "effort", + "interactionMode", + "runtimeMode", + "activeWorkers", + "maxActiveWorkers", + "backgroundTasks", + "ownerUserId", + "senderUserId", + "sessionSource", + "creationSource", + "createdBy", + "resumedAfterRestart", + "backfill", + "snoozed", + "ageMs", + "status", + "queueWaitMs", + "runDurationMs", + "workDurationMs", + "attempts", + "failureClass", + "requestKind", + "waitMs", + "resolution", + "answeredByUserId", + "subagentId", + "driver", + "origin", + "childThreadPresent", + "decision", + "scope", + "count", + "runningRuns", + "queuedRuns", + "waitingHuman", + "waitingWorkers", + "activeSubagents", +] as const; + +// --------------------------------------------------------------------------- +// Payload readers. Payloads arrive decoded (live) or as parsed JSON (backfill), +// so every read is defensive and typed `unknown` until checked. + +type Rec = Readonly>; + +function asRec(value: unknown): Rec | undefined { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Rec) + : undefined; +} + +function str(record: Rec | undefined, key: string): string | undefined { + const value = record?.[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +/** Epoch milliseconds of an ISO string, epoch number or Effect DateTime. */ +export function toEpochMs(value: unknown): number | undefined { + if (typeof value === "number") return Number.isFinite(value) ? value : undefined; + if (typeof value === "string" || DateTime.isDateTime(value)) { + return Option.match(DateTime.make(value), { + onNone: () => undefined, + onSome: DateTime.toEpochMillis, + }); + } + return undefined; +} + +function isoOf(ms: number): string { + return DateTime.formatIso(DateTime.makeUnsafe(ms)); +} + +const DAY_MS = 86_400_000; + +const ACTIVE_RUN_STATUSES = new Set(["preparing", "starting", "running"]); +const FINAL_RUN_STATUSES = new Set([ + "completed", + "failed", + "interrupted", + "cancelled", + "rolled_back", +]); +const ACTIVE_WORK_STATUSES = new Set(["pending", "running", "waiting"]); +const FINAL_SUBAGENT_STATUSES = new Set(["completed", "failed", "cancelled", "interrupted"]); +/** Request kinds a client tool answers, not a person. */ +const NON_HUMAN_REQUEST_KINDS = new Set(["dynamic_tool_call"]); + +function modelInfo(selection: unknown): { + readonly instanceId?: string; + readonly model?: string; + readonly effort?: string; +} { + const record = asRec(selection); + if (record === undefined) return {}; + let effort: string | undefined; + const options = record.options; + if (Array.isArray(options)) { + for (const option of options) { + const entry = asRec(option); + if (entry?.id === "effort" && typeof entry.value === "string") effort = entry.value; + } + } else { + const value = asRec(options)?.effort; + if (typeof value === "string") effort = value; + } + const instanceId = str(record, "instanceId"); + const model = str(record, "model"); + return { + ...(instanceId === undefined ? {} : { instanceId }), + ...(model === undefined ? {} : { model }), + ...(effort === undefined ? {} : { effort }), + }; +} + +function isProviderNativeSubagentThread(payload: Rec | undefined): boolean { + return ( + asRec(payload?.lineage)?.relationshipToParent === "subagent" && + payload?.creationSource === "provider" + ); +} + +function hasLinearLink(payload: Rec): boolean { + if (str(payload, "linearIssueUrl") !== undefined) return true; + return Array.isArray(payload.linearLinks) && payload.linearLinks.length > 0; +} + +/** + * Best-effort origin of a session, from its first thread payload: + * fork lineage, delegate_task child, Mattermost-bound, Linear-tagged and + * created by a server or MCP client (the Linear runner), web or mobile UI, + * MCP, or a server-created session with a parent (another agent's + * t3_create_session). A scheduled first message upgrades it to `schedule`. + */ +function sessionSourceOf(payload: Rec): SessionSource { + const relationship = asRec(payload.lineage)?.relationshipToParent; + if (relationship === "fork") return "fork"; + if (relationship === "subagent") return "delegate"; + if (payload.forkedFrom !== null && payload.forkedFrom !== undefined) return "fork"; + if (str(payload, "mattermostThreadUrl") !== undefined) return "mattermost"; + const creationSource = str(payload, "creationSource"); + if (hasLinearLink(payload) && (creationSource === "server" || creationSource === "mcp")) { + return "linear_runner"; + } + if (creationSource === "web" || creationSource === "mobile") return "ui"; + if (creationSource === "mcp") return "mcp"; + if (creationSource === "server" && str(payload, "parentThreadId") !== undefined) return "mcp"; + return "unknown"; +} + +function runTriggerOf(message: Rec): RunTrigger { + if (message.delegatedCompletion !== undefined && message.delegatedCompletion !== null) { + return "delegated_result"; + } + if (message.notification !== undefined && message.notification !== null) return "wake"; + if (str(message, "scheduledTaskId") !== undefined) return "schedule"; + if (str(message, "id")?.startsWith("async-answer:") === true) return "async_answer"; + if (str(message, "senderThreadId") !== undefined) return "agent"; + const createdBy = str(message, "createdBy"); + if (createdBy === "agent") return "agent"; + if (createdBy === "user") return "human"; + return "unknown"; +} + +function waitKindOfRequest(kind: string): SessionWaitKind { + return kind === "user_input" || kind === "mcp-elicitation" ? "question" : "approval"; +} + +function resolutionOf(status: string, kind: string, decision: string | undefined): string { + if (status === "expired") return "expired"; + if (status === "cancelled") return "cancelled"; + if (kind === "user_input" || kind === "mcp-elicitation") return "answered"; + switch (decision) { + case "accept": + case "acceptForSession": + case "acceptAlways": + return "approved"; + case "decline": + return "declined"; + case "cancel": + return "cancelled"; + default: + return "resolved"; + } +} + +/** Provider roster tasks the agent waits on; plain commands do not hold it. */ +function holdingRosterTaskCount(tasks: unknown): number { + if (!Array.isArray(tasks)) return 0; + let count = 0; + for (const task of tasks) { + const entry = asRec(task); + if (entry === undefined) continue; + if (entry.kind !== "command" || entry.wakesAgent === true) count += 1; + } + return count; +} + +// --------------------------------------------------------------------------- +// Tracker state. + +interface RunInfo { + readonly id: string; + ordinal?: number; + status: string; + requestedAtMs?: number; + startedAtMs?: number; + completedAtMs?: number; + workStartedAtMs?: number; + userMessageId?: string; + providerInstanceId?: string; + model?: string; + effort?: string; + attempts: number; + failureClass?: string; + senderUserId?: string; + trigger?: RunTrigger; + restartContinuation: boolean; + finished: boolean; + /** An imported run finished later than the event that created it: finish it then. */ + deferredFinal?: { readonly status: string; readonly atMs: number }; +} + +interface RequestInfo { + readonly kind: string; + readonly waitKind: SessionWaitKind; + readonly createdAtMs: number; + readonly runId?: string; + readonly live: boolean; +} + +interface PlanInfo { + readonly runId?: string; + readonly atMs: number; + waitingSinceMs?: number; +} + +interface SubagentInfo { + readonly startedAtMs: number; +} + +interface MessageInfo { + readonly senderUserId?: string; + readonly trigger: RunTrigger; +} + +interface OpenSpan { + readonly key: string; + readonly state: SessionState; + readonly waitKind?: SessionWaitKind; + readonly snoozed: boolean; + readonly run?: RunInfo; + readonly startMs: number; + readonly resumed: boolean; + maxWorkers: number; +} + +interface ThreadTrack { + readonly threadId: string; + known: boolean; + projectId?: string; + parentThreadId?: string; + ownerUserId?: string; + createdAtMs?: number; + creationSource?: string; + createdBy?: string; + sessionSource: SessionSource; + providerInstanceId?: string; + provider?: string; + model?: string; + effort?: string; + interactionMode?: string; + runtimeMode?: string; + activeProviderThreadId?: string; + archived: boolean; + snoozed: boolean; + readonly runs: Map; + lastRun?: RunInfo; + readonly requests: Map; + readonly plans: Map; + readonly decidedPlans: Set; + readonly subagents: Map; + readonly finishedSubagents: Set; + readonly backgroundTools: Set; + readonly rosterTasks: Map; + /** Workers a restart seed counted without ids; cleared by the next worker event. */ + seededWorkers: number; + readonly messages: Map; + lastSenderUserId?: string; + span?: OpenSpan; + resumeNextSpan: boolean; +} + +const MAX_TRACKED_MESSAGES = 64; +const MAX_FINISHED_IDS = 512; + +function newThread(threadId: string): ThreadTrack { + return { + threadId, + known: false, + sessionSource: "unknown", + archived: false, + snoozed: false, + runs: new Map(), + requests: new Map(), + plans: new Map(), + decidedPlans: new Set(), + subagents: new Map(), + finishedSubagents: new Set(), + backgroundTools: new Set(), + rosterTasks: new Map(), + seededWorkers: 0, + messages: new Map(), + resumeNextSpan: false, + }; +} + +function rememberBounded(set: Set, id: string): void { + set.add(id); + if (set.size > MAX_FINISHED_IDS) { + const oldest = set.values().next(); + if (!oldest.done) set.delete(oldest.value); + } +} + +/** Event types that may introduce a session the tracker has not seen yet. */ +function introducesSession(type: string): boolean { + return ( + type.startsWith("thread.") || + type === "run.created" || + type === "run.updated" || + type === "runtime-request.updated" || + type === "message.updated" || + type === "plan.updated" + ); +} + +/** The current-state picture used by the reactor's periodic snapshot. */ +interface StateCounts { + readonly byOwnerState: Map; + runningRuns: number; + queuedRuns: number; + waitingHuman: number; + waitingWorkers: number; + activeSubagents: number; +} + +export interface SessionActivityTracker { + /** Feed one event, in order. Returns the analytics events it produced. */ + readonly apply: (event: SessionActivityEvent) => ReadonlyArray; + /** + * Seed a session from a projection shell after a restart. Emits nothing; the + * span it opens at `nowMs` carries `resumedAfterRestart: true`. + */ + readonly seed: (shell: unknown, nowMs: number) => void; + /** Close every open span, for example with `server_stopped` on shutdown. */ + readonly closeAll: (nowMs: number, endReason: string) => ReadonlyArray; + /** Point-in-time counts: one event per (owner, state), plus one totals event. */ + readonly snapshot: (nowMs: number) => ReadonlyArray; +} + +export function makeSessionActivityTracker( + options: SessionActivityTrackerOptions, +): SessionActivityTracker { + const threads = new Map(); + /** Threads that are not sessions (provider-native subagent transcripts, deleted). */ + const ignored = new Set(); + + const emission = ( + t: ThreadTrack | undefined, + event: string, + atMs: number, + insertId: string, + properties: Record, + userId?: string, + ): SessionActivityEmission => { + const clean: Record = {}; + for (const [key, value] of Object.entries(properties)) { + if (value !== undefined) clean[key] = value; + } + clean.backfill = options.backfill; + const attributed = userId ?? t?.ownerUserId ?? t?.lastSenderUserId; + return { + event, + properties: clean, + ...(attributed === undefined ? {} : { userId: attributed }), + timestamp: isoOf(atMs), + insertId, + }; + }; + + const updateThreadFields = (t: ThreadTrack, payload: Rec, atMs: number) => { + const firstPayload = !t.known; + t.known = true; + const projectId = str(payload, "projectId"); + if (projectId !== undefined) t.projectId = projectId; + const parentThreadId = + str(payload, "parentThreadId") ?? str(asRec(payload.lineage), "parentThreadId"); + if (parentThreadId !== undefined) t.parentThreadId = parentThreadId; + const ownerUserId = str(payload, "ownerUserId"); + if (ownerUserId !== undefined) t.ownerUserId = ownerUserId; + t.createdAtMs ??= toEpochMs(payload.createdAt) ?? atMs; + const creationSource = str(payload, "creationSource"); + if (creationSource !== undefined) t.creationSource = creationSource; + const createdBy = str(payload, "createdBy"); + if (createdBy !== undefined) t.createdBy = createdBy; + if (firstPayload) t.sessionSource = sessionSourceOf(payload); + const providerInstanceId = str(payload, "providerInstanceId"); + if (providerInstanceId !== undefined) t.providerInstanceId = providerInstanceId; + const selection = modelInfo(payload.modelSelection); + if (selection.model !== undefined) t.model = selection.model; + if (payload.modelSelection !== undefined) { + if (selection.effort === undefined) delete t.effort; + else t.effort = selection.effort; + } + const interactionMode = str(payload, "interactionMode"); + if (interactionMode !== undefined) t.interactionMode = interactionMode; + const runtimeMode = str(payload, "runtimeMode"); + if (runtimeMode !== undefined) t.runtimeMode = runtimeMode; + if ("activeProviderThreadId" in payload) { + const active = str(payload, "activeProviderThreadId"); + if (active === undefined) delete t.activeProviderThreadId; + else t.activeProviderThreadId = active; + } + if ("archivedAt" in payload) t.archived = toEpochMs(payload.archivedAt) !== undefined; + if ("snoozedUntil" in payload) t.snoozed = toEpochMs(payload.snoozedUntil) !== undefined; + }; + + const baseProps = (t: ThreadTrack) => ({ + threadId: t.threadId, + projectId: t.projectId, + parentThreadId: t.parentThreadId, + isChildSession: t.parentThreadId !== undefined, + ownerUserId: t.ownerUserId, + sessionSource: t.sessionSource, + }); + + const activeWorkers = (t: ThreadTrack) => t.subagents.size + t.seededWorkers; + + const rosterCount = (t: ThreadTrack) => { + if (t.activeProviderThreadId !== undefined) { + return t.rosterTasks.get(t.activeProviderThreadId) ?? 0; + } + let total = 0; + for (const count of t.rosterTasks.values()) total += count; + return total; + }; + + const runById = (t: ThreadTrack, runId: string | undefined): RunInfo | undefined => { + if (runId === undefined) return undefined; + return t.runs.get(runId) ?? (t.lastRun?.id === runId ? t.lastRun : undefined); + }; + + const derive = ( + t: ThreadTrack, + ): { state: SessionState; waitKind?: SessionWaitKind; run?: RunInfo } => { + if (t.archived) return { state: "archived" }; + if (t.requests.size > 0) { + let latest: RequestInfo | undefined; + for (const request of t.requests.values()) { + if (latest === undefined || request.createdAtMs >= latest.createdAtMs) latest = request; + } + const run = runById(t, latest?.runId); + return { + state: "waiting_human", + waitKind: latest?.waitKind ?? "approval", + ...(run === undefined ? {} : { run }), + }; + } + let active: RunInfo | undefined; + let queued: RunInfo | undefined; + for (const run of t.runs.values()) { + if (ACTIVE_RUN_STATUSES.has(run.status)) { + if (active === undefined || (run.ordinal ?? 0) > (active.ordinal ?? 0)) active = run; + } else if (run.status === "queued") { + if (queued === undefined || (run.ordinal ?? 0) < (queued.ordinal ?? 0)) queued = run; + } + } + if (active !== undefined) return { state: "agent_running", run: active }; + if (t.plans.size > 0) { + let latestPlan: PlanInfo | undefined; + for (const plan of t.plans.values()) { + if (latestPlan === undefined || plan.atMs >= latestPlan.atMs) latestPlan = plan; + } + const run = runById(t, latestPlan?.runId); + return { state: "waiting_human", waitKind: "plan", ...(run === undefined ? {} : { run }) }; + } + if (activeWorkers(t) > 0 || t.backgroundTools.size > 0 || rosterCount(t) > 0) { + return { state: "waiting_workers", ...(t.lastRun === undefined ? {} : { run: t.lastRun }) }; + } + if (queued !== undefined) return { state: "queued", run: queued }; + return { state: "idle" }; + }; + + const spanEmissions = ( + t: ThreadTrack, + span: OpenSpan, + endMs: number, + endReason: string, + ): ReadonlyArray => { + const end = Math.max(endMs, span.startMs); + const run = span.run; + const common = { + ...baseProps(t), + state: span.state, + waitKind: span.waitKind, + endReason, + spanStartedAt: isoOf(span.startMs), + runId: run?.id, + runOrdinal: run?.ordinal, + runTrigger: run?.trigger, + provider: t.provider, + providerInstanceId: run?.providerInstanceId ?? t.providerInstanceId, + model: run?.model ?? t.model, + effort: run?.effort ?? t.effort, + interactionMode: t.interactionMode, + runtimeMode: t.runtimeMode, + activeWorkers: activeWorkers(t), + maxActiveWorkers: Math.max(span.maxWorkers, activeWorkers(t)), + backgroundTasks: t.backgroundTools.size + rosterCount(t), + senderUserId: run?.senderUserId ?? t.lastSenderUserId, + creationSource: t.creationSource, + resumedAfterRestart: span.resumed, + snoozed: span.state === "idle" ? span.snoozed : undefined, + }; + const out: Array = []; + let segmentStart = span.startMs; + let segmentIndex = 0; + do { + const nextMidnight = (Math.floor(segmentStart / DAY_MS) + 1) * DAY_MS; + const segmentEnd = Math.min(end, nextMidnight); + const last = segmentEnd >= end; + out.push( + emission( + t, + "session.state.span", + segmentEnd, + `session.state.span|${t.threadId}|${segmentStart}`, + { + ...common, + durationMs: segmentEnd - segmentStart, + startedAt: isoOf(segmentStart), + endedAt: isoOf(segmentEnd), + segmentIndex, + // A segment cut at midnight continues in the same state. + endReason: last ? endReason : span.state, + }, + ), + ); + segmentStart = segmentEnd; + segmentIndex += 1; + } while (segmentStart < end); + return out; + }; + + /** Recompute the state; close the open span and open a new one when it changed. */ + const reconcile = (t: ThreadTrack, atMs: number, out: Array) => { + const next = derive(t); + if (next.state === "waiting_human" && next.waitKind === "plan") { + for (const plan of t.plans.values()) plan.waitingSinceMs ??= atMs; + } + const snoozed = next.state === "idle" && t.snoozed; + const key = `${next.state}|${next.waitKind ?? ""}|${snoozed}|${next.run?.id ?? ""}`; + const workers = activeWorkers(t); + if (t.span !== undefined && t.span.key === key) { + t.span.maxWorkers = Math.max(t.span.maxWorkers, workers); + return; + } + if (t.span !== undefined) out.push(...spanEmissions(t, t.span, atMs, next.state)); + t.span = { + key, + state: next.state, + ...(next.waitKind === undefined ? {} : { waitKind: next.waitKind }), + snoozed, + ...(next.run === undefined ? {} : { run: next.run }), + startMs: atMs, + resumed: t.resumeNextSpan, + maxWorkers: workers, + }; + t.resumeNextSpan = false; + }; + + const decidePlan = ( + t: ThreadTrack, + planId: string, + plan: PlanInfo, + atMs: number, + decision: string, + out: Array, + ) => { + t.plans.delete(planId); + rememberBounded(t.decidedPlans, planId); + out.push( + emission(t, "plan.decided", atMs, `plan.decided|${t.threadId}|${planId}`, { + ...baseProps(t), + runId: plan.runId, + decision, + // Zero when the plan was decided before its run settled. + waitMs: plan.waitingSinceMs === undefined ? 0 : Math.max(0, atMs - plan.waitingSinceMs), + }), + ); + }; + + const decidePlans = ( + t: ThreadTrack, + atMs: number, + decisionFor: (planId: string) => string, + out: Array, + ) => { + for (const [planId, plan] of [...t.plans]) { + decidePlan(t, planId, plan, atMs, decisionFor(planId), out); + } + }; + + const closeRequest = ( + t: ThreadTrack, + requestId: string, + request: RequestInfo, + atMs: number, + resolution: string, + resolvedAtMs: number, + answeredByUserId: string | undefined, + out: Array, + ) => { + t.requests.delete(requestId); + out.push( + emission( + t, + "human_request.resolved", + atMs, + `human_request.resolved|${t.threadId}|${requestId}`, + { + ...baseProps(t), + runId: request.runId, + requestKind: request.kind, + waitKind: request.waitKind, + waitMs: Math.max(0, resolvedAtMs - request.createdAtMs), + resolution, + answeredByUserId, + }, + ), + ); + }; + + const finishRun = ( + t: ThreadTrack, + run: RunInfo, + atMs: number, + out: Array, + ) => { + run.finished = true; + t.runs.delete(run.id); + t.lastRun = run; + const completedAtMs = run.completedAtMs ?? atMs; + out.push( + emission(t, "run.finished", completedAtMs, `run.finished|${t.threadId}|${run.id}`, { + ...baseProps(t), + runId: run.id, + runOrdinal: run.ordinal, + runTrigger: run.trigger, + status: run.status, + queueWaitMs: + run.startedAtMs !== undefined && run.requestedAtMs !== undefined + ? Math.max(0, run.startedAtMs - run.requestedAtMs) + : undefined, + runDurationMs: + run.startedAtMs !== undefined ? Math.max(0, completedAtMs - run.startedAtMs) : undefined, + workDurationMs: + run.workStartedAtMs !== undefined + ? Math.max(0, completedAtMs - run.workStartedAtMs) + : undefined, + attempts: run.attempts, + provider: t.provider, + providerInstanceId: run.providerInstanceId ?? t.providerInstanceId, + model: run.model ?? t.model, + effort: run.effort ?? t.effort, + interactionMode: t.interactionMode, + runtimeMode: t.runtimeMode, + senderUserId: run.senderUserId, + failureClass: run.status === "failed" ? (run.failureClass ?? "unknown") : undefined, + }), + ); + // A live request cannot be answered once its run has ended. + for (const [requestId, request] of t.requests) { + if (request.live && request.runId === run.id) { + closeRequest(t, requestId, request, atMs, "abandoned", atMs, undefined, out); + } + } + }; + + const applyRun = ( + t: ThreadTrack, + event: SessionActivityEvent, + payload: Rec, + atMs: number, + out: Array, + ) => { + const id = str(payload, "id"); + if (id === undefined) return; + if (t.lastRun?.id === id && t.lastRun.finished) return; + let run = t.runs.get(id); + const firstSeen = run === undefined; + if (run === undefined) { + run = { id, status: "queued", attempts: 0, restartContinuation: false, finished: false }; + t.runs.set(id, run); + } + const status = str(payload, "status"); + if (status !== undefined) run.status = status; + if (typeof payload.ordinal === "number") run.ordinal = payload.ordinal; + const requestedAtMs = toEpochMs(payload.requestedAt); + if (requestedAtMs !== undefined) run.requestedAtMs = requestedAtMs; + const startedAtMs = toEpochMs(payload.startedAt); + if (startedAtMs !== undefined) run.startedAtMs = startedAtMs; + const completedAtMs = toEpochMs(payload.completedAt); + if (completedAtMs !== undefined) run.completedAtMs = completedAtMs; + const workStartedAtMs = toEpochMs(payload.workStartedAt); + if (workStartedAtMs !== undefined) run.workStartedAtMs = workStartedAtMs; + const providerInstanceId = str(payload, "providerInstanceId"); + if (providerInstanceId !== undefined) run.providerInstanceId = providerInstanceId; + const selection = modelInfo(payload.modelSelection); + if (selection.model !== undefined) run.model = selection.model; + if (selection.effort !== undefined) run.effort = selection.effort; + if (str(payload, "restartContinuationOfRunId") !== undefined) run.restartContinuation = true; + const userMessageId = str(payload, "userMessageId"); + if (userMessageId !== undefined && run.userMessageId === undefined) { + run.userMessageId = userMessageId; + const message = t.messages.get(userMessageId); + if (message !== undefined) { + t.messages.delete(userMessageId); + if (message.senderUserId !== undefined) run.senderUserId = message.senderUserId; + run.trigger = message.trigger; + } + } + if (run.restartContinuation) run.trigger = "restart"; + if (event.driver !== undefined) t.provider = event.driver; + + if (event.type === "run.created") { + t.seededWorkers = 0; + if (t.plans.size > 0) { + const sourcePlanId = str(asRec(payload.sourcePlanRef), "planId"); + decidePlans( + t, + atMs, + (planId) => (planId === sourcePlanId ? "implemented" : "follow_up"), + out, + ); + } + // A newer run means an older run's live requests are orphaned. + for (const [requestId, request] of t.requests) { + if (request.live && request.runId !== undefined && request.runId !== id) { + const owner = runById(t, request.runId); + if (owner === undefined || owner.finished) { + closeRequest(t, requestId, request, atMs, "abandoned", atMs, undefined, out); + } + } + } + } + // Imported history writes a run once, already finished, at its completion + // time. Replay its running time from the recorded start instead. + if (firstSeen && FINAL_RUN_STATUSES.has(run.status) && run.startedAtMs !== undefined) { + const finalStatus = run.status; + run.status = "running"; + const runningFromMs = Math.max(run.startedAtMs, t.span?.startMs ?? run.startedAtMs); + if (runningFromMs < atMs) reconcile(t, runningFromMs, out); + if (run.completedAtMs !== undefined && run.completedAtMs > atMs) { + run.deferredFinal = { status: finalStatus, atMs: run.completedAtMs }; + return; + } + run.status = finalStatus; + } + if (FINAL_RUN_STATUSES.has(run.status)) { + delete run.deferredFinal; + finishRun(t, run, atMs, out); + } + }; + + /** Finishes imported runs whose recorded completion time has passed. */ + const settleDeferredRuns = ( + t: ThreadTrack, + atMs: number, + out: Array, + ) => { + const due = [...t.runs.values()] + .filter((run) => run.deferredFinal !== undefined && run.deferredFinal.atMs <= atMs) + .sort((a, b) => (a.deferredFinal?.atMs ?? 0) - (b.deferredFinal?.atMs ?? 0)); + for (const run of due) { + const final = run.deferredFinal; + if (final === undefined) continue; + delete run.deferredFinal; + run.status = final.status; + finishRun(t, run, final.atMs, out); + reconcile(t, final.atMs, out); + } + }; + + const applyRequest = ( + t: ThreadTrack, + event: SessionActivityEvent, + payload: Rec, + atMs: number, + out: Array, + ) => { + const id = str(payload, "id"); + const kind = str(payload, "kind"); + const status = str(payload, "status"); + if (id === undefined || kind === undefined || status === undefined) return; + if (NON_HUMAN_REQUEST_KINDS.has(kind)) return; + if (status === "pending") { + if (!t.requests.has(id)) { + t.requests.set(id, { + kind, + waitKind: waitKindOfRequest(kind), + createdAtMs: toEpochMs(payload.createdAt) ?? atMs, + ...(event.runId === undefined ? {} : { runId: event.runId }), + live: asRec(payload.responseCapability)?.type === "live", + }); + } + return; + } + const request = t.requests.get(id); + if (request === undefined) return; + closeRequest( + t, + id, + request, + atMs, + resolutionOf(status, kind, str(payload, "decision")), + toEpochMs(payload.resolvedAt) ?? atMs, + status === "resolved" ? event.actorUserId : undefined, + out, + ); + }; + + const applySubagent = ( + t: ThreadTrack, + event: SessionActivityEvent, + payload: Rec, + atMs: number, + out: Array, + ) => { + const id = str(payload, "id"); + const status = str(payload, "status"); + if (id === undefined || status === undefined) return; + t.seededWorkers = 0; + if (ACTIVE_WORK_STATUSES.has(status)) { + if (!t.subagents.has(id) && !t.finishedSubagents.has(id)) { + t.subagents.set(id, { startedAtMs: toEpochMs(payload.startedAt) ?? atMs }); + } + return; + } + const info = t.subagents.get(id); + t.subagents.delete(id); + if (!FINAL_SUBAGENT_STATUSES.has(status) || t.finishedSubagents.has(id)) return; + rememberBounded(t.finishedSubagents, id); + const startedAtMs = toEpochMs(payload.startedAt) ?? info?.startedAtMs; + const completedAtMs = toEpochMs(payload.completedAt) ?? atMs; + out.push( + emission(t, "subagent.finished", completedAtMs, `subagent.finished|${t.threadId}|${id}`, { + ...baseProps(t), + runId: str(payload, "runId") ?? event.runId, + subagentId: id, + status, + origin: str(payload, "origin"), + driver: str(payload, "driver"), + durationMs: + startedAtMs === undefined ? undefined : Math.max(0, completedAtMs - startedAtMs), + childThreadPresent: str(payload, "childThreadId") !== undefined, + }), + ); + }; + + const applyTurnItem = (t: ThreadTrack, event: SessionActivityEvent, payload: Rec) => { + const type = payload.type; + if (type === "dynamic_tool") { + const id = str(payload, "id"); + if (id === undefined) return; + const persistent = asRec(payload.input)?.persistent === true; + const status = str(payload, "status"); + if (!persistent && status !== undefined && ACTIVE_WORK_STATUSES.has(status)) { + t.backgroundTools.add(id); + } else { + t.backgroundTools.delete(id); + } + return; + } + if (type === "error") { + const failureClass = str(asRec(payload.failure), "class"); + const run = runById(t, str(payload, "runId") ?? event.runId); + if (failureClass !== undefined && run !== undefined) run.failureClass = failureClass; + } + }; + + const applyPlan = ( + t: ThreadTrack, + payload: Rec, + atMs: number, + out: Array, + ) => { + if (payload.kind !== "proposed_plan") return; + const id = str(payload, "id"); + const status = str(payload, "status"); + if (id === undefined || status === undefined) return; + if (status === "draft" || status === "active") { + if (!t.decidedPlans.has(id) && !t.plans.has(id)) { + const runId = str(payload, "runId"); + t.plans.set(id, { atMs, ...(runId === undefined ? {} : { runId }) }); + } + return; + } + // completed or superseded + const plan = t.plans.get(id); + if (plan !== undefined) decidePlan(t, id, plan, atMs, status, out); + }; + + const applyMessage = (t: ThreadTrack, payload: Rec) => { + if (payload.role !== "user") return; + const id = str(payload, "id"); + if (id === undefined) return; + const senderUserId = str(payload, "sentByUserId"); + const trigger = runTriggerOf(payload); + if (trigger === "schedule" && t.sessionSource === "unknown") t.sessionSource = "schedule"; + if (senderUserId !== undefined && trigger === "human") t.lastSenderUserId = senderUserId; + // The run may have been created before its message reached the log. + for (const run of t.runs.values()) { + if (run.userMessageId === id && run.trigger === undefined) { + if (senderUserId !== undefined) run.senderUserId = senderUserId; + run.trigger = run.restartContinuation ? "restart" : trigger; + return; + } + } + t.messages.set(id, { trigger, ...(senderUserId === undefined ? {} : { senderUserId }) }); + if (t.messages.size > MAX_TRACKED_MESSAGES) { + const oldest = t.messages.keys().next(); + if (!oldest.done) t.messages.delete(oldest.value); + } + }; + + const applyThreadEvent = ( + t: ThreadTrack, + event: SessionActivityEvent, + payload: Rec | undefined, + atMs: number, + out: Array, + ) => { + const wasKnown = t.known; + const wasArchived = t.archived; + const wasSnoozed = t.snoozed; + if (payload !== undefined) updateThreadFields(t, payload, atMs); + if (event.driver !== undefined) t.provider = event.driver; + const ageMs = t.createdAtMs === undefined ? undefined : Math.max(0, atMs - t.createdAtMs); + const lifecycle = (name: string) => + out.push( + emission(t, name, atMs, `${name}|${t.threadId}|${atMs}`, { ...baseProps(t), ageMs }), + ); + switch (event.type) { + case "thread.created": + if (!wasKnown) { + out.push( + emission(t, "session.created", atMs, `session.created|${t.threadId}`, { + ...baseProps(t), + provider: t.provider, + providerInstanceId: t.providerInstanceId, + model: t.model, + effort: t.effort, + interactionMode: t.interactionMode, + runtimeMode: t.runtimeMode, + creationSource: t.creationSource, + createdBy: t.createdBy, + }), + ); + } + return; + case "thread.archived": + t.archived = true; + if (!wasArchived) { + lifecycle("session.archived"); + if (t.plans.size > 0) decidePlans(t, atMs, () => "archived", out); + } + return; + case "thread.unarchived": + t.archived = false; + if (wasArchived) lifecycle("session.unarchived"); + return; + case "thread.snoozed": + t.snoozed = true; + if (!wasSnoozed) lifecycle("session.snoozed"); + return; + case "thread.unsnoozed": + t.snoozed = false; + if (wasSnoozed) lifecycle("session.unsnoozed"); + return; + default: + return; + } + }; + + const apply: SessionActivityTracker["apply"] = (event) => { + if (ignored.has(event.threadId)) return []; + const atMs = toEpochMs(event.occurredAt); + if (atMs === undefined) return []; + const payload = asRec(event.payload); + let t = threads.get(event.threadId); + if (t === undefined) { + if (!introducesSession(event.type)) return []; + if (event.type.startsWith("thread.") && isProviderNativeSubagentThread(payload)) { + ignored.add(event.threadId); + return []; + } + t = newThread(event.threadId); + threads.set(event.threadId, t); + } + const out: Array = []; + settleDeferredRuns(t, atMs, out); + + if (event.type === "thread.deleted") { + if (t.span !== undefined) out.push(...spanEmissions(t, t.span, atMs, "deleted")); + threads.delete(event.threadId); + ignored.add(event.threadId); + return out; + } + + if (event.type.startsWith("thread.")) { + applyThreadEvent(t, event, payload, atMs, out); + } else if (payload !== undefined) { + switch (event.type) { + case "run.created": + case "run.updated": + applyRun(t, event, payload, atMs, out); + break; + case "run-attempt.created": { + const run = runById(t, str(payload, "runId")); + if (run !== undefined) run.attempts += 1; + if (event.driver !== undefined) t.provider = event.driver; + break; + } + case "runtime-request.updated": + applyRequest(t, event, payload, atMs, out); + break; + case "subagent.updated": + applySubagent(t, event, payload, atMs, out); + break; + case "turn-item.updated": + applyTurnItem(t, event, payload); + break; + case "provider-thread.updated": { + const id = str(payload, "id"); + if (id === undefined) break; + t.seededWorkers = 0; + const count = holdingRosterTaskCount(payload.pendingBackgroundTasks); + if (count > 0) t.rosterTasks.set(id, count); + else t.rosterTasks.delete(id); + break; + } + case "plan.updated": + applyPlan(t, payload, atMs, out); + break; + case "message.updated": + applyMessage(t, payload); + break; + default: + break; + } + } + reconcile(t, atMs, out); + return out; + }; + + const seed: SessionActivityTracker["seed"] = (shell, nowMs) => { + const payload = asRec(shell); + const threadId = str(payload, "id"); + if (payload === undefined || threadId === undefined) return; + if (isProviderNativeSubagentThread(payload) || toEpochMs(payload.deletedAt) !== undefined) { + ignored.add(threadId); + return; + } + const t = newThread(threadId); + threads.set(threadId, t); + updateThreadFields(t, payload, nowMs); + const activeRunId = str(payload, "activeRunId"); + const activityStatus = str(payload, "activityRunStatus"); + if ( + activeRunId !== undefined && + activityStatus !== undefined && + ACTIVE_RUN_STATUSES.has(activityStatus) + ) { + const startedAtMs = toEpochMs(payload.activityRunStartedAt); + t.runs.set(activeRunId, { + id: activeRunId, + status: activityStatus, + attempts: 0, + restartContinuation: false, + finished: false, + ...(startedAtMs === undefined ? {} : { startedAtMs }), + }); + } + const latestRunId = str(payload, "latestRunId"); + if (latestRunId !== undefined && payload.status === "queued" && !t.runs.has(latestRunId)) { + t.runs.set(latestRunId, { + id: latestRunId, + status: "queued", + attempts: 0, + restartContinuation: false, + finished: false, + }); + } + const pending = asRec(payload.pendingRuntimeRequest); + const requestId = str(pending, "id"); + const requestKind = str(pending, "kind"); + if ( + requestId !== undefined && + requestKind !== undefined && + !NON_HUMAN_REQUEST_KINDS.has(requestKind) + ) { + t.requests.set(requestId, { + kind: requestKind, + waitKind: waitKindOfRequest(requestKind), + createdAtMs: toEpochMs(pending?.createdAt) ?? nowMs, + // Live requests did not survive the restart; message-answered ones did. + live: false, + }); + } + const subagentCount = + typeof payload.activeSubagentCount === "number" ? payload.activeSubagentCount : 0; + t.seededWorkers = subagentCount + holdingRosterTaskCount(payload.pendingBackgroundTasks); + t.resumeNextSpan = true; + reconcile(t, nowMs, []); + }; + + const closeAll: SessionActivityTracker["closeAll"] = (nowMs, endReason) => { + const out: Array = []; + for (const t of threads.values()) { + if (t.span === undefined) continue; + out.push(...spanEmissions(t, t.span, nowMs, endReason)); + delete t.span; + } + return out; + }; + + const snapshot: SessionActivityTracker["snapshot"] = (nowMs) => { + const counts: StateCounts = { + byOwnerState: new Map(), + runningRuns: 0, + queuedRuns: 0, + waitingHuman: 0, + waitingWorkers: 0, + activeSubagents: 0, + }; + for (const t of threads.values()) { + const span = t.span; + if (span === undefined || span.state === "archived") continue; + const key = `${t.ownerUserId ?? ""}|${span.state}`; + const entry = counts.byOwnerState.get(key); + if (entry === undefined) { + counts.byOwnerState.set(key, { + ...(t.ownerUserId === undefined ? {} : { ownerUserId: t.ownerUserId }), + state: span.state, + count: 1, + }); + } else { + entry.count += 1; + } + if (span.state === "agent_running") counts.runningRuns += 1; + if (span.state === "waiting_human") counts.waitingHuman += 1; + if (span.state === "waiting_workers") counts.waitingWorkers += 1; + for (const run of t.runs.values()) if (run.status === "queued") counts.queuedRuns += 1; + counts.activeSubagents += activeWorkers(t); + } + const out: Array = []; + for (const entry of counts.byOwnerState.values()) { + out.push( + emission( + undefined, + "session.state.snapshot", + nowMs, + `session.state.snapshot|${entry.ownerUserId ?? ""}|${entry.state}|${nowMs}`, + { scope: "user", ownerUserId: entry.ownerUserId, state: entry.state, count: entry.count }, + entry.ownerUserId, + ), + ); + } + out.push({ + event: "session.state.snapshot", + properties: { + scope: "total", + runningRuns: counts.runningRuns, + queuedRuns: counts.queuedRuns, + waitingHuman: counts.waitingHuman, + waitingWorkers: counts.waitingWorkers, + activeSubagents: counts.activeSubagents, + backfill: options.backfill, + }, + timestamp: isoOf(nowMs), + insertId: `session.state.snapshot|total|${nowMs}`, + }); + return out; + }; + + return { apply, seed, closeAll, snapshot }; +} diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index e793d4ddf5fb5..decc4ed912d76 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -262,6 +262,7 @@ import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts"; import * as HostResources from "./resourceTelemetry/HostResources.ts"; import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; +import { clientConnectionMode } from "./telemetry/serverEvents.ts"; // T3-CUSTOM(expbkt3) import * as UsageService from "./usage/UsageService.ts"; import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; import * as PullRequestService from "./pullRequest/PullRequestService.ts"; @@ -1262,10 +1263,15 @@ const layerWsRpc = ( // client starts is credited to its surface + app version. Best-effort: // attribution must never fail the user's command. const originProps = clientAnalyticsProps; + // T3-CUSTOM(expbkt3): client events carry the signed-in user (TEC-1631). + const analyticsUser = + currentSession.userId === null ? undefined : { userId: currentSession.userId }; const recordClientCommandAnalytics = (command: OrchestrationV2Command) => { switch (command.type) { case "message.dispatch": - return analytics.record("client.turn.requested", originProps).pipe(Effect.ignore); + return analytics + .record("client.turn.requested", originProps, analyticsUser) + .pipe(Effect.ignore); default: return Effect.void; } @@ -2492,13 +2498,14 @@ const layerWsRpc = ( ) .pipe( Effect.tap(() => + // T3-CUSTOM(expbkt3): client events carry the signed-in user (TEC-1631). analytics - .record("client.thread.started", originProps) + .record("client.thread.started", originProps, analyticsUser) .pipe( Effect.andThen( input.initialMessage === undefined ? Effect.void - : analytics.record("client.turn.requested", originProps), + : analytics.record("client.turn.requested", originProps, analyticsUser), ), Effect.ignore, ), @@ -3899,7 +3906,18 @@ export const layer = Layer.unwrap( const clientOrigin = readClientConnectionOrigin(request); const clientAnalyticsProps = readClientAnalyticsProps(request); yield* sessions.recordClientConnection(session.sessionId, clientOrigin); - yield* analytics.record("client.connected", clientAnalyticsProps); + // T3-CUSTOM(expbkt3): the signed-in user and how the client reached us (TEC-1631). + yield* analytics.record( + "client.connected", + { + ...clientAnalyticsProps, + connectionMode: clientConnectionMode({ + ...clientAnalyticsProps, + host: request.headers.host, + }), + }, + session.userId === null ? undefined : { userId: session.userId }, + ); const rpcWebSocketHttpEffect = yield* Effect.gen(function* () { const { protocol, httpEffect } = yield* RpcServer.makeProtocolWithHttpEffectWebsocket; yield* RpcServer.make(ServerWsRpcGroup, WS_RPC_SERVER_OPTIONS).pipe( diff --git a/apps/web/package.json b/apps/web/package.json index 2df68088f3dc8..a209cf55d1e5e 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -68,6 +68,7 @@ "mermaid": "^11.17.2", "morphicons": "^1.7.1", "platejs": "53.3.3", + "posthog-js": "^1.438.9", "react": "19.2.6", "react-dom": "19.2.6", "react-markdown": "^10.1.0", diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index e2d496d9bf025..382845b9339c0 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -632,6 +632,8 @@ const EMPTY_PROVIDERS: ServerProvider[] = []; const EMPTY_PROVIDER_MODELS: ServerProvider["models"] = []; const EMPTY_USAGE_LIMIT_SOURCES: UsageLimitSourceSnapshots = []; import type { CodexArtifactTemplate } from "@t3tools/client-runtime/codex-artifact-templates"; +// T3-CUSTOM(expbkt3): product analytics (TEC-1631). +import { lastInputWasKeyboard, trackProductEvent } from "../lib/analytics/productAnalytics"; const TIMELINE_SCROLL_CANCEL_SENTINEL = Object.freeze({}); const EMPTY_FEEDBACK_SUBMISSIONS: ReadonlyArray = []; @@ -9228,6 +9230,11 @@ export default function ChatView(props: ChatViewProps) { promptRef.current = ""; clearComposerDraftContent(composerDraftTarget); composerRef.current?.resetCursorState(); + trackProductEvent("plan.action", { + threadId: activeThread.id, + action: followUp.interactionMode === "plan" ? "revise" : "approve", + source: "composer", + }); const followUpSent = await onSubmitPlanFollowUp({ text: followUp.text, context: buildMessageContext({ @@ -9466,6 +9473,12 @@ export default function ChatView(props: ChatViewProps) { setThreadError(threadIdForSend, attachmentCapabilitiesBeforeUpload.fileBlockReason); return; } + trackProductEvent("composer.sent", { + threadId: threadIdForSend, + interactionMode: sendInteractionMode, + hasAttachments: composerAttachmentsSnapshot.length > 0, + viaKeyboard: lastInputWasKeyboard(), + }); const turnUsesAttachmentUploads = composerFilesSnapshot.length > 0 ? attachmentCapabilitiesBeforeUpload.supportsAttachmentUploads @@ -10237,11 +10250,13 @@ export default function ChatView(props: ChatViewProps) { async (requestId: RuntimeRequestId, decision: ProviderApprovalDecision) => { if (!activeThreadId || !readEnvironmentScope(environmentId, AuthOrchestrationOperateScope)) return; - if ( - pendingApprovals.find((approval) => approval.requestId === requestId) - ?.responseCapability !== "live" - ) - return; + const pendingApproval = pendingApprovals.find((approval) => approval.requestId === requestId); + if (pendingApproval?.responseCapability !== "live") return; + trackProductEvent("approval.answered_in_ui", { + threadId: activeThreadId, + kind: pendingApproval.requestKind, + decision, + }); setRespondingRequestIds((existing) => existing.includes(requestId) ? existing : [...existing, requestId], @@ -10304,6 +10319,10 @@ export default function ChatView(props: ChatViewProps) { ); } userInputResponsesInFlight.current.add(responseKey); + trackProductEvent("question.answered_in_ui", { + threadId: activeThreadId, + kind: "user_input", + }); setRespondingUserInputRequestIds((existing) => existing.includes(requestId) ? existing : [...existing, requestId], @@ -10681,6 +10700,11 @@ export default function ChatView(props: ChatViewProps) { const nextThreadTitle = truncate(buildPlanImplementationThreadTitle(planMarkdown)); const nextThreadModelSelection: ModelSelection = ctxSelectedModelSelection; + trackProductEvent("plan.action", { + threadId: activeThread.id, + action: "approve", + source: "new_thread", + }); sendInFlightRef.current = true; beginLocalDispatch({ preparingWorktree: false }); const finish = () => { diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index 749b8b933d82b..2fb3c78976394 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -224,6 +224,9 @@ import { import type { Project } from "../types"; import { PullRequestGlyph } from "~/components/pullRequest/pullRequestIcons"; import { readPullRequestListPreferences } from "~/components/pullRequest/pullRequestListPreferences"; +// T3-CUSTOM(expbkt3): product analytics (TEC-1631). +import { trackProductEvent } from "~/lib/analytics/productAnalytics"; +import { commandIdForPaletteItem } from "~/lib/analytics/productAnalytics.logic"; const EMPTY_BROWSE_ENTRIES: FilesystemBrowseResult["entries"] = []; @@ -3200,6 +3203,9 @@ function OpenCommandPaletteDialog(props: { setOpen(false); } + trackProductEvent("command_palette.used", { + commandId: commandIdForPaletteItem(item), + }); void item.run().catch((error: unknown) => { toastManager.add( stackedThreadToast({ diff --git a/apps/web/src/components/planreview/PlanReviewPanel.tsx b/apps/web/src/components/planreview/PlanReviewPanel.tsx index 299118f7531f0..bb31e24a3580d 100644 --- a/apps/web/src/components/planreview/PlanReviewPanel.tsx +++ b/apps/web/src/components/planreview/PlanReviewPanel.tsx @@ -54,6 +54,7 @@ import { usePlanReviewScrollSurfaceRef, } from "../../fork/planReviewComposerDock"; import { canHidePlanGlobalNote, shouldShowPlanGlobalNote } from "../../fork/planReviewGlobalNote"; +import { trackProductEvent } from "../../lib/analytics/productAnalytics"; interface PlanReviewPanelProps { readonly environmentId: EnvironmentId; @@ -354,6 +355,11 @@ export default function PlanReviewPanel({ const handleSubmit = useCallback( (decision: "approved" | "changes-requested" | "discarded") => { + trackProductEvent("plan.action", { + action: + decision === "approved" ? "approve" : decision === "discarded" ? "discard" : "revise", + source: "plan_review", + }); void submit({ environmentId, input: { diff --git a/apps/web/src/hooks/useSettings.ts b/apps/web/src/hooks/useSettings.ts index 04fab1c1d1c0b..57e86ff4cd1a5 100644 --- a/apps/web/src/hooks/useSettings.ts +++ b/apps/web/src/hooks/useSettings.ts @@ -53,6 +53,8 @@ import { useAtomCommand } from "~/state/use-atom-command"; import { useTheme } from "./useTheme"; import { environmentSession, readEnvironmentScope, useEnvironmentScope } from "~/state/session"; import { appAtomRegistry } from "~/rpc/atomRegistry"; +// T3-CUSTOM(expbkt3): product analytics (TEC-1631). +import { trackSettingsChanged } from "~/lib/analytics/productAnalytics"; const CLIENT_SETTINGS_PERSISTENCE_ERROR_SCOPE = "[CLIENT_SETTINGS]"; @@ -477,6 +479,7 @@ function useUpdateSettingsTarget(environmentId: EnvironmentId | null) { const sharedSettingsSyncTargetIds = useSharedSettingsSyncTargetIds(true); const updateSettings = useCallback( (patch: UnifiedSettingsPatch) => { + trackSettingsChanged(Object.keys(patch)); const { serverPatch, clientPatch } = splitPatch(patch); const canWriteServerPatch = @@ -577,6 +580,7 @@ export function useUpdatePrimarySettings() { export function useUpdateClientSettings() { return useCallback((patch: ClientSettingsPatch) => { + trackSettingsChanged(Object.keys(patch)); return persistClientSettingsPatch(patch); }, []); } diff --git a/apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx b/apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx new file mode 100644 index 0000000000000..1caea6c6c607c --- /dev/null +++ b/apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx @@ -0,0 +1,125 @@ +/** + * T3-CUSTOM(expbkt3): wires client product analytics to app state (TEC-1631). + * + * Mounted once under the router. It hands the primary environment's analytics + * config to the service and, only while that is on, mounts the trackers for + * identity, views, opened threads and app visibility. With analytics off it + * renders nothing and subscribes to nothing beyond the primary environment. + * + * @module ProductAnalyticsCoordinator + */ +import { useParams, useRouterState } from "@tanstack/react-router"; +import { useEffect, useMemo, useRef } from "react"; + +import { APP_VERSION } from "../../branding"; +import { isElectron } from "../../env"; +import { usePrimaryEnvironment } from "../../state/environments"; +import { useCurrentUserId } from "../../state/identity"; +import { useOrgMembers } from "../../state/orgMembers"; +import { productAnalytics, trackProductEvent } from "./productAnalytics"; +import { + createVisibilityReporter, + resolveConnectionMode, + viewNameFromRouteId, +} from "./productAnalytics.logic"; + +/** A visibility change must hold this long before it is reported. */ +const VISIBILITY_SETTLE_MS = 5_000; + +let appOpenedSent = false; + +export function ProductAnalyticsCoordinator() { + const primary = usePrimaryEnvironment(); + const analytics = primary?.serverConfig?.environment.analytics ?? null; + const target = primary?.entry.target ?? null; + const key = analytics !== null && analytics.enabled ? analytics.posthogKey : null; + const host = analytics !== null && analytics.enabled ? analytics.posthogHost : null; + const instance = analytics?.instance ?? null; + const connectionMode = useMemo(() => resolveConnectionMode(target), [target]); + + useEffect(() => { + productAnalytics.configure({ + config: + key === null || host === null + ? null + : { + enabled: true, + posthogKey: key, + posthogHost: host, + ...(instance === null ? {} : { instance }), + }, + surface: isElectron ? "desktop" : "web", + appVersion: APP_VERSION, + connectionMode, + }); + if (key !== null && !appOpenedSent) { + appOpenedSent = true; + trackProductEvent("app.opened"); + } + }, [key, host, instance, connectionMode]); + + if (key === null) return null; + return ( + <> + + + + + ); +} + +/** Identifies by work email; falls back to the server user id when it is unknown. */ +function ProductAnalyticsIdentity() { + const userId = useCurrentUserId(); + const { resolveUser } = useOrgMembers(); + const email = userId === null ? null : resolveUser(userId).email; + const distinctId = email ?? userId; + useEffect(() => { + productAnalytics.identify(distinctId); + }, [distinctId]); + return null; +} + +function ProductAnalyticsRouteTracker() { + const routeId = useRouterState({ select: (state) => state.matches.at(-1)?.routeId ?? null }); + const { threadId: routeThreadId } = useParams({ strict: false }); + const threadId = typeof routeThreadId === "string" ? routeThreadId : null; + const view = viewNameFromRouteId(routeId); + const lastThreadId = useRef(null); + + useEffect(() => { + if (view !== null) trackProductEvent("view.changed", { view }); + }, [view]); + + useEffect(() => { + if (threadId === null || threadId === lastThreadId.current) return; + lastThreadId.current = threadId; + trackProductEvent("session.opened", { threadId }); + }, [threadId]); + return null; +} + +/** App visibility (settled, so window peeks send nothing) and last input kind. */ +function ProductAnalyticsActivityTracker() { + useEffect(() => { + const reporter = createVisibilityReporter({ + delayMs: VISIBILITY_SETTLE_MS, + emit: (state) => + trackProductEvent(state === "foreground" ? "app.foregrounded" : "app.backgrounded"), + }); + const onVisibility = () => + reporter.update(document.visibilityState === "hidden" ? "background" : "foreground"); + const onKeyDown = () => productAnalytics.noteInput("keyboard"); + const onPointerDown = () => productAnalytics.noteInput("pointer"); + document.addEventListener("visibilitychange", onVisibility); + window.addEventListener("keydown", onKeyDown, { capture: true, passive: true }); + window.addEventListener("pointerdown", onPointerDown, { capture: true, passive: true }); + return () => { + reporter.dispose(); + document.removeEventListener("visibilitychange", onVisibility); + window.removeEventListener("keydown", onKeyDown, { capture: true }); + window.removeEventListener("pointerdown", onPointerDown, { capture: true }); + }; + }, []); + return null; +} diff --git a/apps/web/src/lib/analytics/productAnalytics.logic.test.ts b/apps/web/src/lib/analytics/productAnalytics.logic.test.ts new file mode 100644 index 0000000000000..79edc68cfea94 --- /dev/null +++ b/apps/web/src/lib/analytics/productAnalytics.logic.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + commandIdForPaletteItem, + commandIdFromPaletteValue, + createKeyThrottle, + createVisibilityReporter, + isAnalyticsSettingKey, + resolveConnectionMode, + sanitizeAnalyticsProperties, + stripUrlProperties, + viewNameFromRouteId, +} from "./productAnalytics.logic"; + +describe("sanitizeAnalyticsProperties", () => { + it("keeps only the event's allowlisted keys", () => { + expect( + sanitizeAnalyticsProperties("composer.sent", { + threadId: "thread-1", + interactionMode: "plan", + hasAttachments: true, + viaKeyboard: false, + prompt: "secret text", + title: "Fix the login page", + }), + ).toEqual({ + threadId: "thread-1", + interactionMode: "plan", + hasAttachments: true, + viaKeyboard: false, + }); + }); + + it("drops objects, long strings and non-finite numbers", () => { + expect( + sanitizeAnalyticsProperties("settings.changed", { settingKey: { nested: "value" } }), + ).toEqual({}); + expect(sanitizeAnalyticsProperties("view.changed", { view: "x".repeat(500) })).toEqual({}); + expect(sanitizeAnalyticsProperties("command_palette.used", { commandId: Number.NaN })).toEqual( + {}, + ); + }); + + it("keeps nothing for unknown events or missing properties", () => { + expect(sanitizeAnalyticsProperties("custom.event", { threadId: "t" })).toEqual({}); + expect(sanitizeAnalyticsProperties("app.opened", { threadId: "t" })).toEqual({}); + expect(sanitizeAnalyticsProperties("composer.sent", undefined)).toEqual({}); + }); +}); + +describe("viewNameFromRouteId", () => { + it("names a view by its route template, never by ids", () => { + expect(viewNameFromRouteId("/_chat/$environmentId/$threadId")).toBe( + "/chat/$environmentId/$threadId", + ); + expect(viewNameFromRouteId("/_chat/draft/$draftId")).toBe("/chat/draft/$draftId"); + expect(viewNameFromRouteId("/_chat/")).toBe("/chat"); + expect(viewNameFromRouteId("/settings/general")).toBe("/settings/general"); + expect(viewNameFromRouteId("__root__")).toBe("/"); + }); + + it("drops segments that look like concrete values", () => { + expect(viewNameFromRouteId("/projects/3f2a9c1e-uuid/thread title")).toBe("/projects"); + expect(viewNameFromRouteId(null)).toBeNull(); + }); +}); + +describe("palette command ids", () => { + it("keeps the command kind and drops environment ids, paths and text", () => { + expect(commandIdFromPaletteValue("action:add-project:env-123:local")).toBe( + "action:add-project", + ); + expect(commandIdFromPaletteValue("appearance:dark")).toBe("appearance:dark"); + expect(commandIdFromPaletteValue("thread:01HZX9")).toBe("thread"); + expect(commandIdFromPaletteValue("/home/user/repo")).toBeNull(); + }); + + it("prefers the keybinding command and hides project script names", () => { + expect(commandIdForPaletteItem({ shortcutCommand: "terminal.toggle", value: "x" })).toBe( + "terminal.toggle", + ); + expect(commandIdForPaletteItem({ shortcutCommand: "script.deploy-prod.run", value: "x" })).toBe( + "script.run", + ); + expect(commandIdForPaletteItem({ value: "action:new-thread" })).toBe("action:new-thread"); + }); +}); + +describe("isAnalyticsSettingKey", () => { + it("accepts identifiers only", () => { + expect(isAnalyticsSettingKey("followUpBehavior")).toBe(true); + expect(isAnalyticsSettingKey("some value with spaces")).toBe(false); + expect(isAnalyticsSettingKey("/path")).toBe(false); + }); +}); + +describe("resolveConnectionMode", () => { + const primary = (httpBaseUrl: string) => + ({ _tag: "PrimaryConnectionTarget", httpBaseUrl }) as const; + + it("tells local, tailnet and remote primaries apart", () => { + expect(resolveConnectionMode(primary("http://localhost:3773"))).toBe("local"); + expect(resolveConnectionMode(primary("http://127.0.0.1:3773"))).toBe("local"); + expect(resolveConnectionMode(primary("https://dev-server-1.tailab6257.ts.net"))).toBe( + "tailnet", + ); + expect(resolveConnectionMode(primary("http://100.101.5.7:3773"))).toBe("tailnet"); + expect(resolveConnectionMode(primary("https://bkt3.example.com"))).toBe("remote"); + }); + + it("maps other targets by kind", () => { + expect(resolveConnectionMode({ _tag: "RelayConnectionTarget" })).toBe("relay"); + expect(resolveConnectionMode({ _tag: "SshConnectionTarget" })).toBe("ssh"); + expect(resolveConnectionMode({ _tag: "BearerConnectionTarget" })).toBe("remote"); + expect(resolveConnectionMode(null)).toBeNull(); + }); +}); + +describe("stripUrlProperties", () => { + it("removes the URL and title properties PostHog adds", () => { + expect( + stripUrlProperties({ + $current_url: "https://bkt3/chat/env/thread", + $pathname: "/chat/env/thread", + $title: "Fix login", + $browser: "Chrome", + view: "/chat", + }), + ).toEqual({ $browser: "Chrome", view: "/chat" }); + }); +}); + +describe("createVisibilityReporter", () => { + function harness() { + const timers = new Map void>(); + let nextId = 0; + const emitted: string[] = []; + const reporter = createVisibilityReporter({ + delayMs: 1_000, + emit: (state) => emitted.push(state), + setTimer: (callback) => { + nextId += 1; + timers.set(nextId, callback); + return nextId; + }, + clearTimer: (handle) => timers.delete(handle as number), + }); + const flush = () => { + for (const [id, callback] of [...timers]) { + timers.delete(id); + callback(); + } + }; + return { reporter, emitted, flush }; + } + + it("sends a settled change once", () => { + const { reporter, emitted, flush } = harness(); + reporter.update("background"); + flush(); + reporter.update("background"); + flush(); + expect(emitted).toEqual(["background"]); + }); + + it("sends nothing for a quick hide and show", () => { + const { reporter, emitted, flush } = harness(); + reporter.update("background"); + reporter.update("foreground"); + flush(); + expect(emitted).toEqual([]); + }); +}); + +describe("createKeyThrottle", () => { + it("allows each key once per window", () => { + let now = 0; + const allow = createKeyThrottle(1_000, () => now); + expect(allow("a")).toBe(true); + expect(allow("a")).toBe(false); + expect(allow("b")).toBe(true); + now = 1_000; + expect(allow("a")).toBe(true); + }); +}); diff --git a/apps/web/src/lib/analytics/productAnalytics.logic.ts b/apps/web/src/lib/analytics/productAnalytics.logic.ts new file mode 100644 index 0000000000000..0f7f6c6730ba8 --- /dev/null +++ b/apps/web/src/lib/analytics/productAnalytics.logic.ts @@ -0,0 +1,230 @@ +/** + * T3-CUSTOM(expbkt3): pure rules for client product analytics (TEC-1631). + * + * Everything that decides *what* leaves the client lives here, so it can be + * tested without PostHog: the named events, the property allowlist per event, + * route → view names, palette command ids, and the URL properties PostHog adds + * on its own that we strip before send. Nothing here sends text, titles, + * paths, branch or repository names, or setting values. + * + * @module productAnalytics.logic + */ + +export type ProductAnalyticsSurface = "web" | "desktop"; +export type ProductAnalyticsConnectionMode = "local" | "tailnet" | "remote" | "relay" | "ssh"; + +/** Properties each event may carry. Anything else is dropped before capture. */ +export const PRODUCT_ANALYTICS_EVENTS = { + "app.opened": [], + "app.foregrounded": [], + "app.backgrounded": [], + "view.changed": ["view"], + "session.opened": ["threadId"], + "composer.sent": ["threadId", "interactionMode", "hasAttachments", "viaKeyboard"], + "command_palette.used": ["commandId"], + "settings.changed": ["settingKey"], + "approval.answered_in_ui": ["threadId", "kind", "decision"], + "question.answered_in_ui": ["threadId", "kind"], + "plan.action": ["threadId", "action", "source"], +} as const satisfies Record>; + +export type ProductAnalyticsEvent = keyof typeof PRODUCT_ANALYTICS_EVENTS; +export type ProductAnalyticsProperties = Readonly>; + +/** Longest string value kept. Ids and enum tokens fit; free text does not. */ +const MAX_STRING_PROPERTY_LENGTH = 96; + +/** + * Keeps only the allowlisted keys of `event` whose values are short strings, + * finite numbers or booleans. Unknown events keep nothing. + */ +export function sanitizeAnalyticsProperties( + event: string, + properties: Readonly> | undefined, +): ProductAnalyticsProperties { + const allowed: ReadonlyArray | undefined = + PRODUCT_ANALYTICS_EVENTS[event as ProductAnalyticsEvent]; + if (allowed === undefined || properties === undefined) return {}; + const result: Record = {}; + for (const key of allowed) { + const value = properties[key]; + if (typeof value === "boolean") result[key] = value; + else if (typeof value === "number" && Number.isFinite(value)) result[key] = value; + else if ( + typeof value === "string" && + value.length > 0 && + value.length <= MAX_STRING_PROPERTY_LENGTH + ) { + result[key] = value; + } + } + return result; +} + +/** + * A view name from a TanStack route id, which is the route *template* + * (`/_chat/$environmentId/$threadId`), never the visited URL. Pathless layout + * segments (`_chat`) lose their underscore; params keep only their name. + */ +export function viewNameFromRouteId(routeId: string | null | undefined): string | null { + if (!routeId) return null; + if (routeId === "__root__") return "/"; + const segments = routeId + .split("/") + .filter((segment) => segment.length > 0) + .map((segment) => (segment.startsWith("_") ? segment.slice(1) : segment)) + .filter((segment) => /^\$?[A-Za-z][\w-]*$/.test(segment)); + return `/${segments.join("/")}`; +} + +const COMMAND_SEGMENT = /^[a-z][a-z-]*$/; + +/** + * A stable command id from a palette item `value` such as + * `action:add-project::local`. Keeps at most the first two + * segments, and stops at the first one that is not a plain lowercase word, so + * ids, paths and search text never leave. + */ +export function commandIdFromPaletteValue(value: string): string | null { + const kept: string[] = []; + for (const segment of value.split(":").slice(0, 2)) { + if (!COMMAND_SEGMENT.test(segment)) break; + kept.push(segment); + } + return kept.length === 0 ? null : kept.join(":"); +} + +/** + * The command id for a palette item: its keybinding command when it has one, + * else the cleaned `value`. Project scripts (`script..run`) collapse to + * `script.run`, since the name is the user's own. + */ +export function commandIdForPaletteItem(item: { + readonly shortcutCommand?: string | undefined; + readonly value: string; +}): string | null { + if (item.shortcutCommand !== undefined) { + return item.shortcutCommand.startsWith("script.") ? "script.run" : item.shortcutCommand; + } + return commandIdFromPaletteValue(item.value); +} + +/** Settings keys are camelCase identifiers; anything else is not a key. */ +export function isAnalyticsSettingKey(key: string): boolean { + return /^[A-Za-z][A-Za-z0-9]{0,63}$/.test(key); +} + +const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]", "::1"]); + +/** How this client reaches its primary environment, from the primary target. */ +export function resolveConnectionMode( + target: + | { readonly _tag: "PrimaryConnectionTarget"; readonly httpBaseUrl: string } + | { readonly _tag: "RelayConnectionTarget" } + | { readonly _tag: "BearerConnectionTarget" } + | { readonly _tag: "SshConnectionTarget" } + | null, +): ProductAnalyticsConnectionMode | null { + if (target === null) return null; + switch (target._tag) { + case "RelayConnectionTarget": + return "relay"; + case "SshConnectionTarget": + return "ssh"; + case "BearerConnectionTarget": + return "remote"; + case "PrimaryConnectionTarget": { + let hostname: string; + try { + hostname = new URL(target.httpBaseUrl).hostname.toLowerCase(); + } catch { + return null; + } + if (LOOPBACK_HOSTS.has(hostname)) return "local"; + if (hostname.endsWith(".ts.net") || isTailnetIpv4(hostname)) return "tailnet"; + return "remote"; + } + } +} + +function isTailnetIpv4(hostname: string): boolean { + const match = /^100\.(\d{1,3})\.\d{1,3}\.\d{1,3}$/.exec(hostname); + if (!match) return false; + const second = Number(match[1]); + return second >= 64 && second <= 127; +} + +/** + * Properties PostHog adds to every event on its own. They carry the visited + * URL (thread ids, draft ids) or the window title (thread titles), so they are + * removed in `before_send`. + */ +export const STRIPPED_POSTHOG_PROPERTIES = [ + "$current_url", + "$pathname", + "$title", + "$referrer", + "$referring_domain", + "$initial_current_url", + "$initial_pathname", + "$initial_referrer", + "$initial_referring_domain", + "$initial_title", + "$prev_pageview_pathname", +] as const; + +export function stripUrlProperties>(properties: T): T { + const next: Record = { ...properties }; + for (const key of STRIPPED_POSTHOG_PROPERTIES) delete next[key]; + return next as T; +} + +/** + * Reports app visibility once it has been stable for `delayMs`. A quick + * hide-then-show (window switch, tab peek) sends nothing; a real change sends + * one event, and the same state is never sent twice in a row. + */ +export function createVisibilityReporter(input: { + readonly delayMs: number; + readonly emit: (state: "foreground" | "background") => void; + readonly setTimer?: (callback: () => void, ms: number) => unknown; + readonly clearTimer?: (handle: unknown) => void; +}) { + const setTimer = + input.setTimer ?? ((callback: () => void, ms: number) => globalThis.setTimeout(callback, ms)); + const clearTimer = + input.clearTimer ?? + ((handle: unknown) => globalThis.clearTimeout(handle as ReturnType)); + let lastEmitted: "foreground" | "background" = "foreground"; + let timer: unknown = null; + return { + update(state: "foreground" | "background"): void { + if (timer !== null) { + clearTimer(timer); + timer = null; + } + if (state === lastEmitted) return; + timer = setTimer(() => { + timer = null; + lastEmitted = state; + input.emit(state); + }, input.delayMs); + }, + dispose(): void { + if (timer !== null) clearTimer(timer); + timer = null; + }, + }; +} + +/** True at most once per `windowMs` for each key. */ +export function createKeyThrottle(windowMs: number, now: () => number = Date.now) { + const lastByKey = new Map(); + return (key: string): boolean => { + const at = now(); + const last = lastByKey.get(key); + if (last !== undefined && at - last < windowMs) return false; + lastByKey.set(key, at); + return true; + }; +} diff --git a/apps/web/src/lib/analytics/productAnalytics.test.ts b/apps/web/src/lib/analytics/productAnalytics.test.ts new file mode 100644 index 0000000000000..ee196034d21bf --- /dev/null +++ b/apps/web/src/lib/analytics/productAnalytics.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + createProductAnalytics, + type ProductAnalyticsClient, + type ProductAnalyticsContext, + type ProductAnalyticsLoader, +} from "./productAnalytics"; + +interface FakeClient { + readonly key: string; + readonly captured: Array<{ event: string; properties: unknown }>; + readonly identified: string[]; + readonly registered: Array>; + optedOut: boolean; +} + +function fakeLoader() { + const clients: FakeClient[] = []; + let loads = 0; + const load: ProductAnalyticsLoader = async (key) => { + loads += 1; + const fake: FakeClient = { + key, + captured: [], + identified: [], + registered: [], + optedOut: false, + }; + clients.push(fake); + const client = { + capture: (event: string, properties?: unknown) => { + fake.captured.push({ event, properties }); + return undefined; + }, + identify: (id?: string) => { + if (id !== undefined) fake.identified.push(id); + }, + register: (properties: Record) => { + fake.registered.push(properties); + }, + opt_out_capturing: () => { + fake.optedOut = true; + }, + } as unknown as ProductAnalyticsClient; + return client; + }; + return { load, clients, loadCount: () => loads }; +} + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + +function context(config: ProductAnalyticsContext["config"]): ProductAnalyticsContext { + return { config, surface: "web", appVersion: "1.2.3", connectionMode: "tailnet" }; +} + +const enabledConfig = { + enabled: true, + posthogKey: "phc_company", + posthogHost: "https://us.i.posthog.com", + instance: "bkt3", +}; + +describe("createProductAnalytics", () => { + it("is a no-op and never loads PostHog when the config is missing", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.track("app.opened"); + analytics.configure(context(null)); + analytics.track("composer.sent", { threadId: "t1" }); + await flush(); + expect(loader.loadCount()).toBe(0); + expect(analytics.isActive()).toBe(false); + }); + + it("is a no-op when the server marks analytics disabled", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.configure(context({ ...enabledConfig, enabled: false })); + analytics.track("app.opened"); + await flush(); + expect(loader.loadCount()).toBe(0); + }); + + it("queues events until loaded, then sends sanitized properties", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.configure(context(enabledConfig)); + analytics.identify("person@beknown.work"); + analytics.track("composer.sent", { threadId: "t1", prompt: "do not send me" }); + await flush(); + const [client] = loader.clients; + expect(client?.captured).toEqual([{ event: "composer.sent", properties: { threadId: "t1" } }]); + expect(client?.identified).toEqual(["person@beknown.work"]); + expect(client?.registered.at(-1)).toEqual({ + surface: "web", + appVersion: "1.2.3", + instance: "bkt3", + connectionMode: "tailnet", + }); + }); + + it("retires the old client when the key changes and stops when disabled", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.configure(context(enabledConfig)); + await flush(); + analytics.configure(context({ ...enabledConfig, posthogKey: "phc_other" })); + await flush(); + expect(loader.clients.map((client) => client.optedOut)).toEqual([true, false]); + analytics.configure(context(null)); + analytics.track("app.opened"); + expect(loader.clients[1]?.optedOut).toBe(true); + expect(loader.clients[1]?.captured).toEqual([]); + }); +}); diff --git a/apps/web/src/lib/analytics/productAnalytics.ts b/apps/web/src/lib/analytics/productAnalytics.ts new file mode 100644 index 0000000000000..eb03a76e45a43 --- /dev/null +++ b/apps/web/src/lib/analytics/productAnalytics.ts @@ -0,0 +1,218 @@ +/** + * T3-CUSTOM(expbkt3): client product analytics (TEC-1631). + * + * Call sites use `trackProductEvent(name, props)`: one line, no awaits, and a + * no-op until the primary environment advertises an analytics config. Only + * then is `posthog-js` loaded, by dynamic import, so the main bundle and + * startup cost do not change while analytics is off. + * + * Only the primary environment's config is used. A client connected to + * several servers never mixes keys: a new key retires the old client first. + * + * @module productAnalytics + */ +import type { ExecutionEnvironmentAnalytics } from "@t3tools/contracts"; +import type { PostHog, PostHogConfig } from "posthog-js"; + +import { + createKeyThrottle, + isAnalyticsSettingKey, + type ProductAnalyticsConnectionMode, + type ProductAnalyticsEvent, + type ProductAnalyticsSurface, + sanitizeAnalyticsProperties, + stripUrlProperties, +} from "./productAnalytics.logic"; + +export interface ProductAnalyticsContext { + readonly config: ExecutionEnvironmentAnalytics | null; + readonly surface: ProductAnalyticsSurface; + readonly appVersion: string; + readonly connectionMode: ProductAnalyticsConnectionMode | null; +} + +/** The part of the PostHog client this module calls. */ +export type ProductAnalyticsClient = Pick< + PostHog, + "capture" | "identify" | "register" | "opt_out_capturing" +>; + +export type ProductAnalyticsLoader = ( + key: string, + options: Partial, + name: string, +) => Promise; + +/** Events captured before `posthog-js` finishes loading. Older ones are dropped. */ +const MAX_PENDING_EVENTS = 50; + +interface ActiveSession { + readonly key: string; + client: ProductAnalyticsClient | null; + closed: boolean; + readonly pending: Array<{ event: string; properties: Record }>; +} + +const loadPostHog: ProductAnalyticsLoader = async (key, options, name) => { + const { default: posthog } = await import("posthog-js"); + return posthog.init(key, options, name); +}; + +function postHogOptions(host: string): Partial { + return { + api_host: host, + autocapture: false, + capture_pageview: false, + capture_pageleave: false, + disable_session_recording: true, + persistence: "localStorage", + person_profiles: "identified_only", + // Named events only: no surveys, flags, heatmaps, vitals or remote scripts. + disable_surveys: true, + advanced_disable_flags: true, + disable_web_experiments: true, + disable_external_dependency_loading: true, + enable_heatmaps: false, + capture_performance: false, + capture_exceptions: false, + capture_dead_clicks: false, + rageclick: false, + before_send: (event) => ({ + ...event, + properties: stripUrlProperties(event.properties), + ...(event.$set === undefined ? {} : { $set: stripUrlProperties(event.$set) }), + ...(event.$set_once === undefined ? {} : { $set_once: stripUrlProperties(event.$set_once) }), + }), + }; +} + +export function createProductAnalytics(load: ProductAnalyticsLoader = loadPostHog) { + let session: ActiveSession | null = null; + let superProperties: Record = {}; + let distinctId: string | null = null; + let identifiedAs: string | null = null; + let sessionCount = 0; + let lastInputWasKeyboard = false; + + const applyIdentity = (client: ProductAnalyticsClient) => { + client.register(superProperties); + if (distinctId !== null && identifiedAs !== distinctId) { + client.identify(distinctId); + identifiedAs = distinctId; + } + }; + + const close = () => { + if (session === null) return; + session.closed = true; + session.client?.opt_out_capturing(); + session = null; + identifiedAs = null; + }; + + return { + /** Apply the primary environment's config. `null` or disabled turns it off. */ + configure(context: ProductAnalyticsContext): void { + const config = context.config?.enabled === true ? context.config : null; + superProperties = { + surface: context.surface, + appVersion: context.appVersion, + ...(config?.instance === undefined ? {} : { instance: config.instance }), + ...(context.connectionMode === null ? {} : { connectionMode: context.connectionMode }), + }; + if (config === null) { + close(); + return; + } + if (session !== null && session.key === config.posthogKey) { + if (session.client !== null) applyIdentity(session.client); + return; + } + close(); + const next: ActiveSession = { + key: config.posthogKey, + client: null, + closed: false, + pending: [], + }; + session = next; + sessionCount += 1; + void load(config.posthogKey, postHogOptions(config.posthogHost), `bkt3_${sessionCount}`) + .then((client) => { + if (next.closed) { + client.opt_out_capturing(); + return; + } + next.client = client; + applyIdentity(client); + for (const { event, properties } of next.pending.splice(0)) { + client.capture(event, properties); + } + }) + .catch(() => { + // Analytics must never break the app; a failed load stays off. + if (session === next) session = null; + }); + }, + + /** The user's work email, or the server user id when the email is unknown. */ + identify(id: string | null): void { + distinctId = id; + if (session?.client) applyIdentity(session.client); + }, + + track(event: ProductAnalyticsEvent, properties?: Readonly>): void { + if (session === null) return; + const clean = sanitizeAnalyticsProperties(event, properties); + if (session.client !== null) { + session.client.capture(event, clean); + return; + } + if (session.pending.length >= MAX_PENDING_EVENTS) session.pending.shift(); + session.pending.push({ event, properties: clean }); + }, + + isActive(): boolean { + return session !== null; + }, + + noteInput(kind: "keyboard" | "pointer"): void { + lastInputWasKeyboard = kind === "keyboard"; + }, + + /** Whether the last user input before now was a key press. */ + lastInputWasKeyboard(): boolean { + return lastInputWasKeyboard; + }, + }; +} + +export type ProductAnalytics = ReturnType; + +export const productAnalytics: ProductAnalytics = createProductAnalytics(); + +/** One-line hook for call sites. A no-op while analytics is off. */ +export function trackProductEvent( + event: ProductAnalyticsEvent, + properties?: Readonly>, +): void { + productAnalytics.track(event, properties); +} + +export function lastInputWasKeyboard(): boolean { + return productAnalytics.lastInputWasKeyboard(); +} + +/** Drags and repeated writes of one setting count once per window. */ +const SETTINGS_CHANGED_WINDOW_MS = 10_000; +const settingsChangedThrottle = createKeyThrottle(SETTINGS_CHANGED_WINDOW_MS); + +/** `settings.changed` per key in a patch. Keys only, never values. */ +export function trackSettingsChanged(keys: ReadonlyArray): void { + if (!productAnalytics.isActive()) return; + for (const settingKey of keys) { + if (isAnalyticsSettingKey(settingKey) && settingsChangedThrottle(settingKey)) { + productAnalytics.track("settings.changed", { settingKey }); + } + } +} diff --git a/apps/web/src/routes/__root.tsx b/apps/web/src/routes/__root.tsx index 5c8be802c5e96..b175eb9a5e836 100644 --- a/apps/web/src/routes/__root.tsx +++ b/apps/web/src/routes/__root.tsx @@ -40,6 +40,8 @@ import { ChatGptWelcomeCoordinator } from "../components/settings/ChatGptWelcome import { PlanAgentSelectionHeal } from "../fork/PlanAgentSelectionHeal"; // T3-CUSTOM(expbkt3): delivery runs independently of the selected thread or draft. import { DurableOutboxCoordinator } from "../fork/DurableOutboxCoordinator"; +// T3-CUSTOM(expbkt3): product analytics, off unless the primary environment enables it. +import { ProductAnalyticsCoordinator } from "../lib/analytics/ProductAnalyticsCoordinator"; import { ProviderAuthCallbackCoordinator } from "../components/settings/ProviderAuthCallbackCoordinator"; import { ThemeEditorHost } from "../components/settings/ThemeEditorHost"; import { useCopyToClipboard } from "../hooks/useCopyToClipboard"; @@ -228,6 +230,7 @@ function RootRouteView() { {/* T3-CUSTOM(expbkt3): heal saved plan-agent options after authenticated settings load. */} {primaryEnvironmentAuthenticated ? : null} + {primaryEnvironmentAuthenticated ? : null} turn on the instance. Trigger a manual bkt3 deploy from the `t3.dev` instance > or from a human shell. +## Product analytics + +bkt3 sends product events to the company PostHog project. The +server reads these variables: + +| Variable | Meaning | Default | +| --------------------------- | --------------------------------------------------------------------------------- | -------------------------- | +| `T3CODE_TELEMETRY_ENABLED` | Set `false` to send nothing. | `true` | +| `T3CODE_POSTHOG_KEY` | PostHog project API key. Without a key, telemetry is off. | none | +| `T3CODE_POSTHOG_HOST` | PostHog ingestion host. | `https://us.i.posthog.com` | +| `T3CODE_ANALYTICS_INSTANCE` | Name of this instance, sent as the PostHog `instance` group (for example `bkt3`). | `unknown` | + +The fork has no built-in key, so it never sends to the upstream maintainers' +project. Never commit the key. Production keeps the values in +`/home/ubuntu/.t3/nextbkt3-dev/posthog.env` (mode `600`, owner `ubuntu`), which +sets `T3CODE_POSTHOG_KEY`, `T3CODE_POSTHOG_HOST` and +`T3CODE_ANALYTICS_INSTANCE=bkt3`. + +The existing drop-in `/etc/systemd/system/t3-nextbkt3.service.d/clerk.conf` sets +`Environment=T3CODE_TELEMETRY_ENABLED=false`. To turn analytics on, add a second +drop-in named `zz-analytics.conf`. systemd reads drop-ins in name order, so this +one comes after `clerk.conf` and its value replaces the `false`: + +```ini +# /etc/systemd/system/t3-nextbkt3.service.d/zz-analytics.conf +[Service] +EnvironmentFile=-/home/ubuntu/.t3/nextbkt3-dev/posthog.env +Environment=T3CODE_TELEMETRY_ENABLED=true +``` + +```bash +sudo systemctl daemon-reload +sudo systemctl show t3-nextbkt3 -p Environment # expect T3CODE_TELEMETRY_ENABLED=true +``` + +The new environment takes effect at the next restart of the service. To turn +analytics off again, remove `zz-analytics.conf`, reload systemd and restart. +Provider and agent processes never inherit these variables: the server masks +them when it builds a provider's environment. + +Events carry ids, enums, numbers and booleans only. Free text (titles, prompts, +messages, paths, branches, errors) is dropped by the allowlist in +`apps/server/src/telemetry/analyticsProperties.ts`. An event with a known user +is sent as that user's `environment_users.primary_email`. + ## Temporary files `start.sh` exports `TMPDIR=/tmp` so the service and every agent session diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 0059cddf08d8d..50ed7f11000a9 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -250,6 +250,23 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ }); export type ExecutionEnvironmentCapabilities = typeof ExecutionEnvironmentCapabilities.Type; +// T3-CUSTOM(expbkt3): BEGIN — product analytics config for the client (TEC-1631). +/** + * Where a client sends its product analytics for this environment. Present + * only when the server has an explicit PostHog project key and telemetry is + * on, so the client bundle never carries a key of its own. `posthogKey` is a + * public write-only `phc_` key. + */ +export const ExecutionEnvironmentAnalytics = Schema.Struct({ + enabled: Schema.Boolean, + posthogKey: TrimmedNonEmptyString, + posthogHost: TrimmedNonEmptyString, + /** Which deployment the events come from, such as `bkt3` or `expbkt3`. */ + instance: Schema.optionalKey(TrimmedNonEmptyString), +}); +export type ExecutionEnvironmentAnalytics = typeof ExecutionEnvironmentAnalytics.Type; +// T3-CUSTOM(expbkt3): END + export const ExecutionEnvironmentDescriptor = Schema.Struct({ environmentId: EnvironmentId, label: TrimmedNonEmptyString, @@ -258,6 +275,8 @@ export const ExecutionEnvironmentDescriptor = Schema.Struct({ /** Absent on hosts from before explicit orchestration protocol negotiation. */ orchestrationProtocolVersion: Schema.optionalKey(Schema.Int), capabilities: ExecutionEnvironmentCapabilities, + // T3-CUSTOM(expbkt3): absent on servers without product analytics (TEC-1631). + analytics: Schema.optionalKey(ExecutionEnvironmentAnalytics), }); export type ExecutionEnvironmentDescriptor = typeof ExecutionEnvironmentDescriptor.Type; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0b4b37d7b91be..aa73fdf2186fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -505,7 +505,7 @@ importers: version: 4.0.1(patch_hash=359f6fb2f7b3ec145bb72208edb9034f02489791aa2491a55cdbd69bd56ee0d2)(@types/node@24.12.4)(@vitest/ui@5.0.1)(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0) '@pierre/trees': specifier: 1.0.0-beta.6 - version: 1.0.0-beta.6(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) + version: 1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) '@types/react': specifier: ~19.3.0 version: 19.3.0 @@ -680,7 +680,7 @@ importers: version: 1.5.2(patch_hash=b33a575eb1382270ed0df956307e4b6ffe4b8e4349cefeb43e14f648c0fdc3b0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) '@pierre/trees': specifier: 1.0.0-beta.6 - version: 1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0) + version: 1.0.0-beta.6(@pierre/theme@2.0.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0) '@platejs/basic-nodes': specifier: 53.0.0 version: 53.0.0(platejs@53.3.3(@types/react@19.3.0)(immer@10.2.0)(react-dom@19.2.6(react@19.2.6))(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.2.6)(utf-8-validate@6.0.6))(react@19.2.6)(scheduler@0.28.0)(use-sync-external-store@1.6.0(react@19.2.6)))(react-dom@19.2.6(react@19.2.6))(react@19.2.6) @@ -804,6 +804,9 @@ importers: platejs: specifier: 53.3.3 version: 53.3.3(@types/react@19.3.0)(immer@10.2.0)(react-dom@19.2.6(react@19.2.6))(react-native@0.88.0-rc.3(@babel/core@7.29.7)(@react-native/metro-config@0.88.0-rc.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.2.6)(utf-8-validate@6.0.6))(react@19.2.6)(scheduler@0.28.0)(use-sync-external-store@1.6.0(react@19.2.6)) + posthog-js: + specifier: ^1.438.9 + version: 1.438.9(@types/react@19.3.0)(preact-render-to-string@6.6.5(preact@11.0.0-beta.0))(react@19.2.6) react: specifier: 19.2.6 version: 19.2.6 @@ -4676,6 +4679,15 @@ packages: '@polka/url@1.0.0-next.29': resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} + '@posthog/browser-common@0.9.4': + resolution: {integrity: sha512-QlG0WZFG9iTp9cmXTypB4cgnxBAD3GImrj9AwmA84uh8uy5vIVgVLjoTTdOAW29XL+kh/eLn/5ZtBQt2JQfR+w==} + + '@posthog/core@1.58.1': + resolution: {integrity: sha512-z0ISzMUoF1izukfLIUSjGZBdO5H2BPdqN8MX27OKYGQ1sDj/r+WW9Nq7m5WsQY56NOxVsFtHClx9iHj6MYM3Tg==} + + '@posthog/types@1.415.3': + resolution: {integrity: sha512-y90ARRBbyXcZDwuXXXvAnbjiuxQ+4FmUI4sasI19AHOPgdWPyd2Kpo0c0B2pKIR+fq+BcRokhF35eAdA54Mfbw==} + '@puppeteer/browsers@3.2.2': resolution: {integrity: sha512-q2BU4YfO9h/Wt7IcWPcggpOOqLk2Tbs1hDwolvKZrweRjy751OJBKMN9zO5bfD0pzU7X/tvKw/exQds4pM/LOg==} engines: {node: '>=22.12.0'} @@ -7161,6 +7173,9 @@ packages: core-js@3.47.0: resolution: {integrity: sha512-c3Q2VVkGAUyupsjRnaNX6u8Dq2vAdzm9iuPj5FW0fRxzlxgq9Q39MDq10IvmQSpLgHQNyQzQmOo6bgGHmH3NNg==} + core-js@3.50.0: + resolution: {integrity: sha512-BRWgOLKkFeCgRudR6zrs8p9XJZcE14grzKMMssoYrk6krtuEZ7MTKPIY5RzOnqsEKIR9kst7wNzphttraT+Yqw==} + core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} @@ -8331,6 +8346,9 @@ packages: ffi-rs@1.3.2: resolution: {integrity: sha512-4s8dX9VbBw/jd5NOuE3EJRqXaIVdjMyiumeeDzrOhtjQRwp6Bz2za7iksWXTnvTQKV/tTdm1s1w7mObe92zPjQ==} + fflate@0.4.9: + resolution: {integrity: sha512-zdxgIEddhfsyCaWpJ2SdXEP8ZMrKJ6+5jl4OupODcywU0IhRk6gdXuVGcPICyfx2H97hVK7xmJtRLPjkxAX8Vw==} + fflate@0.8.3: resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} @@ -10115,6 +10133,17 @@ packages: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} + posthog-js@1.438.9: + resolution: {integrity: sha512-/KTZQF3sPlQbgLAlSncxv3z8dz4yex+JmIbCmGKIYkIVupdqESiqXx3SadPkWKgqgHtnQGbPSqRVe4vgpXnrGw==} + peerDependencies: + '@types/react': '>=16.8.0' + react: '>=16.8.0' + peerDependenciesMeta: + '@types/react': + optional: true + react: + optional: true + postject@1.0.0-alpha.6: resolution: {integrity: sha512-b9Eb8h2eVqNE8edvKdwqkrY6O7kAwmI8kcnBv1NScolYJbo59XUF0noFq+lxbC1yN20bmC0WBEbDC5H/7ASb0A==} engines: {node: '>=14.0.0'} @@ -10125,6 +10154,14 @@ packages: peerDependencies: preact: '>=10 || >= 11.0.0-0' + preact@10.29.8: + resolution: {integrity: sha512-ej2aVZ+vZ8WO7tvlQWRM9N63A0KzF9q4mWJfDUHgYaIofWY9hu74QdnQrjoPMmZi2/nZ5gN0bJCQF49xQqx09Q==} + peerDependencies: + preact-render-to-string: '>=5' + peerDependenciesMeta: + preact-render-to-string: + optional: true + preact@11.0.0-beta.0: resolution: {integrity: sha512-IcODoASASYwJ9kxz7+MJeiJhvLriwSb4y4mHIyxdgaRZp6kPUud7xytrk/6GZw8U3y6EFJaRb5wi9SrEK+8+lg==} @@ -10252,6 +10289,9 @@ packages: resolution: {integrity: sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==} engines: {node: '>=0.6'} + query-selector-shadow-dom@1.0.1: + resolution: {integrity: sha512-lT5yCqEBgfoMYpf3F2xQRK7zEr1rhIIZuceDK6+xRkJQ4NMbHTwXqk4NkwDwQMNqXgG9r9fyHnzwNVs6zV5KRw==} + query-string@7.1.3: resolution: {integrity: sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==} engines: {node: '>=6'} @@ -11741,6 +11781,12 @@ packages: web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + web-vitals@6.2.1: + resolution: {integrity: sha512-rLcLXA2sx6+9dE88NHFubwTtGxpK4yYBLj6qHPdFoCaLr0cXGb4efOqtKLlm4loGA4OEKHIQKMKzZkKyOh5ctw==} + + web-vitals@6.2.3: + resolution: {integrity: sha512-Dmy+vdO1UPMqXiAtZSNayhl4f794Bxfd94W4HIj2HlYtczEQfEOon969memlcpcHrTuOKLa5baFjX7WbgmOq+A==} + webcrypto-core@1.9.2: resolution: {integrity: sha512-gsXecm82UQNlTBURJGuqOWy1Ww08S3kZUcr3aOJS02Pk0xLtkfeUAVC0u0xhgdonFme80edSJUIJyuvL/7250Q==} @@ -15358,20 +15404,20 @@ snapshots: '@pierre/theme@2.0.0': {} - '@pierre/theming@1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0)': + '@pierre/theming@1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': optionalDependencies: '@pierre/theme': 1.1.0 - '@shikijs/themes': 4.3.0 - react: 19.2.6 - react-dom: 19.2.6(react@19.2.6) + '@shikijs/themes': 4.2.0 + react: 19.3.0 + react-dom: 19.3.0(react@19.3.0) shiki: 4.2.0 - '@pierre/theming@1.0.0(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': + '@pierre/theming@1.0.0(@pierre/theme@2.0.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0)': optionalDependencies: '@pierre/theme': 2.0.0 - '@shikijs/themes': 4.2.0 - react: 19.3.0 - react-dom: 19.3.0(react@19.3.0) + '@shikijs/themes': 4.3.0 + react: 19.2.6 + react-dom: 19.2.6(react@19.2.6) shiki: 4.2.0 '@pierre/theming@1.0.1(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': @@ -15390,25 +15436,25 @@ snapshots: react-dom: 19.2.6(react@19.2.6) shiki: 4.2.0 - '@pierre/trees@1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0)': + '@pierre/trees@1.0.0-beta.6(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': dependencies: - '@pierre/theming': 1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0) + '@pierre/theming': 1.0.0(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) preact: 11.0.0-beta.0 preact-render-to-string: 6.6.5(preact@11.0.0-beta.0) - react: 19.2.6 - react-dom: 19.2.6(react@19.2.6) + react: 19.3.0 + react-dom: 19.3.0(react@19.3.0) transitivePeerDependencies: - '@pierre/theme' - '@shikijs/themes' - shiki - '@pierre/trees@1.0.0-beta.6(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0)': + '@pierre/trees@1.0.0-beta.6(@pierre/theme@2.0.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0)': dependencies: - '@pierre/theming': 1.0.0(@pierre/theme@2.0.0)(@shikijs/themes@4.2.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0)(shiki@4.2.0) + '@pierre/theming': 1.0.0(@pierre/theme@2.0.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0) preact: 11.0.0-beta.0 preact-render-to-string: 6.6.5(preact@11.0.0-beta.0) - react: 19.3.0 - react-dom: 19.3.0(react@19.3.0) + react: 19.2.6 + react-dom: 19.2.6(react@19.2.6) transitivePeerDependencies: - '@pierre/theme' - '@shikijs/themes' @@ -15585,6 +15631,17 @@ snapshots: '@polka/url@1.0.0-next.29': {} + '@posthog/browser-common@0.9.4': + dependencies: + '@posthog/core': 1.58.1 + '@posthog/types': 1.415.3 + + '@posthog/core@1.58.1': + dependencies: + '@posthog/types': 1.415.3 + + '@posthog/types@1.415.3': {} + '@puppeteer/browsers@3.2.2(yauzl@3.4.0)': dependencies: modern-tar: 0.8.5 @@ -16016,7 +16073,7 @@ snapshots: '@shikijs/types@4.2.0': dependencies: '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 '@shikijs/types@4.3.0': dependencies: @@ -18097,6 +18154,8 @@ snapshots: core-js@3.47.0: {} + core-js@3.50.0: {} + core-util-is@1.0.2: {} cors@2.8.6: @@ -19309,10 +19368,6 @@ snapshots: dependencies: walk-up-path: 4.0.0 - fdir@6.5.0(picomatch@4.0.4): - optionalDependencies: - picomatch: 4.0.4 - fdir@6.5.0(picomatch@4.0.7): optionalDependencies: picomatch: 4.0.7 @@ -19333,6 +19388,8 @@ snapshots: '@yuuang/ffi-rs-win32-ia32-msvc': 1.3.2 '@yuuang/ffi-rs-win32-x64-msvc': 1.3.2 + fflate@0.4.9: {} + fflate@0.8.3: {} filelist@1.0.6: @@ -21668,6 +21725,24 @@ snapshots: xtend: 4.0.2 optional: true + posthog-js@1.438.9(@types/react@19.3.0)(preact-render-to-string@6.6.5(preact@11.0.0-beta.0))(react@19.2.6): + dependencies: + '@posthog/browser-common': 0.9.4 + '@posthog/core': 1.58.1 + '@posthog/types': 1.415.3 + core-js: 3.50.0 + dompurify: 3.4.16 + fflate: 0.4.9 + preact: 10.29.8(preact-render-to-string@6.6.5(preact@11.0.0-beta.0)) + query-selector-shadow-dom: 1.0.1 + web-vitals: 6.2.3 + web-vitals-soft-navs: web-vitals@6.2.1 + optionalDependencies: + '@types/react': 19.3.0 + react: 19.2.6 + transitivePeerDependencies: + - preact-render-to-string + postject@1.0.0-alpha.6: dependencies: commander: 9.5.0 @@ -21677,6 +21752,10 @@ snapshots: dependencies: preact: 11.0.0-beta.0 + preact@10.29.8(preact-render-to-string@6.6.5(preact@11.0.0-beta.0)): + optionalDependencies: + preact-render-to-string: 6.6.5(preact@11.0.0-beta.0) + preact@11.0.0-beta.0: {} prettier@3.8.3: {} @@ -21832,6 +21911,8 @@ snapshots: es-define-property: 1.0.1 side-channel: 1.1.1 + query-selector-shadow-dom@1.0.1: {} + query-string@7.1.3: dependencies: decode-uri-component: 0.2.2 @@ -22587,7 +22668,7 @@ snapshots: '@shikijs/themes': 4.2.0 '@shikijs/types': 4.2.0 '@shikijs/vscode-textmate': 10.0.2 - '@types/hast': 3.0.4 + '@types/hast': 3.0.5 should-equal@2.0.0: dependencies: @@ -22971,8 +23052,8 @@ snapshots: tinyglobby@0.2.17: dependencies: - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 tinypool@2.1.2: {} @@ -23573,6 +23654,10 @@ snapshots: web-namespaces@2.0.1: {} + web-vitals@6.2.1: {} + + web-vitals@6.2.3: {} + webcrypto-core@1.9.2: dependencies: '@peculiar/asn1-schema': 2.8.0 From 71fa6f3db5bec5bf0515a5918eaed340f6c72212 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Wed, 14 Oct 2026 10:26:37 +0000 Subject: [PATCH 06/20] chore(deploy): script for bkt3 PostHog dashboards and alerts (TEC-1631) Co-Authored-By: Claude Opus 5.5 --- deploy/bkt3/posthog-dashboards.py | 357 ++++++++++++++++++++++++++++++ 1 file changed, 357 insertions(+) create mode 100644 deploy/bkt3/posthog-dashboards.py diff --git a/deploy/bkt3/posthog-dashboards.py b/deploy/bkt3/posthog-dashboards.py new file mode 100644 index 0000000000000..f905be391d8c0 --- /dev/null +++ b/deploy/bkt3/posthog-dashboards.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +"""Create the bkt3 product-metrics dashboards and alerts in PostHog (TEC-1631). + +Reads POSTHOG_PERSONAL_API_KEY and POSTHOG_HOST from the environment +(source ~/.config/posthog/dev-server-1.env) and POSTHOG_PROJECT_ID +(source ~/.t3/nextbkt3-dev/posthog.env). Idempotent: a dashboard that +already exists is left alone unless --replace is given. + + python3 deploy/bkt3/posthog-dashboards.py --check # run every query once + python3 deploy/bkt3/posthog-dashboards.py # create what is missing + python3 deploy/bkt3/posthog-dashboards.py --replace # delete and recreate + +Every query filters on the `instance` property, so other apps that share the +PostHog project do not leak into these numbers. +""" + +import json +import os +import sys +import urllib.error +import urllib.request + +INSTANCE = os.environ.get("T3CODE_ANALYTICS_INSTANCE", "bkt3") +F = f"properties.instance = '{INSTANCE}'" +TZ = "Asia/Kolkata" +DAYS = 90 + +# Time in each state comes from session.state.span. Spans are split at UTC +# midnight, so summing durationMs per day is exact. +SPAN = f"event = 'session.state.span' AND {F} AND timestamp > now() - INTERVAL {DAYS} DAY" + + +def table(sql): + return {"display": "ActionsTable", "sql": sql} + + +def line(sql): + return {"display": "ActionsLineGraph", "sql": sql} + + +def bar(sql): + return {"display": "ActionsStackedBar", "sql": sql} + + +DASHBOARDS = { + "bkt3 · Usage": { + "Daily active people": line(f""" + SELECT toDate(timestamp) AS day, count(DISTINCT distinct_id) AS people + FROM events + WHERE {F} AND timestamp > now() - INTERVAL {DAYS} DAY + AND event IN ('session.created', 'run.finished', 'composer.sent', + 'human_request.resolved', 'app.opened') + AND distinct_id LIKE '%@%' + GROUP BY day ORDER BY day"""), + "Sessions created per day, by person": bar(f""" + SELECT toDate(timestamp) AS day, distinct_id AS person, count() AS sessions + FROM events + WHERE event = 'session.created' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, person ORDER BY day"""), + "Runs per day, by provider": bar(f""" + SELECT toDate(timestamp) AS day, properties.providerInstanceId AS provider, + count() AS runs + FROM events + WHERE event = 'run.finished' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, provider ORDER BY day"""), + "Runs by hour of day (IST) and weekday, last 30 days": table(f""" + SELECT toDayOfWeek(toTimeZone(timestamp, '{TZ}')) AS weekday, + toHour(toTimeZone(timestamp, '{TZ}')) AS hour_ist, count() AS runs + FROM events + WHERE event = 'run.finished' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY weekday, hour_ist ORDER BY weekday, hour_ist"""), + "Sessions by source per day": bar(f""" + SELECT toDate(timestamp) AS day, properties.sessionSource AS source, + count() AS sessions + FROM events + WHERE event = 'session.created' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, source ORDER BY day"""), + "Client surfaces opened per day": bar(f""" + SELECT toDate(timestamp) AS day, properties.surface AS surface, count() AS opens + FROM events + WHERE event = 'app.opened' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, surface ORDER BY day"""), + }, + "bkt3 · Session time": { + "Hours per state per day": bar(f""" + SELECT toDate(timestamp) AS day, + if(properties.state = 'waiting_human', + concat('waiting_human:', toString(properties.waitKind)), + toString(properties.state)) AS state, + round(sum(toFloat(properties.durationMs)) / 3600000, 1) AS hours + FROM events + WHERE {SPAN} AND properties.state NOT IN ('idle', 'archived') + GROUP BY day, state ORDER BY day"""), + "Agent running hours per person per day": bar(f""" + SELECT toDate(timestamp) AS day, distinct_id AS person, + round(sum(toFloat(properties.durationMs)) / 3600000, 1) AS hours + FROM events + WHERE {SPAN} AND properties.state = 'agent_running' + GROUP BY day, person ORDER BY day"""), + "Time waiting on a person, by person and kind (last 30 days)": table(f""" + SELECT distinct_id AS person, properties.waitKind AS kind, + round(sum(toFloat(properties.durationMs)) / 3600000, 1) AS hours, + count() AS waits + FROM events + WHERE {SPAN} AND properties.state = 'waiting_human' + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY person, kind ORDER BY hours DESC"""), + "Time waiting on background workers per day": line(f""" + SELECT toDate(timestamp) AS day, + round(sum(toFloat(properties.durationMs)) / 3600000, 1) AS hours + FROM events + WHERE {SPAN} AND properties.state = 'waiting_workers' + GROUP BY day ORDER BY day"""), + "Person answer time (minutes), last 30 days": table(f""" + SELECT coalesce(properties.answeredByUserId, distinct_id) AS person, + properties.requestKind AS kind, count() AS answered, + round(quantile(0.5)(toFloat(properties.waitMs)) / 60000, 1) AS p50_min, + round(quantile(0.9)(toFloat(properties.waitMs)) / 60000, 1) AS p90_min + FROM events + WHERE event = 'human_request.resolved' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY person, kind ORDER BY answered DESC"""), + "Share of active time by state, per person (last 30 days)": table(f""" + SELECT distinct_id AS person, + round(sumIf(toFloat(properties.durationMs), properties.state = 'agent_running') / 3600000, 1) AS running_h, + round(sumIf(toFloat(properties.durationMs), properties.state = 'waiting_human') / 3600000, 1) AS waiting_human_h, + round(sumIf(toFloat(properties.durationMs), properties.state = 'waiting_workers') / 3600000, 1) AS waiting_workers_h, + round(sumIf(toFloat(properties.durationMs), properties.state = 'queued') / 3600000, 2) AS queued_h + FROM events + WHERE {SPAN} AND timestamp > now() - INTERVAL 30 DAY + GROUP BY person ORDER BY running_h DESC"""), + }, + "bkt3 · Agent quality": { + "Run outcome per day": bar(f""" + SELECT toDate(timestamp) AS day, properties.status AS status, count() AS runs + FROM events + WHERE event = 'run.finished' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, status ORDER BY day"""), + "Run time and failure rate by model (last 30 days)": table(f""" + SELECT properties.model AS model, count() AS runs, + round(quantile(0.5)(toFloat(properties.runDurationMs)) / 60000, 1) AS p50_min, + round(quantile(0.95)(toFloat(properties.runDurationMs)) / 60000, 1) AS p95_min, + round(100 * countIf(properties.status = 'failed') / count(), 1) AS failed_pct, + round(100 * countIf(properties.status = 'interrupted') / count(), 1) AS interrupted_pct + FROM events + WHERE event = 'run.finished' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY model ORDER BY runs DESC"""), + "Failure classes (last 30 days)": table(f""" + SELECT properties.failureClass AS failure, properties.providerInstanceId AS provider, + count() AS runs + FROM events + WHERE event = 'run.finished' AND properties.status = 'failed' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY failure, provider ORDER BY runs DESC"""), + "Queue wait p95 (seconds) per day": line(f""" + SELECT toDate(timestamp) AS day, + round(quantile(0.95)(toFloat(properties.queueWaitMs)) / 1000, 1) AS p95_s + FROM events + WHERE event = 'run.finished' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day ORDER BY day"""), + "Session recoveries per day": bar(f""" + SELECT toDate(timestamp) AS day, properties.outcome AS outcome, count() AS recoveries + FROM events + WHERE event = 'session.recovery.finished' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, outcome ORDER BY day"""), + "Subagents per day, by status": bar(f""" + SELECT toDate(timestamp) AS day, properties.status AS status, count() AS subagents + FROM events + WHERE event = 'subagent.finished' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, status ORDER BY day"""), + }, + "bkt3 · Load": { + "Peak concurrent sessions per hour, by state": line(f""" + SELECT toStartOfHour(timestamp) AS hour, + max(toFloat(properties.runningRuns)) AS running, + max(toFloat(properties.waitingHuman)) AS waiting_human, + max(toFloat(properties.waitingWorkers)) AS waiting_workers, + max(toFloat(properties.queuedRuns)) AS queued, + max(toFloat(properties.activeSubagents)) AS subagents + FROM events + WHERE event = 'session.state.snapshot' AND properties.scope = 'total' AND {F} + AND timestamp > now() - INTERVAL 14 DAY + GROUP BY hour ORDER BY hour"""), + "Peak running sessions per person per day": table(f""" + SELECT toDate(timestamp) AS day, distinct_id AS person, + max(toFloat(properties.count)) AS peak_running + FROM events + WHERE event = 'session.state.snapshot' AND properties.scope = 'user' + AND properties.state = 'agent_running' AND {F} + AND timestamp > now() - INTERVAL 14 DAY + GROUP BY day, person ORDER BY day DESC, peak_running DESC"""), + "Connected clients per day, by connection mode": bar(f""" + SELECT toDate(timestamp) AS day, properties.connectionMode AS mode, count() AS connects + FROM events + WHERE event = 'client.connected' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, mode ORDER BY day"""), + }, + "bkt3 · Automation": { + "MCP tool calls (last 30 days)": table(f""" + SELECT properties.toolName AS tool, properties.callerKind AS caller, count() AS calls, + round(100 * countIf(properties.ok = false) / count(), 1) AS error_pct, + round(quantile(0.95)(toFloat(properties.durationMs))) AS p95_ms + FROM events + WHERE event = 'mcp.tool.called' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY tool, caller ORDER BY calls DESC LIMIT 50"""), + "Scheduled task runs per day, by status": bar(f""" + SELECT toDate(timestamp) AS day, properties.status AS status, count() AS runs + FROM events + WHERE event = 'scheduled_task.ran' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, status ORDER BY day"""), + "Webhook deliveries per day": bar(f""" + SELECT toDate(timestamp) AS day, properties.kind AS kind, properties.status AS status, + count() AS deliveries + FROM events + WHERE event = 'webhook.delivered' AND {F} + AND timestamp > now() - INTERVAL {DAYS} DAY + GROUP BY day, kind, status ORDER BY day"""), + "Plans: decisions and wait (last 30 days)": table(f""" + SELECT properties.decision AS decision, count() AS plans, + round(quantile(0.5)(toFloat(properties.waitMs)) / 60000, 1) AS p50_wait_min + FROM events + WHERE event = 'plan.decided' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY decision ORDER BY plans DESC"""), + "Top client views (last 30 days)": table(f""" + SELECT properties.view AS view, count() AS views, count(DISTINCT distinct_id) AS people + FROM events + WHERE event = 'view.changed' AND {F} + AND timestamp > now() - INTERVAL 30 DAY + GROUP BY view ORDER BY views DESC LIMIT 30"""), + }, +} + +# Alerts need a trends insight. Thresholds come from 30 days of history: +# a normal day has under 40 failed runs; the server sends a snapshot every 5 min. +ALERTS = [ + { + "insight": "Alert · server snapshots per hour", + "event": "session.state.snapshot", + "interval": "hour", + "properties": [{"key": "scope", "value": ["total"], "operator": "exact", "type": "event"}], + "alert": "bkt3 server stopped sending metrics", + "bounds": {"lower": 1}, + "calculation_interval": "hourly", + }, + { + "insight": "Alert · failed runs per day", + "event": "run.finished", + "interval": "day", + "properties": [{"key": "status", "value": ["failed"], "operator": "exact", "type": "event"}], + "alert": "bkt3 failed runs above 40 in a day", + "bounds": {"upper": 40}, + "calculation_interval": "daily", + }, +] + + +def api(method, path, body=None): + host = os.environ["POSTHOG_HOST"].rstrip("/") + req = urllib.request.Request( + f"{host}{path}", + method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={ + "Authorization": f"Bearer {os.environ['POSTHOG_PERSONAL_API_KEY']}", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(req, timeout=60) as res: + raw = res.read() + return json.loads(raw) if raw else None + except urllib.error.HTTPError as err: + raise SystemExit(f"{method} {path} -> {err.code}: {err.read().decode()[:500]}") + + +def hogql(sql): + return { + "kind": "DataVisualizationNode", + "source": {"kind": "HogQLQuery", "query": " ".join(sql.split())}, + } + + +def main(): + pid = os.environ["POSTHOG_PROJECT_ID"] + base = f"/api/projects/{pid}" + args = set(sys.argv[1:]) + + if "--check" in args: + for dash, insights in DASHBOARDS.items(): + for name, spec in insights.items(): + res = api("POST", f"{base}/query/", {"query": hogql(spec["sql"])["source"]}) + print(f"ok {dash} / {name}: {len(res.get('results') or [])} rows") + return + + existing = {d["name"]: d for d in api("GET", f"{base}/dashboards/?limit=200")["results"]} + for dash, insights in DASHBOARDS.items(): + if dash in existing: + if "--replace" not in args: + print(f"skip {dash} (exists, id {existing[dash]['id']})") + continue + api("PATCH", f"{base}/dashboards/{existing[dash]['id']}/", {"deleted": True}) + d = api("POST", f"{base}/dashboards/", {"name": dash, "description": "TEC-1631 bkt3 product metrics"}) + for name, spec in insights.items(): + query = hogql(spec["sql"]) + query["display"] = spec["display"] + api("POST", f"{base}/insights/", {"name": name, "query": query, "dashboards": [d["id"]]}) + print(f"made {dash} -> {os.environ['POSTHOG_HOST'].rstrip('/')}/project/{pid}/dashboard/{d['id']}") + + me = api("GET", "/api/users/@me/") + alerts = {a["name"] for a in (api("GET", f"{base}/alerts/") or {}).get("results", [])} + for spec in ALERTS: + if spec["alert"] in alerts: + print(f"skip alert {spec['alert']} (exists)") + continue + insight = api("POST", f"{base}/insights/", { + "name": spec["insight"], + "query": { + "kind": "InsightVizNode", + "source": { + "kind": "TrendsQuery", + "series": [{"kind": "EventsNode", "event": spec["event"], "math": "total"}], + "interval": spec["interval"], + "dateRange": {"date_from": "-7d"}, + "properties": [{"key": "instance", "value": [INSTANCE], "operator": "exact", "type": "event"}] + + spec["properties"], + }, + }, + }) + api("POST", f"{base}/alerts/", { + "name": spec["alert"], + "insight": insight["id"], + "subscribed_users": [me["id"]], + "threshold": {"configuration": {"type": "absolute", "bounds": spec["bounds"]}}, + "condition": {"type": "absolute_value"}, + "config": {"type": "TrendsAlertConfig", "series_index": 0}, + "calculation_interval": spec["calculation_interval"], + "enabled": True, + }) + print(f"made alert {spec['alert']}") + + +if __name__ == "__main__": + main() From 3585a02bb03fddb9a9915110daf0ae0dae06875d Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Wed, 14 Oct 2026 10:35:00 +0000 Subject: [PATCH 07/20] fix(server,web): review fixes for bkt3 analytics (TEC-1631) - No zero-length spans; span id includes state, waitKind and run id. - Ignore late updates of finished runs; close only active spans at shutdown. - Backfill: normalised emails, --until required for --send. - Web: no persistent opt-out, strip all URL-derived properties, identify only by normalised email. - Register backfill script with knip; scope one Effect diagnostic. Co-Authored-By: Claude Opus 5.5 --- apps/server/scripts/analytics-backfill.ts | 44 +++++++++-- .../src/mcp/mcpToolAnalytics.expbkt3.ts | 3 +- .../sessionActivity/SessionActivityReactor.ts | 7 +- .../SessionActivityReducer.test.ts | 74 ++++++++++++++++++- .../sessionActivity/SessionActivityReducer.ts | 57 ++++++++++---- .../analytics/ProductAnalyticsCoordinator.tsx | 10 ++- .../analytics/productAnalytics.logic.test.ts | 20 +++++ .../lib/analytics/productAnalytics.logic.ts | 70 +++++++++++++++--- .../lib/analytics/productAnalytics.test.ts | 66 ++++++++++++++--- .../web/src/lib/analytics/productAnalytics.ts | 70 ++++++++++-------- knip.jsonc | 1 + 11 files changed, 339 insertions(+), 83 deletions(-) diff --git a/apps/server/scripts/analytics-backfill.ts b/apps/server/scripts/analytics-backfill.ts index 95404f72c9081..1e969e3ac5ecd 100644 --- a/apps/server/scripts/analytics-backfill.ts +++ b/apps/server/scripts/analytics-backfill.ts @@ -16,8 +16,11 @@ * --out JSONL output path (default analytics-backfill.jsonl) * --send POST to ${T3CODE_POSTHOG_HOST}/batch/; the key comes * only from env T3CODE_POSTHOG_KEY - * --until ignore events after this time. Set it to the moment - * the live reactor started, so the two never overlap. + * --until ignore events after this time; required with --send. + * Set it to the moment the live reactor started, so + * the two never overlap. + * + * Events with no user whose email is known are skipped (see toBatchItem). * --report-days print per-state hours for the last n days (default 7) * * Events are processed one thread at a time, sorted by `occurredAt`: imported @@ -239,7 +242,9 @@ function loadUserEmails(db: NodeSqlite.DatabaseSync): Map { for (const row of db .prepare("SELECT user_id, primary_email FROM environment_users") .all() as Array<{ user_id: string; primary_email: string | null }>) { - if (row.primary_email) emails.set(row.user_id, row.primary_email); + // Same normalisation as analyticsIdentity.ts, so live and backfill name one person. + const email = row.primary_email?.trim().toLowerCase() ?? ""; + if (email !== "") emails.set(row.user_id, email); } return emails; } @@ -252,22 +257,34 @@ interface BatchItem { readonly properties: Rec; } +/** + * The PostHog item for an emission, or undefined when no person is known. + * + * Live sends an unattributed event under the server install's anonymous id, + * which comes from that machine's T3 home, not from the database, so a copy + * cannot reproduce it. Inventing another id would create a fake person, so + * the backfill skips these events instead. Nearly every session has an owner, + * so very little is lost. `--until` keeps the backfill and live from covering + * the same time, so this cannot break live/backfill de-duplication; re-runs of + * the backfill stay de-duplicated by the stable uuid. + */ function toBatchItem( emission: SessionActivityEmission, emails: ReadonlyMap, instance: string, -): BatchItem { +): BatchItem | undefined { const email = emission.userId === undefined ? undefined : emails.get(emission.userId); + if (email === undefined) return undefined; return { uuid: uuidFromInsertId(emission.insertId), event: emission.event, - distinct_id: email ?? `bkt3-instance:${instance}`, + distinct_id: email, timestamp: emission.timestamp, properties: { ...emission.properties, backfill: true, $insert_id: emission.insertId, - $process_person_profile: email !== undefined, + $process_person_profile: true, $groups: { instance }, instance, }, @@ -302,6 +319,12 @@ async function main() { const apiKey = process.env.T3CODE_POSTHOG_KEY ?? ""; const instance = process.env.T3CODE_ANALYTICS_INSTANCE ?? "unknown"; if (options.send && apiKey.trim() === "") fail("--send needs env T3CODE_POSTHOG_KEY"); + if (options.send && options.untilMs === undefined) { + fail( + "--send needs --until : the time the live reactor started. Without it the backfill " + + "and the live server can both report the same time.", + ); + } const db = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true }); const emails = loadUserEmails(db); @@ -318,6 +341,7 @@ async function main() { let latestMs = 0; let pending: Array = []; let sent = 0; + let unattributed = 0; const flush = async () => { if (pending.length === 0) return; @@ -342,7 +366,12 @@ async function main() { stateHours.set(state, (stateHours.get(state) ?? 0) + hours); } } - pending.push(toBatchItem(emission, emails, instance)); + const item = toBatchItem(emission, emails, instance); + if (item === undefined) { + unattributed += 1; + continue; + } + pending.push(item); if (pending.length >= BATCH_SIZE) await flush(); } }; @@ -390,6 +419,7 @@ async function main() { console.log(`threads replayed: ${threads}`); console.log(`latest event: ${latestMs > 0 ? new Date(latestMs).toISOString() : "none"}`); console.log(options.send ? `sent: ${sent} events` : `wrote: ${options.outPath}`); + console.log(`skipped, no known user email: ${unattributed}`); console.log("events by type:"); for (const [event, count] of [...counts].sort((a, b) => b[1] - a[1])) { console.log(` ${event}: ${count}`); diff --git a/apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts b/apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts index 8a7555d9c3fa1..7d34948b314ed 100644 --- a/apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts +++ b/apps/server/src/mcp/mcpToolAnalytics.expbkt3.ts @@ -30,7 +30,8 @@ export function withMcpToolCallAnalytics(handlers: Handlers): Handlers ? (...args: ReadonlyArray) => { const result = (handler as (...params: ReadonlyArray) => unknown)(...args); return Effect.isEffect(result) - ? withMcpToolAnalytics(toolName, currentCaller, result) + ? // @effect-diagnostics-next-line anyUnknownInErrorContext:off -- handlers are heterogeneous; types pass through unchanged. + withMcpToolAnalytics(toolName, currentCaller, result) : result; } : handler; diff --git a/apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts b/apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts index f5460974ce2a0..c8fcb8d28a6c3 100644 --- a/apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts +++ b/apps/server/src/telemetry/sessionActivity/SessionActivityReactor.ts @@ -5,8 +5,9 @@ * (`SessionActivityReducer.ts`) and records what it emits through * `AnalyticsService`. On boot it seeds open sessions from one shell snapshot * read, so their first span carries `resumedAfterRestart: true`. On shutdown it - * closes every open span with `endReason: "server_stopped"`. Every five - * minutes it records a `session.state.snapshot` for concurrency and peak load. + * closes the spans of active sessions (not idle or archived ones) with + * `endReason: "server_stopped"`. Every five minutes it records a + * `session.state.snapshot` for concurrency and peak load. * * It does nothing, not even subscribe, when telemetry is off or no PostHog key * is set. @@ -90,7 +91,7 @@ export const layer = Layer.effectDiscard( // Runs before AnalyticsService's own finalizer, which flushes the buffer. yield* Effect.addFinalizer(() => Clock.currentTimeMillis.pipe( - Effect.flatMap((nowMs) => record(tracker.closeAll(nowMs, "server_stopped"))), + Effect.flatMap((nowMs) => record(tracker.closeActive(nowMs, "server_stopped"))), ), ); }), diff --git a/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts index dda19f2f74873..c7250bd5b77b0 100644 --- a/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts +++ b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.test.ts @@ -361,6 +361,76 @@ describe("SessionActivityReducer", () => { assert.strictEqual(decided?.properties.waitMs, 600_000); }); + it("keeps both spans when one run ends and the next starts in the same millisecond", () => { + const runB = (status: string, fields: Record = {}) => + runPayload(status, { id: "run-B", ordinal: 74, requestedAt: at("09:03:00.000"), ...fields }); + const { out } = feed([ + created, + event("run.created", at("09:01:00.000"), startedRun("09:01:00.000")), + event("run.created", at("09:03:00.000"), runB("queued")), + event( + "run.updated", + at("09:05:00.000"), + runPayload("completed", { startedAt: at("09:01:00.000"), completedAt: at("09:05:00.000") }), + ), + event("run.updated", at("09:05:00.000"), runB("running", { startedAt: at("09:05:00.000") })), + event( + "run.updated", + at("09:07:00.000"), + runB("completed", { startedAt: at("09:05:00.000"), completedAt: at("09:07:00.000") }), + ), + ]); + const emitted = spans(out); + assert.deepStrictEqual( + emitted.map((span) => [ + span.properties.state, + span.properties.runId, + span.properties.durationMs, + ]), + [ + ["idle", undefined, 60_000], + ["agent_running", RUN, 240_000], + ["agent_running", "run-B", 120_000], + ], + ); + assert.ok(emitted.every((span) => Number(span.properties.durationMs) > 0)); + assert.strictEqual(new Set(emitted.map((span) => span.insertId)).size, emitted.length); + }); + + it("ignores later updates of a run it already reported", () => { + const doneA = runPayload("completed", { + startedAt: at("09:01:00.000"), + completedAt: at("09:05:00.000"), + }); + const { out } = feed([ + created, + event("run.created", at("09:01:00.000"), startedRun("09:01:00.000")), + event("run.updated", at("09:05:00.000"), doneA), + event( + "run.created", + at("09:10:00.000"), + runPayload("completed", { + id: "run-B", + ordinal: 74, + startedAt: at("09:10:00.000"), + completedAt: at("09:10:00.000"), + }), + ), + // A delivery acknowledgement re-writes run A long after it finished. + event("run.updated", at("09:30:00.000"), doneA), + event("thread.visited", at("09:40:00.000"), threadPayload()), + ]); + const finishedA = out.filter( + (emission) => emission.event === "run.finished" && emission.properties.runId === RUN, + ); + assert.strictEqual(finishedA.length, 1); + const runningSpans = spans(out).filter((span) => span.properties.state === "agent_running"); + assert.deepStrictEqual( + runningSpans.map((span) => span.properties.runId), + [RUN], + ); + }); + it("splits a span at UTC midnight", () => { const tracker = makeSessionActivityTracker({ backfill: false }); tracker.apply(event("thread.created", "2026-10-09T22:00:00.000Z", threadPayload())); @@ -446,7 +516,7 @@ describe("SessionActivityReducer", () => { assert.strictEqual(perUser?.userId, OWNER); assert.strictEqual(perUser?.properties.count, 1); - const closed = tracker.closeAll(bootMs + 60_000, "server_stopped"); + const closed = tracker.closeActive(bootMs + 60_000, "server_stopped"); assert.strictEqual(closed.length, 1); assert.strictEqual(closed[0]?.properties.state, "agent_running"); assert.strictEqual(closed[0]?.properties.resumedAfterRestart, true); @@ -496,7 +566,7 @@ describe("SessionActivityReducer", () => { const all = [ ...out, ...tracker.snapshot(epoch(at("11:00:00.000"))), - ...tracker.closeAll(epoch(at("12:00:00.000")), "server_stopped"), + ...tracker.closeActive(epoch(at("12:00:00.000")), "server_stopped"), ]; const allowed = new Set(SESSION_ACTIVITY_PROPERTY_KEYS); for (const emission of all) { diff --git a/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts index db3541fb341c2..5cce5b2c955ec 100644 --- a/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts +++ b/apps/server/src/telemetry/sessionActivity/SessionActivityReducer.ts @@ -39,9 +39,11 @@ * (Claude Agent tool transcripts) are not sessions; their time is the parent's * `agent_running` or `waiting_workers` time. * - * Spans are split at UTC midnight, so per-day sums are exact. A span that - * started before a server restart is closed with `endReason: "server_stopped"`; - * the next server opens a new one with `resumedAfterRestart: true`. + * Spans are split at UTC midnight, so per-day sums are exact. A state that + * lasts no time (several events share one millisecond) emits no span. At + * shutdown, spans of active states close with `endReason: "server_stopped"`; + * idle and archived spans are dropped. The next server seeds every open + * session and opens new spans with `resumedAfterRestart: true`. * * Properties hold only ids, enums, counts, durations and timestamps. Never add * prompt, message, title, question or answer text, paths, branches or @@ -418,6 +420,8 @@ interface ThreadTrack { readonly decidedPlans: Set; readonly subagents: Map; readonly finishedSubagents: Set; + /** Runs already reported; later metadata-only updates of them are ignored. */ + readonly finishedRuns: Set; readonly backgroundTools: Set; readonly rosterTasks: Map; /** Workers a restart seed counted without ids; cleared by the next worker event. */ @@ -444,6 +448,7 @@ function newThread(threadId: string): ThreadTrack { decidedPlans: new Set(), subagents: new Map(), finishedSubagents: new Set(), + finishedRuns: new Set(), backgroundTools: new Set(), rosterTasks: new Map(), seededWorkers: 0, @@ -490,8 +495,16 @@ export interface SessionActivityTracker { * span it opens at `nowMs` carries `resumedAfterRestart: true`. */ readonly seed: (shell: unknown, nowMs: number) => void; - /** Close every open span, for example with `server_stopped` on shutdown. */ - readonly closeAll: (nowMs: number, endReason: string) => ReadonlyArray; + /** + * Close the open spans of active states (everything but `idle` and + * `archived`), for example with `server_stopped` on shutdown. Idle and + * archived time is cheap to lose and is resumed by the next boot's seed, so + * shutdown records only the few spans that matter within its time budget. + */ + readonly closeActive: ( + nowMs: number, + endReason: string, + ) => ReadonlyArray; /** Point-in-time counts: one event per (owner, state), plus one totals event. */ readonly snapshot: (nowMs: number) => ReadonlyArray; } @@ -635,7 +648,10 @@ export function makeSessionActivityTracker( endMs: number, endReason: string, ): ReadonlyArray => { - const end = Math.max(endMs, span.startMs); + // Several events can share one millisecond. A state that lasted no time is + // not a span, and emitting it would also reuse the next span's insert id. + if (endMs <= span.startMs) return []; + const end = endMs; const run = span.run; const common = { ...baseProps(t), @@ -672,7 +688,14 @@ export function makeSessionActivityTracker( t, "session.state.span", segmentEnd, - `session.state.span|${t.threadId}|${segmentStart}`, + [ + "session.state.span", + t.threadId, + span.state, + span.waitKind ?? "", + run?.id ?? "", + segmentStart, + ].join("|"), { ...common, durationMs: segmentEnd - segmentStart, @@ -703,7 +726,10 @@ export function makeSessionActivityTracker( t.span.maxWorkers = Math.max(t.span.maxWorkers, workers); return; } - if (t.span !== undefined) out.push(...spanEmissions(t, t.span, atMs, next.state)); + const previous = t.span; + if (previous !== undefined) out.push(...spanEmissions(t, previous, atMs, next.state)); + // A replaced zero-length span hands on its restart flag. + const carriesResume = previous !== undefined && previous.startMs >= atMs && previous.resumed; t.span = { key, state: next.state, @@ -711,7 +737,7 @@ export function makeSessionActivityTracker( snoozed, ...(next.run === undefined ? {} : { run: next.run }), startMs: atMs, - resumed: t.resumeNextSpan, + resumed: t.resumeNextSpan || carriesResume, maxWorkers: workers, }; t.resumeNextSpan = false; @@ -786,6 +812,7 @@ export function makeSessionActivityTracker( out: Array, ) => { run.finished = true; + rememberBounded(t.finishedRuns, run.id); t.runs.delete(run.id); t.lastRun = run; const completedAtMs = run.completedAtMs ?? atMs; @@ -834,7 +861,9 @@ export function makeSessionActivityTracker( ) => { const id = str(payload, "id"); if (id === undefined) return; - if (t.lastRun?.id === id && t.lastRun.finished) return; + // A finished run keeps receiving updates (delivery acknowledgements and + // the like). Re-reading one would report it twice, and as an imported run. + if (t.finishedRuns.has(id)) return; let run = t.runs.get(id); const firstSeen = run === undefined; if (run === undefined) { @@ -1263,10 +1292,12 @@ export function makeSessionActivityTracker( reconcile(t, nowMs, []); }; - const closeAll: SessionActivityTracker["closeAll"] = (nowMs, endReason) => { + const closeActive: SessionActivityTracker["closeActive"] = (nowMs, endReason) => { const out: Array = []; for (const t of threads.values()) { - if (t.span === undefined) continue; + if (t.span === undefined || t.span.state === "idle" || t.span.state === "archived") { + continue; + } out.push(...spanEmissions(t, t.span, nowMs, endReason)); delete t.span; } @@ -1332,5 +1363,5 @@ export function makeSessionActivityTracker( return out; }; - return { apply, seed, closeAll, snapshot }; + return { apply, seed, closeActive, snapshot }; } diff --git a/apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx b/apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx index 1caea6c6c607c..f55f354e4d6e7 100644 --- a/apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx +++ b/apps/web/src/lib/analytics/ProductAnalyticsCoordinator.tsx @@ -68,15 +68,17 @@ export function ProductAnalyticsCoordinator() { ); } -/** Identifies by work email; falls back to the server user id when it is unknown. */ +/** + * Identifies by work email only, as the server does. Without an email the + * events stay anonymous, and posthog-js merges them once the email arrives. + */ function ProductAnalyticsIdentity() { const userId = useCurrentUserId(); const { resolveUser } = useOrgMembers(); const email = userId === null ? null : resolveUser(userId).email; - const distinctId = email ?? userId; useEffect(() => { - productAnalytics.identify(distinctId); - }, [distinctId]); + productAnalytics.identify(email); + }, [email]); return null; } diff --git a/apps/web/src/lib/analytics/productAnalytics.logic.test.ts b/apps/web/src/lib/analytics/productAnalytics.logic.test.ts index 79edc68cfea94..024b68bf22592 100644 --- a/apps/web/src/lib/analytics/productAnalytics.logic.test.ts +++ b/apps/web/src/lib/analytics/productAnalytics.logic.test.ts @@ -128,6 +128,26 @@ describe("stripUrlProperties", () => { }), ).toEqual({ $browser: "Chrome", view: "/chat" }); }); + + it("removes every session-entry, initial and previous-pageview property by prefix", () => { + expect( + stripUrlProperties({ + $session_entry_url: "https://bkt3/pull-requests?repository=owner/repo", + $session_entry_pathname: "/pull-requests", + $session_entry_referrer: "https://github.com/owner/repo", + $session_entry_host: "bkt3", + $session_entry_some_future_key: "x", + $initial_person_info: { u: "https://bkt3/chat/env/thread", r: "$direct" }, + $initial_referrer_info: { $referrer: "https://github.com" }, + $prev_pageview_pathname: "/chat/env/thread", + $host: "dev-server-1.tailab6257.ts.net", + utm_source: "newsletter", + gclid: "abc", + $session_id: "s1", + instance: "bkt3", + }), + ).toEqual({ $session_id: "s1", instance: "bkt3" }); + }); }); describe("createVisibilityReporter", () => { diff --git a/apps/web/src/lib/analytics/productAnalytics.logic.ts b/apps/web/src/lib/analytics/productAnalytics.logic.ts index 0f7f6c6730ba8..399d12f19dd4d 100644 --- a/apps/web/src/lib/analytics/productAnalytics.logic.ts +++ b/apps/web/src/lib/analytics/productAnalytics.logic.ts @@ -155,27 +155,75 @@ function isTailnetIpv4(hostname: string): boolean { } /** - * Properties PostHog adds to every event on its own. They carry the visited - * URL (thread ids, draft ids) or the window title (thread titles), so they are - * removed in `before_send`. + * Properties PostHog adds on its own that carry the visited URL (thread ids, + * `?repository=owner/repo`), the host, the referrer or the window title + * (thread titles). They are removed in `before_send`, from event properties + * and from `$set` / `$set_once`. Checked against posthog-js 1.438. */ export const STRIPPED_POSTHOG_PROPERTIES = [ "$current_url", "$pathname", + "$host", "$title", "$referrer", "$referring_domain", - "$initial_current_url", - "$initial_pathname", - "$initial_referrer", - "$initial_referring_domain", - "$initial_title", - "$prev_pageview_pathname", + "$external_click_url", + "$el_text", + "$elements", + "$elements_chain", + // Campaign and click ids are read from the URL query string. + "utm_source", + "utm_medium", + "utm_campaign", + "utm_content", + "utm_term", + "gad_source", + "mc_cid", + "gclid", + "gclsrc", + "dclid", + "gbraid", + "wbraid", + "fbclid", + "msclkid", + "twclid", + "li_fat_id", + "igshid", + "ttclid", + "rdt_cid", + "epik", + "qclid", + "sccid", + "oppref", + "irclid", + "_kx", ] as const; +/** + * Prefixes of PostHog property families built from the URL or referrer: + * `$initial_*` (first-touch person info), `$session_entry_*` (the session's + * entry URL, path, host and referrer), `$prev_pageview_*` (the previous path). + */ +export const STRIPPED_POSTHOG_PROPERTY_PREFIXES = [ + "$initial_", + "$session_entry_", + "$prev_pageview_", +] as const; + +const STRIPPED_PROPERTY_SET: ReadonlySet = new Set(STRIPPED_POSTHOG_PROPERTIES); + +function isStrippedProperty(key: string): boolean { + return ( + STRIPPED_PROPERTY_SET.has(key) || + STRIPPED_POSTHOG_PROPERTY_PREFIXES.some((prefix) => key.startsWith(prefix)) + ); +} + export function stripUrlProperties>(properties: T): T { - const next: Record = { ...properties }; - for (const key of STRIPPED_POSTHOG_PROPERTIES) delete next[key]; + const next: Record = {}; + for (const [key, value] of Object.entries(properties)) { + if (!isStrippedProperty(key)) next[key] = value; + } return next as T; } diff --git a/apps/web/src/lib/analytics/productAnalytics.test.ts b/apps/web/src/lib/analytics/productAnalytics.test.ts index ee196034d21bf..b71151b9db272 100644 --- a/apps/web/src/lib/analytics/productAnalytics.test.ts +++ b/apps/web/src/lib/analytics/productAnalytics.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test"; import { createProductAnalytics, + normalizeAnalyticsEmail, type ProductAnalyticsClient, type ProductAnalyticsContext, type ProductAnalyticsLoader, @@ -12,22 +13,23 @@ interface FakeClient { readonly captured: Array<{ event: string; properties: unknown }>; readonly identified: string[]; readonly registered: Array>; + readonly optIns: unknown[]; optedOut: boolean; } function fakeLoader() { const clients: FakeClient[] = []; - let loads = 0; const load: ProductAnalyticsLoader = async (key) => { - loads += 1; const fake: FakeClient = { key, captured: [], identified: [], registered: [], + optIns: [], optedOut: false, }; clients.push(fake); + // opt_out_capturing is present so a regression that calls it is visible. const client = { capture: (event: string, properties?: unknown) => { fake.captured.push({ event, properties }); @@ -39,13 +41,16 @@ function fakeLoader() { register: (properties: Record) => { fake.registered.push(properties); }, + opt_in_capturing: (options?: unknown) => { + fake.optIns.push(options); + }, opt_out_capturing: () => { fake.optedOut = true; }, } as unknown as ProductAnalyticsClient; return client; }; - return { load, clients, loadCount: () => loads }; + return { load, clients }; } const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); @@ -69,7 +74,7 @@ describe("createProductAnalytics", () => { analytics.configure(context(null)); analytics.track("composer.sent", { threadId: "t1" }); await flush(); - expect(loader.loadCount()).toBe(0); + expect(loader.clients).toEqual([]); expect(analytics.isActive()).toBe(false); }); @@ -79,19 +84,20 @@ describe("createProductAnalytics", () => { analytics.configure(context({ ...enabledConfig, enabled: false })); analytics.track("app.opened"); await flush(); - expect(loader.loadCount()).toBe(0); + expect(loader.clients).toEqual([]); }); it("queues events until loaded, then sends sanitized properties", async () => { const loader = fakeLoader(); const analytics = createProductAnalytics(loader.load); analytics.configure(context(enabledConfig)); - analytics.identify("person@beknown.work"); + analytics.identify(" Person@Beknown.Work "); analytics.track("composer.sent", { threadId: "t1", prompt: "do not send me" }); await flush(); const [client] = loader.clients; expect(client?.captured).toEqual([{ event: "composer.sent", properties: { threadId: "t1" } }]); expect(client?.identified).toEqual(["person@beknown.work"]); + expect(client?.optIns).toEqual([{ captureEventName: null }]); expect(client?.registered.at(-1)).toEqual({ surface: "web", appVersion: "1.2.3", @@ -100,17 +106,55 @@ describe("createProductAnalytics", () => { }); }); - it("retires the old client when the key changes and stops when disabled", async () => { + it("never opts out: a config flap stops forwarding and reuses the same client", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.configure(context(enabledConfig)); + await flush(); + analytics.configure(context(null)); + analytics.track("app.opened"); + analytics.configure(context(enabledConfig)); + await flush(); + analytics.track("app.foregrounded"); + expect(loader.clients).toHaveLength(1); + expect(loader.clients[0]?.optedOut).toBe(false); + expect(loader.clients[0]?.captured.map((entry) => entry.event)).toEqual(["app.foregrounded"]); + }); + + it("sends to the new key only after a key change, without opting the old one out", async () => { const loader = fakeLoader(); const analytics = createProductAnalytics(loader.load); analytics.configure(context(enabledConfig)); await flush(); analytics.configure(context({ ...enabledConfig, posthogKey: "phc_other" })); await flush(); - expect(loader.clients.map((client) => client.optedOut)).toEqual([true, false]); - analytics.configure(context(null)); analytics.track("app.opened"); - expect(loader.clients[1]?.optedOut).toBe(true); - expect(loader.clients[1]?.captured).toEqual([]); + expect(loader.clients.map((client) => client.optedOut)).toEqual([false, false]); + expect(loader.clients[0]?.captured).toEqual([]); + expect(loader.clients[1]?.captured.map((entry) => entry.event)).toEqual(["app.opened"]); + }); + + it("stays anonymous until an email is known, so one person is never split", async () => { + const loader = fakeLoader(); + const analytics = createProductAnalytics(loader.load); + analytics.configure(context(enabledConfig)); + analytics.identify(null); + analytics.identify("user_2abc"); + analytics.track("app.opened"); + await flush(); + const [client] = loader.clients; + expect(client?.identified).toEqual([]); + expect(client?.captured.map((entry) => entry.event)).toEqual(["app.opened"]); + analytics.identify("person@beknown.work"); + analytics.identify("PERSON@beknown.work"); + expect(client?.identified).toEqual(["person@beknown.work"]); + }); +}); + +describe("normalizeAnalyticsEmail", () => { + it("trims and lower-cases emails and rejects anything else", () => { + expect(normalizeAnalyticsEmail(" A@B.co ")).toBe("a@b.co"); + expect(normalizeAnalyticsEmail("user_2abc")).toBeNull(); + expect(normalizeAnalyticsEmail(null)).toBeNull(); }); }); diff --git a/apps/web/src/lib/analytics/productAnalytics.ts b/apps/web/src/lib/analytics/productAnalytics.ts index eb03a76e45a43..ee4464c4276e8 100644 --- a/apps/web/src/lib/analytics/productAnalytics.ts +++ b/apps/web/src/lib/analytics/productAnalytics.ts @@ -34,7 +34,7 @@ export interface ProductAnalyticsContext { /** The part of the PostHog client this module calls. */ export type ProductAnalyticsClient = Pick< PostHog, - "capture" | "identify" | "register" | "opt_out_capturing" + "capture" | "identify" | "register" | "opt_in_capturing" >; export type ProductAnalyticsLoader = ( @@ -49,7 +49,6 @@ const MAX_PENDING_EVENTS = 50; interface ActiveSession { readonly key: string; client: ProductAnalyticsClient | null; - closed: boolean; readonly pending: Array<{ event: string; properties: Record }>; } @@ -86,28 +85,45 @@ function postHogOptions(host: string): Partial { }; } +/** Matches the server's distinct id (`analyticsIdentity.ts`): trimmed, lower case. */ +export function normalizeAnalyticsEmail(email: string | null | undefined): string | null { + const normalized = email?.trim().toLowerCase() ?? ""; + return normalized.includes("@") ? normalized : null; +} + export function createProductAnalytics(load: ProductAnalyticsLoader = loadPostHog) { + // One PostHog instance per key for the page's life. Turning analytics off only + // stops forwarding: `opt_out_capturing` would persist for a year per token, + // so a brief config flap must never reach it. + const clients = new Map>(); + const identifiedAs = new WeakMap(); let session: ActiveSession | null = null; let superProperties: Record = {}; - let distinctId: string | null = null; - let identifiedAs: string | null = null; - let sessionCount = 0; + let email: string | null = null; let lastInputWasKeyboard = false; const applyIdentity = (client: ProductAnalyticsClient) => { client.register(superProperties); - if (distinctId !== null && identifiedAs !== distinctId) { - client.identify(distinctId); - identifiedAs = distinctId; + // Until the email is known, events stay anonymous; identify(email) later + // merges them into the person. Identifying by a user id first would split it. + if (email !== null && identifiedAs.get(client) !== email) { + client.identify(email); + identifiedAs.set(client, email); } }; - const close = () => { - if (session === null) return; - session.closed = true; - session.client?.opt_out_capturing(); - session = null; - identifiedAs = null; + const clientFor = (key: string, host: string): Promise => { + const existing = clients.get(key); + if (existing !== undefined) return existing; + const created = load(key, postHogOptions(host), `bkt3_${clients.size + 1}`).then((client) => { + // Clears an opt-out a build that still called opt_out_capturing persisted. + client.opt_in_capturing({ captureEventName: null }); + return client; + }); + clients.set(key, created); + // A failed load may be retried by the next configure. + created.catch(() => clients.delete(key)); + return created; }; return { @@ -121,28 +137,18 @@ export function createProductAnalytics(load: ProductAnalyticsLoader = loadPostHo ...(context.connectionMode === null ? {} : { connectionMode: context.connectionMode }), }; if (config === null) { - close(); + session = null; return; } if (session !== null && session.key === config.posthogKey) { if (session.client !== null) applyIdentity(session.client); return; } - close(); - const next: ActiveSession = { - key: config.posthogKey, - client: null, - closed: false, - pending: [], - }; + const next: ActiveSession = { key: config.posthogKey, client: null, pending: [] }; session = next; - sessionCount += 1; - void load(config.posthogKey, postHogOptions(config.posthogHost), `bkt3_${sessionCount}`) + clientFor(config.posthogKey, config.posthogHost) .then((client) => { - if (next.closed) { - client.opt_out_capturing(); - return; - } + if (session !== next) return; next.client = client; applyIdentity(client); for (const { event, properties } of next.pending.splice(0)) { @@ -155,9 +161,11 @@ export function createProductAnalytics(load: ProductAnalyticsLoader = loadPostHo }); }, - /** The user's work email, or the server user id when the email is unknown. */ - identify(id: string | null): void { - distinctId = id; + /** The user's work email. Anything that is not an email is ignored. */ + identify(rawEmail: string | null): void { + const normalized = normalizeAnalyticsEmail(rawEmail); + if (normalized === null) return; + email = normalized; if (session?.client) applyIdentity(session.client); }, diff --git a/knip.jsonc b/knip.jsonc index 8e1b9a45161e2..55125e838996c 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -37,6 +37,7 @@ "entry": [ "src/bin.ts!", "src/claude-history-worker.ts!", + "scripts/analytics-backfill.ts", "scripts/cli.ts", "scripts/evaluate-thread-titles.ts", "scripts/measure-pr-preview.ts", From b8a1fc0a36ea12bf52f6b52a9e3e55f08fa39c1a Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Wed, 14 Oct 2026 10:37:17 +0000 Subject: [PATCH 08/20] chore(web): mark fork analytics edits (TEC-1631) Co-Authored-By: Claude Opus 5.5 --- apps/web/src/components/ChatView.tsx | 10 ++++++++++ apps/web/src/components/CommandPalette.tsx | 2 ++ apps/web/src/hooks/useSettings.ts | 2 ++ knip.jsonc | 1 + 4 files changed, 15 insertions(+) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 382845b9339c0..42b61847c5f7e 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -9230,11 +9230,13 @@ export default function ChatView(props: ChatViewProps) { promptRef.current = ""; clearComposerDraftContent(composerDraftTarget); composerRef.current?.resetCursorState(); + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics trackProductEvent("plan.action", { threadId: activeThread.id, action: followUp.interactionMode === "plan" ? "revise" : "approve", source: "composer", }); + // T3-CUSTOM(expbkt3): END const followUpSent = await onSubmitPlanFollowUp({ text: followUp.text, context: buildMessageContext({ @@ -9473,12 +9475,14 @@ export default function ChatView(props: ChatViewProps) { setThreadError(threadIdForSend, attachmentCapabilitiesBeforeUpload.fileBlockReason); return; } + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics trackProductEvent("composer.sent", { threadId: threadIdForSend, interactionMode: sendInteractionMode, hasAttachments: composerAttachmentsSnapshot.length > 0, viaKeyboard: lastInputWasKeyboard(), }); + // T3-CUSTOM(expbkt3): END const turnUsesAttachmentUploads = composerFilesSnapshot.length > 0 ? attachmentCapabilitiesBeforeUpload.supportsAttachmentUploads @@ -10252,11 +10256,13 @@ export default function ChatView(props: ChatViewProps) { return; const pendingApproval = pendingApprovals.find((approval) => approval.requestId === requestId); if (pendingApproval?.responseCapability !== "live") return; + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics trackProductEvent("approval.answered_in_ui", { threadId: activeThreadId, kind: pendingApproval.requestKind, decision, }); + // T3-CUSTOM(expbkt3): END setRespondingRequestIds((existing) => existing.includes(requestId) ? existing : [...existing, requestId], @@ -10319,10 +10325,12 @@ export default function ChatView(props: ChatViewProps) { ); } userInputResponsesInFlight.current.add(responseKey); + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics trackProductEvent("question.answered_in_ui", { threadId: activeThreadId, kind: "user_input", }); + // T3-CUSTOM(expbkt3): END setRespondingUserInputRequestIds((existing) => existing.includes(requestId) ? existing : [...existing, requestId], @@ -10700,11 +10708,13 @@ export default function ChatView(props: ChatViewProps) { const nextThreadTitle = truncate(buildPlanImplementationThreadTitle(planMarkdown)); const nextThreadModelSelection: ModelSelection = ctxSelectedModelSelection; + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics trackProductEvent("plan.action", { threadId: activeThread.id, action: "approve", source: "new_thread", }); + // T3-CUSTOM(expbkt3): END sendInFlightRef.current = true; beginLocalDispatch({ preparingWorktree: false }); const finish = () => { diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index 2fb3c78976394..b7fb3dbaa2906 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -3203,9 +3203,11 @@ function OpenCommandPaletteDialog(props: { setOpen(false); } + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics trackProductEvent("command_palette.used", { commandId: commandIdForPaletteItem(item), }); + // T3-CUSTOM(expbkt3): END void item.run().catch((error: unknown) => { toastManager.add( stackedThreadToast({ diff --git a/apps/web/src/hooks/useSettings.ts b/apps/web/src/hooks/useSettings.ts index 57e86ff4cd1a5..64f209840ccae 100644 --- a/apps/web/src/hooks/useSettings.ts +++ b/apps/web/src/hooks/useSettings.ts @@ -479,6 +479,7 @@ function useUpdateSettingsTarget(environmentId: EnvironmentId | null) { const sharedSettingsSyncTargetIds = useSharedSettingsSyncTargetIds(true); const updateSettings = useCallback( (patch: UnifiedSettingsPatch) => { + // T3-CUSTOM(expbkt3): TEC-1631 product analytics trackSettingsChanged(Object.keys(patch)); const { serverPatch, clientPatch } = splitPatch(patch); @@ -580,6 +581,7 @@ export function useUpdatePrimarySettings() { export function useUpdateClientSettings() { return useCallback((patch: ClientSettingsPatch) => { + // T3-CUSTOM(expbkt3): TEC-1631 product analytics trackSettingsChanged(Object.keys(patch)); return persistClientSettingsPatch(patch); }, []); diff --git a/knip.jsonc b/knip.jsonc index 55125e838996c..607baf19f5d4f 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -37,6 +37,7 @@ "entry": [ "src/bin.ts!", "src/claude-history-worker.ts!", + // T3-CUSTOM(expbkt3): TEC-1631 product analytics "scripts/analytics-backfill.ts", "scripts/cli.ts", "scripts/evaluate-thread-titles.ts", From a3ac8b63aa3df31491b1e596ccc586d390cc2f91 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Wed, 14 Oct 2026 10:37:33 +0000 Subject: [PATCH 09/20] chore(web): mark fork analytics edits (TEC-1631) Co-Authored-By: Claude Opus 5.5 --- apps/web/src/components/ChatView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 42b61847c5f7e..b9621544fd7c4 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -10254,9 +10254,9 @@ export default function ChatView(props: ChatViewProps) { async (requestId: RuntimeRequestId, decision: ProviderApprovalDecision) => { if (!activeThreadId || !readEnvironmentScope(environmentId, AuthOrchestrationOperateScope)) return; + // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics const pendingApproval = pendingApprovals.find((approval) => approval.requestId === requestId); if (pendingApproval?.responseCapability !== "live") return; - // T3-CUSTOM(expbkt3): BEGIN TEC-1631 product analytics trackProductEvent("approval.answered_in_ui", { threadId: activeThreadId, kind: pendingApproval.requestKind, From 20539ad467b6117234eb69e247ebe9b2f13ea42a Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Wed, 14 Oct 2026 11:39:30 +0000 Subject: [PATCH 10/20] fix(lint): analytics naming for repo lint rules (TEC-1631) Co-Authored-By: Claude Opus 5.5 --- apps/server/scripts/analytics-backfill.ts | 14 +++++++------- apps/web/src/lib/analytics/productAnalytics.ts | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/apps/server/scripts/analytics-backfill.ts b/apps/server/scripts/analytics-backfill.ts index 1e969e3ac5ecd..6a096d73e49fa 100644 --- a/apps/server/scripts/analytics-backfill.ts +++ b/apps/server/scripts/analytics-backfill.ts @@ -30,8 +30,8 @@ * `$insert_id` and uuid, so a re-run does not duplicate events. */ import * as NodeCrypto from "node:crypto"; -import * as NodeFs from "node:fs"; -import * as NodeOs from "node:os"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import * as NodeSqlite from "node:sqlite"; @@ -107,8 +107,8 @@ function parseArgs(argv: ReadonlyArray): Options { /** Refuses any state file under a T3 home `userdata` directory: those are live. */ function assertNotLiveState(dbPath: string): string { - const resolved = NodeFs.realpathSync(dbPath); - const t3Home = NodePath.join(NodeOs.homedir(), ".t3"); + const resolved = NodeFS.realpathSync(dbPath); + const t3Home = NodePath.join(NodeOS.homedir(), ".t3"); const relative = NodePath.relative(t3Home, resolved); const insideT3Home = !relative.startsWith("..") && !NodePath.isAbsolute(relative); if (insideT3Home && relative.split(NodePath.sep).includes("userdata")) { @@ -330,7 +330,7 @@ async function main() { const emails = loadUserEmails(db); const tracker = makeSessionActivityTracker({ backfill: true }); - const out = options.send ? undefined : NodeFs.openSync(options.outPath, "w"); + const out = options.send ? undefined : NodeFS.openSync(options.outPath, "w"); const counts = new Map(); const stateHours = new Map(); const newest = db @@ -351,7 +351,7 @@ async function main() { await sendBatch(host, apiKey, batch); sent += batch.length; } else if (out !== undefined) { - NodeFs.writeSync(out, batch.map((item) => JSON.stringify(item)).join("\n") + "\n"); + NodeFS.writeSync(out, batch.map((item) => JSON.stringify(item)).join("\n") + "\n"); } }; @@ -413,7 +413,7 @@ async function main() { } if (threadId !== undefined) await replayThread(); await flush(); - if (out !== undefined) NodeFs.closeSync(out); + if (out !== undefined) NodeFS.closeSync(out); db.close(); console.log(`threads replayed: ${threads}`); diff --git a/apps/web/src/lib/analytics/productAnalytics.ts b/apps/web/src/lib/analytics/productAnalytics.ts index ee4464c4276e8..7fa36d37edefe 100644 --- a/apps/web/src/lib/analytics/productAnalytics.ts +++ b/apps/web/src/lib/analytics/productAnalytics.ts @@ -48,7 +48,7 @@ const MAX_PENDING_EVENTS = 50; interface ActiveSession { readonly key: string; - client: ProductAnalyticsClient | null; + posthog: ProductAnalyticsClient | null; readonly pending: Array<{ event: string; properties: Record }>; } @@ -141,15 +141,15 @@ export function createProductAnalytics(load: ProductAnalyticsLoader = loadPostHo return; } if (session !== null && session.key === config.posthogKey) { - if (session.client !== null) applyIdentity(session.client); + if (session.posthog !== null) applyIdentity(session.posthog); return; } - const next: ActiveSession = { key: config.posthogKey, client: null, pending: [] }; + const next: ActiveSession = { key: config.posthogKey, posthog: null, pending: [] }; session = next; clientFor(config.posthogKey, config.posthogHost) .then((client) => { if (session !== next) return; - next.client = client; + next.posthog = client; applyIdentity(client); for (const { event, properties } of next.pending.splice(0)) { client.capture(event, properties); @@ -166,14 +166,14 @@ export function createProductAnalytics(load: ProductAnalyticsLoader = loadPostHo const normalized = normalizeAnalyticsEmail(rawEmail); if (normalized === null) return; email = normalized; - if (session?.client) applyIdentity(session.client); + if (session?.posthog) applyIdentity(session.posthog); }, track(event: ProductAnalyticsEvent, properties?: Readonly>): void { if (session === null) return; const clean = sanitizeAnalyticsProperties(event, properties); - if (session.client !== null) { - session.client.capture(event, clean); + if (session.posthog !== null) { + session.posthog.capture(event, clean); return; } if (session.pending.length >= MAX_PENDING_EVENTS) session.pending.shift(); From 564ef6f9a688bae11275da7de52778cd64f1d336 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Thu, 15 Oct 2026 15:38:24 +0000 Subject: [PATCH 11/20] fix(web): the BK sidebar drops the PLAN tag once plan mode is off A thread kept its PLAN pill, Plan Ready group and needs-human count for as long as its plan stayed active, even after plan mode was turned off. A plan now waits only while its thread is in plan mode, matching upstream's Plan Ready rule, so stale tags clear without a migration. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e925a5bf529057fad55bf1b68b436b4a34c223d8) --- .../sidebar/PhaseGroupedSidebar.logic.test.ts | 73 ++++++++++++++----- .../sidebar/PhaseSidebarTree.logic.test.ts | 20 ++++- .../sidebar/sidebarSessionCounters.test.ts | 14 +++- .../client-runtime/src/state/phaseSidebar.ts | 34 ++++++--- 4 files changed, 109 insertions(+), 32 deletions(-) diff --git a/apps/web/src/components/sidebar/PhaseGroupedSidebar.logic.test.ts b/apps/web/src/components/sidebar/PhaseGroupedSidebar.logic.test.ts index 289dc04aa8b38..16e63abd789de 100644 --- a/apps/web/src/components/sidebar/PhaseGroupedSidebar.logic.test.ts +++ b/apps/web/src/components/sidebar/PhaseGroupedSidebar.logic.test.ts @@ -809,7 +809,11 @@ describe("phase sidebar lifecycle", () => { ).toBe("implementing"); expect( resolvePhaseSidebarPhase( - makeThread({ hasActionableProposedPlan: true, backgroundLiveness: null }), + makeThread({ + interactionMode: "plan", + hasActionableProposedPlan: true, + backgroundLiveness: null, + }), ), ).toBe("plan_ready"); }); @@ -853,15 +857,29 @@ describe("phase sidebar lifecycle", () => { ]); }); - // T3-CUSTOM(expbkt3): BEGIN — Plan Ready means a plan is waiting, in any mode. - it("files a plan submitted from a default-mode turn under Plan Ready", () => { - // `t3_submit_plan` never changes interactionMode, so gating the group on - // plan mode hid every plan an agent submitted mid-turn. + // T3-CUSTOM(expbkt3): BEGIN — Plan Ready means a plan is waiting, and a plan + // waits only while its thread is in plan mode. + it("files a held plan under Plan Ready only while the thread is in plan mode", () => { expect( resolvePhaseSidebarPhase( - makeThread({ interactionMode: "default", hasActionableProposedPlan: true }), + makeThread({ interactionMode: "plan", hasActionableProposedPlan: true }), ), ).toBe("plan_ready"); + // Turning plan mode off clears a stale plan out of the group. + expect( + resolvePhaseSidebarPhase( + makeThread({ interactionMode: "default", hasActionableProposedPlan: true }), + ), + ).toBe("ready"); + expect( + resolvePhaseSidebarPhase( + makeThread({ + interactionMode: "default", + hasActionableProposedPlan: true, + runtime: makeSession("error"), + }), + ), + ).toBe("ready"); }); it("keeps a waiting plan out of Plan Ready while the agent is still running", () => { @@ -979,6 +997,7 @@ describe("phase sidebar lifecycle", () => { expect( resolvePhaseSidebarPhase( makeThread({ + interactionMode: "plan", hasPendingAsyncUserInput: true, hasActionableProposedPlan: true, }), @@ -991,7 +1010,11 @@ describe("phase sidebar lifecycle", () => { // this deliberately stays "plan": both badges have to survive together. expect( resolvePhaseSidebarAttentionKind( - makeThread({ hasActionableProposedPlan: true, hasPendingAsyncUserInput: true }), + makeThread({ + interactionMode: "plan", + hasActionableProposedPlan: true, + hasPendingAsyncUserInput: true, + }), ), ).toBe("plan"); expect(resolvePhaseSidebarAttentionKind(makeThread({ hasPendingAsyncUserInput: true }))).toBe( @@ -1074,35 +1097,47 @@ describe("phase sidebar attention badges", () => { // T3-CUSTOM(expbkt3): BEGIN — a plan waiting on a human is attention too. it("reports a waiting plan as attention, ranked below every blocking kind", () => { - expect(resolvePhaseSidebarAttentionKind(makeThread({ hasActionableProposedPlan: true }))).toBe( - "plan", - ); + const plan = { interactionMode: "plan", hasActionableProposedPlan: true } as const; + expect(resolvePhaseSidebarAttentionKind(makeThread(plan))).toBe("plan"); // A question, an approval and a failure each outrank it. expect( - resolvePhaseSidebarAttentionKind( - makeThread({ hasActionableProposedPlan: true, hasPendingUserInput: true }), - ), + resolvePhaseSidebarAttentionKind(makeThread({ ...plan, hasPendingUserInput: true })), ).toBe("input"); + expect( + resolvePhaseSidebarAttentionKind(makeThread({ ...plan, hasPendingApprovals: true })), + ).toBe("approval"); + expect( + resolvePhaseSidebarAttentionKind(makeThread({ ...plan, runtime: makeSession("error") })), + ).toBe("error"); + }); + + it("drops the plan badge once plan mode is off", () => { expect( resolvePhaseSidebarAttentionKind( - makeThread({ hasActionableProposedPlan: true, hasPendingApprovals: true }), + makeThread({ interactionMode: "default", hasActionableProposedPlan: true }), ), - ).toBe("approval"); + ).toBeNull(); + // An async question on the same thread still flies its own badge. expect( resolvePhaseSidebarAttentionKind( makeThread({ + interactionMode: "default", hasActionableProposedPlan: true, - runtime: makeSession("error"), + hasPendingAsyncUserInput: true, }), ), - ).toBe("error"); + ).toBe("ask"); }); it("badges a plan that arrives while the agent is still working", () => { - // The row stays in Implementing, but it still has to say a plan is there. + // The row stays in Planning, but it still has to say a plan is there. expect( resolvePhaseSidebarAttentionKind( - makeThread({ hasActionableProposedPlan: true, runtime: makeSession("running") }), + makeThread({ + interactionMode: "plan", + hasActionableProposedPlan: true, + runtime: makeSession("running"), + }), ), ).toBe("plan"); }); diff --git a/apps/web/src/components/sidebar/PhaseSidebarTree.logic.test.ts b/apps/web/src/components/sidebar/PhaseSidebarTree.logic.test.ts index 06b78eaa65758..a64377ff6fbc8 100644 --- a/apps/web/src/components/sidebar/PhaseSidebarTree.logic.test.ts +++ b/apps/web/src/components/sidebar/PhaseSidebarTree.logic.test.ts @@ -71,7 +71,7 @@ function makeRow( readonly archived?: boolean; readonly pendingApproval?: boolean; readonly unread?: boolean; - // T3-CUSTOM(expbkt3): the row holds an actionable plan. + // T3-CUSTOM(expbkt3): the row holds an actionable plan, in plan mode. readonly plan?: boolean; } = {}, ): PhaseSidebarRow { @@ -87,7 +87,7 @@ function makeRow( : {}), ...(options.archived ? { archivedAt: now } : {}), ...(options.pendingApproval ? { hasPendingApprovals: true } : {}), - ...(options.plan ? { hasActionableProposedPlan: true } : {}), + ...(options.plan ? { interactionMode: "plan" as const, hasActionableProposedPlan: true } : {}), }); return { thread, @@ -455,6 +455,22 @@ describe("resolvePhaseSidebarTreePhase", () => { expect(resolvePhaseSidebarTreePhase(tree[0] as never)).toBe("implementing"); }); + it("does not hoist a stale plan once the child left plan mode", () => { + // Turning plan mode off files the child under Ready, so the collapsed + // parent must not keep reporting a plan underneath it. + const child = makeRow("child", { parent: "parent", phaseId: "ready", plan: true }); + const tree = buildPhaseSidebarTree( + [ + makeRow("parent", { phaseId: "ready" }), + { ...child, thread: { ...child.thread, interactionMode: "default" } }, + ], + { compareSiblings: byId }, + ); + + expect(tree[0]?.descendantAttention).toBeNull(); + expect(resolvePhaseSidebarTreePhase(tree[0] as never)).toBe("ready"); + }); + it("still goes red when the subtree holds both a plan and a question", () => { const tree = buildPhaseSidebarTree( [ diff --git a/apps/web/src/components/sidebar/sidebarSessionCounters.test.ts b/apps/web/src/components/sidebar/sidebarSessionCounters.test.ts index abe9669b6bf29..df5cfc6b789b3 100644 --- a/apps/web/src/components/sidebar/sidebarSessionCounters.test.ts +++ b/apps/web/src/components/sidebar/sidebarSessionCounters.test.ts @@ -102,10 +102,22 @@ describe("sidebar session counters", () => { it("classifies approvals, input, plans, and failures as human attention", () => { expect(threadNeedsHumanAttention(makeThread({ hasPendingApprovals: true }))).toBe(true); expect(threadNeedsHumanAttention(makeThread({ hasPendingUserInput: true }))).toBe(true); - expect(threadNeedsHumanAttention(makeThread({ hasActionableProposedPlan: true }))).toBe(true); + expect( + threadNeedsHumanAttention( + makeThread({ interactionMode: "plan", hasActionableProposedPlan: true }), + ), + ).toBe(true); expect(threadNeedsHumanAttention(makeThread({ runtime: makeSession("error") }))).toBe(true); }); + it("stops counting a held plan once plan mode is off", () => { + expect( + threadNeedsHumanAttention( + makeThread({ interactionMode: "default", hasActionableProposedPlan: true }), + ), + ).toBe(false); + }); + it("counts starting and running sessions as running", () => { expect(threadIsRunning(makeThread({ runtime: makeSession("running") }))).toBe(true); expect(threadIsRunning(makeThread({ runtime: makeSession("starting") }))).toBe(true); diff --git a/packages/client-runtime/src/state/phaseSidebar.ts b/packages/client-runtime/src/state/phaseSidebar.ts index bd8801ca02e97..1660d7b2ddbde 100644 --- a/packages/client-runtime/src/state/phaseSidebar.ts +++ b/packages/client-runtime/src/state/phaseSidebar.ts @@ -970,6 +970,17 @@ export function phaseSidebarHasAsyncQuestion( return thread.hasPendingAsyncUserInput === true; } +/** + * T3-CUSTOM(expbkt3): a plan waits on a human only while its thread is in plan + * mode. Turning plan mode off clears the PLAN pill, the Plan Ready group and + * the needs-human count, the same rule as upstream's Plan Ready pill. + */ +export function phaseSidebarHasWaitingPlan( + thread: Pick, +): boolean { + return thread.hasActionableProposedPlan && thread.interactionMode === "plan"; +} + /** * T3-CUSTOM(expbkt3): is an agent actually working on this thread right now? * @@ -1011,6 +1022,8 @@ export function resolvePhaseSidebarAttentionKind( | "hasPendingApprovals" | "hasPendingUserInput" | "hasActionableProposedPlan" + // T3-CUSTOM(expbkt3): a plan waits only in plan mode (`phaseSidebarHasWaitingPlan`). + | "interactionMode" // T3-CUSTOM(expbkt3): an async question is attention that leaves the agent running. | "hasPendingAsyncUserInput" // T3-CUSTOM(expbkt3): a session error is a failure the badge has to show. @@ -1035,7 +1048,7 @@ export function resolvePhaseSidebarAttentionKind( // It deliberately loses to `plan` in the row badges rather than replacing it: // callers render ASK from `phaseSidebarHasAsyncQuestion` alongside whatever // this returns, so a thread holding both flies both badges. - if (thread.hasActionableProposedPlan) return "plan"; + if (phaseSidebarHasWaitingPlan(thread)) return "plan"; if (phaseSidebarHasAsyncQuestion(thread)) return "ask"; return null; } @@ -1070,8 +1083,8 @@ export function resolvePhaseSidebarPhase( // Sidebar V2's reliability ordering: a failure must not be hidden by // liveness that can linger while background work winds down. if (hasFailure) { - // T3-CUSTOM(expbkt3): a failure still files a held plan under Plan Ready. - return thread.hasActionableProposedPlan ? "plan_ready" : "ready"; + // T3-CUSTOM(expbkt3): a failure still files a waiting plan under Plan Ready. + return phaseSidebarHasWaitingPlan(thread) ? "plan_ready" : "ready"; } // A settled foreground turn can still own native subagents, workflows, or @@ -1085,12 +1098,12 @@ export function resolvePhaseSidebarPhase( return thread.interactionMode === "plan" ? "planning" : "implementing"; } - // T3-CUSTOM(expbkt3): Plan Ready means a plan is waiting for a human, in any - // interaction mode. `t3_submit_plan` never changes interactionMode, so gating - // this on plan mode hid every plan submitted from a default-mode turn. A - // settled turn holding an actionable plan outranks a failure: the row's whole - // job is the decision, and the failure still flies its own badge. - if (thread.hasActionableProposedPlan) return "plan_ready"; + // T3-CUSTOM(expbkt3): Plan Ready means a plan is waiting for a human, and a + // plan waits only while its thread is in plan mode + // (`phaseSidebarHasWaitingPlan`). Turning plan mode off files a stale plan + // under Ready. A settled turn holding a waiting plan outranks a failure: the + // row's whole job is the decision, and the failure still flies its own badge. + if (phaseSidebarHasWaitingPlan(thread)) return "plan_ready"; // T3-CUSTOM(expbkt3): an idle plan-mode thread with nothing to decide is just // idle. It used to land in Plan Ready, which made the group unscannable — the @@ -1513,7 +1526,8 @@ export function threadNeedsHumanAttention(thread: ThreadShell): boolean { // T3-CUSTOM(expbkt3): an async question is a human step too, even though it // leaves the agent running. phaseSidebarHasAsyncQuestion(thread) || - thread.hasActionableProposedPlan || + // T3-CUSTOM(expbkt3): a plan counts only while the thread is in plan mode. + phaseSidebarHasWaitingPlan(thread) || thread.runtime?.status === "failed" ); } From e418cdfee5e8b70fe4f0f3b6e9a7fb935a53b89c Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Thu, 15 Oct 2026 15:38:26 +0000 Subject: [PATCH 12/20] feat(web): plan cards render like chat, go wide, and take comments - Remove the plan card's Preview button; Expand plan stays. - New header: icon, title, state, step count, reading time, and a Widen plan toggle that lets cards use the free width beside the chat. - Plan bodies get the chat's markdown features (skills, runnable shell commands, image preview, artifact templates). - HTML in a plan renders as a live themed page, like html_render: a whole HTML plan, a raw HTML document, or an html-render fence. - Selecting plan text offers Cite and the comment actions; highlights, pins and scroll-to-quote work on plans, and the panel labels them. - Plan-mode turns tell the agent how to add an html-render visual. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 0d144384c4dc684befbe9137d602916c3b3b03e5) --- .../orchestration-v2/RunExecutionService.ts | 6 +- .../forkAgentPlanInstructions.expbkt3.test.ts | 21 ++- .../forkAgentPlanInstructions.expbkt3.ts | 25 ++- apps/web/src/components/ChatView.tsx | 8 - .../src/components/chat/MessagesTimeline.tsx | 37 ++-- .../src/components/chat/ProposedPlanCard.tsx | 83 +++------ .../chat/useAssistantCitationTarget.ts | 19 ++- .../web/src/fork/planCard/PlanCardHeading.tsx | 96 +++++++++++ apps/web/src/fork/planCard/PlanCardRow.tsx | 94 +++++++++++ apps/web/src/fork/planCard/PlanHtmlFrame.tsx | 45 +++++ apps/web/src/fork/planCard/PlanMarkdown.tsx | 66 ++++++++ apps/web/src/fork/planCard/planCard.css | 25 +++ apps/web/src/fork/planCard/planCard.test.ts | 159 ++++++++++++++++++ apps/web/src/fork/planCard/planCardContext.ts | 36 ++++ apps/web/src/fork/planCard/planCardLayout.ts | 44 +++++ apps/web/src/fork/planCard/planCardSeams.ts | 57 +++++++ .../src/fork/planCard/planCardWidthStore.ts | 36 ++++ apps/web/src/fork/planCard/planContent.ts | 114 +++++++++++++ apps/web/src/fork/planCard/planHtml.ts | 32 ++++ apps/web/src/fork/planCard/planSummary.ts | 56 ++++++ .../threadComments/ThreadCommentsPanel.tsx | 4 +- .../web/src/fork/threadComments/model.test.ts | 11 ++ apps/web/src/fork/threadComments/model.ts | 14 +- docs/operations/expbkt3-customizations.md | 3 +- docs/user/chat-comments.md | 4 +- docs/user/plan-review.md | 22 ++- 26 files changed, 995 insertions(+), 122 deletions(-) create mode 100644 apps/web/src/fork/planCard/PlanCardHeading.tsx create mode 100644 apps/web/src/fork/planCard/PlanCardRow.tsx create mode 100644 apps/web/src/fork/planCard/PlanHtmlFrame.tsx create mode 100644 apps/web/src/fork/planCard/PlanMarkdown.tsx create mode 100644 apps/web/src/fork/planCard/planCard.css create mode 100644 apps/web/src/fork/planCard/planCard.test.ts create mode 100644 apps/web/src/fork/planCard/planCardContext.ts create mode 100644 apps/web/src/fork/planCard/planCardLayout.ts create mode 100644 apps/web/src/fork/planCard/planCardSeams.ts create mode 100644 apps/web/src/fork/planCard/planCardWidthStore.ts create mode 100644 apps/web/src/fork/planCard/planContent.ts create mode 100644 apps/web/src/fork/planCard/planHtml.ts create mode 100644 apps/web/src/fork/planCard/planSummary.ts diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index db5ea97545e20..e14a84f27dd39 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1387,7 +1387,11 @@ export const layer: Layer.Layer< : // T3-CUSTOM(expbkt3): command history stays unchanged; the agent gets the plan policy. { ...input.message, - text: appendAgentPlanInstructions(comments.text, agentPlanSubmissionEnabled), + text: appendAgentPlanInstructions( + comments.text, + agentPlanSubmissionEnabled, + input.runtimePolicy.interactionMode, + ), }; // A context read that fails costs the agent the apps' notes for // this turn, not the turn itself. diff --git a/apps/server/src/orchestration-v2/forkAgentPlanInstructions.expbkt3.test.ts b/apps/server/src/orchestration-v2/forkAgentPlanInstructions.expbkt3.test.ts index 9cf970267b8d0..9644a8afea953 100644 --- a/apps/server/src/orchestration-v2/forkAgentPlanInstructions.expbkt3.test.ts +++ b/apps/server/src/orchestration-v2/forkAgentPlanInstructions.expbkt3.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "vite-plus/test"; -import { appendAgentPlanInstructions } from "./forkAgentPlanInstructions.expbkt3.ts"; +import { + appendAgentPlanInstructions, + PLAN_FORMAT_INSTRUCTIONS, +} from "./forkAgentPlanInstructions.expbkt3.ts"; describe("server agent plan instructions", () => { it("keeps the complete provider message unchanged when submission is enabled", () => { @@ -16,10 +19,24 @@ describe("server agent plan instructions", () => { expect(result).toContain("Do not call t3_submit_plan"); }); + it("tells a plan-mode turn how to put live HTML in the plan card", () => { + const result = appendAgentPlanInstructions("Plan the change.", true, "plan"); + expect(result).toBe(`Plan the change.\n\n${PLAN_FORMAT_INSTRUCTIONS}`); + expect(result).toContain("`html-render`"); + }); + + it("adds the plan format only in plan mode, after the submission policy", () => { + expect(appendAgentPlanInstructions("Build it.", true, "default")).toBe("Build it."); + const result = appendAgentPlanInstructions("Plan it.", false, "plan"); + expect(result.indexOf("")).toBeLessThan( + result.indexOf(""), + ); + }); + it.each(["/compact", " /review current changes", "/model"])( "keeps the native provider command %s unchanged", (text) => { - expect(appendAgentPlanInstructions(text, false)).toBe(text); + expect(appendAgentPlanInstructions(text, false, "plan")).toBe(text); }, ); }); diff --git a/apps/server/src/orchestration-v2/forkAgentPlanInstructions.expbkt3.ts b/apps/server/src/orchestration-v2/forkAgentPlanInstructions.expbkt3.ts index dbdbd0f359e5a..9a3aadda2a1f8 100644 --- a/apps/server/src/orchestration-v2/forkAgentPlanInstructions.expbkt3.ts +++ b/apps/server/src/orchestration-v2/forkAgentPlanInstructions.expbkt3.ts @@ -1,6 +1,25 @@ // T3-CUSTOM(expbkt3): one plan policy reaches every provider through native turn input. -export function appendAgentPlanInstructions(text: string, submissionEnabled: boolean): string { + +/** What a plan in plan mode may contain; the web plan card renders these parts. */ +export const PLAN_FORMAT_INSTRUCTIONS = [ + "", + "The T3 plan card renders your plan's Markdown like a chat reply. To show a visual inside the plan (a chart, diagram, comparison or mockup), add a fenced code block whose language is `html-render` and that holds one self-contained HTML document. The card shows it as a live page in T3's theme: style it with var(--foreground), var(--muted-foreground), var(--border), var(--card), var(--accent), var(--chart-1) to var(--chart-6) and var(--font-sans), leave the page background transparent, and let the content set the height. A plan written entirely as one HTML document also renders as a page. Plain `html` code blocks stay as code.", + "", +].join("\n"); + +export function appendAgentPlanInstructions( + text: string, + submissionEnabled: boolean, + interactionMode?: string, +): string { // Native slash commands must stay intact for the provider CLI. - if (submissionEnabled || text.trimStart().startsWith("/")) return text; - return `${text}\n\n\nThe T3 plan submission tool is disabled for this server. Write the complete plan directly in your main chat response. Do not call t3_submit_plan or use another T3 tool to create a plan panel.\n`; + if (text.trimStart().startsWith("/")) return text; + const blocks: string[] = []; + if (!submissionEnabled) { + blocks.push( + "\nThe T3 plan submission tool is disabled for this server. Write the complete plan directly in your main chat response. Do not call t3_submit_plan or use another T3 tool to create a plan panel.\n", + ); + } + if (interactionMode === "plan") blocks.push(PLAN_FORMAT_INSTRUCTIONS); + return blocks.length === 0 ? text : `${text}\n\n${blocks.join("\n\n")}`; } diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index b9621544fd7c4..4720b95e6fa2d 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -2,7 +2,6 @@ import { PlanReviewPanel, useOpenPlanReviewDocumentId } from "../fork/planReviewSurface"; import { PlanReviewTakeover } from "../fork/planReviewTakeover"; import { PlanReviewComposerDock } from "../fork/planReviewComposerDock"; -import { usePlanReviewTakeoverStore } from "../planReviewTakeoverStore"; import { THREAD_COMMENTS_EMPTY_SEND_TEXT, ThreadCommentsPanel, @@ -3494,10 +3493,6 @@ export default function ChatView(props: ChatViewProps) { } return findLatestProposedPlan(serverProjection, activeLatestRun?.runId ?? null); }, [activeLatestRun?.runId, latestRunSettled, serverProjection]); - // T3-CUSTOM(expbkt3): a native plan opens the review takeover. - const openPlanReviewSurface = useCallback((documentId: string) => { - usePlanReviewTakeoverStore.getState().undismiss(documentId); - }, []); const planReviewDocumentId = useOpenPlanReviewDocumentId( activeThreadRef?.environmentId ?? null, activeThreadRef?.threadId ?? null, @@ -11555,9 +11550,6 @@ export default function ChatView(props: ChatViewProps) { {/* Messages — LegendList handles virtualization and scrolling internally */} void) | undefined; - planReviewDocumentId: string | null | undefined; citationRequest: AssistantCitationTarget | null; listRef: React.RefObject; timestampFormat: TimestampFormat; @@ -450,9 +448,6 @@ export interface MessagesTimelineHistoryControls { } interface MessagesTimelineProps { - // T3-CUSTOM(expbkt3): a card opens the shared native review. - onOpenPlanReview?: ((documentId: string) => void) | undefined; - planReviewDocumentId?: string | null | undefined; citationRequest?: AssistantCitationRequest | null; citationHistoryLoading?: boolean; onCiteAssistantText?: ( @@ -548,9 +543,6 @@ interface MessagesTimelineProps { // --------------------------------------------------------------------------- export const MessagesTimeline = memo(function MessagesTimeline({ - // T3-CUSTOM(expbkt3): callback keeps the review controller outside virtual rows. - onOpenPlanReview, - planReviewDocumentId, citationRequest = null, citationHistoryLoading = false, onCiteAssistantText, @@ -1224,9 +1216,6 @@ export const MessagesTimeline = memo(function MessagesTimeline({ const sharedState = useMemo( () => ({ - // T3-CUSTOM(expbkt3): keep card callbacks stable across native item updates. - onOpenPlanReview, - planReviewDocumentId, citationRequest: readyCitationRequest, listRef, timestampFormat, @@ -1265,8 +1254,6 @@ export const MessagesTimeline = memo(function MessagesTimeline({ workGroupViewState, }), [ - onOpenPlanReview, - planReviewDocumentId, readyCitationRequest, listRef, timestampFormat, @@ -2790,17 +2777,17 @@ function ProposedPlanTimelineRow({ return (
- + {/* T3-CUSTOM(expbkt3): BEGIN — plans share the chat's markdown, citations and comments, and can go wide. */} + + + + {/* T3-CUSTOM(expbkt3): END */}
); } diff --git a/apps/web/src/components/chat/ProposedPlanCard.tsx b/apps/web/src/components/chat/ProposedPlanCard.tsx index b19f28f4a908b..b03449f184ac9 100644 --- a/apps/web/src/components/chat/ProposedPlanCard.tsx +++ b/apps/web/src/components/chat/ProposedPlanCard.tsx @@ -9,20 +9,25 @@ import { type ScopedThreadRef, } from "@t3tools/contracts"; import { - buildCollapsedProposedPlanPreviewMarkdown, buildProposedPlanMarkdownFilename, downloadPlanAsTextFile, normalizePlanMarkdownForExport, proposedPlanTitle, stripDisplayedPlanMarkdown, } from "../../proposedPlan"; -import ChatMarkdown from "../ChatMarkdown"; -import { ArrowRightIcon, EllipsisIcon, LoaderCircleIcon } from "lucide-react"; // T3-CUSTOM(expbkt3): icons for the native plan review entry point. +// T3-CUSTOM(expbkt3): BEGIN — plans render like chat (HTML parts, citations, comments) and stay mounted when folded. +import ChatMarkdown from "../../fork/planCard/PlanMarkdown"; +import { + buildCollapsedProposedPlanPreviewMarkdown, + useExpandPlanForCitation, +} from "../../fork/planCard/planCardSeams"; +import { PlanCardHeading } from "../../fork/planCard/PlanCardHeading"; +// T3-CUSTOM(expbkt3): END +import { EllipsisIcon } from "lucide-react"; import { Button } from "../ui/button"; import { Input } from "../ui/input"; import { Menu, MenuItem, MenuPopup, MenuTrigger } from "../ui/menu"; import { cn } from "~/lib/utils"; -import { Badge } from "../ui/badge"; import { Dialog, DialogDescription, @@ -44,24 +49,15 @@ export const ProposedPlanCard = memo(function ProposedPlanCard({ threadRef, cwd, workspaceRoot, - // T3-CUSTOM(expbkt3): BEGIN — native timeline plan review entry point. - onOpenPlanReview, - planReviewDocumentId, - reviewable = false, - // T3-CUSTOM(expbkt3): END }: { planMarkdown: string; environmentId: EnvironmentId; threadRef?: ScopedThreadRef | undefined; cwd: string | undefined; workspaceRoot: string | undefined; - // T3-CUSTOM(expbkt3): BEGIN — native timeline plan review entry point. - onOpenPlanReview?: ((documentId: string) => void) | undefined; - planReviewDocumentId?: string | null | undefined; - reviewable?: boolean | undefined; - // T3-CUSTOM(expbkt3): END }) { const [expanded, setExpanded] = useState(false); + useExpandPlanForCitation(setExpanded); // T3-CUSTOM(expbkt3): a quote inside a folded plan opens it. const canWriteFiles = useEnvironmentScope(environmentId, AuthFilesystemWriteScope); const [isSaveDialogOpen, setIsSaveDialogOpen] = useState(false); const [savePath, setSavePath] = useState(""); @@ -165,12 +161,8 @@ export const ProposedPlanCard = memo(function ProposedPlanCard({ return (
-
- Plan - {/* Same heading level as the message author headings in the timeline, - so a plan's own headings nest beneath it in the outline. */} -

{title}

-
+ {/* T3-CUSTOM(expbkt3): icon, title, state, size and the width control. */} + } @@ -216,47 +208,16 @@ export const ProposedPlanCard = memo(function ProposedPlanCard({
) : null}
- {canCollapse || (reviewable && onOpenPlanReview) ? ( // T3-CUSTOM(expbkt3): native timeline plan review entry point. -
- {/* T3-CUSTOM(expbkt3): BEGIN — layout host for the plan review button alongside the collapse button. */} - {canCollapse ? ( - - ) : null} - {/* T3-CUSTOM(expbkt3): END */} - {/* T3-CUSTOM(expbkt3): BEGIN — native timeline plan review entry point. */} - {reviewable && onOpenPlanReview ? ( - planReviewDocumentId ? ( - - ) : ( - - ) - ) : null} - {/* T3-CUSTOM(expbkt3): END */} + {canCollapse ? ( +
+
) : null}
diff --git a/apps/web/src/components/chat/useAssistantCitationTarget.ts b/apps/web/src/components/chat/useAssistantCitationTarget.ts index 958e2531eeee0..f89b6dfaccf1b 100644 --- a/apps/web/src/components/chat/useAssistantCitationTarget.ts +++ b/apps/web/src/components/chat/useAssistantCitationTarget.ts @@ -5,6 +5,12 @@ import type { TimelineEntry } from "../../session-logic"; import type { MessagesTimelineRow } from "./MessagesTimeline.logic"; import type { AssistantCitationRequest, AssistantCitationTarget } from "./AssistantCitationSource"; import { toastManager } from "../ui/toast"; +// T3-CUSTOM(expbkt3): a citation or comment can also quote a plan card. +import { + citationSourceRunId, + isCitableSource, + isCitationSource, +} from "../../fork/planCard/planCardSeams"; export interface CitationHistoryPage { readonly loading: boolean; @@ -84,8 +90,7 @@ export function useAssistantCitationTarget({ toastManager.add({ type: "warning", title, description }); }; const source = entries.find( - (entry) => - entry.kind === "message" && entry.message.id === navigation.target.citation.messageId, + (entry) => isCitationSource(entry, navigation.target.citation.messageId), // T3-CUSTOM(expbkt3): plans too. ); if (!source) { if (loadEarlier) { @@ -108,7 +113,8 @@ export function useAssistantCitationTarget({ ); return; } - if (source.kind !== "message" || source.message.role !== "assistant") { + if (!isCitableSource(source)) { + // T3-CUSTOM(expbkt3): an assistant response or a plan. fail( "The citation does not refer to an assistant response", "The selected text is still saved in your citation.", @@ -116,10 +122,11 @@ export function useAssistantCitationTarget({ return; } const index = rows.findIndex( - (row) => row.kind === "message" && row.message.id === navigation.target.citation.messageId, + (row) => isCitationSource(row, navigation.target.citation.messageId), // T3-CUSTOM(expbkt3): plans too. ); if (index < 0) { - if (source.message.runId) onExpandTurn(source.message.runId); + const runId = citationSourceRunId(source); // T3-CUSTOM(expbkt3): a plan's run too. + if (runId) onExpandTurn(runId); // T3-CUSTOM(expbkt3): a plan's run too. return; } if (listLoaded && listRef.current) setReady(navigation.target); @@ -180,7 +187,7 @@ export function useAssistantCitationTarget({ const positioning = request !== null && finishedKey !== request.key; const sourceRow = target && positioning - ? rows.find((row) => row.kind === "message" && row.message.id === target.citation.messageId) + ? rows.find((row) => isCitationSource(row, target.citation.messageId)) // T3-CUSTOM(expbkt3): plans too. : undefined; return { target, diff --git a/apps/web/src/fork/planCard/PlanCardHeading.tsx b/apps/web/src/fork/planCard/PlanCardHeading.tsx new file mode 100644 index 0000000000000..d7487f867c384 --- /dev/null +++ b/apps/web/src/fork/planCard/PlanCardHeading.tsx @@ -0,0 +1,96 @@ +/** + * T3-CUSTOM(expbkt3): the left side of a plan card's header. + * + * Replaces upstream's "Plan" badge and title with an icon, the title, the + * plan's state, its size, and the control that widens or narrows the card. + * It takes the header's free width, so the width control sits just before + * upstream's actions menu. + */ +import { FoldHorizontalIcon, ListChecksIcon, UnfoldHorizontalIcon } from "lucide-react"; +import { use, useMemo } from "react"; + +import { Badge } from "../../components/ui/badge"; +import { Button } from "../../components/ui/button"; +import { Tooltip, TooltipPopup, TooltipTrigger } from "../../components/ui/tooltip"; +import { PlanCardContext } from "./planCardContext"; +import { usePlanCardWidthStore } from "./planCardWidthStore"; +import { + PLAN_STATUS_LABEL, + PLAN_STATUS_VARIANT, + planCardTitle, + summarizePlan, +} from "./planSummary"; + +export function PlanCardHeading({ + planMarkdown, + title, +}: { + readonly planMarkdown: string; + readonly title: string; +}) { + const card = use(PlanCardContext); + const summary = useMemo(() => summarizePlan(planMarkdown), [planMarkdown]); + const status = card?.plan.status; + return ( +
+ +
+ {/* Same heading level as the message author headings in the timeline, + so a plan's own headings nest beneath it in the outline. */} +

+ {planCardTitle(planMarkdown, title)} +

+

+ Plan + {summary.steps > 0 ? ( + + · {summary.steps} {summary.steps === 1 ? "step" : "steps"} + + ) : null} + · {summary.readMinutes} min read +

+
+ {status ? ( + {PLAN_STATUS_LABEL[status]} + ) : null} + {card ? : null} +
+ ); +} + +function PlanCardWidthToggle({ canWiden }: { readonly canWiden: boolean }) { + const width = usePlanCardWidthStore((state) => state.width); + const toggle = usePlanCardWidthStore((state) => state.toggle); + const wide = width === "wide"; + // A narrow window has no room beside the chat column, so there is nothing to toggle. + if (!wide && !canWiden) return null; + const label = wide ? "Fit plan to the chat column" : "Widen plan"; + return ( + + + } + > + {wide ? ( + + {label} + + ); +} diff --git a/apps/web/src/fork/planCard/PlanCardRow.tsx b/apps/web/src/fork/planCard/PlanCardRow.tsx new file mode 100644 index 0000000000000..8244bac72f6a5 --- /dev/null +++ b/apps/web/src/fork/planCard/PlanCardRow.tsx @@ -0,0 +1,94 @@ +/** + * T3-CUSTOM(expbkt3): the timeline row seam for plan cards. + * + * Wraps upstream's `ProposedPlanCard` to give it the chat's timeline state + * (see `planCardContext`) and to draw it wide when the reader asks. A wide + * card leaves the chat column for the free width of the conversation area; + * the column itself is measured from the DOM, because the timeline centres it + * inside padding that the chat canvas changes as floating cards come and go. + */ +import type { ReactNode } from "react"; +import { useLayoutEffect, useMemo, useRef, useState } from "react"; + +import { useChatCanvas } from "../../components/chat/ChatCanvasContext"; +import type { ProposedPlan } from "../../types"; +import "./planCard.css"; +import { PlanCardContext, type PlanCardTimeline } from "./planCardContext"; +import { resolveWidePlanBox, type WidePlanBox } from "./planCardLayout"; +import { usePlanCardWidthStore } from "./planCardWidthStore"; + +// Floating things a wide card must stop before, inside the chat canvas. +const OBSTACLE_SELECTOR = '[data-thread-details-panel="inline"], [data-preview-mini-player]'; + +function sameBox(left: WidePlanBox | null, right: WidePlanBox | null) { + return left?.offset === right?.offset && left?.width === right?.width; +} + +export function PlanCardRow({ + plan, + itemKey, + timeline, + children, +}: { + readonly plan: ProposedPlan; + readonly itemKey: string; + readonly timeline: PlanCardTimeline; + readonly children: ReactNode; +}) { + const width = usePlanCardWidthStore((state) => state.width); + const canvas = useChatCanvas(); + const ref = useRef(null); + const [box, setBox] = useState(null); + + // The canvas context changes whenever the column, a floating card or the + // preview moves, so it re-runs the measurement along with resizes. + useLayoutEffect(() => { + const element = ref.current; + const column = element?.parentElement; + const area = element?.closest(".messages-timeline-scroll"); + if (!element || !column || !area) return; + const measure = () => { + const columnRect = column.getBoundingClientRect(); + const columnStyle = getComputedStyle(column); + const areaRect = area.getBoundingClientRect(); + const areaLeft = areaRect.left + area.clientLeft; + const obstacles = Array.from( + element.closest("[data-chat-canvas]")?.querySelectorAll(OBSTACLE_SELECTOR) ?? [], + (obstacle) => obstacle.getBoundingClientRect().left, + ); + const next = resolveWidePlanBox({ + lane: { + left: columnRect.left + Number.parseFloat(columnStyle.paddingLeft), + right: columnRect.right - Number.parseFloat(columnStyle.paddingRight), + }, + area: { left: areaLeft, right: areaLeft + area.clientWidth }, + padding: canvas?.lane.padding ?? 12, + obstacles, + }); + setBox((current) => (sameBox(current, next) ? current : next)); + }; + measure(); + const observer = new ResizeObserver(measure); + observer.observe(column); + observer.observe(area); + return () => observer.disconnect(); + }, [canvas]); + + const wide = width === "wide" && box !== null; + const value = useMemo( + () => ({ plan, itemKey, timeline, canWiden: box !== null }), + [plan, itemKey, timeline, box], + ); + return ( + +
+ {children} +
+ + ); +} diff --git a/apps/web/src/fork/planCard/PlanHtmlFrame.tsx b/apps/web/src/fork/planCard/PlanHtmlFrame.tsx new file mode 100644 index 0000000000000..e1a7cadbda57d --- /dev/null +++ b/apps/web/src/fork/planCard/PlanHtmlFrame.tsx @@ -0,0 +1,45 @@ +/** + * T3-CUSTOM(expbkt3): one HTML part of a plan, shown like an agent's HTML render. + * + * It reuses the chat's `HtmlRenderDocument`, so the page gets the same sandbox, + * the app's theme variables before first paint, live theme changes, link + * handling, and a frame that grows to the page's height. The plan's HTML has + * no stored attachment, so the bootstrapped page is served from a blob URL that + * lives as long as the frame. + */ +import { useEffect, useState } from "react"; + +import { HtmlRenderDocument } from "../../components/files/BrowserDocumentFrame"; +import { planHtmlDocument, planHtmlFrameHeight } from "./planHtml"; + +export function PlanHtmlFrame({ html, title }: { readonly html: string; readonly title: string }) { + const [src, setSrc] = useState(null); + const [contentHeight, setContentHeight] = useState(); + useEffect(() => { + const url = URL.createObjectURL( + new Blob([planHtmlDocument(html)], { type: "text/html;charset=utf-8" }), + ); + // oxlint-disable-next-line react/set-state-in-effect -- The URL exists only while this effect owns it. + setSrc(url); + return () => URL.revokeObjectURL(url); + }, [html]); + + return ( +
+ {src !== null ? ( + + ) : null} +
+ ); +} diff --git a/apps/web/src/fork/planCard/PlanMarkdown.tsx b/apps/web/src/fork/planCard/PlanMarkdown.tsx new file mode 100644 index 0000000000000..cf19daae37aeb --- /dev/null +++ b/apps/web/src/fork/planCard/PlanMarkdown.tsx @@ -0,0 +1,66 @@ +/** + * T3-CUSTOM(expbkt3): the plan card's body, a drop-in for `ChatMarkdown`. + * + * Upstream's `ProposedPlanCard` imports this under the name `ChatMarkdown`, so + * its JSX stays unchanged. Inside a timeline row it renders the plan the way + * an assistant message renders: the same Markdown features (skills, runnable + * shell commands, image preview, artifact templates), HTML parts as live + * themed pages, and a citation source so a selection offers Cite and the + * comment actions, with comment highlights painted back on the text. + */ +import { MessageId } from "@t3tools/contracts"; +import { type ComponentProps, use, useMemo } from "react"; + +import { AssistantCitationSource } from "../../components/chat/AssistantCitationSource"; +import { shouldPreserveAssistantLineBreaks } from "../../components/chat/MessagesTimeline.logic"; +import ChatMarkdown from "../../components/ChatMarkdown"; +import { ThreadCommentMarks } from "../threadComments/threadCommentsSurface"; +import { PlanCardContext } from "./planCardContext"; +import { splitPlanSegments } from "./planContent"; +import { PlanHtmlFrame } from "./PlanHtmlFrame"; + +export default function PlanMarkdown(props: ComponentProps) { + const card = use(PlanCardContext); + const segments = useMemo(() => splitPlanSegments(props.text), [props.text]); + const timeline = card?.timeline; + const body = segments.map((segment, index) => + segment.kind === "html" ? ( + + ) : ( + + ), + ); + if (!card || !timeline) return <>{body}; + + return ( + // Comment marks position against this box, and a collapsed card clips them + // together with the text they mark. +
+ + {body} + + {timeline.threadRef ? ( + + ) : null} +
+ ); +} diff --git a/apps/web/src/fork/planCard/planCard.css b/apps/web/src/fork/planCard/planCard.css new file mode 100644 index 0000000000000..731e9df88a754 --- /dev/null +++ b/apps/web/src/fork/planCard/planCard.css @@ -0,0 +1,25 @@ +/* T3-CUSTOM(expbkt3): plan card styles. Fork-owned, loaded by PlanCardRow. */ + +/* A wide plan card leaves the chat column, so its timeline row must not clip it. + The card stays inside the scrolling area, which still clips horizontally. */ +[data-timeline-root]:has([data-plan-card-width="wide"]) { + overflow-x: visible; +} + +/* Plans read as a sequence of sections: each top-level section heading after + the first gets a quiet rule above it. */ +[data-plan-card-body] .chat-markdown > :is(h1, h2):not(:first-child) { + margin-top: 1.5rem; + padding-top: 1rem; + border-top: 1px solid color-mix(in srgb, var(--border) 70%, transparent); +} + +/* Numbered steps are the plan's spine: their numbers carry the accent. */ +[data-plan-card-body] .chat-markdown > ol > li::marker { + color: var(--info-foreground); + font-weight: 600; +} + +[data-plan-card-body] .chat-markdown > ol > li + li { + margin-top: 0.35rem; +} diff --git a/apps/web/src/fork/planCard/planCard.test.ts b/apps/web/src/fork/planCard/planCard.test.ts new file mode 100644 index 0000000000000..ce3aa57a8af61 --- /dev/null +++ b/apps/web/src/fork/planCard/planCard.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { resolveWidePlanBox } from "./planCardLayout"; +import { isHtmlPlan, planHtmlTitle, splitPlanSegments } from "./planContent"; +import { planHtmlDocument, planHtmlFrameHeight } from "./planHtml"; +import { planCardTitle, summarizePlan } from "./planSummary"; + +const PAGE = + "\nRollout & risks

x

"; + +describe("splitPlanSegments", () => { + it("keeps a Markdown plan as one Markdown part", () => { + const plan = "# Ship it\n\n1. Build\n2. Test"; + expect(splitPlanSegments(plan)).toEqual([{ kind: "markdown", text: plan }]); + }); + + it("renders a whole-document plan as one page titled from the document", () => { + expect(splitPlanSegments(PAGE)).toEqual([ + { kind: "html", html: PAGE, title: "Rollout & risks" }, + ]); + expect(planCardTitle(PAGE, "Proposed plan")).toBe("Rollout & risks"); + }); + + it("treats a balanced root element as an HTML plan, but not an inline example", () => { + expect(isHtmlPlan('

Steps

')).toBe(true); + expect(isHtmlPlan("
example
\n\nThen change the parser.")).toBe(false); + }); + + it("splits html-render fences out of Markdown and leaves plain html fences as code", () => { + const plan = [ + "# Plan", + "", + "```html-render", + "

Chart

", + "```", + "", + "```html", + "

markup the plan proposes

", + "```", + ].join("\n"); + expect(splitPlanSegments(plan)).toEqual([ + { kind: "markdown", text: "# Plan\n" }, + { kind: "html", html: "

Chart

", title: "Chart" }, + { + kind: "markdown", + text: "\n```html\n

markup the plan proposes

\n```", + }, + ]); + }); + + it("lifts a raw HTML document out of a Markdown plan up to its closing tag", () => { + const plan = `# Title\n\nIntro.\n\n${PAGE}\n\nAfter.`; + const segments = splitPlanSegments(plan); + expect(segments.map((segment) => segment.kind)).toEqual(["markdown", "html", "markdown"]); + expect(segments[1]).toMatchObject({ kind: "html", html: PAGE }); + }); + + it("does not lift a document that sits inside a code fence", () => { + const plan = `Example:\n\n\`\`\`\n${PAGE}\n\`\`\``; + expect(splitPlanSegments(plan)).toEqual([{ kind: "markdown", text: plan }]); + }); + + it("falls back to the given title for an untitled page", () => { + expect(planHtmlTitle("

no title

")).toBeNull(); + expect(splitPlanSegments("```html-render\n

x

\n```", "Plan")).toEqual([ + { kind: "html", html: "

x

", title: "Plan" }, + ]); + }); +}); + +describe("planHtmlDocument", () => { + it("adds the chat theme bootstrap, then a transparent page background the page can override", () => { + const document = planHtmlDocument( + "", + ); + const theme = document.indexOf('"; + +/** The page as the frame loads it: the chat's theme bootstrap, then the plan's base style. */ +export function planHtmlDocument(html: string): string { + const document = injectHtmlRenderBootstrap(html); + const theme = document.indexOf('" }, + }); + const open = document.indexOf('", open)); + expect(document).not.toContain("--bad name"); + }); + + it("keys pages by their markup and remembers their last height", () => { + expect(planHtmlKey("

a

")).toBe(planHtmlKey("

a

")); + expect(planHtmlKey("

a

")).not.toBe(planHtmlKey("

b

")); + const key = planHtmlKey("

remembered

"); + expect(readPlanHtmlHeight(key)).toBeUndefined(); + rememberPlanHtmlHeight(key, 640); + expect(readPlanHtmlHeight(key)).toBe(640); + }); + it("fits the frame to the page within sane bounds", () => { expect(planHtmlFrameHeight(undefined)).toBe(320); expect(planHtmlFrameHeight(10)).toBe(80); @@ -126,6 +153,25 @@ describe("resolveWidePlanBox", () => { ).toEqual({ offset: -272, width: 1280 }); }); + it("starts after the minimap on the left so it keeps taking its own clicks", () => { + expect( + resolveWidePlanBox({ + lane: { left: 432, right: 1168 }, + area, + padding: 20, + leftObstacles: [64], + }), + ).toEqual({ offset: -272, width: 1280 }); + expect( + resolveWidePlanBox({ + lane: { left: 232, right: 968 }, + area: { left: 0, right: 1200 }, + padding: 20, + leftObstacles: [64], + }), + ).toEqual({ offset: -148, width: 1096 }); + }); + it("has nothing to offer when the column already fills the area", () => { expect(resolveWidePlanBox({ lane: { left: 20, right: 1580 }, area, padding: 20 })).toBeNull(); }); diff --git a/apps/web/src/fork/planCard/planCardLayout.ts b/apps/web/src/fork/planCard/planCardLayout.ts index 5664fdba13693..dd5c7a6217bc9 100644 --- a/apps/web/src/fork/planCard/planCardLayout.ts +++ b/apps/web/src/fork/planCard/planCardLayout.ts @@ -22,13 +22,23 @@ export function resolveWidePlanBox(input: { readonly area: { readonly left: number; readonly right: number }; /** The conversation's side padding. */ readonly padding: number; - /** Left edges of floating things the card must not slide under. */ + /** Left edges of floating things on the right the card must not slide under. */ readonly obstacles?: ReadonlyArray; + /** Right edges of floating things on the left, such as the timeline minimap. */ + readonly leftObstacles?: ReadonlyArray; readonly maxWidth?: number; }): WidePlanBox | null { const { lane, area, padding } = input; const laneWidth = lane.right - lane.left; - const bandLeft = Math.min(lane.left, area.left + padding); + const bandLeft = Math.min( + lane.left, + Math.max( + area.left + padding, + ...(input.leftObstacles ?? []) + .filter((right) => right <= lane.left) + .map((right) => right + padding), + ), + ); // Floating things keep the same gap from the card as the area's edge does. const blockers = (input.obstacles ?? []) .filter((left) => left >= lane.right) diff --git a/apps/web/src/fork/planCard/planHtml.ts b/apps/web/src/fork/planCard/planHtml.ts index 97807958bb2f9..e9f999fba4c97 100644 --- a/apps/web/src/fork/planCard/planHtml.ts +++ b/apps/web/src/fork/planCard/planHtml.ts @@ -3,7 +3,11 @@ * * Pure, so it is testable without the React frame that shows it. */ -import { HTML_RENDER_MIN_HEIGHT, injectHtmlRenderBootstrap } from "@t3tools/shared/htmlRender"; +import { + HTML_RENDER_MIN_HEIGHT, + injectHtmlRenderBootstrap, + type HtmlRenderTheme, +} from "@t3tools/shared/htmlRender"; // A plan page can be long. The cap only stops a page with viewport-based // heights from growing its frame without end. @@ -13,15 +17,37 @@ const PLAN_HTML_INITIAL_HEIGHT = 320; // The chat's base style paints the thread background; a plan page sits on the // card instead, so it starts transparent. The page's own CSS still wins. const PLAN_PAGE_STYLE = ""; +const THEME_STYLE_OPEN = '", open); + const scriptEnd = document.indexOf("", open); + if (close === -1 || scriptEnd === -1) return document; const at = scriptEnd + "".length; - return document.slice(0, at) + PLAN_PAGE_STYLE + document.slice(at); + return ( + document.slice(0, close) + + (theme ? themeRule(theme) : "") + + document.slice(close, at) + + PLAN_PAGE_STYLE + + document.slice(at) + ); } export function planHtmlFrameHeight(contentHeight: number | undefined): number { @@ -30,3 +56,30 @@ export function planHtmlFrameHeight(contentHeight: number | undefined): number { Math.max(HTML_RENDER_MIN_HEIGHT, Math.round(contentHeight ?? PLAN_HTML_INITIAL_HEIGHT)), ); } + +// Plan rows are virtualised, so a frame remounts as it scrolls back into view. +// Starting it at the height it last reported keeps the list from jumping. +const MEASURED_HEIGHT_LIMIT = 200; +const measuredHeights = new Map(); + +/** A short, stable key for one page's markup. */ +export function planHtmlKey(html: string): string { + let hash = 0x811c9dc5; + for (let index = 0; index < html.length; index += 1) { + hash = Math.imul(hash ^ html.charCodeAt(index), 0x01000193); + } + return `${html.length}:${(hash >>> 0).toString(36)}`; +} + +export function readPlanHtmlHeight(key: string): number | undefined { + return measuredHeights.get(key); +} + +export function rememberPlanHtmlHeight(key: string, height: number): void { + measuredHeights.delete(key); + measuredHeights.set(key, height); + if (measuredHeights.size > MEASURED_HEIGHT_LIMIT) { + const oldest = measuredHeights.keys().next().value; + if (oldest !== undefined) measuredHeights.delete(oldest); + } +} diff --git a/docs/user/plan-review.md b/docs/user/plan-review.md index 4a6e5aaf3f7fd..dc1423174c956 100644 --- a/docs/user/plan-review.md +++ b/docs/user/plan-review.md @@ -29,10 +29,10 @@ reading time. **Expand plan** shows a long plan in full. ## Opening a plan -By default a plan opens for review as soon as it is ready. Turn that off with -**Settings → BK Add-ons → Open a ready plan automatically**; the plan then waits -behind the floating **Open the plan in preview** button above the composer, -which appears once an agent has proposed a plan that has not been implemented yet. +By default a plan opens for review as soon as it is ready. Closing the review +keeps that plan closed; it stays readable as its card in the conversation. Turn +automatic opening off with **Settings → BK Add-ons → Open a ready plan +automatically**; ready plans then stay as cards. The panel opens beside the chat as a tab, so the conversation stays visible. Use the maximise control in the panel header for a full-width read. diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 67641cc018aaf..2de5c503c09e0 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -492,12 +492,12 @@ export const ClientSettingsSchema = Schema.Struct({ // old keys, so everyone, including prior beta opt-outs, resets to the new // default sidebar. legacySidebarEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), - // T3-CUSTOM(expbkt3): native plan review. On by default; turning it off hides - // the Preview entry points. + // T3-CUSTOM(expbkt3): native plan review. On by default; turning it off leaves + // plans as cards in the conversation. nativePlanReviewEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), // T3-CUSTOM(expbkt3): a reviewable plan takes over the transcript as soon as - // it lands, so a plan gate cannot be missed. Off leaves the Preview button as - // the only way in. Requires nativePlanReviewEnabled. + // it lands, so a plan gate cannot be missed. Off leaves a ready plan as its + // conversation card. Requires nativePlanReviewEnabled. planReviewAutoOpenEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), // T3-CUSTOM(expbkt3): agent-rendered UI surfaces in the chat transcript. While // off, a `t3_show_ui` call stays an ordinary collapsed tool row. From 06e40b27d86cd2bd304d235f2cea3b9e5f5c163d Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Thu, 15 Oct 2026 18:40:25 +0000 Subject: [PATCH 16/20] fix(server,web): questions left by a failed run can be answered or dismissed When a run fails while the agent waits on a question (the provider stream closes, an event fails to ingest), the run ends and its provider session is detached, but the question stays pending with a live capability. Submit then failed with "Provider session ... was not found", dismiss failed with "This question needs an answer", and Stop was gone. A request whose session is detached or whose run has ended is now handled in message mode: an answer resolves it and reaches the agent as the next message, a dismissal closes it, and an approval closes without a reply. The composer marks such a question dismissible. Multi-select answers join their choices when sent as a message. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e8ee65f5ba2d75c5fae89d08819d32449ecdf23e) --- .../src/orchestration-v2/Orchestrator.ts | 21 +- .../orphanedRuntimeRequest.expbkt3.test.ts | 252 ++++++++++++++++++ .../orphanedRuntimeRequest.expbkt3.ts | 81 ++++++ apps/web/src/components/ChatView.tsx | 7 +- apps/web/src/fork/orphanedUserInputs.test.ts | 47 ++++ apps/web/src/fork/orphanedUserInputs.ts | 46 ++++ docs/operations/expbkt3-customizations.md | 1 + docs/user/t3-mcp-control.md | 3 + 8 files changed, 454 insertions(+), 4 deletions(-) create mode 100644 apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.test.ts create mode 100644 apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.ts create mode 100644 apps/web/src/fork/orphanedUserInputs.test.ts create mode 100644 apps/web/src/fork/orphanedUserInputs.ts diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 461a918dd6490..001512652c861 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -148,6 +148,7 @@ import { planThreadDeletion } from "./ThreadDeletion.ts"; // T3-CUSTOM(expbkt3): durable fork command decisions and projection metadata. import { forkAsyncInputChange, updatePendingAsyncUserInputIds } from "./forkAsyncInput.expbkt3.ts"; import { findForkAsyncRequest, forkAsyncAnswer } from "./forkAsyncResponse.expbkt3.ts"; +import { messageAnswerText, respondableRuntimeRequest } from "./orphanedRuntimeRequest.expbkt3.ts"; import * as Result from "effect/Result"; import { legacyActivityItem } from "./ForkLegacyHistory.expbkt3.ts"; import { isForkThreadCommand, planForkThreadCommand } from "./forkCommands.expbkt3.ts"; @@ -7299,7 +7300,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio .pipe( Effect.mapError(() => new OrchestratorProjectionError({ threadId: command.threadId })), ); - const runtimeRequest = context.request; + // T3-CUSTOM(expbkt3): a request its provider outlived is answered in message mode. + const runtimeRequest = yield* respondableRuntimeRequest({ + projectionStore, + threadId: command.threadId, + context, + }).pipe( + Effect.mapError(() => new OrchestratorProjectionError({ threadId: command.threadId })), + ); if (runtimeRequest === undefined) { return yield* new OrchestratorDispatchError({ commandId: command.commandId, @@ -7422,6 +7430,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } if (runtimeRequest.responseCapability.type === "message") { if (resolvedNodeStatus === "cancelled") return; + // T3-CUSTOM(expbkt3): an approval its provider outlived closes without a reply. + if (runtimeRequest.kind !== "user_input") return; if (approvalTurnItem?.type !== "user_input_request") { return yield* new OrchestratorDispatchError({ commandId: command.commandId, @@ -7431,7 +7441,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio } const replies: string[] = []; for (const question of approvalTurnItem.questions) { - const answer = command.answers?.[question.id]; + // T3-CUSTOM(expbkt3): a multi-select answer lists its choices. + const answer = messageAnswerText(command.answers?.[question.id]); if (typeof answer !== "string" || answer.trim().length === 0) { if (question.required === false) continue; return yield* new OrchestratorDispatchError({ @@ -7525,9 +7536,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio Effect.gen(function* () { // T3-CUSTOM(expbkt3): dismissing a fork question does not interrupt its provider. if (yield* dispatchForkAsyncResponse(command, events, effects)) return; + // T3-CUSTOM(expbkt3): a question its provider outlived can be dismissed. const request = yield* projectionStore - .getRuntimeRequest(command.threadId, command.requestId) + .getRuntimeResponseContext(command.threadId, command.requestId) .pipe( + Effect.flatMap((context) => + respondableRuntimeRequest({ projectionStore, threadId: command.threadId, context }), + ), Effect.mapError(() => new OrchestratorProjectionError({ threadId: command.threadId })), ); if (request === undefined || request.status !== "pending") { diff --git a/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.test.ts b/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.test.ts new file mode 100644 index 0000000000000..b16164f7f74ba --- /dev/null +++ b/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.test.ts @@ -0,0 +1,252 @@ +import { assert, describe, it } from "@effect/vitest"; +import { + CommandId, + EventId, + NodeId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ProviderSessionId, + RuntimeRequestId, + ThreadId, + TurnItemId, + type OrchestrationV2RuntimeRequest, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import { isOrphanedRuntimeRequest, messageAnswerText } from "./orphanedRuntimeRequest.expbkt3.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import type { ProviderAdapterV2Shape } from "./ProviderAdapter.ts"; +import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; +import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; + +const instanceId = ProviderInstanceId.make("codex"); +const modelSelection = { instanceId, model: "gpt-5.1-codex" }; +const adapter = { + instanceId, + driver: ProviderDriverKind.make("codex"), + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }), + openSession: () => Effect.die("No provider process needed for orphaned requests"), +} as ProviderAdapterV2Shape; +const layerDatabase = SqlitePersistence.layerMemory; +const layerTest = Layer.mergeAll( + layerDatabase, + ProjectionStore.layer.pipe(Layer.provide(layerDatabase)), + ProviderReplayHarness.layerWithRegistry( + { name: "orphaned-requests" }, + ProviderAdapterRegistry.layerFromAdapters([adapter]), + { databaseLayer: layerDatabase, runEffectWorker: false }, + ), +); + +const liveRequest = (status: OrchestrationV2RuntimeRequest["status"] = "pending") => + ({ + id: RuntimeRequestId.make("request"), + nodeId: NodeId.make("node"), + providerTurnId: null, + nativeRequestRef: null, + kind: "user_input", + status, + responseCapability: { type: "live", providerSessionId: ProviderSessionId.make("session") }, + createdAt: DateTime.makeUnsafe(0), + resolvedAt: null, + }) satisfies OrchestrationV2RuntimeRequest; + +describe("isOrphanedRuntimeRequest", () => { + it("is orphaned once its session is detached or its run has ended", () => { + const request = liveRequest(); + assert.isTrue(isOrphanedRuntimeRequest({ request, sessionBound: false, runStatus: "running" })); + assert.isTrue(isOrphanedRuntimeRequest({ request, sessionBound: true, runStatus: "failed" })); + assert.isFalse(isOrphanedRuntimeRequest({ request, sessionBound: true, runStatus: "running" })); + assert.isFalse(isOrphanedRuntimeRequest({ request, sessionBound: true, runStatus: undefined })); + }); + + it("leaves answered and message-mode requests alone", () => { + assert.isFalse( + isOrphanedRuntimeRequest({ + request: liveRequest("resolved"), + sessionBound: false, + runStatus: "failed", + }), + ); + assert.isFalse( + isOrphanedRuntimeRequest({ + request: { ...liveRequest(), responseCapability: { type: "message" } }, + sessionBound: false, + runStatus: "failed", + }), + ); + }); +}); + +describe("messageAnswerText", () => { + it("lists a multi-select answer's choices and keeps a text answer", () => { + assert.equal(messageAnswerText([" Later ", "", "Now"]), "Later, Now"); + assert.isUndefined(messageAnswerText([])); + assert.equal(messageAnswerText("Later"), "Later"); + }); +}); + +it.effect("answers, dismisses and closes requests whose provider session is gone", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread:orphaned-requests"); + const now = yield* DateTime.now; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create-orphaned"), + threadId, + projectId: ProjectId.make("project:orphaned-requests"), + title: "Orphaned", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + // The session the requests name was never bound to the thread, as after a + // failed run detaches it. + const goneSession = ProviderSessionId.make("session:gone"); + let ordinal = 0; + const seed = (name: string, kind: OrchestrationV2RuntimeRequest["kind"]) => + Effect.gen(function* () { + const requestId = RuntimeRequestId.make(`request:${name}`); + const nodeId = NodeId.make(`node:${name}`); + yield* projections.apply({ + id: EventId.make(`request:${name}`), + type: "runtime-request.updated", + threadId, + occurredAt: now, + payload: { + id: requestId, + nodeId, + providerTurnId: null, + nativeRequestRef: null, + kind, + status: "pending", + responseCapability: { type: "live", providerSessionId: goneSession }, + createdAt: now, + resolvedAt: null, + }, + }); + yield* projections.apply({ + id: EventId.make(`node:${name}`), + type: "node.updated", + threadId, + occurredAt: now, + payload: { + id: nodeId, + threadId, + runId: null, + parentNodeId: null, + rootNodeId: nodeId, + kind: kind === "user_input" ? "user_input_request" : "approval_request", + status: "waiting", + countsForRun: false, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + runtimeRequestId: requestId, + checkpointScopeId: null, + startedAt: now, + completedAt: null, + }, + }); + if (kind === "user_input") { + yield* projections.apply({ + id: EventId.make(`item:${name}`), + type: "turn-item.updated", + threadId, + occurredAt: now, + payload: { + id: TurnItemId.make(`item:${name}`), + threadId, + runId: null, + nodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: ++ordinal, + status: "waiting", + title: null, + startedAt: now, + completedAt: null, + updatedAt: now, + type: "user_input_request", + requestId, + questions: [ + { + id: "next", + header: "TEC-1657", + question: "What should happen next?", + options: [ + { label: "Later", description: "Change nothing now." }, + { label: "Now", description: "Start now." }, + ], + }, + ], + }, + }); + } + return { requestId, nodeId }; + }); + + // An answer is recorded and reaches the agent as the next message. + const answered = yield* seed("answered", "user_input"); + yield* orchestrator.dispatch({ + type: "runtime-request.respond", + commandId: CommandId.make("respond-orphaned"), + threadId, + requestId: answered.requestId, + answers: { next: "Later" }, + }); + const answeredRequest = yield* projections.getRuntimeRequest(threadId, answered.requestId); + assert.equal(answeredRequest?.status, "resolved"); + assert.equal(answeredRequest?.responseCapability.type, "message"); + const { messages } = yield* projections.getThreadRecords(threadId, ["messages"]); + assert.isTrue( + messages.some( + (message) => message.role === "user" && message.text === "What should happen next?\nLater", + ), + ); + + // A dismissal closes the question without a reply. + const dismissed = yield* seed("dismissed", "user_input"); + yield* orchestrator.dispatch({ + type: "thread.user-input.dismiss", + commandId: CommandId.make("dismiss-orphaned"), + threadId, + requestId: dismissed.requestId, + }); + const dismissedContext = yield* projections.getRuntimeResponseContext( + threadId, + dismissed.requestId, + ); + assert.equal(dismissedContext.request?.status, "resolved"); + assert.equal(dismissedContext.node?.status, "cancelled"); + + // An approval closes; no provider can act on its decision. + const approval = yield* seed("approval", "command"); + yield* orchestrator.dispatch({ + type: "runtime-request.respond", + commandId: CommandId.make("approve-orphaned"), + threadId, + requestId: approval.requestId, + decision: "accept", + }); + assert.equal( + (yield* projections.getRuntimeRequest(threadId, approval.requestId))?.status, + "resolved", + ); + }).pipe(Effect.provide(layerTest)), +); diff --git a/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.ts b/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.ts new file mode 100644 index 0000000000000..b1f793a72b0d0 --- /dev/null +++ b/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.ts @@ -0,0 +1,81 @@ +// T3-CUSTOM(expbkt3): a question or approval that outlived the provider that asked it. +/** + * A provider question (Claude's AskUserQuestion) or approval is answered live: + * the answer goes back to the callback the provider is blocked on. When the run + * fails instead (the provider stream closes, an event fails to ingest), the run + * ends and its provider session is detached, but the request stays `pending` + * with a `live` capability that nothing can reach. Answering it then fails + * ("Provider session … was not found"), dismissing it fails ("This question + * needs an answer"), and Stop is gone because the run already ended. + * + * Such a request is handled the way a message-mode question is: an answer is + * recorded and reaches the agent as the next message, a dismissal closes it, and + * an approval closes without a reply, since no provider can act on it now. + */ +import type { + OrchestrationV2Run, + OrchestrationV2RuntimeRequest, + ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import type { + ProjectionRuntimeResponseContext, + ProjectionStoreV2Shape, +} from "./ProjectionStore.ts"; + +/** Run states after which no turn of the run waits for an answer. */ +const ENDED_RUN_STATUSES: ReadonlySet = new Set([ + "completed", + "failed", + "interrupted", + "cancelled", +]); + +/** Whether a pending live request can no longer reach the provider that asked it. */ +export function isOrphanedRuntimeRequest(input: { + readonly request: OrchestrationV2RuntimeRequest; + readonly sessionBound: boolean; + readonly runStatus: OrchestrationV2Run["status"] | undefined; +}): boolean { + return ( + input.request.status === "pending" && + input.request.responseCapability.type === "live" && + (!input.sessionBound || + (input.runStatus !== undefined && ENDED_RUN_STATUSES.has(input.runStatus))) + ); +} + +/** + * The request as a response should treat it: unchanged while its provider can + * still answer, else in message mode. The run is read only when the session is + * still bound, which is the uncommon case. + */ +export const respondableRuntimeRequest = (input: { + readonly projectionStore: Pick; + readonly threadId: ThreadId; + readonly context: ProjectionRuntimeResponseContext; +}) => + Effect.gen(function* () { + const { request, node, session } = input.context; + if (request === undefined || request.status !== "pending") return request; + if (request.responseCapability.type !== "live") return request; + let runStatus: OrchestrationV2Run["status"] | undefined; + const runId = node?.runId ?? null; + if (session !== undefined && runId !== null) { + const { runs } = yield* input.projectionStore.getThreadRecords(input.threadId, ["runs"]); + runStatus = runs.find((run) => run.id === runId)?.status; + } + return isOrphanedRuntimeRequest({ request, sessionBound: session !== undefined, runStatus }) + ? { ...request, responseCapability: { type: "message" as const } } + : request; + }); + +/** One question's answer as message text; a multi-select answer lists its choices. */ +export function messageAnswerText(answer: unknown): unknown { + if (!Array.isArray(answer)) return answer; + const choices = answer.filter( + (choice): choice is string => typeof choice === "string" && choice.trim().length > 0, + ); + return choices.length === 0 ? undefined : choices.map((choice) => choice.trim()).join(", "); +} diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 4720b95e6fa2d..9a8c144299ca2 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -15,6 +15,7 @@ import { useRightPanelMaximizedPreference } from "../rightPanelLayoutPreference" import { useCurrentUserId } from "../state/identity"; import { useDurableThreadOutbox } from "../fork/useDurableThreadOutbox"; import { DurableOutboxRecovery } from "../fork/DurableOutboxRecovery"; +import { releaseEndedRunUserInputs } from "../fork/orphanedUserInputs"; import { resolveThreadCreationProfile, threadCreationForkFields, @@ -3338,7 +3339,11 @@ export default function ChatView(props: ChatViewProps) { () => serverProjection === null ? { approvals: [], userInputs: [] } - : derivePendingThreadRequests(serverProjection), + : // T3-CUSTOM(expbkt3): a question whose run ended can be dismissed. + releaseEndedRunUserInputs( + derivePendingThreadRequests(serverProjection), + serverProjection, + ), [serverProjection], ); const pendingApprovals = useMemo( diff --git a/apps/web/src/fork/orphanedUserInputs.test.ts b/apps/web/src/fork/orphanedUserInputs.test.ts new file mode 100644 index 0000000000000..4ec2c66751710 --- /dev/null +++ b/apps/web/src/fork/orphanedUserInputs.test.ts @@ -0,0 +1,47 @@ +import type { PendingThreadRequests } from "@t3tools/client-runtime/state/thread-requests"; +import type { OrchestrationV2ThreadProjection, RuntimeRequestId } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { releaseEndedRunUserInputs } from "./orphanedUserInputs"; + +type Projection = Pick; + +const pending = (requestId: string): PendingThreadRequests => ({ + approvals: [], + userInputs: [ + { + requestId: requestId as RuntimeRequestId, + createdAt: "2026-10-15T15:28:34.029Z", + questions: [], + responseCapability: "live", + dismissible: false, + }, + ], +}); + +const projection = (runStatus: string): Projection => + ({ + runs: [{ id: "run:9", status: runStatus }], + turnItems: [{ type: "user_input_request", requestId: "request", runId: "run:9" }], + }) as unknown as Projection; + +describe("releaseEndedRunUserInputs", () => { + it("makes a question whose run failed answerable by message and dismissible", () => { + const [input] = releaseEndedRunUserInputs(pending("request"), projection("failed")).userInputs; + expect(input).toMatchObject({ + responseCapability: "message", + responseMode: "message", + dismissible: true, + }); + }); + + it("leaves a question alone while its run is still going", () => { + const requests = pending("request"); + expect(releaseEndedRunUserInputs(requests, projection("running"))).toBe(requests); + }); + + it("leaves a question from another run alone", () => { + const requests = pending("other"); + expect(releaseEndedRunUserInputs(requests, projection("failed"))).toBe(requests); + }); +}); diff --git a/apps/web/src/fork/orphanedUserInputs.ts b/apps/web/src/fork/orphanedUserInputs.ts new file mode 100644 index 0000000000000..7adb2e33a47e8 --- /dev/null +++ b/apps/web/src/fork/orphanedUserInputs.ts @@ -0,0 +1,46 @@ +/** + * T3-CUSTOM(expbkt3): a question whose run already ended can be dismissed. + * + * When a run fails while the agent waits on a question, the question stays + * pending with a live capability that no provider can answer. The server + * answers such a question in message mode (`orphanedRuntimeRequest.expbkt3.ts` + * in apps/server); this lets the composer offer Dismiss for it too, instead + * of only a Submit that the dead turn never receives. + */ +import type { PendingThreadRequests } from "@t3tools/client-runtime/state/thread-requests"; +import type { OrchestrationV2ThreadProjection } from "@t3tools/contracts"; + +const ENDED_RUN_STATUSES: ReadonlySet = new Set([ + "completed", + "failed", + "interrupted", + "cancelled", +]); + +export function releaseEndedRunUserInputs( + pending: PendingThreadRequests, + projection: Pick, +): PendingThreadRequests { + if (!pending.userInputs.some((input) => input.responseCapability === "live")) return pending; + const endedRunIds = new Set( + projection.runs.filter((run) => ENDED_RUN_STATUSES.has(run.status)).map((run) => run.id), + ); + if (endedRunIds.size === 0) return pending; + let changed = false; + const userInputs = pending.userInputs.map((input) => { + if (input.responseCapability !== "live") return input; + const item = projection.turnItems.findLast( + (candidate) => + candidate.type === "user_input_request" && candidate.requestId === input.requestId, + ); + if (item?.runId == null || !endedRunIds.has(item.runId)) return input; + changed = true; + return { + ...input, + responseCapability: "message" as const, + responseMode: "message" as const, + dismissible: true, + }; + }); + return changed ? { ...pending, userInputs } : pending; +} diff --git a/docs/operations/expbkt3-customizations.md b/docs/operations/expbkt3-customizations.md index 8bdec1e99aa08..9f3b776e221a1 100644 --- a/docs/operations/expbkt3-customizations.md +++ b/docs/operations/expbkt3-customizations.md @@ -180,6 +180,7 @@ are the identity and credential-actor restart predicate in | Chat comments | `packages/contracts/src/threadComments.ts`, `apps/server/src/threadcomments/`, `packages/client-runtime/src/state/threadComments.ts`, `apps/web/src/fork/threadComments/`, `apps/web/src/state/threadComments.ts`, `docs/user/chat-comments.md` | fork RPC methods in `rpcFork.ts` + one subscription tag in `client-runtime/src/rpc/client.ts`, `threadComments` capability, `chatCommentsEnabled` setting, `comments` right-panel kind (store/tabs), `ChatView.tsx` panel branch + banner + empty-send gate, `ChatComposer.tsx` send state + placeholder, `AssistantSelectionToolbar.tsx` group + `extraActions`, `MessagesTimeline.tsx` wrap, `ProviderCommandReactor.ts` seam | | Native plan review | `apps/server/src/planreview/`, `persistence/PlanReviewDocuments.ts`, migration 1009, `packages/shared/src/planReview.ts`, `packages/contracts/src/planReview.ts`, `apps/web/src/components/planreview/`, `apps/web/src/fork/planReviewSurface.tsx` | fork RPC group + scopes + handlers, one `ws.ts` dep, right-panel store/tabs, `ChatView.tsx` branch, Experiments settings toggles, nullable comment range in `reviewCommentContext.ts` + `reviewCommentSelection.ts` + `nativeReviewDiffAdapter.ts`, plan-comment card in `MessagesTimeline.tsx` + `ThreadFeed.tsx` | | Plan cards | `apps/web/src/fork/planCard/` (header, width preference, HTML parts, citation and comment source), `apps/server/src/orchestration-v2/forkAgentPlanInstructions.expbkt3.ts` (plan-mode `html-render` hint) | `ProposedPlanCard.tsx` imports (`ChatMarkdown` alias, folded preview, citation expand) + header element, `MessagesTimeline.tsx` `PlanCardRow` wrap, `useAssistantCitationTarget.ts` source predicates, `RunExecutionService.ts` interaction-mode argument | +| Orphaned questions | `apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.ts`, `apps/web/src/fork/orphanedUserInputs.ts` | `Orchestrator.ts` respond and dismiss read the request through `respondableRuntimeRequest`, one approval line and one multi-select line in the message branch; `ChatView.tsx` wraps `derivePendingThreadRequests` | | Session identity | `apps/server/src/identity/SessionIdentityEnvironment.ts`, `apps/server/src/provider/claudeSessionIdentity.expbkt3.ts` | `ProviderCommandReactor.ts` execution-options seam, `ProviderService.ts` adapter-spawn seam, `identityEnvironment` on `ProviderSessionExecutionOptions`, conditional scrub in `SourceControlExecutionEnvironment.ts`, `server.ts` layer, `ClaudeAdapter.ts` system-prompt append + `UserPromptSubmit` hook seams | | Agent views in chat | `apps/server/src/agentui/`, `persistence/AgentUiRenders.ts`, migration 1022, `packages/contracts/src/agentUi.ts`, `packages/client-runtime/src/state/agentUi.ts`, `apps/web/src/fork/agentUiSurface.tsx`, `apps/web/src/state/agentUi.ts` | `t3_show_ui` in the MCP control toolkit, fork RPC group + scopes + handlers, one `ws.ts` dep, one `server.ts` layer, marked handle passthrough in `ActivityPayloadProjection.ts`, `agentUi` field + read in `session-logic.ts`, one import and early return in `MessagesTimeline.tsx`, Experiments toggle + `settingsSearch.ts` entry | | User presence | `apps/server/src/presence/` (rules, tracker, message query, `presenceHttp.expbkt3.ts`, `presenceEnvironment.expbkt3.ts`, `backgroundPolicyRedaction.expbkt3.ts`), `t3_user_presence` tool, web + mobile `useThreadPresenceScope` | `userPresence` capability, one route + layer in `server.ts`, `BK_T3_PRESENCE_URL` beside the bearer in `ClaudeAdapter.ts` + `CodexAdapter.ts`, lease redaction at both background-policy RPCs in `ws.ts`, exported `retainBackgroundScope` + one `ChatView.tsx` hook call, `retainMobileBackgroundScope` + one `ThreadRouteScreen.tsx` hook call | diff --git a/docs/user/t3-mcp-control.md b/docs/user/t3-mcp-control.md index 12e98e06aed3c..7297ee150f437 100644 --- a/docs/user/t3-mcp-control.md +++ b/docs/user/t3-mcp-control.md @@ -167,6 +167,9 @@ question pauses the agent until someone answers it. A non-blocking question, such as a Codex async question, did not stop the agent: the answer reaches it as a new message, in its running turn or as a new turn, and `t3_answer` with `dismiss: true` closes the question without an answer. +A blocking question whose run already ended, for example after a provider +error, works the same way: the answer starts a new turn, `dismiss: true` closes +it, and an approval left open by such a run closes when answered. `t3_send_message` refuses only while a blocking question or an approval is open. Pass the `requestId` that `t3_inbox` shows when a session has more than one open question, and with every `clientRequestId`, so a retry cannot answer a From 5837955e289adbbb2bd9a35b9e291882028e8c45 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Thu, 15 Oct 2026 18:47:17 +0000 Subject: [PATCH 17/20] chore(web): mark the plan review lookup as a fork edit Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 2a6abc88e04fdbad6397c0be905bfedd37129636) --- apps/web/src/components/ChatView.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 9a8c144299ca2..945a7c19c3ebd 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -3498,6 +3498,7 @@ export default function ChatView(props: ChatViewProps) { } return findLatestProposedPlan(serverProjection, activeLatestRun?.runId ?? null); }, [activeLatestRun?.runId, latestRunSettled, serverProjection]); + // T3-CUSTOM(expbkt3): the plan review takeover reads the thread's open review document. const planReviewDocumentId = useOpenPlanReviewDocumentId( activeThreadRef?.environmentId ?? null, activeThreadRef?.threadId ?? null, From b1e5f2264c67430ed3811be9ae8f84f3961e4677 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Thu, 15 Oct 2026 18:56:39 +0000 Subject: [PATCH 18/20] fix(server,web): review fixes for orphaned questions - A completed run no longer counts as ended: its session stays live and a background subagent can still wait on an approval. Failed, interrupted and cancelled runs do. - Read only the request's own run, and take it from the timeline item when the node has none. - MCP lists a question whose run failed or stopped as non-blocking, so t3_answer can dismiss it and t3_send_message no longer refuses. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 56e13066dcd6e4c8fba30e13707c9aeca3859abe) --- .../src/mcp/toolkits/agent/pending.test.ts | 35 ++++++++++++++++++- apps/server/src/mcp/toolkits/agent/pending.ts | 21 +++++++++-- .../orphanedRuntimeRequest.expbkt3.test.ts | 4 +++ .../orphanedRuntimeRequest.expbkt3.ts | 28 ++++++++++----- apps/web/src/fork/orphanedUserInputs.test.ts | 3 +- apps/web/src/fork/orphanedUserInputs.ts | 8 ++--- docs/user/t3-mcp-control.md | 6 ++-- 7 files changed, 83 insertions(+), 22 deletions(-) diff --git a/apps/server/src/mcp/toolkits/agent/pending.test.ts b/apps/server/src/mcp/toolkits/agent/pending.test.ts index 08eada90b61ac..6763db5d10261 100644 --- a/apps/server/src/mcp/toolkits/agent/pending.test.ts +++ b/apps/server/src/mcp/toolkits/agent/pending.test.ts @@ -57,13 +57,14 @@ const shell = (fields: Partial>): SessionShell => const callerWith = (records: { readonly runtimeRequests: ReadonlyArray; readonly turnItems: ReadonlyArray; + readonly runs?: ReadonlyArray; }) => { const reads: Array = []; const caller = { threads: { getThreadRecords: (_threadId: unknown, _fields: unknown, filter: unknown) => { reads.push(filter); - return Effect.succeed(records); + return Effect.succeed({ runs: [], ...records }); }, }, } as unknown as Caller; @@ -149,6 +150,38 @@ describe("loadPendingDetail", () => { }), ); + it.effect("lets a question whose run failed be dismissed and answered by message", () => + Effect.gen(function* () { + const { caller, reads } = callerWith({ + runtimeRequests: [ + runtimeRequest({ + id: BLOCKING_ID, + responseCapability: { type: "live", providerSessionId: "provider-session-1" }, + }), + ], + turnItems: [questionItem(BLOCKING_ID, "Blocking?")], + runs: [{ id: "run-1", status: "failed" }], + }); + const detail = yield* loadPendingDetail( + caller, + shell({ + pendingRuntimeRequest: { + id: BLOCKING_ID, + kind: "user_input", + createdAt: DateTime.makeUnsafe(ASKED), + }, + }), + { includePlan: false }, + ); + expect(reads).toContainEqual({ runIds: ["run-1"] }); + expect(detail.question).toMatchObject({ + requestId: BLOCKING_ID, + async: true, + textOnly: true, + }); + }), + ); + it.effect("keeps an approval's details while a non-blocking question is open", () => Effect.gen(function* () { const approvalId = diff --git a/apps/server/src/mcp/toolkits/agent/pending.ts b/apps/server/src/mcp/toolkits/agent/pending.ts index 7e70242cffdcf..8f8d9aa703a93 100644 --- a/apps/server/src/mcp/toolkits/agent/pending.ts +++ b/apps/server/src/mcp/toolkits/agent/pending.ts @@ -24,6 +24,7 @@ import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import { findForkAsyncRequest } from "../../../orchestration-v2/forkAsyncResponse.expbkt3.ts"; +import { runEndedWithoutAnswer } from "../../../orchestration-v2/orphanedRuntimeRequest.expbkt3.ts"; import type { Caller } from "../../threadAccess.ts"; import type { QuestionView } from "./answers.ts"; import { shortRef, toIso, type SessionShell } from "./view.ts"; @@ -147,6 +148,13 @@ const approvalSubject = (caller: Caller, shell: SessionShell, runId: RunId | nul type TurnItemType = OrchestrationV2TurnItem["type"]; +/** Whether the run that asked a live question has failed or stopped, which the server answers by message. */ +const runEndedFor = (caller: Caller, shell: SessionShell, runId: RunId) => + caller.threads.getThreadRecords(shell.id, ["runs"], { runIds: [runId] }).pipe( + Effect.map(({ runs }) => runEndedWithoutAnswer(runs.find((run) => run.id === runId)?.status)), + Effect.orElseSucceed(() => false), + ); + /** * Reads the details for the waits the shell reports. `includePlan` also reads * the newest proposed plan, which only a plan review needs. @@ -196,11 +204,20 @@ export const loadPendingDetail = Effect.fn("agentToolkit.loadPendingDetail")(fun if (secretItem?.type === "secret_request") { secret = { label: secretItem.label, reason: secretItem.reason }; } else if (item?.type === "user_input_request") { + // A question whose run failed or stopped no longer blocks the agent: the + // server sends its answer as a new turn and lets it be dismissed. + const orphaned = + runtime?.responseCapability.type === "live" && + item.runId !== null && + (yield* runEndedFor(caller, shell, item.runId)); question = { requestId: request.id, questions: item.questions, - textOnly: runtime?.responseCapability.type === "message" || item.responseMode === "message", - async: false, + textOnly: + orphaned || + runtime?.responseCapability.type === "message" || + item.responseMode === "message", + async: orphaned, askedAt: toIso(request.createdAt), }; } else if (item?.type === "approval_request") { diff --git a/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.test.ts b/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.test.ts index b16164f7f74ba..f75a375f1cb2c 100644 --- a/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.test.ts +++ b/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.test.ts @@ -65,6 +65,10 @@ describe("isOrphanedRuntimeRequest", () => { assert.isTrue(isOrphanedRuntimeRequest({ request, sessionBound: true, runStatus: "failed" })); assert.isFalse(isOrphanedRuntimeRequest({ request, sessionBound: true, runStatus: "running" })); assert.isFalse(isOrphanedRuntimeRequest({ request, sessionBound: true, runStatus: undefined })); + // A completed run's session stays live for its background subagents. + assert.isFalse( + isOrphanedRuntimeRequest({ request, sessionBound: true, runStatus: "completed" }), + ); }); it("leaves answered and message-mode requests alone", () => { diff --git a/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.ts b/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.ts index b1f793a72b0d0..54d000be55419 100644 --- a/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.ts +++ b/apps/server/src/orchestration-v2/orphanedRuntimeRequest.expbkt3.ts @@ -24,14 +24,22 @@ import type { ProjectionStoreV2Shape, } from "./ProjectionStore.ts"; -/** Run states after which no turn of the run waits for an answer. */ +/** + * Run states after which no turn of the run waits for an answer. A completed + * run is left out: its session stays live, and a background subagent of it + * can still be waiting on an approval. + */ const ENDED_RUN_STATUSES: ReadonlySet = new Set([ - "completed", "failed", "interrupted", "cancelled", ]); +/** Whether a run ended without its provider waiting for answers any longer. */ +export function runEndedWithoutAnswer(status: OrchestrationV2Run["status"] | undefined): boolean { + return status !== undefined && ENDED_RUN_STATUSES.has(status); +} + /** Whether a pending live request can no longer reach the provider that asked it. */ export function isOrphanedRuntimeRequest(input: { readonly request: OrchestrationV2RuntimeRequest; @@ -41,15 +49,14 @@ export function isOrphanedRuntimeRequest(input: { return ( input.request.status === "pending" && input.request.responseCapability.type === "live" && - (!input.sessionBound || - (input.runStatus !== undefined && ENDED_RUN_STATUSES.has(input.runStatus))) + (!input.sessionBound || runEndedWithoutAnswer(input.runStatus)) ); } /** * The request as a response should treat it: unchanged while its provider can - * still answer, else in message mode. The run is read only when the session is - * still bound, which is the uncommon case. + * still answer, else in message mode. Only the request's own run is read, and + * only while its session is still bound. */ export const respondableRuntimeRequest = (input: { readonly projectionStore: Pick; @@ -57,13 +64,16 @@ export const respondableRuntimeRequest = (input: { readonly context: ProjectionRuntimeResponseContext; }) => Effect.gen(function* () { - const { request, node, session } = input.context; + const { request, node, item, session } = input.context; if (request === undefined || request.status !== "pending") return request; if (request.responseCapability.type !== "live") return request; let runStatus: OrchestrationV2Run["status"] | undefined; - const runId = node?.runId ?? null; + // The node names the run; older requests carry it only on their timeline item. + const runId = node?.runId ?? item?.runId ?? null; if (session !== undefined && runId !== null) { - const { runs } = yield* input.projectionStore.getThreadRecords(input.threadId, ["runs"]); + const { runs } = yield* input.projectionStore.getThreadRecords(input.threadId, ["runs"], { + runIds: [runId], + }); runStatus = runs.find((run) => run.id === runId)?.status; } return isOrphanedRuntimeRequest({ request, sessionBound: session !== undefined, runStatus }) diff --git a/apps/web/src/fork/orphanedUserInputs.test.ts b/apps/web/src/fork/orphanedUserInputs.test.ts index 4ec2c66751710..369d23e6dd6c8 100644 --- a/apps/web/src/fork/orphanedUserInputs.test.ts +++ b/apps/web/src/fork/orphanedUserInputs.test.ts @@ -35,9 +35,10 @@ describe("releaseEndedRunUserInputs", () => { }); }); - it("leaves a question alone while its run is still going", () => { + it("leaves a question alone while its run is still going or has completed", () => { const requests = pending("request"); expect(releaseEndedRunUserInputs(requests, projection("running"))).toBe(requests); + expect(releaseEndedRunUserInputs(requests, projection("completed"))).toBe(requests); }); it("leaves a question from another run alone", () => { diff --git a/apps/web/src/fork/orphanedUserInputs.ts b/apps/web/src/fork/orphanedUserInputs.ts index 7adb2e33a47e8..3e110c8da7d4a 100644 --- a/apps/web/src/fork/orphanedUserInputs.ts +++ b/apps/web/src/fork/orphanedUserInputs.ts @@ -10,12 +10,8 @@ import type { PendingThreadRequests } from "@t3tools/client-runtime/state/thread-requests"; import type { OrchestrationV2ThreadProjection } from "@t3tools/contracts"; -const ENDED_RUN_STATUSES: ReadonlySet = new Set([ - "completed", - "failed", - "interrupted", - "cancelled", -]); +// The server's rule (`runEndedWithoutAnswer`): a completed run's session stays live. +const ENDED_RUN_STATUSES: ReadonlySet = new Set(["failed", "interrupted", "cancelled"]); export function releaseEndedRunUserInputs( pending: PendingThreadRequests, diff --git a/docs/user/t3-mcp-control.md b/docs/user/t3-mcp-control.md index 7297ee150f437..c1a547c0eea92 100644 --- a/docs/user/t3-mcp-control.md +++ b/docs/user/t3-mcp-control.md @@ -167,9 +167,9 @@ question pauses the agent until someone answers it. A non-blocking question, such as a Codex async question, did not stop the agent: the answer reaches it as a new message, in its running turn or as a new turn, and `t3_answer` with `dismiss: true` closes the question without an answer. -A blocking question whose run already ended, for example after a provider -error, works the same way: the answer starts a new turn, `dismiss: true` closes -it, and an approval left open by such a run closes when answered. +A blocking question whose run failed or was stopped, for example after a +provider error, works the same way: the answer starts a new turn, `dismiss: +true` closes it, and an approval left open by such a run closes when answered. `t3_send_message` refuses only while a blocking question or an approval is open. Pass the `requestId` that `t3_inbox` shows when a session has more than one open question, and with every `clientRequestId`, so a retry cannot answer a From d0956359db663a78c99210f7e43890f71ae99ed3 Mon Sep 17 00:00:00 2001 From: Bk Agent Date: Fri, 16 Oct 2026 18:12:06 +0530 Subject: [PATCH 19/20] fix(server): fast session-webhook ack (Toolyard callbacks no longer time out) (#318) * fix(server): fast webhook ack and personal-token toolyard proxy Session webhook receive: the admission transaction called the Toolyard integration's owner check, which reads environment users through Effect.runPromise on a fresh fiber. The SQLite client has one connection behind a one-permit semaphore held by the open transaction, so that read waited for the transaction that waited for it. Every push callback hung until Toolyard's 20 s client timeout, the request was interrupted and the insert rolled back. The transaction now runs SQL only; the binding is checked before it and again by the dispatch loop. Upstream MCP proxy: a personal T3 token (t3usr_) may call /mcp/upstream/toolyard as the token's own user, with that user's managed Toolyard credential, when the server and user external MCP switches are on. No x-t3-session-id is sent for these calls. Provider sessions are unchanged. Co-Authored-By: Claude Opus 5.5 * test(server): fix the session-webhook owner-check test for lint and the live clock - Reuse one payload for the duplicate delivery. Under it.live a second h.event() carries a new timestamp, so the retry was a 409 content conflict instead of a duplicate. - Keep the fresh-fiber Effect.runPromise read, which replicates the production owner check, behind a reasoned oxlint suppression. Co-Authored-By: Claude Opus 5.5 * revert(server): drop the personal-token toolyard proxy from this PR The host relay timer that needed a personal t3usr_ token on /mcp/upstream/toolyard was cancelled (2026-10-15), so the new access path has no user. Keep only the fast session-webhook ack. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../SessionWebhookService.test.ts | 39 ++++++++++++++++++- .../session-webhooks/SessionWebhookService.ts | 17 ++++---- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/apps/server/src/session-webhooks/SessionWebhookService.test.ts b/apps/server/src/session-webhooks/SessionWebhookService.test.ts index 5d25910ce3367..e18fbe813f838 100644 --- a/apps/server/src/session-webhooks/SessionWebhookService.test.ts +++ b/apps/server/src/session-webhooks/SessionWebhookService.test.ts @@ -67,6 +67,7 @@ const harness = (mode: "push" | "pull" = "push") => const historyEntered = yield* Deferred.make(); const twoHistoriesEntered = yield* Deferred.make(); const fourHistoriesEntered = yield* Deferred.make(); + const sql = yield* SqlClient.SqlClient; const secrets = new Map(); const receipts = new Map(); const state = { @@ -119,7 +120,17 @@ const harness = (mode: "push" | "pull" = "push") => dispatches: 0, registrations: 0, notifications: [] as string[], + // Production checks the owner with `Effect.runPromise(users.get(...))`: + // a fresh fiber that reads the same SQLite client outside any transaction. + ownerCheckReadsDatabase: false, }; + // oxlint-disable-next-line t3code/no-manual-effect-runtime-in-tests -- Replicates the production owner check's own root fiber; a forked fiber would inherit the transaction connection and hide the deadlock. + const readOnFreshFiber = () => Effect.runPromise(sql`SELECT 1`); + const ownerCheckRead = Effect.suspend(() => + state.ownerCheckReadsDatabase + ? Effect.promise(readOnFreshFiber).pipe(Effect.asVoid) + : Effect.void, + ); const shell = (id = threadId) => ({ id, @@ -198,7 +209,7 @@ const harness = (mode: "push" | "pull" = "push") => : null, ), callbackBinding: () => - Effect.sync(() => + Effect.map(ownerCheckRead, () => state.integrationEnabled ? { instanceId: state.instanceId, @@ -215,7 +226,7 @@ const harness = (mode: "push" | "pull" = "push") => ), assertConnectionOwner: () => state.userEnabled - ? Effect.void + ? ownerCheckRead : Effect.fail( new PersonalMcpSettingsError({ operation: "test", message: "owner-disabled" }), ), @@ -1357,6 +1368,30 @@ describe("durable session webhooks", () => { ).pipe(Effect.provide(NodeHttpServer.layerTest)), ), ); + it.live("acknowledges a push callback when the owner check reads the database", () => + scenario((h) => + Effect.gen(function* () { + h.state.ownerCheckReadsDatabase = true; + // Before the fix the admission transaction called the owner check, which + // waited for the connection that transaction held, until the sender gave up. + // One payload for both deliveries: the live clock would give a second + // `h.event()` a new timestamp, which is a content conflict, not a retry. + const payload = h.event(); + const first = yield* h.receive(payload).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(first)); + assert.strictEqual(Option.getOrThrow(first).duplicate, false); + const repeat = yield* h.receive(payload).pipe(Effect.timeoutOption("2 seconds")); + assert.strictEqual(Option.getOrThrow(repeat).duplicate, true); + assert.strictEqual(h.state.dispatches, 0); + assert.strictEqual( + (yield* h.service.inspect(owner, h.webhook.id)).deliveries[0]?.state, + "queued", + ); + yield* h.service.drain(); + assert.strictEqual(h.state.dispatches, 1); + }), + ), + ); it.effect("returns an existing destination without revealing secrets", () => scenario((h) => Effect.gen(function* () { diff --git a/apps/server/src/session-webhooks/SessionWebhookService.ts b/apps/server/src/session-webhooks/SessionWebhookService.ts index 91e1b787e1c6d..49d6e83cb724c 100644 --- a/apps/server/src/session-webhooks/SessionWebhookService.ts +++ b/apps/server/src/session-webhooks/SessionWebhookService.ts @@ -420,6 +420,14 @@ export const make = Effect.gen(function* () { catch: () => fail(400, "invalid-callback-payload"), }); const fingerprint = callbackFingerprint(body); + // Only SQL runs inside the admission transaction. The SQLite client has one + // connection behind a one-permit semaphore, and the transaction holds that + // permit until it commits. The integration checks the owner with + // `Effect.runPromise(users.get(...))`, a fresh fiber outside this + // transaction, so calling it here waited for the permit this fiber held: + // every push callback hung until the sender's 20 s timeout and rolled back. + // The binding was verified above; the drain loop re-checks it before dispatch + // and `retireReplacedReceiver` ends queued events of a replaced receiver. return yield* sql.withTransaction( Effect.gen(function* () { const existing = @@ -431,14 +439,7 @@ export const make = Effect.gen(function* () { } const current = (yield* sql`SELECT * FROM session_webhooks WHERE id=${id}`)[0]; if (current?.status !== "active") return yield* fail(410, "webhook-disabled-or-removed"); - const currentBinding = yield* integration.callbackBinding( - UserId.make(current.owner_user_id), - ); - if ( - !currentBinding || - !currentBinding.enabled || - receiverTrustBinding(currentBinding) !== current.trust_binding - ) + if (current.trust_binding !== row.trust_binding) return yield* fail(410, "integration-disabled-or-replaced"); const cutoff = DateTime.formatIso(DateTime.makeUnsafe(millis() - 60_000)); const count = From e6c620a3ec5dad2245c24944df0ea0510dd177c5 Mon Sep 17 00:00:00 2001 From: bk-agent-01 Date: Mon, 19 Oct 2026 10:49:08 +0000 Subject: [PATCH 20/20] fix(server): stop the pull-request sweep and plan-flag reads from freezing bkt3 The minute pull-request sweep checked all 1,100 unsettled threads on bkt3. Each check starts 6-7 git processes and every start forks the 1.4 GB server on the event loop, so a sweep ran 50-95 s of every two minutes with the main thread saturated (55% of samples in uv_spawn). Replies took 504 ms median during a sweep against 2 ms outside it, and desktop connections hit their 15 s setup limit. A thread with no run or message for a day is now checked once every 30 passes; its own events still queue an immediate check. The agent toolkit's plan-flag correction read a run's turn items with an `IN (...) OR (...)` filter, so SQLite walked every item of the thread in ordinal order: 0.3-2.6 s per cold 30,000-item thread, measured as a 2.5 s event-loop stall at 10:32 UTC. Run filters now read through the (thread_id, run_id) index: 4 ms median on the same cold threads. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5afbacbb0126e76e659213b74db18d59cc5ab609) --- .../src/orchestration-v2/ProjectionStore.ts | 17 +- .../ThreadPullRequestService.ts | 15 ++ .../threadPullRequestSweep.expbkt3.test.ts | 196 ++++++++++++++++++ .../threadPullRequestSweep.expbkt3.ts | 57 +++++ 4 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 apps/server/src/orchestration-v2/threadPullRequestSweep.expbkt3.test.ts create mode 100644 apps/server/src/orchestration-v2/threadPullRequestSweep.expbkt3.ts diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 2ded30eabfcd7..5b3a82f1ea04a 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -2716,18 +2716,31 @@ export const layer: Layer.Layer = return yield* new ProjectionStoreThreadNotFoundError({ threadId }); } + // T3-CUSTOM(expbkt3): a run filter reads only that run's items through the + // (thread_id, run_id) index. In ordinal order SQLite walked every item of the + // thread instead, which took 0.3-2.6 s on a cold 30,000-item thread and froze + // the server (agent toolkit plan-flag checks run it for up to 40 sessions). + const turnItemRunIdsWithoutNull = + filter?.turnItemRunIds === undefined || filter.turnItemRunIds.includes(null) + ? undefined + : filter.turnItemRunIds.filter((id): id is RunId => id !== null); + const turnItemRunIndex = + filter?.turnItemRunId !== undefined || turnItemRunIdsWithoutNull !== undefined + ? sql`INDEXED BY orchestration_v2_projection_turn_items_thread_run_idx` + : sql``; + const boundedTurnItemRows = fields !== undefined && !fields.includes("turnItems") ? [] : window === undefined ? yield* sql` SELECT payload_json - FROM orchestration_v2_projection_turn_items + FROM orchestration_v2_projection_turn_items ${turnItemRunIndex} /* T3-CUSTOM(expbkt3): run filters use the run index. */ WHERE thread_id = ${threadId} ${filter?.turnItemTypes === undefined ? sql`` : sql`AND type IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemTypes)}))`} ${filter?.turnItemStatuses === undefined ? sql`` : sql`AND status IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemStatuses)}))`} ${filter?.turnItemRunId === undefined ? sql`` : sql`AND run_id = ${filter.turnItemRunId}`} - ${filter?.turnItemRunIds === undefined ? sql`` : sql`AND (run_id IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemRunIds.filter((id): id is RunId => id !== null))})) OR (${filter.turnItemRunIds.includes(null) ? 1 : 0} = 1 AND run_id IS NULL))`} + ${filter?.turnItemRunIds === undefined ? sql`` : turnItemRunIdsWithoutNull !== undefined ? sql`AND run_id IN (SELECT value FROM json_each(${encodeIdList(turnItemRunIdsWithoutNull)}))` : sql`AND (run_id IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemRunIds.filter((id): id is RunId => id !== null))})) OR (${filter.turnItemRunIds.includes(null) ? 1 : 0} = 1 AND run_id IS NULL))`} /* T3-CUSTOM(expbkt3): an OR here defeats the run index. */ ORDER BY ordinal ASC, turn_item_id ASC ` : yield* sql` diff --git a/apps/server/src/orchestration-v2/ThreadPullRequestService.ts b/apps/server/src/orchestration-v2/ThreadPullRequestService.ts index cd88d93977768..a1dabb267b2f8 100644 --- a/apps/server/src/orchestration-v2/ThreadPullRequestService.ts +++ b/apps/server/src/orchestration-v2/ThreadPullRequestService.ts @@ -12,6 +12,8 @@ import { } from "@t3tools/contracts"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import * as Cause from "effect/Cause"; +// T3-CUSTOM(expbkt3): the idle-thread rotation reads the clock. +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; @@ -28,6 +30,8 @@ import * as PullRequestService from "../pullRequest/PullRequestService.ts"; import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; import { forkParked } from "../serverActivation.ts"; import * as Orchestrator from "./Orchestrator.ts"; +// T3-CUSTOM(expbkt3): idle threads ride a slow rotation in the periodic sweep. +import { includeInPeriodicSweep } from "./threadPullRequestSweep.expbkt3.ts"; class ThreadPullRequestServiceV2 extends Context.Service< ThreadPullRequestServiceV2, @@ -106,6 +110,8 @@ export const make = Effect.gen(function* () { const crypto = yield* Crypto.Crypto; const fileSystem = yield* FileSystem.FileSystem; const pendingBackfill = new Map(); + // T3-CUSTOM(expbkt3): counts all-thread passes for the idle-thread rotation. + let periodicPass = 0; const finishBackfill = (threads: ReadonlyArray>) => { for (const thread of threads) pendingBackfill.delete(thread.id); @@ -167,8 +173,17 @@ export const make = Effect.gen(function* () { for (const threadId of checkedIds) { if (!visibleThreadIds.has(threadId)) pendingBackfill.delete(threadId); } + // T3-CUSTOM(expbkt3): BEGIN — an all-thread pass checks idle threads on a rotation. + const nowMs = yield* Clock.currentTimeMillis; + const pass = periodicPass; + if (request.threadId === null) periodicPass += 1; + // T3-CUSTOM(expbkt3): END const threads = threadSnapshot.threads.filter( (thread) => + // T3-CUSTOM(expbkt3): a one-thread check and backfill are never rotated. + (request.threadId !== null || + pendingBackfill.has(thread.id) || + includeInPeriodicSweep(thread, nowMs, pass)) && thread.archivedAt === null && ((thread.settledOverride !== "settled" && thread.settledAt === null) || request.threadId !== null || diff --git a/apps/server/src/orchestration-v2/threadPullRequestSweep.expbkt3.test.ts b/apps/server/src/orchestration-v2/threadPullRequestSweep.expbkt3.test.ts new file mode 100644 index 0000000000000..cd5fda6ed57d2 --- /dev/null +++ b/apps/server/src/orchestration-v2/threadPullRequestSweep.expbkt3.test.ts @@ -0,0 +1,196 @@ +import { + ProjectId, + ProviderInstanceId, + ThreadId, + type OrchestrationProjectShell, + type OrchestrationV2ThreadShell, +} from "@t3tools/contracts"; +import { describe, expect, it } from "@effect/vitest"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; + +import * as GitManager from "../git/GitManager.ts"; +import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; +import * as PullRequestService from "../pullRequest/PullRequestService.ts"; +import * as ServerActivation from "../serverActivation.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import * as ProjectStore from "./ProjectStore.ts"; +import * as ThreadPullRequestService from "./ThreadPullRequestService.ts"; +import { + IDLE_SWEEP_EVERY_PASSES, + idleSweepSlot, + includeInPeriodicSweep, + isIdleForSweep, +} from "./threadPullRequestSweep.expbkt3.ts"; + +const NOW = DateTime.makeUnsafe("2026-10-19T10:00:00.000Z"); +const NOW_MS = DateTime.toEpochMillis(NOW); +const hoursAgo = (hours: number) => DateTime.makeUnsafe(NOW_MS - hours * 60 * 60 * 1000); + +const threadShell = ( + id: string, + overrides: Partial = {}, +): OrchestrationV2ThreadShell => { + const threadId = ThreadId.make(id); + return { + id: threadId, + projectId: ProjectId.make(`project-${id}`), + title: id, + providerInstanceId: ProviderInstanceId.make("codex"), + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: `feature/${id}`, + worktreePath: null, + activeProviderThreadId: null, + lineage: { rootThreadId: threadId, parentThreadId: null, relationshipToParent: null }, + forkedFrom: null, + createdBy: "user", + creationSource: "web", + activeRunId: null, + latestRunId: null, + status: "idle", + pendingRuntimeRequest: null, + latestVisibleMessage: null, + latestUserMessageAt: null, + hasActionableProposedPlan: false, + itemCount: 0, + visibleItemCount: 0, + createdAt: hoursAgo(24 * 30), + updatedAt: NOW, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + ...overrides, + }; +}; + +describe("periodic pull-request sweep rotation", () => { + it("treats a thread as idle only after a day with no run, message or live work", () => { + expect(isIdleForSweep(threadShell("old", { latestRunCompletedAt: hoursAgo(25) }), NOW_MS)).toBe( + true, + ); + expect( + isIdleForSweep(threadShell("recent", { latestRunCompletedAt: hoursAgo(2) }), NOW_MS), + ).toBe(false); + expect( + isIdleForSweep(threadShell("messaged", { latestUserMessageAt: hoursAgo(1) }), NOW_MS), + ).toBe(false); + expect(isIdleForSweep(threadShell("new", { createdAt: hoursAgo(3) }), NOW_MS)).toBe(false); + expect( + isIdleForSweep( + threadShell("waiting", { + latestRunCompletedAt: hoursAgo(48), + activityRunStatus: "waiting", + }), + NOW_MS, + ), + ).toBe(false); + }); + + it("checks every idle thread exactly once per rotation and active threads on every pass", () => { + const idle = Array.from({ length: 200 }, (_, index) => + threadShell(`idle-${index}`, { latestRunCompletedAt: hoursAgo(72) }), + ); + const active = threadShell("active", { latestRunCompletedAt: hoursAgo(1) }); + const checks = new Map(); + for (let pass = 0; pass < IDLE_SWEEP_EVERY_PASSES; pass += 1) { + for (const thread of [...idle, active]) { + if (includeInPeriodicSweep(thread, NOW_MS, pass)) { + checks.set(thread.id, (checks.get(thread.id) ?? 0) + 1); + } + } + } + expect(idle.every((thread) => checks.get(thread.id) === 1)).toBe(true); + expect(checks.get(active.id)).toBe(IDLE_SWEEP_EVERY_PASSES); + // The slots spread the idle threads instead of bunching them on one pass. + const perSlot = new Set(idle.map((thread) => idleSweepSlot(thread.id))); + expect(perSlot.size).toBeGreaterThan(IDLE_SWEEP_EVERY_PASSES / 2); + }); + + it.effect( + "the service looks up an idle thread once per rotation and an active one every pass", + () => + Effect.scoped( + Effect.gen(function* () { + const idle = threadShell("idle", { latestRunCompletedAt: hoursAgo(72) }); + const active = threadShell("active", { latestRunCompletedAt: hoursAgo(1) }); + const project = (thread: OrchestrationV2ThreadShell): OrchestrationProjectShell => ({ + id: thread.projectId, + title: thread.title, + workspaceRoot: `/workspace/${thread.id}`, + defaultModelSelection: null, + scripts: [], + repositoryIdentity: null, + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }); + const activation = yield* Deferred.make(); + const passes = yield* Queue.unbounded(); + const lookups = new Map(); + const layerDependencies = Layer.mergeAll( + Layer.mock(Orchestrator.OrchestratorV2)({ + streamDomainEvents: Stream.never, + getShellSnapshot: () => + Queue.offer(passes, undefined).pipe( + Effect.as({ + schemaVersion: 2, + snapshotSequence: 1, + threads: [idle, active], + archivedThreads: [], + }), + ), + }), + Layer.mock(ProjectStore.ProjectStoreV2)({ + listShells: () => Effect.succeed([project(idle), project(active)]), + }), + Layer.mock(GitManager.GitManager)({}), + Layer.mock(PullRequestService.PullRequestService)({}), + // No repository identity: the group ends right after this lookup. + Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({ + resolve: (root) => + Effect.sync(() => { + lookups.set(root, (lookups.get(root) ?? 0) + 1); + return null; + }), + }), + Layer.succeed(ServerActivation.ServerActivation, Deferred.await(activation)), + Layer.succeed( + Crypto.Crypto, + Crypto.make({ + randomBytes: (size) => new Uint8Array(size).fill(1), + digest: (_algorithm, data) => Effect.succeed(data), + }), + ), + FileSystem.layerNoop({}), + ); + + yield* Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const service = yield* ThreadPullRequestService.make; + yield* service.start(); + yield* Deferred.succeed(activation, undefined); + // The startup pass, then one pass a minute. + yield* Queue.take(passes); + yield* service.drain; + for (let pass = 1; pass < IDLE_SWEEP_EVERY_PASSES; pass += 1) { + yield* TestClock.adjust("1 minute"); + yield* Queue.take(passes); + yield* service.drain; + } + expect(lookups.get("/workspace/active")).toBe(IDLE_SWEEP_EVERY_PASSES); + expect(lookups.get("/workspace/idle")).toBe(1); + }).pipe(Effect.provide(layerDependencies)); + }), + ), + ); +}); diff --git a/apps/server/src/orchestration-v2/threadPullRequestSweep.expbkt3.ts b/apps/server/src/orchestration-v2/threadPullRequestSweep.expbkt3.ts new file mode 100644 index 0000000000000..b2014a0f10c30 --- /dev/null +++ b/apps/server/src/orchestration-v2/threadPullRequestSweep.expbkt3.ts @@ -0,0 +1,57 @@ +/** + * T3-CUSTOM(expbkt3): which threads a periodic pull-request sweep checks. + * + * Each check starts several git processes, and every start forks the whole + * server process on the event loop. With 1,100 unsettled threads the sweep ran + * for 50-95 s of every two minutes and kept the loop saturated, so clients + * timed out while connecting. A thread with no run or message for a day is + * now checked on a rotation, once every IDLE_SWEEP_EVERY_PASSES all-thread + * passes. Its own events (a run ending, a checkpoint, an unsettle) still queue + * an immediate one-thread check, so activity is never waited on. + */ +import type { OrchestrationV2ThreadShell } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; + +export const IDLE_SWEEP_AFTER_MS = 24 * 60 * 60 * 1000; +export const IDLE_SWEEP_EVERY_PASSES = 30; + +type SweepThread = Pick< + OrchestrationV2ThreadShell, + | "id" + | "activeRunId" + | "activityRunStatus" + | "createdAt" + | "latestUserMessageAt" + | "latestRunRequestedAt" + | "latestRunStartedAt" + | "latestRunCompletedAt" +>; + +const latestActivityMs = (thread: SweepThread) => + Math.max( + ...[ + thread.createdAt, + thread.latestUserMessageAt, + thread.latestRunRequestedAt, + thread.latestRunStartedAt, + thread.latestRunCompletedAt, + ].map((at) => (at == null ? Number.NEGATIVE_INFINITY : DateTime.toEpochMillis(at))), + ); + +export const isIdleForSweep = (thread: SweepThread, nowMs: number) => + thread.activeRunId === null && + thread.activityRunStatus == null && + nowMs - latestActivityMs(thread) >= IDLE_SWEEP_AFTER_MS; + +/** The pass, modulo IDLE_SWEEP_EVERY_PASSES, on which an idle thread is checked. */ +export const idleSweepSlot = (threadId: string) => { + let hash = 0; + for (let index = 0; index < threadId.length; index += 1) { + hash = (Math.imul(hash, 31) + threadId.charCodeAt(index)) >>> 0; + } + return hash % IDLE_SWEEP_EVERY_PASSES; +}; + +/** Whether all-thread pass number `pass` checks this thread. */ +export const includeInPeriodicSweep = (thread: SweepThread, nowMs: number, pass: number) => + !isIdleForSweep(thread, nowMs) || idleSweepSlot(thread.id) === pass % IDLE_SWEEP_EVERY_PASSES;