diff --git a/knip.json b/knip.json index 8f8f03a3..a40c6a84 100644 --- a/knip.json +++ b/knip.json @@ -3,6 +3,8 @@ "entry": [ "src/main/index.ts", "src/main/standalone.ts", + "src/cli/analyzeSession.ts", + "src/cli/sessionInventory.ts", "src/preload/index.ts", "src/renderer/main.tsx", "electron.vite.config.ts", diff --git a/package.json b/package.json index a99f83e8..2c98a675 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,8 @@ "test:coverage": "vitest run --coverage", "test:coverage:critical": "vitest run --coverage --config vitest.critical.config.ts", "standalone": "tsx src/main/standalone.ts", + "analyze:session": "tsx src/cli/analyzeSession.ts", + "analyze:sessions": "tsx src/cli/sessionInventory.ts", "standalone:build": "electron-vite build && vite build --config vite.standalone.config.ts", "standalone:start": "node dist-standalone/index.cjs" }, diff --git a/src/cli/analyzeSession.ts b/src/cli/analyzeSession.ts new file mode 100644 index 00000000..77246a8f --- /dev/null +++ b/src/cli/analyzeSession.ts @@ -0,0 +1,715 @@ +/** + * Session token audit CLI — where do billed tokens go, and what was wasted. + * + * Numbers only: exact per-turn/per-round usage from JSONL (input / cache_read / + * cache_write / output), waste findings, slow-subagent table. No transcript. + * + * Usage: + * pnpm analyze:session [flags] + * pnpm analyze:session --project --last [flags] + * Flags: + * --rounds N rounds table length (default 20) + * --subagent-min-minutes N slow-subagent threshold (default 5) + * --min-severity S low | medium | high (default low = all) + * --json machine-readable output + */ + +import { ProjectScanner, SubagentResolver } from '@main/services/discovery'; +import { isParsedUserChunkMessage } from '@main/types'; +import { deduplicateByRequestId, getTaskCalls, parseJsonlFile } from '@main/utils/jsonl'; +import { encodePath, extractSessionId, getProjectsBasePath } from '@main/utils/pathDecoder'; +import { parseModelString } from '@shared/utils/modelParser'; +import { + estimateTokens, + formatTokensCompact, + formatTokensDetailed, +} from '@shared/utils/tokenFormatting'; +import * as fs from 'fs'; +import * as path from 'path'; +import { pathToFileURL } from 'url'; + +import type { ParsedMessage, Process } from '@main/types'; + +// ponytail: single-file pricing table; extract to a module when something else needs it +const PRICE_PER_MTOK: Record = { + opus: [15, 75, 1.5, 18.75], + sonnet: [3, 15, 0.3, 3.75], + haiku: [1, 5, 0.1, 1.25], +}; + +// ponytail: local stopgap for short ids (claude-sonnet-5) while upstream PR #235 is unmerged +export function priceFamily(model: string): string | null { + const info = parseModelString(model); + if (info) return info.family; + const m = /claude-(opus|sonnet|haiku)/.exec(model.toLowerCase()); + return m ? m[1] : null; +} + +export type BillingScheme = 'anthropic-style' | 'router-style' | 'no-cache' | 'mixed'; + +// anthropic-style billing has cache_write > 0 on rounds; routers typically report +// cache_read without any write counter — cw=0 with cr>0 is the router signature +export function detectBillingScheme( + rounds: { cacheReadTokens: number; cacheCreationTokens: number }[] +): BillingScheme { + let sawWrite = false; + let sawRead = false; + for (const r of rounds) { + if (r.cacheCreationTokens > 0) sawWrite = true; + else if (r.cacheReadTokens > 0) sawRead = true; + } + return billingFromFlags(sawWrite, sawRead); +} + +export function billingFromFlags(sawWrite: boolean, sawRead: boolean): BillingScheme { + if (sawWrite && sawRead) return 'mixed'; + if (sawWrite) return 'anthropic-style'; + if (sawRead) return 'router-style'; + return 'no-cache'; +} + +export const WASTE_THRESHOLDS = { + oversizedOutputTokens: 8000, + contextSpikeTokens: 30000, + cacheDeadContextTokens: 20000, + thinkingHeavyTokens: 8000, +} as const; + +// Tool results that look like errors but are normal flow (user said no / aborted) +const REJECTION_PATTERNS = [ + "The user doesn't want to proceed with this tool use", + '[Request interrupted by user', +]; + +export type FindingType = + | 'duplicate_call' + | 'failed_call' + | 'oversized_output' + | 'context_spike' + | 'cache_dead' + | 'thinking_heavy'; + +export interface Finding { + type: FindingType; + severity: 'low' | 'medium' | 'high'; + tokensWasted: number; + turnIndex?: number; + summary: string; +} + +export interface RoundRow { + index: number; + timestamp: Date; + turnIndex: number; + model: string; + inputTokens: number; + cacheReadTokens: number; + cacheCreationTokens: number; + outputTokens: number; + contextSize: number; + contextDelta: number; + thinkingTokens: number; + tools: string[]; +} + +export interface TurnRow { + index: number; + start: Date; +} + +export interface SessionLedger { + turns: TurnRow[]; + rounds: RoundRow[]; + totals: { + inputTokens: number; + cacheReadTokens: number; + cacheCreationTokens: number; + outputTokens: number; + billedTokens: number; + rereadShare: number; + thinkingTokens: number; + costUsd?: number; + costPartial?: boolean; + }; + models: string[]; + durationMs: number; + billing: BillingScheme; +} + +// ============================================================================= +// Ledger (port of the validated prototype) +// ============================================================================= + +function thinkingTokensOf(msg: ParsedMessage): number { + if (!Array.isArray(msg.content)) return 0; + let chars = 0; + for (const block of msg.content) { + if (block.type === 'thinking') chars += (block.thinking ?? '').length; + } + return Math.ceil(chars / 4); +} + +export function buildLedger(allMessages: ParsedMessage[]): SessionLedger { + const messages = deduplicateByRequestId(allMessages); + + const turns: TurnRow[] = []; + const rounds: RoundRow[] = []; + const models = new Set(); + let currentTurn: TurnRow | null = null; + let costUsd = 0; + let unpriced = false; + let prevContext = 0; + let minTs = Number.POSITIVE_INFINITY; + let maxTs = Number.NEGATIVE_INFINITY; + + const newTurn = (ts: Date): TurnRow => { + const turn: TurnRow = { index: turns.length + 1, start: ts }; + turns.push(turn); + currentTurn = turn; + return turn; + }; + + for (const msg of messages) { + minTs = Math.min(minTs, msg.timestamp.getTime()); + maxTs = Math.max(maxTs, msg.timestamp.getTime()); + + if (isParsedUserChunkMessage(msg)) { + newTurn(msg.timestamp); + continue; + } + if (msg.type !== 'assistant' || msg.isSidechain) continue; + if (!msg.usage || msg.model === '') continue; + + const turn = currentTurn ?? newTurn(msg.timestamp); + const u = msg.usage; + const input = u.input_tokens ?? 0; + const cacheRead = u.cache_read_input_tokens ?? 0; + const cacheWrite = u.cache_creation_input_tokens ?? 0; + const output = u.output_tokens ?? 0; + const contextSize = input + cacheRead + cacheWrite; + const think = thinkingTokensOf(msg); + const model = msg.model ?? 'unknown'; + const tools = msg.toolCalls.map((tc) => tc.name); + + const round: RoundRow = { + index: rounds.length + 1, + timestamp: msg.timestamp, + turnIndex: turn.index, + model, + inputTokens: input, + cacheReadTokens: cacheRead, + cacheCreationTokens: cacheWrite, + outputTokens: output, + contextSize, + contextDelta: rounds.length === 0 ? 0 : contextSize - prevContext, + thinkingTokens: think, + tools, + }; + prevContext = contextSize; + rounds.push(round); + models.add(model); + + const price = PRICE_PER_MTOK[priceFamily(model) ?? '']; + if (price) { + costUsd += + (input * price[0] + output * price[1] + cacheRead * price[2] + cacheWrite * price[3]) / 1e6; + } else { + unpriced = true; + } + } + + const t = { + inputTokens: 0, + cacheReadTokens: 0, + cacheCreationTokens: 0, + outputTokens: 0, + billedTokens: 0, + rereadShare: 0, + thinkingTokens: 0, + }; + for (const r of rounds) { + t.inputTokens += r.inputTokens; + t.cacheReadTokens += r.cacheReadTokens; + t.cacheCreationTokens += r.cacheCreationTokens; + t.outputTokens += r.outputTokens; + t.thinkingTokens += r.thinkingTokens; + } + t.billedTokens = t.inputTokens + t.cacheReadTokens + t.cacheCreationTokens + t.outputTokens; + t.rereadShare = t.billedTokens > 0 ? t.cacheReadTokens / t.billedTokens : 0; + + return { + turns, + rounds, + totals: { ...t, ...(costUsd > 0 ? { costUsd, costPartial: unpriced } : {}) }, + models: [...models], + durationMs: Number.isFinite(minTs) ? Math.max(0, maxTs - minTs) : 0, + billing: detectBillingScheme(rounds), + }; +} + +// ============================================================================= +// Findings +// ============================================================================= + +// ponytail: normalization is a heuristic — same command/file/pattern counts as a repeat +function asText(v: unknown): string { + if (typeof v === 'string') return v; + if (v === undefined || v === null) return ''; + return JSON.stringify(v); +} + +const squash = (v: unknown): string => asText(v).replace(/\s+/g, ' ').trim(); + +export function normalizeCallKey(name: string, input: Record): string { + switch (name) { + case 'Bash': + return `Bash|${squash(input.command)}`; + case 'Read': + case 'Write': + case 'Edit': + case 'NotebookEdit': + return `${name}|${asText(input.file_path)}`; + case 'Grep': + case 'Glob': + return `${name}|${asText(input.pattern)}|${asText(input.path)}`; + case 'Skill': + return `Skill|${asText(input.skill)}`; + case 'Task': + case 'Agent': + return `Task|${asText(input.description) || asText(input.prompt)}`; + default: + // own top-level keys, sorted — a replacer array would recurse and flatten + // nested objects to {}, colliding keys of calls differing only in nesting + return `${name}|${JSON.stringify( + Object.fromEntries( + Object.keys(input) + .sort((a, b) => a.localeCompare(b)) + .map((k) => [k, input[k]]) + ) + )}`; + } +} + +function resultText(content: string | unknown[]): string { + return typeof content === 'string' ? content : JSON.stringify(content); +} + +export function computeFindings(messages: ParsedMessage[], ledger: SessionLedger): Finding[] { + const findings: Finding[] = []; + const th = WASTE_THRESHOLDS; + + const results = new Map(); + for (const msg of messages) { + if (msg.isSidechain) continue; + for (const r of msg.toolResults) { + results.set(r.toolUseId, { content: r.content, isError: r.isError }); + } + } + + // duplicate / failed / oversized — walk tool calls in order + const seen = new Map(); + for (const msg of messages) { + if (msg.isSidechain) continue; + for (const call of msg.toolCalls) { + const key = normalizeCallKey(call.name, call.input); + const result = results.get(call.id); + const text = result ? resultText(result.content) : ''; + const resultTok = estimateTokens(text); + + if (result?.isError) { + const rejected = REJECTION_PATTERNS.some((p) => text.includes(p)); + if (!rejected) { + const inputTok = estimateTokens(JSON.stringify(call.input)); + findings.push({ + type: 'failed_call', + severity: 'medium', + tokensWasted: inputTok + resultTok, + summary: `${call.name} failed: ${short(text, 80)}`, + }); + } + } + if (resultTok > th.oversizedOutputTokens) { + findings.push({ + type: 'oversized_output', + severity: 'medium', + tokensWasted: resultTok, + summary: `${call.name} returned ~${formatTokensCompact(resultTok)} tok: ${short(call.name === 'Bash' ? asText(call.input.command) : JSON.stringify(call.input), 70)}`, + }); + } + + const prev = seen.get(key); + if (prev) { + prev.count += 1; + prev.tokens += resultTok; + } else { + seen.set(key, { count: 1, tokens: resultTok, first: resultTok }); + } + } + } + for (const [key, { count, tokens, first }] of seen) { + if (count > 1) { + const reread = tokens - first; // repeats only — the first read was legitimate + findings.push({ + type: 'duplicate_call', + severity: 'medium', + tokensWasted: reread, + summary: `${short(key, 90)} — called ${count}x (~${formatTokensCompact(reread)} tok of results re-read)`, + }); + } + } + + // context-side findings from the ledger + for (const r of ledger.rounds) { + if (r.contextDelta > th.contextSpikeTokens) { + findings.push({ + type: 'context_spike', + severity: 'high', + tokensWasted: r.contextDelta, + turnIndex: r.turnIndex, + summary: `context +${formatTokensCompact(r.contextDelta)} in one round (${formatTokensCompact(r.contextSize - r.contextDelta)} → ${formatTokensCompact(r.contextSize)}), tools: ${r.tools.join(', ') || 'none'}`, + }); + } + if ( + r.contextSize > th.cacheDeadContextTokens && + r.cacheReadTokens === 0 && + r.cacheCreationTokens === 0 + ) { + findings.push({ + type: 'cache_dead', + severity: 'high', + tokensWasted: r.contextSize, + turnIndex: r.turnIndex, + summary: `no prompt caching: ${formatTokensCompact(r.contextSize)} tok billed as fresh input (model ${r.model})`, + }); + } + } + for (const turn of ledger.turns) { + const think = ledger.rounds + .filter((r) => r.turnIndex === turn.index) + .reduce((s, r) => s + r.thinkingTokens, 0); + if (think > th.thinkingHeavyTokens) { + findings.push({ + type: 'thinking_heavy', + severity: 'low', + tokensWasted: think, + turnIndex: turn.index, + summary: `thinking ~${formatTokensCompact(think)} tok in turn ${turn.index}`, + }); + } + } + + const order = { high: 0, medium: 1, low: 2 } as const; + findings.sort((a, b) => order[b.severity] - order[a.severity] || b.tokensWasted - a.tokensWasted); + return findings; +} + +export function short(s: string, n: number): string { + const flat = s.replace(/\s+/g, ' ').trim(); + return flat.length <= n ? flat : `${flat.slice(0, n - 1)}…`; +} + +// ============================================================================= +// CLI plumbing +// ============================================================================= + +interface CliOpts { + sessionPath?: string; + projectArg?: string; + useLast: boolean; + rounds: number; + subagentMinMinutes: number; + minSeverity: 'low' | 'medium' | 'high'; + json: boolean; +} + +// the next token is a flag's value unless missing or itself a flag +// (--rounds --json must not swallow --json) +export function takeFlagValue(argv: string[], i: number): { value: string; next: number } { + const hasArg = i + 1 < argv.length; + const isValue = hasArg && !argv[i + 1].startsWith('--'); + return isValue ? { value: argv[i + 1], next: i + 2 } : { value: '', next: i + 1 }; +} + +export function parseArgs(argv: string[]): CliOpts { + const opts: CliOpts = { + useLast: false, + rounds: 20, + subagentMinMinutes: 5, + minSeverity: 'low', + json: false, + }; + let i = 0; + while (i < argv.length) { + const a = argv[i]; + const { value, next } = takeFlagValue(argv, i); + if (a === '--project') { + opts.projectArg = value; + i = next; + continue; + } + if (a === '--rounds') { + opts.rounds = parseInt(value, 10) || 20; + i = next; + continue; + } + if (a === '--subagent-min-minutes') { + opts.subagentMinMinutes = parseInt(value, 10) || 5; + i = next; + continue; + } + if (a === '--min-severity') { + opts.minSeverity = value === 'medium' || value === 'high' ? value : 'low'; + i = next; + continue; + } + if (a === '--last') { + opts.useLast = true; + i += 1; + continue; + } + if (a === '--json') { + opts.json = true; + i += 1; + continue; + } + if (!a.startsWith('--')) opts.sessionPath = a; + i += 1; + } + return opts; +} + +export function pickNewestSessionFile(dir: string): string | null { + if (!fs.existsSync(dir)) return null; + let newest: { file: string; mtime: number } | null = null; + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + if (!e.isFile() || !e.name.endsWith('.jsonl') || e.name.startsWith('agent-')) continue; + const file = path.join(dir, e.name); + let mtime: number; + try { + mtime = fs.statSync(file).mtimeMs; + } catch { + continue; // vanished between readdir and stat + } + if (!newest || mtime > newest.mtime) newest = { file, mtime }; + } + return newest?.file ?? null; +} + +export function resolveProjectDir(arg: string): string { + const projectsRoot = getProjectsBasePath(); + // already encoded (full path or bare name) → use under the projects root + if (path.basename(arg).startsWith('-')) { + return path.isAbsolute(arg) ? arg : path.join(projectsRoot, arg); + } + // plain filesystem path of a project → its encoded sessions dir + return path.join(projectsRoot, encodePath(path.resolve(arg))); +} + +function splitSessionPath(file: string): { projectId: string; sessionId: string } { + const rel = path.relative(getProjectsBasePath(), path.resolve(file)); + const [projectId, sessionId] = rel.split(path.sep); + return { projectId, sessionId: sessionId ? extractSessionId(sessionId) : '' }; +} + +async function resolveSubagentsFor( + projectId: string, + sessionId: string, + messages: ParsedMessage[] +): Promise { + const scanner = new ProjectScanner(); + const resolver = new SubagentResolver(scanner); + return resolver.resolveSubagents(projectId, sessionId, getTaskCalls(messages), messages); +} + +// ============================================================================= +// Output +// ============================================================================= + +export const pad = (s: string, n: number): string => + s.length >= n ? s : s + ' '.repeat(n - s.length); +export const padL = (s: string, n: number): string => + s.length >= n ? s : ' '.repeat(n - s.length) + s; +const fmt = (n: number): string => formatTokensDetailed(n); +const hhmm = (d: Date): string => + `${String(d.getHours()).padStart(2, '0')}:${String(d.getMinutes()).padStart(2, '0')}`; +export const dur = (ms: number): string => + `${Math.floor(ms / 3600000)}h ${String(Math.floor((ms % 3600000) / 60000)).padStart(2, '0')}m`; + +function printReport( + file: string, + ledger: SessionLedger, + findings: Finding[], + subagents: Process[], + opts: CliOpts +): void { + const t = ledger.totals; + const activeTurns = new Set(ledger.rounds.map((r) => r.turnIndex)).size; + console.log('=== SESSION ==='); + console.log('file :', path.basename(file)); + console.log( + `turns: ${activeTurns} rounds: ${ledger.rounds.length} duration: ${dur(ledger.durationMs)}` + ); + console.log('models:', ledger.models.join(', ') || 'n/a'); + console.log('billing:', ledger.billing); + if (t.costUsd !== undefined) { + const partial = t.costPartial ? ' (partial — unpriced models excluded)' : ''; + console.log('est. cost: $' + t.costUsd.toFixed(2) + partial); + } + console.log(); + console.log('=== TOKENS (billed) ==='); + console.log(`input (uncached) : ${fmt(t.inputTokens)}`); + console.log( + `cache_read (reread) : ${fmt(t.cacheReadTokens)} <- whole context re-read EVERY round` + ); + console.log(`cache_write (new) : ${fmt(t.cacheCreationTokens)}`); + console.log(`output : ${fmt(t.outputTokens)}`); + console.log(`thinking (estimate) : ~${fmt(t.thinkingTokens)}`); + console.log(`TOTAL : ${fmt(t.billedTokens)}`); + console.log(`reread share : ${Math.round(t.rereadShare * 100)}%`); + console.log(); + console.log('=== BY TURN ==='); + console.log( + `${pad('#', 3)} ${pad('time', 6)} ${padL('context', 9)} ${padL('reread', 9)} ${padL('new', 8)} ${padL('out', 7)} ${pad('think%', 7)} tools` + ); + for (const turn of ledger.turns) { + const rs = ledger.rounds.filter((r) => r.turnIndex === turn.index); + if (rs.length === 0) continue; // trailing user msg / empty implicit turn + const ctx = rs.at(-1)?.contextSize ?? 0; + const reread = rs.reduce((s, r) => s + r.cacheReadTokens, 0); + const fresh = rs.reduce((s, r) => s + r.inputTokens + r.cacheCreationTokens, 0); + const out = rs.reduce((s, r) => s + r.outputTokens, 0); + const think = rs.reduce((s, r) => s + r.thinkingTokens, 0); + const genTotal = think + out; + const thinkPct = genTotal > 0 ? Math.round((think / genTotal) * 100) : 0; + const toolCounts = new Map(); + for (const r of rs) + for (const name of r.tools) toolCounts.set(name, (toolCounts.get(name) ?? 0) + 1); + const tools = [...toolCounts].map(([n, c]) => `${n} x${c}`).join(', '); + console.log( + `${pad(String(turn.index), 3)} ${pad(hhmm(turn.start), 6)} ${padL(fmt(ctx), 9)} ${padL(fmt(reread), 9)} ${padL(fmt(fresh), 8)} ${padL(fmt(out), 7)} ${pad(thinkPct + '%', 7)} ${short(tools, 60)}` + ); + } + console.log(); + console.log(`=== ROUNDS (last ${opts.rounds}) ===`); + console.log( + `${pad('#', 4)} ${pad('time', 6)} ${padL('context', 9)} ${padL('delta', 9)} ${padL('in', 7)} ${padL('cr', 8)} ${padL('cw', 6)} ${padL('out', 6)} model` + ); + for (const r of ledger.rounds.slice(-opts.rounds)) { + console.log( + `${pad(String(r.index), 4)} ${pad(hhmm(r.timestamp), 6)} ${padL(fmt(r.contextSize), 9)} ${padL((r.contextDelta >= 0 ? '+' : '') + fmt(r.contextDelta), 9)} ${padL(fmt(r.inputTokens), 7)} ${padL(fmt(r.cacheReadTokens), 8)} ${padL(fmt(r.cacheCreationTokens), 6)} ${padL(fmt(r.outputTokens), 6)} ${short(r.model, 28)}` + ); + } + console.log(); + + if (subagents.length > 0) { + console.log('=== SUBAGENTS ==='); + for (const s of subagents) { + const slow = s.durationMs > opts.subagentMinMinutes * 60000; + const flag = slow ? ` !SLOW >${opts.subagentMinMinutes}min` : ''; + const ongoing = s.isOngoing ? ' (ongoing)' : ''; + console.log( + `${pad(short(s.description ?? s.subagentType ?? s.id, 52), 52)} ${pad(s.subagentType ?? '', 10)} ${padL(dur(s.durationMs), 8)}${flag}${ongoing} ~${formatTokensCompact(s.metrics?.totalTokens ?? 0)} tok` + ); + } + console.log(); + } + + const severityRank = { low: 0, medium: 1, high: 2 } as const; + const visible = findings.filter( + (f) => severityRank[f.severity] >= severityRank[opts.minSeverity] + ); + console.log('=== FINDINGS ==='); + if (visible.length === 0) { + console.log('(none — clean session)'); + } else { + for (const f of visible) { + const turnTag = f.turnIndex === undefined ? '' : ', turn ' + String(f.turnIndex); + console.log(`[${f.type}${turnTag}] ${f.summary}`); + } + } + console.log(); + console.log( + 'note: this is observation, not prevention — add deny rules for culprit commands in Claude Code settings.' + ); +} + +async function main(): Promise { + const raw = process.argv.slice(2); + if (raw.includes('--help') || raw.includes('-h')) { + console.log( + 'usage: pnpm analyze:session | --project --last [--rounds N] [--subagent-min-minutes N] [--min-severity low|medium|high] [--json]' + ); + return; + } + const opts = parseArgs(raw); + + let sessionFile = opts.sessionPath ? path.resolve(opts.sessionPath) : null; + if (!sessionFile && opts.projectArg) { + if (!opts.useLast) { + console.error('--project requires --last'); + process.exitCode = 1; + return; + } + const dir = resolveProjectDir(opts.projectArg); + if (!fs.existsSync(dir)) { + console.error(`project dir not found: ${dir}`); + process.exitCode = 1; + return; + } + sessionFile = pickNewestSessionFile(dir); + if (!sessionFile) { + console.error(`no .jsonl sessions found in ${dir}`); + process.exitCode = 1; + return; + } + } + if (!sessionFile || !fs.existsSync(sessionFile)) { + console.error( + 'usage: pnpm analyze:session | pnpm analyze:session --project --last' + ); + process.exitCode = 1; + return; + } + + const { projectId, sessionId } = splitSessionPath(sessionFile); + const messages = await parseJsonlFile(sessionFile); + const ledger = buildLedger(messages); + const findings = computeFindings(messages, ledger); + + let subagents: Process[] = []; + try { + subagents = await resolveSubagentsFor(projectId, sessionId, messages); + } catch (err) { + console.error(`(subagent resolution unavailable: ${String(err)})`); + } + + if (opts.json) { + // Process carries the full parsed transcript — strip it for the JSON dump + const subagentSummaries = subagents.map((p) => ({ + id: p.id, + description: p.description, + subagentType: p.subagentType, + durationMs: p.durationMs, + totalTokens: p.metrics?.totalTokens ?? 0, + isOngoing: p.isOngoing, + })); + console.log( + JSON.stringify( + { file: sessionFile, projectId, sessionId, ledger, findings, subagents: subagentSummaries }, + null, + 2 + ) + ); + return; + } + printReport(sessionFile, ledger, findings, subagents, opts); +} + +// run only when executed directly: sessionInventory imports this module for +// helpers, vitest imports it for the pure functions — neither may trigger main +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + void main().catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/src/cli/sessionInventory.ts b/src/cli/sessionInventory.ts new file mode 100644 index 00000000..97fd5b1f --- /dev/null +++ b/src/cli/sessionInventory.ts @@ -0,0 +1,296 @@ +/** + * Sessions inventory CLI — duration, models, token totals across all sessions. + * Answers "which sessions ran 2h+, on which model" without opening the app. + * + * Usage: + * pnpm analyze:sessions [--project ] [--min-minutes N] [--sort duration|tokens|date] [--limit N] [--json] + */ + +import { decodePath, extractSessionId, getProjectsBasePath } from '@main/utils/pathDecoder'; +import { formatTokensCompact } from '@shared/utils/tokenFormatting'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import * as readline from 'readline'; +import { pathToFileURL } from 'url'; + +import { + billingFromFlags, + type BillingScheme, + dur, + pad, + padL, + resolveProjectDir, + short, + takeFlagValue, +} from './analyzeSession'; + +interface RawUsage { + input_tokens?: number; + output_tokens?: number; + cache_read_input_tokens?: number; + cache_creation_input_tokens?: number; +} + +interface ScanEntry { + type?: string; + timestamp?: string; + requestId?: string; + isSidechain?: boolean; + message?: { model?: string; usage?: RawUsage }; +} + +function mergeUsage(a: RawUsage, b: RawUsage): RawUsage { + return { + input_tokens: (a.input_tokens ?? 0) + (b.input_tokens ?? 0), + output_tokens: (a.output_tokens ?? 0) + (b.output_tokens ?? 0), + cache_read_input_tokens: (a.cache_read_input_tokens ?? 0) + (b.cache_read_input_tokens ?? 0), + cache_creation_input_tokens: + (a.cache_creation_input_tokens ?? 0) + (b.cache_creation_input_tokens ?? 0), + }; +} + +export interface InventoryEntry { + projectId: string; + sessionId: string; + filePath: string; + durationMs: number; + lastTs: Date | null; + models: string[]; + messageCount: number; + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cacheCreationTokens: number; + totalTokens: number; + sizeBytes: number; + billing: BillingScheme; +} + +// ponytail: own streaming pass instead of parseJsonlFile — it materializes every +// ParsedMessage and that measurably blows up on 1200+ files incl. 66MB ones +export async function scanSessionFile(filePath: string): Promise { + const rl = readline.createInterface({ + input: fs.createReadStream(filePath, { encoding: 'utf8' }), + crlfDelay: Infinity, + }); + + let firstTs: number | null = null; + let lastTs: number | null = null; + let messageCount = 0; + const models = new Set(); + // anthropic-style rounds report cache writes; routers report cache_read with cw=0 + let sawWrite = false; + let sawRead = false; + const usageByRequestId = new Map(); + let directUsage: RawUsage = {}; + + for await (const line of rl) { + if (line === '') continue; + let e: ScanEntry; + try { + e = JSON.parse(line) as ScanEntry; + } catch { + continue; + } + if (!e.timestamp) continue; + const ts = new Date(e.timestamp).getTime(); + if (Number.isNaN(ts)) continue; + if (firstTs === null || ts < firstTs) firstTs = ts; + if (lastTs === null || ts > lastTs) lastTs = ts; + if (e.type === 'user' || e.type === 'assistant') messageCount++; + + // sidechain (subagent) entries stay out of totals/models/billing — same + // accounting as buildLedger; timestamps and message count cover the file + if ( + e.type === 'assistant' && + !e.isSidechain && + e.message?.usage && + e.message.model !== '' + ) { + if (e.message.model) models.add(e.message.model); + if (e.message.usage.cache_creation_input_tokens) sawWrite = true; + else if (e.message.usage.cache_read_input_tokens) sawRead = true; + // streaming writes several entries per request — the last one has final counts + if (e.requestId) usageByRequestId.set(e.requestId, e.message.usage); + else directUsage = mergeUsage(directUsage, e.message.usage); + } + } + + if (firstTs === null || lastTs === null) return null; + + let totals: RawUsage = directUsage; + for (const u of usageByRequestId.values()) totals = mergeUsage(totals, u); + + const rel = path.relative(getProjectsBasePath(), path.resolve(filePath)); + const [projectId = '', sessionId = ''] = rel.split(path.sep); + + const inputTokens = totals.input_tokens ?? 0; + const outputTokens = totals.output_tokens ?? 0; + const cacheReadTokens = totals.cache_read_input_tokens ?? 0; + const cacheCreationTokens = totals.cache_creation_input_tokens ?? 0; + + return { + projectId, + sessionId: extractSessionId(sessionId), + filePath, + durationMs: Math.max(0, lastTs - firstTs), + lastTs: new Date(lastTs), + models: [...models], + messageCount, + inputTokens, + outputTokens, + cacheReadTokens, + cacheCreationTokens, + totalTokens: inputTokens + outputTokens + cacheReadTokens + cacheCreationTokens, + sizeBytes: fs.statSync(filePath).size, + billing: billingFromFlags(sawWrite, sawRead), + }; +} + +async function listSessionFiles(projectDir: string): Promise { + const out: string[] = []; + for (const e of await fs.promises.readdir(projectDir, { withFileTypes: true })) { + if (e.isFile() && e.name.endsWith('.jsonl') && !e.name.startsWith('agent-')) { + out.push(path.join(projectDir, e.name)); + } + } + return out; +} + +export async function mapWithConcurrency( + items: T[], + limit: number, + fn: (x: T) => Promise +): Promise<(R | null)[]> { + // ponytail: result order is nondeterministic — rows are sorted downstream anyway + const results: (R | null)[] = []; + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const item = items[next]; + next += 1; + try { + results.push(await fn(item)); + } catch (err) { + // one unreadable file must not abort a 1200+-file scan + console.error(`(skipped ${String(item)}: ${String(err)})`); + results.push(null); + } + } + }); + await Promise.all(workers); + return results; +} + +async function collect(projectsRoot: string, projectArg?: string): Promise { + let dirs: string[]; + if (projectArg) { + dirs = [projectArg]; + } else { + const es = await fs.promises.readdir(projectsRoot, { withFileTypes: true }); + dirs = es.filter((e) => e.isDirectory()).map((e) => path.join(projectsRoot, e.name)); + } + const fileGroups = await mapWithConcurrency(dirs, 8, listSessionFiles); + const files = fileGroups.filter((g): g is string[] => g !== null).flat(); + // ponytail: no mtime cache yet — first full scan is IO-bound seconds, add one if it hurts + const scanned = await mapWithConcurrency(files, 8, scanSessionFile); + return scanned.filter((e): e is InventoryEntry => e !== null); +} + +const shortenHome = (p: string): string => { + const home = os.homedir(); + return p.startsWith(home) ? `~${p.slice(home.length)}` : p; +}; + +async function main(): Promise { + const argv = process.argv.slice(2); + if (argv.includes('--help') || argv.includes('-h')) { + console.log( + 'usage: pnpm analyze:sessions [--project ] [--min-minutes N] [--sort duration|tokens|date] [--limit N] [--json]' + ); + return; + } + let projectArg: string | undefined; + let minMinutes = 0; + let sort: 'duration' | 'tokens' | 'date' = 'duration'; + let limit = Number.POSITIVE_INFINITY; + let json = false; + let i = 0; + while (i < argv.length) { + const a = argv[i]; + const { value, next } = takeFlagValue(argv, i); + if (a === '--project') { + projectArg = value; + i = next; + continue; + } + if (a === '--min-minutes') { + minMinutes = parseInt(value, 10) || 0; + i = next; + continue; + } + if (a === '--sort') { + sort = value === 'tokens' || value === 'date' ? value : 'duration'; + i = next; + continue; + } + if (a === '--limit') { + const n = parseInt(value, 10); + limit = n > 0 ? n : Number.POSITIVE_INFINITY; + i = next; + continue; + } + if (a === '--json') { + json = true; + i += 1; + continue; + } + i += 1; + } + + let projectDir: string | undefined; + if (projectArg) { + projectDir = resolveProjectDir(projectArg); + if (!fs.existsSync(projectDir)) { + console.error(`project dir not found: ${projectDir}`); + process.exitCode = 1; + return; + } + } + + const entries = await collect(getProjectsBasePath(), projectDir); + entries.sort((a, b) => { + if (sort === 'tokens') return b.totalTokens - a.totalTokens; + if (sort === 'date') return (b.lastTs?.getTime() ?? 0) - (a.lastTs?.getTime() ?? 0); + return b.durationMs - a.durationMs; + }); + const shown = entries.filter((e) => e.durationMs >= minMinutes * 60000).slice(0, limit); + + if (json) { + console.log(JSON.stringify(shown, null, 2)); + return; + } + + const minNote = minMinutes > 0 ? ` (>= ${String(minMinutes)} min)` : ''; + console.log(`sessions: ${entries.length} total, showing ${shown.length}${minNote}`); + console.log(); + console.log( + `${padL('duration', 9)} ${pad('date', 11)} ${pad('file', 9)} ${pad('project', 42)} ${pad('models', 30)} ${padL('tokens', 9)} ${padL('msgs', 6)} ${pad('billing', 15)}` + ); + for (const e of shown) { + const project = shortenHome(decodePath(e.projectId)); + const models = e.models.join(', '); + console.log( + `${padL(dur(e.durationMs), 9)} ${pad(e.lastTs ? e.lastTs.toISOString().slice(0, 10) : 'n/a', 11)} ${pad(e.sessionId.slice(0, 8), 9)} ${pad(short(project, 42), 42)} ${pad(short(models, 30), 30)} ${padL(formatTokensCompact(e.totalTokens), 9)} ${padL(String(e.messageCount), 6)} ${pad(e.billing, 15)}` + ); + } +} + +// run only when executed directly — vitest imports this file for scanSessionFile +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + void main().catch((err) => { + console.error(err); + process.exitCode = 1; + }); +} diff --git a/test/main/cli/analyzeSession.test.ts b/test/main/cli/analyzeSession.test.ts new file mode 100644 index 00000000..ddd82d40 --- /dev/null +++ b/test/main/cli/analyzeSession.test.ts @@ -0,0 +1,334 @@ +/** + * Tests for the token-analytics CLI (src/cli/*). + * Covers the ported prototype logic: turn boundaries, requestId dedup, + * duplicate-call keys, waste findings, slow-subagent math, inventory scan. + */ +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import * as path from 'path'; + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { + buildLedger, + computeFindings, + detectBillingScheme, + normalizeCallKey, + parseArgs, + priceFamily, +} from '../../../src/cli/analyzeSession'; +import { mapWithConcurrency, scanSessionFile } from '../../../src/cli/sessionInventory'; +import { estimateTokens } from '../../../src/shared/utils/tokenFormatting'; +import type { ParsedMessage } from '../../../src/main/types'; + +let seq = 0; +function makeMsg( + overrides: Partial & { type: ParsedMessage['type'] } +): ParsedMessage { + return { + uuid: `u${++seq}`, + parentUuid: null, + timestamp: new Date('2026-09-20T10:00:00Z'), + content: '', + toolCalls: [], + toolResults: [], + isSidechain: false, + isMeta: false, + ...overrides, + }; +} + +const usage = (input: number, cr: number, cw: number, out: number) => ({ + input_tokens: input, + output_tokens: out, + cache_read_input_tokens: cr, + cache_creation_input_tokens: cw, +}); + +afterEach(async () => { + seq = 0; +}); + +describe('buildLedger', () => { + it('starts turns only on real user messages, not isMeta tool-result carriers', () => { + const messages = [ + makeMsg({ type: 'user', content: 'hello' }), + makeMsg({ type: 'assistant', model: 'm1', usage: usage(100, 1000, 50, 10) }), + makeMsg({ + type: 'user', + isMeta: true, + content: [{ type: 'tool_result', tool_use_id: 't1', content: 'ok' } as never], + toolResults: [{ toolUseId: 't1', content: 'ok', isError: false }], + }), + makeMsg({ + type: 'assistant', + model: 'm1', + usage: usage(120, 1100, 0, 20), + toolCalls: [{ id: 't2', name: 'Bash', input: { command: 'ls' }, isTask: false }], + }), + ]; + + const ledger = buildLedger(messages); + expect(ledger.turns).toHaveLength(1); + expect(ledger.rounds).toHaveLength(2); + expect(ledger.rounds[1].contextDelta).toBe(120 + 1100 - (100 + 1000 + 50)); + expect(ledger.rounds[1].tools).toEqual(['Bash']); + expect(ledger.totals.outputTokens).toBe(30); + }); + + it('deduplicates streaming entries by requestId keeping the last usage', () => { + const messages = [ + makeMsg({ type: 'user', content: 'go' }), + makeMsg({ + type: 'assistant', + model: 'm1', + requestId: 'r1', + usage: usage(100, 0, 0, 5), + }), + makeMsg({ + type: 'assistant', + model: 'm1', + requestId: 'r1', + usage: usage(100, 0, 0, 25), + }), + ]; + + const ledger = buildLedger(messages); + expect(ledger.rounds).toHaveLength(1); + expect(ledger.totals.outputTokens).toBe(25); + }); + + it('excludes sidechain and assistant messages', () => { + const messages = [ + makeMsg({ type: 'user', content: 'go' }), + makeMsg({ type: 'assistant', model: '', usage: usage(9, 0, 0, 1) }), + makeMsg({ type: 'assistant', isSidechain: true, model: 'm1', usage: usage(50, 0, 0, 5) }), + makeMsg({ type: 'assistant', model: 'm1', usage: usage(10, 0, 0, 2) }), + ]; + + const ledger = buildLedger(messages); + expect(ledger.rounds).toHaveLength(1); + expect(ledger.totals.inputTokens).toBe(10); + }); +}); + +describe('normalizeCallKey', () => { + it('collapses whitespace in Bash commands', () => { + expect(normalizeCallKey('Bash', { command: 'pnpm test' })).toBe( + normalizeCallKey('Bash', { command: 'pnpm test' }) + ); + }); + + it('distinguishes different files for Read', () => { + expect(normalizeCallKey('Read', { file_path: '/a' })).not.toBe( + normalizeCallKey('Read', { file_path: '/b' }) + ); + }); + + it('keeps nested object inputs distinct in the default branch', () => { + expect(normalizeCallKey('TodoWrite', { todos: [{ content: 'a' }] })).not.toBe( + normalizeCallKey('TodoWrite', { todos: [{ content: 'b' }] }) + ); + }); + + it('does not swallow a following flag as a value', () => { + const opts = parseArgs(['--rounds', '--json', 'x.jsonl']); + expect(opts.rounds).toBe(20); + expect(opts.json).toBe(true); + expect(opts.sessionPath).toBe('x.jsonl'); + }); +}); + +describe('computeFindings', () => { + it('flags duplicate calls, real failures, and skips user rejections', () => { + const messages = [ + makeMsg({ type: 'user', content: 'go' }), + makeMsg({ + type: 'assistant', + usage: usage(10, 0, 0, 1), + toolCalls: [ + { id: 't1', name: 'Bash', input: { command: 'pytest -q' }, isTask: false }, + { id: 't2', name: 'Bash', input: { command: 'pytest -q' }, isTask: false }, + { id: 't3', name: 'Bash', input: { command: 'boom' }, isTask: false }, + { id: 't4', name: 'Bash', input: { command: 'declined' }, isTask: false }, + ], + }), + makeMsg({ + type: 'user', + isMeta: true, + toolResults: [ + { toolUseId: 't1', content: 'all passed', isError: false }, + { toolUseId: 't2', content: 'all passed', isError: false }, + { toolUseId: 't3', content: 'Traceback: boom', isError: true }, + { + toolUseId: 't4', + content: "The user doesn't want to proceed with this tool use.", + isError: true, + }, + ], + }), + ]; + const ledger = buildLedger(messages); + const findings = computeFindings(messages, ledger); + const types = findings.map((f) => f.type); + + expect(types).toContain('duplicate_call'); + expect(types).toContain('failed_call'); + const failed = findings.filter((f) => f.type === 'failed_call'); + expect(failed).toHaveLength(1); // only the real failure, not the rejection + const dup = findings.find((f) => f.type === 'duplicate_call'); + expect(dup?.tokensWasted).toBe(estimateTokens('all passed')); // one re-read, not both results + }); + + it('flags context spikes and dead caching from ledger rounds', () => { + const messages = [ + makeMsg({ type: 'user', content: 'go' }), + makeMsg({ type: 'assistant', model: 'm1', usage: usage(1000, 0, 0, 1) }), + makeMsg({ type: 'assistant', model: 'm1', usage: usage(45000, 0, 0, 1) }), + ]; + + const ledger = buildLedger(messages); + const findings = computeFindings(messages, ledger); + expect(findings.some((f) => f.type === 'context_spike')).toBe(true); + expect(findings.some((f) => f.type === 'cache_dead')).toBe(true); + }); +}); + +describe('scanSessionFile', () => { + it('computes duration, dedups usage per requestId, excludes synthetic and sidechain usage', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'devtools-inv-')); + try { + const file = path.join(dir, 'session1.jsonl'); + await writeFile( + file, + [ + JSON.stringify({ type: 'user', uuid: '1', timestamp: '2026-09-20T10:00:00Z' }), + JSON.stringify({ + type: 'assistant', + uuid: '2', + timestamp: '2026-09-20T10:05:00Z', + requestId: 'r1', + message: { + model: 'claude-sonnet-5', + usage: { input_tokens: 10, output_tokens: 1, cache_read_input_tokens: 100 }, + }, + }), + JSON.stringify({ + type: 'assistant', + uuid: '3', + timestamp: '2026-09-20T10:06:00Z', + requestId: 'r1', + message: { + model: 'claude-sonnet-5', + usage: { input_tokens: 10, output_tokens: 5, cache_read_input_tokens: 100 }, + }, + }), + JSON.stringify({ + type: 'assistant', + uuid: '4', + timestamp: '2026-09-20T10:07:00Z', + message: { model: '', usage: { input_tokens: 9, output_tokens: 1 } }, + }), + JSON.stringify({ + type: 'assistant', + uuid: '5', + timestamp: '2026-09-20T10:08:00Z', + isSidechain: true, + requestId: 'r2', + message: { + model: 'claude-haiku-4-5', + usage: { input_tokens: 500, output_tokens: 50, cache_creation_input_tokens: 30 }, + }, + }), + ].join('\n') + ); + + const entry = await scanSessionFile(file); + expect(entry).not.toBeNull(); + // sidechain timestamps span the file, but its tokens/models/billing stay out + expect(entry?.durationMs).toBe(8 * 60 * 1000); + expect(entry?.models).toEqual(['claude-sonnet-5']); + expect(entry?.inputTokens).toBe(10); + expect(entry?.outputTokens).toBe(5); + expect(entry?.cacheReadTokens).toBe(100); + expect(entry?.cacheCreationTokens).toBe(0); + expect(entry?.messageCount).toBe(5); + expect(entry?.billing).toBe('router-style'); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); + +describe('mapWithConcurrency', () => { + it('isolates per-item failures as null entries', async () => { + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + const results = await mapWithConcurrency([1, 2, 3], 2, async (n) => { + if (n === 2) throw new Error('EACCES: permission denied'); + return n * 10; + }); + expect(results).toHaveLength(3); + expect(results).toContain(10); + expect(results).toContain(30); + expect(results).toContain(null); + } finally { + errSpy.mockRestore(); + } + }); +}); + +describe('priceFamily and billing scheme', () => { + it('extracts claude family from short ids without date', () => { + expect(priceFamily('claude-sonnet-5')).toBe('sonnet'); + expect(priceFamily('claude-sonnet-5-20250929')).toBe('sonnet'); + expect(priceFamily('glm-5.3-flash')).toBeNull(); + }); + + it('detects billing scheme from round signatures', () => { + const w = { cacheReadTokens: 0, cacheCreationTokens: 100 }; + const r = { cacheReadTokens: 500, cacheCreationTokens: 0 }; + const n = { cacheReadTokens: 0, cacheCreationTokens: 0 }; + expect(detectBillingScheme([w, r])).toBe('mixed'); + expect(detectBillingScheme([w])).toBe('anthropic-style'); + expect(detectBillingScheme([r, r])).toBe('router-style'); + expect(detectBillingScheme([n])).toBe('no-cache'); + }); + + it('computes cost for anthropic-style sessions logged with short model ids', () => { + const messages = [ + makeMsg({ type: 'user', content: 'go' }), + makeMsg({ + type: 'assistant', + model: 'claude-sonnet-5', + usage: usage(100, 1000, 200, 50), + }), + ]; + const ledger = buildLedger(messages); + expect(ledger.billing).toBe('anthropic-style'); + expect(ledger.totals.costUsd).toBeDefined(); + expect(ledger.totals.costUsd).toBeGreaterThan(0); + expect(ledger.totals.costPartial).toBe(false); + }); + + it('labels router-style sessions without cost', () => { + const messages = [ + makeMsg({ type: 'user', content: 'go' }), + makeMsg({ type: 'assistant', model: 'glm-5.3-flash', usage: usage(100, 4000, 0, 10) }), + ]; + const ledger = buildLedger(messages); + expect(ledger.billing).toBe('router-style'); + expect(ledger.totals.costUsd).toBeUndefined(); + }); + + it('marks cost partial when the session mixes priced and unpriced models', () => { + const messages = [ + makeMsg({ type: 'user', content: 'go' }), + makeMsg({ type: 'assistant', model: 'claude-sonnet-5', usage: usage(100, 0, 0, 10) }), + makeMsg({ type: 'assistant', model: 'glm-5.3-flash', usage: usage(100, 0, 0, 10) }), + ]; + const ledger = buildLedger(messages); + expect(ledger.totals.costUsd).toBeDefined(); + expect(ledger.totals.costPartial).toBe(true); + }); +});