Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions eslint.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,7 @@ export default defineConfig([
'billedTotalTokens',
'firstAssistantTotalTokens',
'inputSideTokens',
'isMainChainAssistantLine',
'isRealUserLine',
'isTeammateRelayLine',
'isTranscriptTurnLine',
Expand Down
7 changes: 6 additions & 1 deletion scripts/turn-accounting.d.mts
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@ export interface AccountableMessage {
messageId?: string;
isMeta?: boolean;
isCompactSummary?: boolean;
isSidechain?: boolean;
content?: unknown;
usage?: UsageLike;
message?: { id?: string; usage?: UsageLike; content?: unknown };
message?: { id?: string; model?: string; usage?: UsageLike; content?: unknown };
}

/** requestId ?? messageId ?? message.id — one billed request per key. */
Expand Down Expand Up @@ -67,6 +68,10 @@ export declare function isTranscriptTurnLine(m: AccountableMessage): boolean;
/** Turn boundary: user-initiated message or compaction marker. */
export declare function isTurnBoundary(m: AccountableMessage): boolean;

/** Main-chain assistant line — the only lines billed to the turn's re-read
* (sidechain/synthetic excluded; detector and panel see the same set). */
export declare function isMainChainAssistantLine(m: AccountableMessage): boolean;

/** Sum input-side spend of the current turn, scanning lines newest-first. */
export declare function analyzeTurn(linesNewestFirst: Iterable<string>): {
spent: number;
Expand Down
13 changes: 12 additions & 1 deletion scripts/turn-accounting.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,17 @@ export function isTurnBoundary(m) {
return isUserChunkLine(m) || m.isCompactSummary === true;
}

/** Main-chain assistant line — the only lines whose usage is billed to the
* turn's re-read. Sidechain (subagent) rounds run in their own context and
* synthetic lines carry no real request; the detector (loopDetection.ts)
* and the panel's aiGroups never see them, so the hook must not either —
* one definition, no drift. */
export function isMainChainAssistantLine(m) {
return (
m.type === 'assistant' && m.isSidechain !== true && m.message?.model !== '<synthetic>'
);
}

/** Sum input-side tokens of the current turn, scanning lines newest-first. */
export function analyzeTurn(linesNewestFirst) {
let spent = 0;
Expand All @@ -225,7 +236,7 @@ export function analyzeTurn(linesNewestFirst) {
boundaryFound = true;
break;
}
if (m.type === 'assistant' && m.message?.usage) {
if (isMainChainAssistantLine(m) && m.message?.usage) {
const key = billedRequestKey(m);
if (key) {
if (billed.has(key)) continue;
Expand Down
3 changes: 3 additions & 0 deletions scripts/turn-budget-hook.d.mts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ export declare function isRealUserLine(m: AccountableMessage): boolean;
/** Turn boundary: real user message or compaction marker. */
export declare function isTurnBoundary(m: AccountableMessage): boolean;

/** Main-chain assistant line — billed to the turn's re-read. */
export declare function isMainChainAssistantLine(m: AccountableMessage): boolean;

/** Sum input-side spend of the current turn, scanning lines newest-first. */
export declare function analyzeTurn(linesNewestFirst: Iterable<string>): {
spent: number;
Expand Down
2 changes: 2 additions & 0 deletions scripts/turn-budget-hook.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import {
billedTotalTokens,
firstAssistantTotalTokens,
inputSideTokens,
isMainChainAssistantLine,
isRealUserLine,
isTurnBoundary,
lastAssistantTotalTokens,
Expand All @@ -59,6 +60,7 @@ export {
billedTotalTokens,
firstAssistantTotalTokens,
inputSideTokens,
isMainChainAssistantLine,
isRealUserLine,
isTurnBoundary,
lastAssistantTotalTokens,
Expand Down
115 changes: 112 additions & 3 deletions src/cli/turnSpendStats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,14 @@ import * as readline from 'readline';

// same turn-boundary predicate the hook enforces — one definition, no drift
import {
analyzeTurn,
billedRequestKey,
inputSideTokens,
isMainChainAssistantLine,
isTurnBoundary,
} from '../../scripts/turn-budget-hook.mjs';

import { wantsHelp } from './args';
import { takeFlagValue, wantsHelp } from './args';

export interface TurnSpend {
file: string;
Expand Down Expand Up @@ -67,7 +69,7 @@ export function feedLine(
return;
}

if (msg.type === 'assistant' && inner.usage && state.current) {
if (isMainChainAssistantLine(msg) && inner.usage && state.current) {
const side = inputSideTokens(inner.usage);
const key = billedRequestKey(msg);
if (key) {
Expand Down Expand Up @@ -102,10 +104,117 @@ export function percentile(sorted: number[], q: number): number {
// CLI
// =============================================================================

/** --audit <file.jsonl>: the three accountings side by side, per turn.
* Every number shown to a human (hook notification, detector bell, panel
* Re-read) must come out equal here — divergence is a bug, this prints it. */
async function auditFile(filePath: string): Promise<void> {
const { sumTurnReread } = await import('../renderer/utils/contextTracker');
const { TurnBudgetDetector } = await import('../main/utils/loopDetection');

const lines = fs
.readFileSync(filePath, 'utf8')
.split('\n')
.filter((l) => l.trim() !== '');
// ParsedMessage shape for the detector: usage/model/toolCalls at the top
const flat = lines.map((l) => {
const m = JSON.parse(l) as Record<string, unknown>;
const inner = m.message as { usage?: unknown; model?: string } | undefined;
return { ...m, usage: m.usage ?? inner?.usage, model: inner?.model, toolCalls: [] };
});

// turn slices (1-based numbering, same as the detector's turnNumber):
// turn 1 opens at the FIRST boundary; metadata before it is not a turn
const slices: string[][] = [];
let cur: string[] = [];
let sawBoundary = false;
for (const line of lines) {
let m: Record<string, unknown> = {};
try {
m = JSON.parse(line) as Record<string, unknown>;
} catch {
continue;
}
if (isTurnBoundary(m)) {
if (sawBoundary && cur.length) slices.push(cur);
cur = [];
sawBoundary = true;
}
if (sawBoundary) cur.push(line);
}
if (cur.length) slices.push(cur);

// detector path (the live bell): line-by-line feed, exactly what FileWatcher
// streams; snapshot each turn's running total right before its boundary resets
// the bucket. Batch-feed equivalence is pinned by the parity test.
const det = new TurnBudgetDetector();
const detTotals: number[] = [];
for (const m of flat) {
const before = det.currentTurnTotal('audit');
if (isTurnBoundary(m as never)) detTotals.push(before);
det.feed('audit', [m as never], Number.MAX_SAFE_INTEGER);
}
detTotals.push(det.currentTurnTotal('audit'));

const fmtK = (n: number): string =>
n >= 1e6 ? (n / 1e6).toFixed(2) + 'M' : n >= 1e3 ? (n / 1e3).toFixed(0) + 'k' : String(n);
console.log('turn start(ts) analyzeTurn detector panelReread rq');
let mismatches = 0;
slices.forEach((slice, i) => {
const hook = analyzeTurn([...slice].reverse()).spent;
// detTotals[0] is the pre-first-boundary prefix; turn i (1-based) = [i+1]
const bell = detTotals[i + 1] ?? 0;
const panel = sumTurnReread(
slice
.map((l) => JSON.parse(l) as Record<string, unknown>)
.filter((m) => m.type === 'assistant')
.map((m) => ({
...m,
usage: m.usage ?? (m.message as { usage?: unknown } | undefined)?.usage,
model: (m.message as { model?: string } | undefined)?.model,
toolCalls: [],
})) as never[]
);
const agree = hook === bell && hook === panel.tokens;
if (!agree) mismatches++;
const startTs = (() => {
try {
return (JSON.parse(slice[0]) as { timestamp?: string }).timestamp ?? '';
} catch {
return '';
}
})();
console.log(
String(i + 1).padStart(4),
startTs.padEnd(24),
fmtK(hook).padStart(11),
fmtK(bell).padStart(10),
fmtK(panel.tokens).padStart(12),
String(panel.requests).padStart(4),
agree ? '' : ' <-- MISMATCH'
);
});
console.log(
mismatches === 0
? `parity OK (${slices.length} turns)`
: `parity BROKEN: ${mismatches}/${slices.length} turns disagree`
);
}

async function main(): Promise<void> {
const argv = process.argv.slice(2);
if (wantsHelp(argv)) {
console.log('Usage: pnpm turn-spend:stats [--p N]');
console.log('Usage: pnpm turn-spend:stats [--p N] | --audit <file.jsonl>');
return;
}
const aIdx = argv.indexOf('--audit');
if (aIdx !== -1) {
const { value } = takeFlagValue(argv, aIdx);
if (!value) {
console.error('--audit requires a .jsonl path');
process.exitCode = 1;
return;
}
await auditFile(path.resolve(value));
return;
}
const pIdx = argv.indexOf('--p');
Expand Down
15 changes: 14 additions & 1 deletion src/main/services/infrastructure/FileWatcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,24 @@ function detectorIncidentToError(args: {
};
if (kind === 'turn_budget') {
const budget = incident as TurnBudgetIncident;
// turn number + wall-clock start — the keys to find the crossing turn in
// the chat and the Visible Context panel (Re-read section shows both)
const turnLabel =
budget.turnNumber > 0
? `turn ${budget.turnNumber}` +
(budget.turnStartTs
? ` (${new Date(budget.turnStartTs).toLocaleTimeString([], {
hour: '2-digit',
minute: '2-digit',
hour12: false,
})})`
: '')
: 'current turn';
return createDetectedError({
...common,
source: 'turn_budget',
message:
`Turn budget · ${formatTokensCompact(budget.spent)} / ` +
`Turn budget · ${turnLabel} · ${formatTokensCompact(budget.spent)} at crossing / ` +
`${formatTokensCompact(budget.budget)} — turn re-read crossed the ` +
'limit; tool calls are being denied',
triggerName: 'Turn budget',
Expand Down
42 changes: 40 additions & 2 deletions src/main/utils/loopDetection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,12 @@
import { type ParsedMessage } from '@main/types';
import { billedRequestKey } from '@main/utils/jsonl';
import { isStalledRound } from '@shared/constants/loopPolicy';
import { billedTotalTokens, inputSideTokens, isTurnBoundary } from '@shared/turnAccounting';
import {
billedTotalTokens,
inputSideTokens,
isTranscriptTurnLine,
isTurnBoundary,
} from '@shared/turnAccounting';
import { bashStem, normalizeCallKey } from '@shared/utils/callKey';

export interface LoopIncident {
Expand Down Expand Up @@ -253,6 +258,11 @@ interface FileBudgetState {
/** this turn already fired its crossing notification */
notified: boolean;
lastToolUseId: string;
/** 1-based ordinal of the current turn (transcript turns: user/relay) */
turnNumber: number;
/** timestamp of the boundary that opened the current turn
* (raw JSONL carries ISO strings, parsed transcripts carry Dates) */
turnStartTs?: string | Date;
cwd?: string;
}

Expand All @@ -262,13 +272,19 @@ const freshBudgetState = (): FileBudgetState => ({
total: 0,
notified: false,
lastToolUseId: '',
turnNumber: 0,
});

export interface TurnBudgetIncident {
/** turn's input-side spend at the crossing */
spent: number;
/** configured per-turn budget (the hook's currency) */
budget: number;
/** 1-based ordinal of the turn that crossed (0 = file opened mid-turn) */
turnNumber: number;
/** when the crossing turn started — the key to find it in the chat
* (raw JSONL carries ISO strings, parsed transcripts carry Dates) */
turnStartTs?: string | Date;
toolUseId: string;
cwd?: string;
batchIndex: number;
Expand Down Expand Up @@ -296,6 +312,12 @@ export class TurnBudgetDetector {
this.perFile.clear();
}

/** Current turn's running input-side total — the parity-test seam: the
* offline accounting (analyzeTurn) must equal this after any feed shape. */
currentTurnTotal(filePath: string): number {
return this.perFile.get(filePath)?.total ?? 0;
}

/** Same contract as LoopDetector.feed; budget = notifications.turnBudget.maxInputTokensPerTurn. */
feed(filePath: string, messages: ParsedMessage[], budget: number): TurnBudgetIncident | null {
let state = this.perFile.get(filePath) ?? freshBudgetState();
Expand All @@ -305,9 +327,23 @@ export class TurnBudgetDetector {

for (let i = 0; i < messages.length; i++) {
const msg = messages[i];
// Numbering follows the chat's Turn N chips (jsonl turnCount canon:
// transcript turns — user OR teammate relay — minus sidechain and
// compact summaries, which masquerade as user lines once parsed), so
// the bell number matches what the chat shows: a compaction must not
// inflate it, a relay must not deflate it. The SPEND bucket still
// resets on the hook boundary (user or compact — pre-compact spend
// must not count).
if (isTranscriptTurnLine(msg) && !msg.isSidechain && !msg.isCompactSummary) {
state.turnNumber += 1;
state.turnStartTs = msg.timestamp;
}
// a new user turn (or compaction) starts a fresh budget bucket
if (isTurnBoundary(msg)) {
state = freshBudgetState();
const numbered = freshBudgetState();
numbered.turnNumber = state.turnNumber;
numbered.turnStartTs = state.turnStartTs;
state = numbered;
this.perFile.set(filePath, state);
}
// main-chain assistant lines only — same accounting as the hook
Expand Down Expand Up @@ -336,6 +372,8 @@ export class TurnBudgetDetector {
incident = {
spent: state.total,
budget,
turnNumber: state.turnNumber,
turnStartTs: state.turnStartTs,
toolUseId: state.lastToolUseId,
cwd: state.cwd,
batchIndex: i,
Expand Down
7 changes: 6 additions & 1 deletion src/renderer/components/chat/ChatHistory.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,12 @@ export const ChatHistory = ({ tabId }: ChatHistoryProps): JSX.Element => {
}

const stats = sessionContextStats.get(targetAiGroupId);
const injections = stats?.accumulatedInjections ?? [];
// reread is session-wide spend (phaseInfo.rereadAll), not phase content —
// a bell notification's turn must be visible without switching phases
const injections = [
...(stats?.accumulatedInjections ?? []).filter((inj) => inj.category !== 'reread'),
...(sessionPhaseInfo?.rereadAll ?? []),
];

// Get total tokens from the target AI group
let totalTokens: number | undefined;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,23 @@ export const RereadSection = ({
<span className="min-w-0 flex-1 truncate text-xs" style={{ color: '#f87171' }}>
Turn {injection.turnIndex + 1}
</span>
{injection.phaseNumber !== undefined && (
<span
className="shrink-0 rounded px-1 text-[9px] tabular-nums"
style={{ backgroundColor: 'rgba(113, 113, 122, 0.2)', color: '#a1a1aa' }}
>
ph{injection.phaseNumber}
</span>
)}
{injection.turnStartTs && (
<span className="shrink-0 text-[10px] tabular-nums" style={{ color: '#71717a' }}>
{new Date(injection.turnStartTs).toLocaleTimeString([], {
Comment thread
axisrow marked this conversation as resolved.
hour: '2-digit',
minute: '2-digit',
hour12: false,
})}
</span>
)}
<span className="shrink-0 text-xs tabular-nums" style={{ color: '#a1a1aa' }}>
{injection.requests} rq
</span>
Expand Down
Loading
Loading