diff --git a/packages/opencode/src/cli/cmd/run-accounting.ts b/packages/opencode/src/cli/cmd/run-accounting.ts index ecd915bcb3..f4cc226d34 100644 --- a/packages/opencode/src/cli/cmd/run-accounting.ts +++ b/packages/opencode/src/cli/cmd/run-accounting.ts @@ -1,9 +1,11 @@ // Fork-only helpers for the `run` command: // - honest turn accounting: compaction-machinery steps must not consume the -// --max-turns budget. `step-start` parts carry only messageID/sessionID, so -// the owning message's summary flag is resolved via a lookup populated from -// `message.updated` events (the assistant message row is persisted — and its -// event published — before its first step-start part streams). +// --max-turns budget, and a turn is charged when a step completes, so a +// failed or retried attempt costs nothing. Step parts carry only +// messageID/sessionID, so the owning message's summary flag is resolved via +// a lookup populated from `message.updated` events (the assistant message +// row is persisted — and its event published — before its first +// step-start part streams). // - dual-attribution termination logging: every run records TWO independent // fields instead of one rc: `why_model_stopped` and `why_harness_stopped`, // so model-looping, tight budgets, and harness errors stop being conflated @@ -92,6 +94,9 @@ export function createRecoverableOverflowTraceErrors() { export function create() { const summaryMessages = new Set() let turnCount = 0 + // altimate_change start — the non-compaction step streaming now, until it finishes + let openAttempt = false + // altimate_change end // altimate_change start — track the active request's committed echo turn allowance let committedAnswer = false // altimate_change end @@ -166,23 +171,50 @@ export function create() { else summaryMessages.delete(info.id) }, isCompactionStep, - /** - * Count a step-start toward the turn budget unless it belongs to a - * compaction-machinery message. Returns true when the step was counted. - */ - onStepStart(messageID: string): boolean { - if (isCompactionStep(messageID)) return false - turnCount++ - return true + // altimate_change start — charge the turn budget on step-finish, not step-start + // A step-start opens an attempt; the processor retries a failed attempt + // (up to SessionRetry.RETRY_MAX_ATTEMPTS per step), and each retry streams + // its own step-start but only the attempt that completes streams a + // step-finish. Charging on start billed every failed attempt as a turn. + // A step that ends without a step-finish is free when a retry or idle + // closes it (a failed or retried attempt). A step still open when compaction + // replaces it, or cut short by a context-overflow error, is charged as + // before, so overflow-and-compact cycles stay bounded by --max-turns + // whichever of the overflow error and the compaction start arrives first. + // The processor's retry cap bounds retried attempts. + // `turnCount` is therefore the number of COMPLETED non-compaction steps; + // harness-injected generations (idle-done challenge, continuation, + // silent-turn reply) are real assistant steps and are charged the same way. + /** Track the step now streaming; a compaction step closes any unfinished one. */ + onStepAttempt(messageID: string) { + if (!isCompactionStep(messageID)) { + openAttempt = true + return + } + // Compaction replacing a step that never finished: that step was cut short. + if (openAttempt) turnCount++ + openAttempt = false + }, + /** No step is streaming any more (the session went idle, or a retry began). */ + closeOpenStep() { + openAttempt = false }, + /** Completed non-compaction assistant steps. */ get turnCount() { return turnCount }, + /** True when `maxTurns` completed steps already ran, so no further step may start. */ + turnBudgetReached(maxTurns: number | undefined): boolean { + return maxTurns !== undefined && maxTurns > 0 && turnCount >= maxTurns + }, onStepFinish(messageID: string, reason: string | undefined) { if (isCompactionStep(messageID)) return + turnCount++ + openAttempt = false lastFinishReason = reason lastFinishMessageID = messageID }, + // altimate_change end onText(messageID: string, text: string, synthetic = false) { if (isCompactionStep(messageID)) return if (synthetic) return @@ -224,6 +256,14 @@ export function create() { } if (errorName === "ContextOverflowError") { pendingContextOverflow = true + // altimate_change start — a step cut short by context overflow still costs its turn + // Counted from the session event only, so the prompt result repeating the + // same overflow cannot charge it again. + if (openAttempt && source === "event") { + turnCount++ + openAttempt = false + } + // altimate_change end return } // the idle-done challenge is delivered by aborting the in-flight diff --git a/packages/opencode/src/cli/cmd/run.ts b/packages/opencode/src/cli/cmd/run.ts index 39ec739b81..6a2fac90b5 100644 --- a/packages/opencode/src/cli/cmd/run.ts +++ b/packages/opencode/src/cli/cmd/run.ts @@ -416,7 +416,8 @@ export const RunCommand = cmd({ // altimate_change start — budget limits for CI/enterprise governance .option("max-turns", { type: "number", - describe: "maximum assistant work turns; allows one committed-answer echo, or one best-effort reply after a budget abort", + describe: + "maximum completed assistant work turns, including harness follow-up turns (a failed or retried attempt is not counted); allows one committed-answer echo, or one best-effort reply after a budget abort", }) // altimate_change start — wall-clock work budget, with a separate finalization allowance .option("time-budget", { @@ -1136,25 +1137,29 @@ You are speaking to a non-technical business executive. Follow these rules stric } if (part.type === "step-start") { + // altimate_change start — one compaction lookup serves output, `step` and the turn budget + const compactionStep = accounting.isCompactionStep(part.messageID) + // altimate_change end // altimate_change start — `--output` keeps only the last step's text (see run-output.ts) - outputParts.stepStart(accounting.isCompactionStep(part.messageID)) + outputParts.stepStart(compactionStep) // altimate_change end // altimate_change start — see `step` (#1334). Compaction steps do not count, // as for the turn budget below: one running after the final answer would // otherwise move `step` past `lastTextStep` and ask for a reply the model // already gave. (release review) - if (!accounting.isCompactionStep(part.messageID)) step++ + if (!compactionStep) step++ // altimate_change end // altimate_change start — enforce max-turns budget // compaction-machinery steps are excluded from turn accounting — // the owning message's agent is resolved via the message.updated lookup // above, so compacting models are not differentially charged turns. - const counted = accounting.onStepStart(part.messageID) + // Turns are charged on step-finish: a step may start only while fewer + // than maxTurns steps have completed, so a retried attempt is free. const requestID = requestOwnership.ownerOf(part.messageID) const committedAnswer = requestID !== undefined && committedRequests.has(requestID) // altimate_change start — the time-budget abort already owns the interrupted work phase if ( - counted && maxTurns && accounting.turnCount > maxTurns && !committedAnswer && !budgetAborted && + !compactionStep && accounting.turnBudgetReached(maxTurns) && !committedAnswer && !budgetAborted && (!timeExhausted() || options?.turnLimit !== undefined) ) { stopTimeBudget() @@ -1169,6 +1174,7 @@ You are speaking to a non-technical business executive. Follow these rules stric // Drain the ordered abort/error/idle tail before starting another phase. continue } + accounting.onStepAttempt(part.messageID) // altimate_change end if (emit("step_start", { part })) continue } @@ -1353,6 +1359,14 @@ You are speaking to a non-technical business executive. Follow these rules stric } // altimate_change end + // altimate_change start — a retry begins: the failed attempt is over and stays uncharged + if ( + event.type === "session.status" && + event.properties.sessionID === sessionID && + event.properties.status.type === "retry" + ) + accounting.closeOpenStep() + // altimate_change end if ( event.type === "session.status" && event.properties.sessionID === sessionID && @@ -1363,6 +1377,7 @@ You are speaking to a non-technical business executive. Follow these rules stric // altimate_change end // altimate_change start — idle gaps are not active work; continuations retain the same start time workActive = false + accounting.closeOpenStep() // altimate_change end break } @@ -2049,7 +2064,8 @@ You are speaking to a non-technical business executive. Follow these rules stric // A silent stop exactly at the work limit has no in-flight generation to // abort. Enter finalization directly instead of spending a reply just to // discover that it exceeds the same budget. - if (!answered() && assistantStarted && !accounting.fatal && args.maxTurns && accounting.turnCount >= args.maxTurns) { + // The same completed-step count the in-loop budget check uses. + if (!answered() && assistantStarted && !accounting.fatal && accounting.turnBudgetReached(args.maxTurns)) { accounting.onBudgetExhausted() error = `Budget exceeded: reached ${args.maxTurns} assistant turn${args.maxTurns !== 1 ? "s" : ""} limit` if (!emit("budget_exhausted", { limit: args.maxTurns })) { diff --git a/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap b/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap index 020fee6fb1..36f831897d 100644 --- a/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap +++ b/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap @@ -135,8 +135,10 @@ Options: statement (1-indexed) [number] --trace enable session tracing (default: true, disable with --no-trace) [boolean] [default: true] - --max-turns maximum assistant work turns; allows one committed-answer - echo, or one best-effort reply after a budget abort [number] + --max-turns maximum completed assistant work turns, including harness + follow-up turns (a failed or retried attempt is not counted); + allows one committed-answer echo, or one best-effort reply + after a budget abort [number] --time-budget maximum time in seconds including startup (env: ALTIMATE_RUN_TIME_BUDGET_S); bounded finalization may follow [number] diff --git a/packages/opencode/test/cli/run-accounting.test.ts b/packages/opencode/test/cli/run-accounting.test.ts index 0338839614..70fe948de5 100644 --- a/packages/opencode/test/cli/run-accounting.test.ts +++ b/packages/opencode/test/cli/run-accounting.test.ts @@ -22,35 +22,142 @@ describe("RunAccounting recoverable overflow trace errors", () => { }) describe("RunAccounting turn accounting", () => { - test("counts ordinary assistant steps", () => { + test("counts ordinary assistant steps when they finish", () => { const acc = RunAccounting.create() acc.onAssistantMessage({ id: "msg_1", agent: "build" }) - expect(acc.onStepStart("msg_1")).toBe(true) - expect(acc.onStepStart("msg_1")).toBe(true) + acc.onStepFinish("msg_1", "tool-calls") + acc.onStepFinish("msg_1", "stop") expect(acc.turnCount).toBe(2) }) + test("a step that fails mid-stream and is retried costs exactly one turn", () => { + const acc = RunAccounting.create() + acc.onAssistantMessage({ id: "msg_1", agent: "build" }) + // Attempt 1 started and its stream broke, so it wrote no step-finish. The + // retry's start is checked against completed steps only, so a one-turn + // budget still lets it run. + acc.onStepAttempt("msg_1") + acc.onStepAttempt("msg_1") + expect(acc.turnCount).toBe(0) + expect(acc.turnBudgetReached(1)).toBe(false) + acc.onStepFinish("msg_1", "stop") + expect(acc.turnCount).toBe(1) + expect(acc.turnBudgetReached(1)).toBe(true) + }) + + test("a session error with no completed step charges no turn and stays fatal", () => { + const acc = RunAccounting.create() + acc.onAssistantMessage({ id: "msg_1", agent: "build" }) + // Every attempt the processor allows failed before finishing. + for (let i = 0; i < 6; i++) acc.onStepAttempt("msg_1") + acc.onSessionError("APIError", "upstream stream reset") + expect(acc.turnCount).toBe(0) + expect(acc.turnBudgetReached(1)).toBe(false) + expect(acc.fatal).toBe(true) + expect(acc.termination().why_harness_stopped).toBe("error") + }) + + test("the budget is reached only after maxTurns steps complete", () => { + const acc = RunAccounting.create() + acc.onAssistantMessage({ id: "msg_1", agent: "build" }) + expect(acc.turnBudgetReached(1)).toBe(false) + acc.onStepFinish("msg_1", "tool-calls") + expect(acc.turnBudgetReached(1)).toBe(true) + expect(acc.turnBudgetReached(2)).toBe(false) + expect(acc.turnBudgetReached(undefined)).toBe(false) + }) + + test("a step cut short by context overflow still costs its turn, once", () => { + const acc = RunAccounting.create() + acc.onAssistantMessage({ id: "msg_1", agent: "build" }) + // An overflow reported before any step started is not charged. + acc.onSessionError("ContextOverflowError", "too long") + expect(acc.turnCount).toBe(0) + acc.onStepAttempt("msg_1") + // The prompt result may report it before the session event; only the event charges. + acc.onPromptResult({ error: { name: "ContextOverflowError" } }) + acc.onSessionError("ContextOverflowError", "too long") + acc.onSessionError("ContextOverflowError", "too long") + expect(acc.turnCount).toBe(1) + }) + + test("a step that compaction replaces is charged once, whichever event arrives first", () => { + for (const overflowFirst of [true, false]) { + const acc = RunAccounting.create() + acc.onAssistantMessage({ id: "msg_1", agent: "build" }) + acc.onAssistantMessage({ id: "msg_compact", agent: "compaction", summary: true }) + acc.onStepAttempt("msg_1") + if (overflowFirst) acc.onSessionError("ContextOverflowError", "too long") + acc.onStepAttempt("msg_compact") + if (!overflowFirst) acc.onSessionError("ContextOverflowError", "too long") + // The summary's own overflow is never charged. + acc.onSessionError("ContextOverflowError", "summary too long") + expect(acc.turnCount).toBe(1) + } + }) + + test("after the session goes idle, a later pre-step overflow is not charged to an interrupted step", () => { + const acc = RunAccounting.create() + acc.onAssistantMessage({ id: "msg_1", agent: "build" }) + acc.onStepAttempt("msg_1") // soft-deadline interrupt: no step-finish + acc.closeOpenStep() + acc.onSessionError("ContextOverflowError", "too long") + expect(acc.turnCount).toBe(0) + }) + + test("an overflow after a finished step is not charged again", () => { + const acc = RunAccounting.create() + acc.onAssistantMessage({ id: "msg_1", agent: "build" }) + acc.onStepAttempt("msg_1") + acc.onStepFinish("msg_1", "tool-calls") + acc.onSessionError("ContextOverflowError", "too long") + expect(acc.turnCount).toBe(1) + }) + + test("an overflow reported only on the prompt result charges nothing", () => { + const acc = RunAccounting.create() + acc.onAssistantMessage({ id: "msg_1", agent: "build" }) + acc.onStepAttempt("msg_1") + acc.onPromptResult({ error: { name: "ContextOverflowError" } }) + expect(acc.turnCount).toBe(0) + }) + + test("once a retry begins, a later pre-step overflow is not charged to the failed attempt", () => { + const acc = RunAccounting.create() + acc.onAssistantMessage({ id: "msg_1", agent: "build" }) + acc.onStepAttempt("msg_1") // fails mid-stream + acc.closeOpenStep() // retry status + acc.onSessionError("ContextOverflowError", "too long") // retry rejected before streaming + expect(acc.turnCount).toBe(0) + }) + test("excludes compaction-machinery steps from turnCount", () => { const acc = RunAccounting.create() acc.onAssistantMessage({ id: "msg_work", agent: "build" }) acc.onAssistantMessage({ id: "msg_compact", agent: "compaction", summary: true }) - expect(acc.onStepStart("msg_work")).toBe(true) - expect(acc.onStepStart("msg_compact")).toBe(false) - expect(acc.onStepStart("msg_compact")).toBe(false) - expect(acc.onStepStart("msg_work")).toBe(true) + expect(acc.isCompactionStep("msg_work")).toBe(false) + acc.onStepFinish("msg_work", "tool-calls") + expect(acc.isCompactionStep("msg_compact")).toBe(true) + acc.onStepFinish("msg_compact", "stop") + expect(acc.isCompactionStep("msg_compact")).toBe(true) + acc.onStepFinish("msg_compact", "stop") + expect(acc.isCompactionStep("msg_work")).toBe(false) + acc.onStepFinish("msg_work", "stop") expect(acc.turnCount).toBe(2) }) test("an explicitly selected agent named compaction still consumes the turn budget", () => { const acc = RunAccounting.create() acc.onAssistantMessage({ id: "msg_user_agent", agent: "compaction", summary: false }) - expect(acc.onStepStart("msg_user_agent")).toBe(true) + expect(acc.isCompactionStep("msg_user_agent")).toBe(false) + acc.onStepFinish("msg_user_agent", "stop") expect(acc.turnCount).toBe(1) }) test("a step whose owning message is unknown is counted (conservative default)", () => { const acc = RunAccounting.create() - expect(acc.onStepStart("msg_unknown")).toBe(true) + expect(acc.isCompactionStep("msg_unknown")).toBe(false) + acc.onStepFinish("msg_unknown", "stop") expect(acc.turnCount).toBe(1) }) @@ -77,7 +184,6 @@ describe("RunAccounting termination attribution (E4)", () => { test("natural finish: model=stop, harness=none", () => { const acc = RunAccounting.create() acc.onAssistantMessage({ id: "m1", agent: "build" }) - acc.onStepStart("m1") acc.onStepFinish("m1", "stop") expect(acc.termination()).toEqual({ why_model_stopped: "stop", why_harness_stopped: "none", done_reason: "none" }) expect(acc.fatal).toBe(false) @@ -86,7 +192,6 @@ describe("RunAccounting termination attribution (E4)", () => { test("model still tool-calling when harness exhausts the budget", () => { const acc = RunAccounting.create() acc.onAssistantMessage({ id: "m1", agent: "build" }) - acc.onStepStart("m1") acc.onStepFinish("m1", "tool-calls") acc.onBudgetExhausted() expect(acc.termination()).toEqual({ @@ -303,7 +408,6 @@ describe("RunAccounting done_reason + idle-done bookkeeping", () => { test("synthetic text appended after DONE does not clear explicit-DONE attribution", () => { const acc = RunAccounting.create() acc.onAssistantMessage({ id: "m1", agent: "plan" }) - acc.onStepStart("m1") acc.onText("m1", "Plan is complete.\nDONE") acc.onText("m1", "altimate-code: plan agent stopped without writing a plan", true) acc.onStepFinish("m1", "stop") @@ -373,17 +477,14 @@ describe("RunAccounting done_reason + idle-done bookkeeping", () => { // the heuristic's doing. const acc = RunAccounting.create() acc.onAssistantMessage({ id: "m1", agent: "build" }) - acc.onStepStart("m1") acc.onIdleDoneChallengeIssued() acc.onIdleDoneChallengeReplySent() acc.onText("m1", "Remaining: wire the config flag. Continuing.") acc.onStepFinish("m1", "stop") acc.onIdleDoneChallengeCompleted() acc.onAssistantMessage({ id: "m2", agent: "build" }) - acc.onStepStart("m2") acc.onStepFinish("m2", "tool-calls") acc.onAssistantMessage({ id: "m3", agent: "build" }) - acc.onStepStart("m3") acc.onText("m3", "All checks green.\nDONE") acc.onStepFinish("m3", "stop") const t = acc.termination() @@ -396,10 +497,8 @@ describe("RunAccounting done_reason + idle-done bookkeeping", () => { acc.onIdleDoneChallengeIssued() acc.onIdleDoneChallengeReplySent() acc.onAssistantMessage({ id: "m1", agent: "build" }) - acc.onStepStart("m1") acc.onStepFinish("m1", "tool-calls") acc.onAssistantMessage({ id: "m2", agent: "build" }) - acc.onStepStart("m2") acc.onText("m2", "Verified.\nDONE") acc.onStepFinish("m2", "stop") acc.onIdleDoneChallengeCompleted() @@ -418,11 +517,9 @@ describe("RunAccounting done_reason + idle-done bookkeeping", () => { test("stale DONE from a tool-calls message is not paired with a later textless stop", () => { const acc = RunAccounting.create() acc.onAssistantMessage({ id: "m1", agent: "build" }) - acc.onStepStart("m1") acc.onText("m1", "Wrapping up.\nDONE") acc.onStepFinish("m1", "tool-calls") acc.onAssistantMessage({ id: "m2", agent: "build" }) - acc.onStepStart("m2") acc.onStepFinish("m2", "stop") // no onText for m2 — no text part at all const t = acc.termination() expect(t.done_reason).toBe("none") diff --git a/packages/opencode/test/cli/run/budget-finalization.test.ts b/packages/opencode/test/cli/run/budget-finalization.test.ts index 90ddbec384..eac6e3983d 100644 --- a/packages/opencode/test/cli/run/budget-finalization.test.ts +++ b/packages/opencode/test/cli/run/budget-finalization.test.ts @@ -11,9 +11,8 @@ import { testProviderConfig } from "../../lib/test-provider" function exhausted() { const accounting = RunAccounting.create() - accounting.onStepStart("work") accounting.onStepFinish("work", "tool-calls") - accounting.onStepStart("interrupted") + // the over-budget step is aborted at its start, so it never completes accounting.onBudgetExhausted() return accounting } @@ -40,13 +39,11 @@ describe("budget finalization accounting", () => { // altimate_change start — distinguish recovered wall-clock limits from turn limits and real failures test("a successful time-budget finalization clears fatal status but keeps budget attribution", () => { const accounting = RunAccounting.create() - accounting.onStepStart("interrupted") accounting.onBudgetExhausted("time") accounting.onSessionError("MessageAbortedError", "intentional time-budget abort") accounting.onPromptResult({ error: { name: "MessageAbortedError" } }) expect(accounting.fatal).toBe(true) - expect(accounting.takeBudgetFinalizationAllowance()).toBe(2) - accounting.onStepStart("finalization") + expect(accounting.takeBudgetFinalizationAllowance()).toBe(1) accounting.onText("finalization", "The evidence supports a limited answer.") accounting.onStepFinish("finalization", "stop") accounting.onPromptResult({ finish: "stop" }) @@ -79,7 +76,7 @@ describe("budget finalization accounting", () => { test("successful turn-budget finalization preserves the existing nonzero exit contract", () => { const accounting = exhausted() - expect(accounting.takeBudgetFinalizationAllowance()).toBe(3) + expect(accounting.takeBudgetFinalizationAllowance()).toBe(2) expect(accounting.completeBudgetFinalization(true)).toBe(false) expect(accounting.fatal).toBe(true) }) @@ -111,15 +108,16 @@ describe("budget finalization accounting", () => { accounting.onSessionError("MessageAbortedError", "aborted by the turn limit") accounting.onPromptResult({ error: { name: "MessageAbortedError" } }) - expect(accounting.takeBudgetFinalizationAllowance()).toBe(3) + expect(accounting.takeBudgetFinalizationAllowance()).toBe(2) expect(accounting.takeBudgetFinalizationAllowance()).toBeUndefined() - expect(accounting.turnCount).toBe(2) + expect(accounting.turnCount).toBe(1) expect(accounting.fatal).toBe(true) - accounting.onStepStart("finalization") + expect(accounting.turnBudgetReached(2)).toBe(false) accounting.onText("finalization", "The supported result is available; verification is incomplete.") accounting.onStepFinish("finalization", "stop") - expect(accounting.turnCount).toBe(3) + expect(accounting.turnCount).toBe(2) + expect(accounting.turnBudgetReached(2)).toBe(true) expect(accounting.termination()).toEqual({ why_model_stopped: "stop", why_harness_stopped: "budget-exhausted", @@ -129,7 +127,6 @@ describe("budget finalization accounting", () => { test("does not grant an allowance before a budget abort", () => { const accounting = RunAccounting.create() - accounting.onStepStart("work") expect(accounting.takeBudgetFinalizationAllowance()).toBeUndefined() expect(accounting.fatal).toBe(false) }) @@ -139,7 +136,7 @@ describe("budget finalization accounting", () => { const accounting = exhausted() accounting.onSessionError("MessageAbortedError", "aborted") accounting.onPromptResult({ finish }) - expect(accounting.takeBudgetFinalizationAllowance()).toBe(3) + expect(accounting.takeBudgetFinalizationAllowance()).toBe(2) }) } @@ -147,7 +144,6 @@ describe("budget finalization accounting", () => { for (const order of ["before", "after"] as const) { test(`${name} ${order} budget exhaustion prevents a finalization despite reason precedence`, () => { const accounting = RunAccounting.create() - accounting.onStepStart("work") if (order === "before") accounting.onSessionError(name, "genuine failure") accounting.onBudgetExhausted() if (order === "after") accounting.onSessionError(name, "genuine failure") @@ -191,7 +187,7 @@ describe("budget finalization accounting", () => { accounting.onPromptResult({ finish: "other" }) accounting.onIdleDoneChallengeReplySent() accounting.onIdleDoneChallengeCompleted() - accounting.onStepStart("continuation") + accounting.onStepFinish("continuation", "tool-calls") accounting.onBudgetExhausted() accounting.onSessionError("MessageAbortedError", "budget interruption") accounting.onPromptResult({ error: { name: "MessageAbortedError" } }) @@ -357,6 +353,10 @@ describe("opencode run: budget finalization subprocess", () => { }, }) } + // The work step completes; only completed steps are charged to the budget. + publish("message.part.updated", { + part: { id: `${messageID}_finish`, messageID, sessionID, type: "step-finish", reason: "tool-calls" }, + }) if (sharedRequest) { // The original echo remains exempt, including synthetic compaction continuation rows. const continuationID = `${parentID}_continuation` @@ -567,7 +567,7 @@ describe("opencode run: budget finalization subprocess", () => { const committedRequests = new Set() ownership.beginRequest("original") ownership.onAssistantMessage({ id: "commit", parentID: "original" }) - accounting.onStepStart("commit") + accounting.onStepFinish("commit", "tool-calls") committedRequests.add(ownership.ownerOf("commit")!) const queue = () => { ownership.onUserMessage("queued") @@ -578,7 +578,8 @@ describe("opencode run: budget finalization subprocess", () => { ownership.onUserMessage("marker", "original") ownership.onAssistantMessage({ id: "summary", parentID: "marker" }) accounting.onAssistantMessage({ id: "summary", summary: true }) - expect(accounting.onStepStart("summary")).toBe(false) + expect(accounting.isCompactionStep("summary")).toBe(true) + accounting.onStepFinish("summary", "stop") if (queued === "during the summary") queue() // SessionCompaction persists this synthetic user before reporting its origin. @@ -589,7 +590,8 @@ describe("opencode run: budget finalization subprocess", () => { ownership.onUserMessage("second-continuation", "second-marker") ownership.onAssistantMessage({ id: "echo", parentID: "second-continuation" }) expect(ownership.ownerOf("echo")).toBe("original") - expect(accounting.onStepStart("echo")).toBe(true) + expect(accounting.isCompactionStep("echo")).toBe(false) + accounting.onStepFinish("echo", "stop") expect(accounting.turnCount).toBe(2) expect(committedRequests.has(ownership.ownerOf("echo")!)).toBe(true) diff --git a/packages/opencode/test/cli/run/turn-budget-retry.test.ts b/packages/opencode/test/cli/run/turn-budget-retry.test.ts new file mode 100644 index 0000000000..c19de2958f --- /dev/null +++ b/packages/opencode/test/cli/run/turn-budget-retry.test.ts @@ -0,0 +1,420 @@ +// altimate_change start — `--max-turns` charges completed steps, not retried attempts +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { SessionRetry } from "../../../src/session/retry" +import { cliIt } from "../../lib/cli-process" +import { raw } from "../../lib/llm-server" + +const chunk = (delta: Record) => ({ + id: "chatcmpl-test", + object: "chat.completion.chunk", + choices: [{ delta }], +}) + +// A provider stream that starts a step, streams some text, then reports an error +// in-band. The error matches the processor's retryable rate-limit detection, so +// the session retries the step. +const failsMidStream = () => + raw({ + head: [ + chunk({ role: "assistant" }), + chunk({ content: "Partial work" }), + { error: { message: "rate limit exceeded", type: "rate_limit_error", code: "rate_limit_exceeded" } }, + ], + }) + +describe("opencode run: --max-turns charges completed steps only", () => { + cliIt.concurrent( + "a step that fails mid-stream and is retried costs one turn", + ({ llm, opencode }) => + Effect.gen(function* () { + yield* llm.push(failsMidStream()) + yield* llm.text("The answer after one retry.") + const result = yield* opencode.run("Answer once.", { + format: "json", + extraArgs: ["--max-turns", "1", "--no-trace"], + timeoutMs: 45_000, + bunRun: true, + }) + const events = opencode.parseJsonEvents(result.stdout) + expect(events.filter((event) => event.type === "budget_exhausted")).toHaveLength(0) + expect(events.filter((event) => event.type === "error")).toHaveLength(0) + expect( + events.filter((event) => event.type === "text").map((event) => (event.part as { text: string }).text), + ).toEqual(["The answer after one retry."]) + // One step-finish for the one completed step, whatever the number of attempts. + expect(events.filter((event) => event.type === "step_finish")).toHaveLength(1) + opencode.expectExit(result, 0) + }), + 60_000, + ) + // Attached-session fake: the CLI sees exactly the events the session publishes for + // a step whose attempts start and fail, without waiting out real retry backoff. + const attached = (behavior: "all-attempts-fail" | "endless-attempts" | "compaction-after-budget" | "overflow-mid-step" | "retry-then-overflow") => + Effect.gen(function* () { + const sessionID = "ses_turn_budget_retry" + let stream: ReadableStreamDefaultController | undefined + let subscriptionReady = Promise.withResolvers() + let finishPrompt: (value: Response) => void = () => {} + let attemptTimer: ReturnType | undefined + const counts = { prompts: 0, aborts: 0, attempts: 0, timerAttempts: 0 } + const encoder = new TextEncoder() + const publish = (type: string, properties: object) => + stream?.enqueue(encoder.encode(`data: ${JSON.stringify({ type, properties })}\n\n`)) + const assistantMessage = (id: string, parentID: string) => { + publish("message.updated", { + info: { + id, + sessionID, + role: "assistant", + parentID, + agent: "build", + modelID: "test-model", + providerID: "test", + time: { created: Date.now() }, + }, + }) + return id + } + // A compaction summary message runs one completed step of its own. + const compactionStep = (messageID: string, parentID: string) => { + const summaryID = `${messageID}zz_summary` + publish("message.updated", { + info: { + id: summaryID, + sessionID, + role: "assistant", + parentID, + agent: "compaction", + summary: true, + modelID: "test-model", + providerID: "test", + time: { created: Date.now() }, + }, + }) + publish("message.part.updated", { + part: { id: `${summaryID}_start`, messageID: summaryID, sessionID, type: "step-start" }, + }) + publish("message.part.updated", { + part: { id: `${summaryID}_finish`, messageID: summaryID, sessionID, type: "step-finish", reason: "stop" }, + }) + } + // The processor counts retries per assistant step, so the fake does too. + const stepAttempts = new Map() + const startAttempt = (messageID: string) => { + counts.attempts++ + const attempt = (stepAttempts.get(messageID) ?? 0) + 1 + stepAttempts.set(messageID, attempt) + // As in the processor, a retry status precedes every attempt of a step after its first. + if (attempt > 1) + publish("session.status", { + sessionID, + status: { type: "retry", attempt: attempt - 1, message: "rate limit exceeded", next: Date.now() + 10 }, + }) + publish("message.part.updated", { + part: { id: `${messageID}_start_${counts.attempts}`, messageID, sessionID, type: "step-start" }, + }) + } + const server = yield* Effect.acquireRelease( + Effect.sync(() => + Bun.serve({ + hostname: "127.0.0.1", + port: 0, + idleTimeout: 0, + async fetch(request) { + const route = new URL(request.url).pathname + if (route === "/provider") + return Response.json({ all: [{ id: "test", models: { "test-model": {} } }], connected: ["test"] }) + if (route === "/config") return Response.json({ share: "disabled", tracing: { enabled: false } }) + if (route === "/event") + return new Response( + new ReadableStream({ + start(controller) { + stream = controller + subscriptionReady.resolve() + publish("server.connected", {}) + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ) + if (route === `/session/${sessionID}/abort`) { + counts.aborts++ + clearInterval(attemptTimer) + const error = { name: "MessageAbortedError", data: { message: "aborted" } } + publish("session.error", { sessionID, error }) + publish("session.status", { sessionID, status: { type: "idle" } }) + finishPrompt(Response.json({ info: { error } })) + return Response.json(true) + } + if (route === `/session/${sessionID}/message` && request.method === "POST") { + const body = (await request.json()) as { messageID: string; noReply?: boolean } + // The soft time-budget reminder is saved without a reply; it is not a prompt. + if (body.noReply) return Response.json({ info: { id: body.messageID, role: "user" } }) + const ready = subscriptionReady + await ready.promise + subscriptionReady = Promise.withResolvers() + counts.prompts++ + const parentID = body.messageID + const messageID = `${parentID}zz_assistant_${counts.prompts}` + publish("session.status", { sessionID, status: { type: "busy" } }) + publish("message.updated", { + info: { + id: messageID, + sessionID, + role: "assistant", + parentID, + agent: "build", + modelID: "test-model", + providerID: "test", + time: { created: Date.now() }, + }, + }) + if (counts.prompts > 1) { + // The time-budget finalization reply completes normally. + startAttempt(messageID) + publish("message.part.updated", { + part: { + id: `${messageID}_text`, + messageID, + sessionID, + type: "text", + text: "Best-effort answer.", + time: { start: 1, end: 2 }, + }, + }) + publish("message.part.updated", { + part: { id: `${messageID}_finish`, messageID, sessionID, type: "step-finish", reason: "stop" }, + }) + publish("session.status", { sessionID, status: { type: "idle" } }) + return Response.json({ info: { finish: "stop" } }) + } + if (behavior === "retry-then-overflow") { + // Attempt 1 fails mid-stream; the retry is rejected for overflow before it + // streams; compaction runs; the continuation step answers. + startAttempt(messageID) + publish("session.status", { + sessionID, + status: { type: "retry", attempt: 1, message: "rate limit exceeded", next: Date.now() + 10 }, + }) + publish("session.error", { + sessionID, + error: { name: "ContextOverflowError", data: { message: "prompt is too long" } }, + }) + compactionStep(messageID, parentID) + publish("session.compacted", { sessionID }) + const nextID = assistantMessage(`${messageID}zzz_next`, parentID) + startAttempt(nextID) + publish("message.part.updated", { + part: { + id: `${nextID}_text`, + messageID: nextID, + sessionID, + type: "text", + text: "The answer after compaction.", + time: { start: 1, end: 2 }, + }, + }) + publish("message.part.updated", { + part: { id: `${nextID}_finish`, messageID: nextID, sessionID, type: "step-finish", reason: "stop" }, + }) + publish("session.status", { sessionID, status: { type: "idle" } }) + return Response.json({ info: { finish: "stop" } }) + } + if (behavior === "overflow-mid-step") { + // A step starts, then overflows; compaction runs and the work continues. + startAttempt(messageID) + publish("session.error", { + sessionID, + error: { name: "ContextOverflowError", data: { message: "prompt is too long" } }, + }) + compactionStep(messageID, parentID) + publish("session.compacted", { sessionID }) + const nextID = assistantMessage(`${messageID}zzz_next`, parentID) + startAttempt(nextID) + return new Promise((resolve) => { + finishPrompt = resolve + }) + } + if (behavior === "compaction-after-budget") { + // The work step answers and completes, using the whole one-turn budget. + startAttempt(messageID) + publish("message.part.updated", { + part: { + id: `${messageID}_text`, + messageID, + sessionID, + type: "text", + text: "The answer.", + time: { start: 1, end: 2 }, + }, + }) + publish("message.part.updated", { + part: { id: `${messageID}_finish`, messageID, sessionID, type: "step-finish", reason: "stop" }, + }) + // Compaction machinery then runs a step of its own; it is never charged or stopped. + compactionStep(messageID, parentID) + publish("session.status", { sessionID, status: { type: "idle" } }) + return Response.json({ info: { finish: "stop" } }) + } + if (behavior === "all-attempts-fail") { + // The first attempt plus every retry the processor allows, none completing. + for (let attempt = 0; attempt <= SessionRetry.RETRY_MAX_ATTEMPTS; attempt++) startAttempt(messageID) + const error = { name: "APIError", data: { message: "rate limit exceeded", isRetryable: true } } + publish("session.error", { sessionID, error }) + publish("session.status", { sessionID, status: { type: "idle" } }) + return Response.json({ info: { error } }) + } + // A step that keeps failing and restarting without ever completing: + // more attempts than one step's retry cap at once, then more until aborted. + for (let attempt = 0; attempt <= SessionRetry.RETRY_MAX_ATTEMPTS + 1; attempt++) startAttempt(messageID) + attemptTimer = setInterval(() => { + counts.timerAttempts++ + startAttempt(messageID) + }, 20) + return new Promise((resolve) => { + finishPrompt = resolve + }) + } + return Response.json({ error: `Unexpected route: ${request.method} ${route}` }, { status: 404 }) + }, + }), + ), + (server) => + Effect.sync(() => { + clearInterval(attemptTimer) + server.stop(true) + }), + ) + return { server, sessionID, counts } + }) + + cliIt.concurrent( + "a step whose every attempt fails costs no turn and surfaces the error", + ({ opencode }) => + Effect.gen(function* () { + const { server, sessionID, counts } = yield* attached("all-attempts-fail") + const result = yield* opencode.run("Answer once.", { + format: "json", + extraArgs: ["--attach", server.url.toString(), "--session", sessionID, "--max-turns", "1", "--no-trace"], + timeoutMs: 20_000, + bunRun: true, + }) + opencode.expectExit(result, 1) + const events = opencode.parseJsonEvents(result.stdout) + expect(events.filter((event) => event.type === "step_start")).toHaveLength(SessionRetry.RETRY_MAX_ATTEMPTS + 1) + // Attempts that never completed did not exhaust the one-turn budget. + expect(events.filter((event) => event.type === "budget_exhausted")).toHaveLength(0) + expect(counts.aborts).toBe(0) + // The provider failure is what the run reports, and nothing re-prompts after it. + expect(events.filter((event) => event.type === "error")).toContainEqual( + expect.objectContaining({ error: expect.objectContaining({ name: "APIError" }) }), + ) + expect(counts.prompts).toBe(1) + expect(events.filter((event) => event.type === "termination")).toMatchObject([ + { why_harness_stopped: "error" }, + ]) + }), + 30_000, + ) + + cliIt.concurrent( + "a failed attempt closed by its retry is not charged for a later overflow", + ({ opencode }) => + Effect.gen(function* () { + const { server, sessionID, counts } = yield* attached("retry-then-overflow") + const result = yield* opencode.run("Answer once.", { + format: "json", + extraArgs: ["--attach", server.url.toString(), "--session", sessionID, "--max-turns", "1", "--no-trace"], + timeoutMs: 20_000, + bunRun: true, + }) + const events = opencode.parseJsonEvents(result.stdout) + expect(events.filter((event) => event.type === "budget_exhausted")).toHaveLength(0) + expect(counts.aborts).toBe(0) + expect( + events.filter((event) => event.type === "text").map((event) => (event.part as { text: string }).text), + ).toEqual(["The answer after compaction."]) + opencode.expectExit(result, 0) + }), + 30_000, + ) + + cliIt.concurrent( + "a step cut short by context overflow still costs its turn", + ({ opencode }) => + Effect.gen(function* () { + const { server, sessionID, counts } = yield* attached("overflow-mid-step") + const result = yield* opencode.run("Answer once.", { + format: "json", + extraArgs: ["--attach", server.url.toString(), "--session", sessionID, "--max-turns", "1", "--no-trace"], + timeoutMs: 20_000, + bunRun: true, + }) + opencode.expectExit(result, 1) + const events = opencode.parseJsonEvents(result.stdout) + // The overflowed step used the one turn, so the continuation's step is stopped. + expect(events.filter((event) => event.type === "budget_exhausted")).toEqual([ + expect.objectContaining({ limit: 1 }), + ]) + expect(counts.aborts).toBe(1) + }), + 30_000, + ) + + cliIt.concurrent( + "a compaction step after the budget is used is neither charged nor stopped", + ({ opencode }) => + Effect.gen(function* () { + const { server, sessionID, counts } = yield* attached("compaction-after-budget") + const result = yield* opencode.run("Answer once.", { + format: "json", + extraArgs: ["--attach", server.url.toString(), "--session", sessionID, "--max-turns", "1", "--no-trace"], + timeoutMs: 20_000, + bunRun: true, + }) + const events = opencode.parseJsonEvents(result.stdout) + expect(events.filter((event) => event.type === "budget_exhausted")).toHaveLength(0) + expect(counts.aborts).toBe(0) + expect(counts.prompts).toBe(1) + opencode.expectExit(result, 0) + }), + 30_000, + ) + + cliIt.concurrent( + "attempts that keep restarting without completing are still bounded by --time-budget", + ({ opencode }) => + Effect.gen(function* () { + const { server, sessionID, counts } = yield* attached("endless-attempts") + const result = yield* opencode.run("Answer once.", { + format: "json", + extraArgs: [ + "--attach", + server.url.toString(), + "--session", + sessionID, + "--max-turns", + "1", + "--time-budget", + "5", + "--no-trace", + ], + timeoutMs: 25_000, + bunRun: true, + }) + const events = opencode.parseJsonEvents(result.stdout) + // The turn budget never fires (no attempt completed); the wall-clock budget ends the work. + expect(events.filter((event) => event.type === "budget_exhausted")).toEqual([ + expect.objectContaining({ limit_seconds: 5 }), + ]) + expect(counts.aborts).toBeGreaterThanOrEqual(1) + // Attempts kept restarting after the initial burst until the deadline stopped them. + expect(counts.timerAttempts).toBeGreaterThan(0) + expect( + events.filter((event) => event.type === "text").map((event) => (event.part as { text: string }).text), + ).toEqual(["Best-effort answer."]) + }), + 40_000, + ) +}) +// altimate_change end diff --git a/packages/opencode/test/session/processor-effect.test.ts b/packages/opencode/test/session/processor-effect.test.ts index 3af5cf283a..44c3a4385e 100644 --- a/packages/opencode/test/session/processor-effect.test.ts +++ b/packages/opencode/test/session/processor-effect.test.ts @@ -17,6 +17,7 @@ import { SessionProcessor } from "../../src/session/processor" import { MessageID, PartID, SessionID } from "../../src/session/schema" import { ProviderID, ModelID } from "@/provider/schema" import { SessionStatus } from "../../src/session/status" +import { SessionRetry } from "../../src/session/retry" import { SessionSummary } from "../../src/session/summary" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" import { testEffect } from "../lib/effect" @@ -1121,6 +1122,61 @@ it.live("session.processor effect tests retry recognized structured json errors" { config: (url) => providerCfg(url) }, ), ) +// altimate_change start — the retry cap bounds a step whose every attempt fails mid-stream. +// `run --max-turns` charges only completed steps, so this cap (not the turn budget) is what +// stops a provider that breaks every stream after the step started. +it.live("session.processor effect tests stop retrying a mid-stream failure at the retry cap", () => + provideTmpdirServerLegacy( + ({ dir, llm }) => + Effect.gen(function* () { + const { processors, session, provider } = yield* boot() + const failsMidStream = () => + raw({ + head: [ + { id: "chatcmpl-test", object: "chat.completion.chunk", choices: [{ delta: { role: "assistant" } }] }, + { id: "chatcmpl-test", object: "chat.completion.chunk", choices: [{ delta: { content: "partial" } }] }, + { error: { message: "rate limit exceeded", type: "rate_limit_error", code: "rate_limit_exceeded" } }, + ], + }) + // One more failing stream than the cap allows; it must never be requested. + for (let attempt = 0; attempt <= SessionRetry.RETRY_MAX_ATTEMPTS + 1; attempt++) yield* llm.push(failsMidStream()) + // Skip the real exponential backoff; the cap, not the delay, is under test. + const delay = spyOn(SessionRetry, "delay").mockReturnValue(1) + yield* Effect.addFinalizer(() => Effect.sync(() => delay.mockRestore())) + + const chat = yield* session.create({}) + const parent = yield* user(chat.id, "retry cap") + const msg = yield* assistant(chat.id, parent.id, path.resolve(dir)) + const mdl = yield* refModel(provider) + const controller = new AbortController() + const handle = yield* processors.create({ + assistantMessage: msg as unknown as MessageV2.Assistant, + sessionID: chat.id, + model: mdl, + abort: controller.signal, + }) + + yield* runProcess(handle, controller, { + user: userInput(parent, chat.id), + sessionID: chat.id, + model: mdl, + agent: agent(), + system: [], + messages: [{ role: "user", content: "retry cap" }], + tools: {}, + }) + + const parts = MessageV2.parts(msg.id) + expect(yield* llm.calls).toBe(SessionRetry.RETRY_MAX_ATTEMPTS + 1) + // Every attempt started a step; none finished, so none would be charged a turn. + expect(parts.filter((part) => part.type === "step-start")).toHaveLength(SessionRetry.RETRY_MAX_ATTEMPTS + 1) + expect(parts.filter((part) => part.type === "step-finish")).toHaveLength(0) + expect(handle.message.error).toBeDefined() + }), + { config: (url) => providerCfg(url) }, + ), +) +// altimate_change end // BUG: REGRESSION: retry attempts no longer reach the EventV2 status listener used by the Effect processor path. // Minimal src fix: packages/opencode/src/session/processor.ts:540 should publish retry through the same // EventV2Bridge/SessionStatus service instance provided to the processor Effect runtime, not an isolated facade.