From fcc9ba517cf76fddcf849ef55552599f29a470fe Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Sun, 15 Nov 2026 06:29:30 +0900 Subject: [PATCH 1/2] fix(execution): SIGTERM follows the work-command cancellation path; bound retained CLI output (AGT-3423) - workCommand: SIGTERM aborts the pipelines like Ctrl-C, so the finally still uninstalls the handler and closes the coordinator; exits 143 (second signal force-quits) - adapters: stdout/stderr per CLI stage are kept as a bounded tail (32 MiB / 1 MiB) starting on a line boundary with a skip-safe marker, so result/cost/error events still parse --- src/adapters/base.ts | 14 +++++--- src/adapters/outputRetention.test.ts | 44 ++++++++++++++++++++++++ src/adapters/outputRetention.ts | 50 ++++++++++++++++++++++++++++ src/cli/workCommand.test.ts | 41 +++++++++++++++++++++++ src/cli/workCommand.ts | 36 ++++++++++++++------ 5 files changed, 170 insertions(+), 15 deletions(-) create mode 100644 src/adapters/outputRetention.test.ts create mode 100644 src/adapters/outputRetention.ts diff --git a/src/adapters/base.ts b/src/adapters/base.ts index c1510f42..a13f404d 100644 --- a/src/adapters/base.ts +++ b/src/adapters/base.ts @@ -3,6 +3,7 @@ // Shared spawn logic for all CLI adapters // ============================================ +import { BoundedOutput, MAX_RETAINED_STDERR_CHARS, MAX_RETAINED_STDOUT_CHARS } from './outputRetention.js'; import { spawn } from 'node:child_process'; import fs from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -249,13 +250,14 @@ export async function spawnCli( }, proc); } - let stdout = ''; - let stderr = ''; + // Bounded: keep the tail, which holds the result/cost/error events. (AGT-3423) + const stdoutBuf = new BoundedOutput(MAX_RETAINED_STDOUT_CHARS); + const stderrBuf = new BoundedOutput(MAX_RETAINED_STDERR_CHARS); let streamBuffer = ''; proc.stdout?.on('data', (data: Buffer) => { const text = data.toString(); - stdout += text; + stdoutBuf.append(text); if (options.onLog && adapter.capabilities.supportsStreaming) { streamBuffer = adapter.parseStreamingChunk ? adapter.parseStreamingChunk(text, options.onLog, streamBuffer) @@ -264,7 +266,7 @@ export async function spawnCli( }); proc.stderr?.on('data', (data: Buffer) => { - stderr += data.toString(); + stderrBuf.append(data.toString()); }); let exitDrainTimer: NodeJS.Timeout | null = null; @@ -280,7 +282,7 @@ export async function spawnCli( cleanupLifecycle(); terminateCliProcessTree(proc); const reason = lifecycleController.signal.reason; - session?.record({ type: 'assistant', rawStdout: stdout, rawStderr: stderr }); + session?.record({ type: 'assistant', rawStdout: stdoutBuf.toString(), rawStderr: stderrBuf.toString() }); session?.close({ outcome: 'aborted', durationMs: Date.now() - startTime, error: reason instanceof Error ? `${reason.name}: ${reason.message}` : String(reason), @@ -300,6 +302,8 @@ export async function spawnCli( : parseCliStreamChunk('\n', options.onLog, streamBuffer); } + const stdout = stdoutBuf.toString(); + const stderr = stderrBuf.toString(); session?.record({ type: 'assistant', rawStdout: stdout, rawStderr: stderr }); session?.close({ outcome: code === 0 || code === null ? 'returned' : 'exit_nonzero', diff --git a/src/adapters/outputRetention.test.ts b/src/adapters/outputRetention.test.ts new file mode 100644 index 00000000..a909d473 --- /dev/null +++ b/src/adapters/outputRetention.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest'; +import { BoundedOutput } from './outputRetention.js'; +import { extractStreamJsonError } from './base.js'; + +describe('BoundedOutput (AGT-3423)', () => { + it('keeps everything under the cap', () => { + const out = new BoundedOutput(100); + out.append('a\n'); + out.append('b\n'); + expect(out.toString()).toBe('a\nb\n'); + expect(out.droppedChars).toBe(0); + }); + + it('never retains more than the cap plus the marker, however much is streamed', () => { + const out = new BoundedOutput(1_000); + for (let i = 0; i < 10_000; i++) out.append(`{"type":"assistant","n":${i}}\n`); + const text = out.toString(); + const body = text.slice(text.indexOf('\n') + 1); + expect(body.length).toBeLessThanOrEqual(1_000); + expect(out.droppedChars).toBeGreaterThan(0); + expect(text.startsWith('[openswarm: ')).toBe(true); + // the newest line survives and the body starts on a whole line + expect(body.trimEnd().endsWith('"n":9999}')).toBe(true); + expect(body.startsWith('{"type":"assistant"')).toBe(true); + }); + + it('keeps the final stream-json result parseable after truncation', () => { + const out = new BoundedOutput(4_096); + for (let i = 0; i < 5_000; i++) out.append(`{"type":"assistant","message":"chunk ${i}"}\n`); + out.append('{"type":"result","is_error":true,"result":"model quota exhausted"}\n'); + expect(extractStreamJsonError(out.toString())).toContain('model quota exhausted'); + }); + + it('still bounds a single huge line with no newline', () => { + const out = new BoundedOutput(1_000); + out.append('x'.repeat(50_000)); + expect(out.toString().length).toBeLessThan(1_100); + }); + + it('rejects an invalid cap', () => { + expect(() => new BoundedOutput(0)).toThrow(); + expect(() => new BoundedOutput(Number.NaN)).toThrow(); + }); +}); diff --git a/src/adapters/outputRetention.ts b/src/adapters/outputRetention.ts new file mode 100644 index 00000000..15db1e3f --- /dev/null +++ b/src/adapters/outputRetention.ts @@ -0,0 +1,50 @@ +// ============================================ +// OpenSwarm - Bounded CLI output retention (AGT-3423) +// A CLI stage's stdout/stderr used to be concatenated without limit for the +// whole run. Keep the most recent output instead: stream-json results, costs +// and error events are at the end, and parsers read line by line, so the +// retained text starts on a line boundary and is prefixed with a plain-text +// marker that JSON-line parsers skip. +// ============================================ + +/** Retained stdout per CLI stage. Far above a normal run; a runaway stream is the case it guards. */ +export const MAX_RETAINED_STDOUT_CHARS = 32 * 1024 * 1024; +/** Retained stderr per CLI stage. */ +export const MAX_RETAINED_STDERR_CHARS = 1024 * 1024; + +export class BoundedOutput { + private text = ''; + private dropped = 0; + + constructor(private readonly maxChars: number) { + if (!Number.isFinite(maxChars) || maxChars < 1) throw new Error(`invalid output cap: ${maxChars}`); + } + + append(chunk: string): void { + this.text += chunk; + if (this.text.length > this.maxChars) this.trim(); + } + + /** Characters dropped from the front so far. */ + get droppedChars(): number { + return this.dropped; + } + + toString(): string { + return this.dropped > 0 + ? `[openswarm: ${this.dropped} earlier characters of output were not retained]\n${this.text}` + : this.text; + } + + // Trim to three quarters of the cap so a steady stream trims once per + // quarter-cap of input rather than on every chunk. + private trim(): void { + const keep = Math.max(1, Math.floor(this.maxChars * 0.75)); + let cut = this.text.length - keep; + // Start on a whole line when one begins reasonably soon after the cut. + const newline = this.text.indexOf('\n', cut); + if (newline !== -1 && newline - cut < keep / 2) cut = newline + 1; + this.dropped += cut; + this.text = this.text.slice(cut); + } +} diff --git a/src/cli/workCommand.test.ts b/src/cli/workCommand.test.ts index 7107729d..b67cbdb0 100644 --- a/src/cli/workCommand.test.ts +++ b/src/cli/workCommand.test.ts @@ -19,6 +19,7 @@ import { summarizeSettled, WORK_EXIT_FAILED, WORK_EXIT_INTERRUPTED, + WORK_EXIT_TERMINATED, WORK_EXIT_NOT_RUN, WORK_EXIT_OK, type WorkCommandDeps, @@ -633,6 +634,46 @@ describe('runWorkCommand — abort propagation (SIGINT)', () => { }); }); +describe('runWorkCommand — SIGTERM takes the same cancellation path (AGT-3423)', () => { + it('SIGTERM aborts the pipeline signal, still closes the coordinator path, and exits 143', async () => { + let handler: ((signal?: 'SIGINT' | 'SIGTERM') => void) | undefined; + let uninstalled = 0; + const observedAborts: Array = []; + const deps = baseDeps({ + installSigintHandler: (fn) => { + handler = fn; + return () => { uninstalled += 1; }; + }, + }); + deps.exec.mockImplementation(async (_ctx, _task, _path, signal?: AbortSignal) => { + handler?.('SIGTERM'); + observedAborts.push(signal?.aborted ? 'aborted' : undefined); + return pipelineResult({ success: false, finalStatus: 'cancelled' }); + }); + const code = await runWorkCommand({ issueIds: ['INT-1'], path: '/repo', yes: true }, deps); + expect(code).toBe(WORK_EXIT_TERMINATED); + expect(observedAborts).toEqual(['aborted']); + expect(uninstalled).toBe(1); + expect(deps.logs.join('\n')).toContain('Interrupted (SIGTERM)'); + }); + + it('default wiring listens for SIGTERM during the run and removes the listener after', async () => { + const beforeTerm = process.listenerCount('SIGTERM'); + const beforeInt = process.listenerCount('SIGINT'); + let duringTerm = -1; + const deps = baseDeps(); + delete deps.installSigintHandler; + deps.exec.mockImplementation(async () => { + duringTerm = process.listenerCount('SIGTERM'); + return pipelineResult({ success: true, finalStatus: 'approved' }); + }); + expect(await runWorkCommand({ issueIds: ['INT-1'], path: '/repo', yes: true }, deps)).toBe(WORK_EXIT_OK); + expect(duringTerm).toBe(beforeTerm + 1); + expect(process.listenerCount('SIGTERM')).toBe(beforeTerm); + expect(process.listenerCount('SIGINT')).toBe(beforeInt); + }); +}); + describe('runWorkCommand — exit code matrix', () => { it('all approved → 0', async () => { const deps = baseDeps(); diff --git a/src/cli/workCommand.ts b/src/cli/workCommand.ts index 64688283..d8b975db 100644 --- a/src/cli/workCommand.ts +++ b/src/cli/workCommand.ts @@ -59,6 +59,9 @@ export const WORK_EXIT_OK = 0; export const WORK_EXIT_FAILED = 1; export const WORK_EXIT_NOT_RUN = 2; export const WORK_EXIT_INTERRUPTED = 130; +/** 128 + SIGTERM: a supervisor (systemd, docker stop, CI) stopped the run. */ +export const WORK_EXIT_TERMINATED = 143; +export type WorkInterruptSignal = 'SIGINT' | 'SIGTERM'; /** The daemon's fan-out admission (autonomousRunner.executeDurably) with the * repository's own openswarm.json automation policy layered in. Worktrees @@ -136,8 +139,8 @@ export interface WorkCommandDeps { deliverEffect?: (effect: EffectClaim, source: ITaskSource | null) => Promise; hasRecoverableWorktree?: (repoPath: string, issueId: string, branchName: string) => Promise; resolveTaskFileScope?: (task: TaskItem, projectPath: string) => Promise; - /** Installs the SIGINT handler; returns the uninstaller. */ - installSigintHandler?: (onSigint: () => void) => () => void; + /** Installs the SIGINT + SIGTERM handler; returns the uninstaller. */ + installSigintHandler?: (onSignal: (signal?: WorkInterruptSignal) => void) => () => void; isTTY?: boolean; /** Progress/diagnostics — stderr, so `--json | jq` parses. */ log?: (line: string) => void; @@ -559,18 +562,31 @@ async function runWorkCommandInner( const exec = deps.executePipeline ?? executePipeline; + // SIGINT (Ctrl-C) and SIGTERM (systemd/docker stop/CI cancel) take the same + // path: abort the pipelines, then the finally below releases the uninstaller + // and closes the coordinator. Without a SIGTERM listener the process-tree + // relay re-raised SIGTERM and Node died before that cleanup ran. const sigint = new AbortController(); let interrupts = 0; + let interruptSignal: WorkInterruptSignal = 'SIGINT'; const forceExit = deps.exit ?? ((code: number) => process.exit(code)); - const onSigint = (): void => { + const exitCodeFor = (signal: WorkInterruptSignal): number => (signal === 'SIGTERM' ? WORK_EXIT_TERMINATED : WORK_EXIT_INTERRUPTED); + const onSigint = (signal: WorkInterruptSignal = 'SIGINT'): void => { interrupts += 1; - if (interrupts > 1) forceExit(WORK_EXIT_INTERRUPTED); - log('Interrupted — aborting running pipelines. Worktrees holding work are preserved; re-run the same command to resume. (Ctrl-C again to force quit)'); - sigint.abort(new Error('SIGINT')); + if (interrupts > 1) forceExit(exitCodeFor(signal)); + interruptSignal = signal; + log(`Interrupted (${signal}) — aborting running pipelines. Worktrees holding work are preserved; re-run the same command to resume. (Ctrl-C again to force quit)`); + sigint.abort(new Error(signal)); }; - const uninstallSigint = (deps.installSigintHandler ?? ((handler: () => void) => { - process.on('SIGINT', handler); - return () => process.removeListener('SIGINT', handler); + const uninstallSigint = (deps.installSigintHandler ?? ((handler: (signal?: WorkInterruptSignal) => void) => { + const onInt = (): void => handler('SIGINT'); + const onTerm = (): void => handler('SIGTERM'); + process.on('SIGINT', onInt); + process.on('SIGTERM', onTerm); + return () => { + process.removeListener('SIGINT', onInt); + process.removeListener('SIGTERM', onTerm); + }; }))(onSigint); let settledRows: Array<{ value?: PipelineResult; error?: unknown }>; @@ -666,7 +682,7 @@ async function runWorkCommandInner( const summaries = plan.map((row, index) => summarizeSettled(row, settledRows[index] ?? {})); const failed = summaries.some((s) => !s.success && s.status !== 'superseded'); const exitCode = interrupts > 0 - ? WORK_EXIT_INTERRUPTED + ? exitCodeFor(interruptSignal) : failed ? WORK_EXIT_FAILED : WORK_EXIT_OK; if (opts.json) { From 57c653c20f03ca593d00168a7ccfc587e9d472c3 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Sun, 15 Nov 2026 06:35:20 +0900 Subject: [PATCH 2/2] test(execution): pin SIGTERM through the default wiring and the spawn-path output bound (AGT-3423) --- src/adapters/base.spawn.test.ts | 30 ++++++++++++++++++++++++++++++ src/cli/workCommand.test.ts | 13 +++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/adapters/base.spawn.test.ts b/src/adapters/base.spawn.test.ts index 2a583e09..2db8deef 100644 --- a/src/adapters/base.spawn.test.ts +++ b/src/adapters/base.spawn.test.ts @@ -240,6 +240,36 @@ describe('argv-safe adapter spawning', () => { } }); + it('retains a bounded tail of stderr and stdout instead of the whole stream (AGT-3423)', async () => { + const proc = Object.assign(new EventEmitter(), { + pid: 122, + stdout: new PassThrough(), stderr: new PassThrough(), + stdin: Object.assign(new EventEmitter(), { end: vi.fn() }), kill: vi.fn(), + }); + const line = `${'e'.repeat(1023)}\n`; + spawnMock.mockImplementationOnce(() => { + queueMicrotask(() => { + for (let i = 0; i < 3 * 1024; i++) proc.stderr.write(line); // 3 MiB of stderr + proc.stderr.end('LAST-STDERR\n'); + proc.stdout.end('{"type":"result","result":"done"}\n'); + setTimeout(() => proc.emit('close', 0), 20); + }); + return proc; + }); + const adapter: CliAdapter = { + name: 'claude', capabilities: { supportsStreaming: false, supportsJsonOutput: false, supportsModelSelection: true, managedGit: false, supportedSkills: [] }, + isAvailable: async () => true, getDefaultModel: async () => 'sonnet', + buildCommand: () => ({ command: 'fixture-cli', args: [] }), + parseWorkerOutput: () => ({ success: true, summary: '', filesChanged: [], commands: [], output: '' }), + parseReviewerOutput: () => ({ decision: 'approve', feedback: '', issues: [], suggestions: [] }), + }; + const result = await spawnCli(adapter, { prompt: 'p', cwd: process.cwd() }); + expect(result.stderr.length).toBeLessThanOrEqual(1024 * 1024 + 200); + expect(result.stderr.startsWith('[openswarm: ')).toBe(true); + expect(result.stderr.trimEnd().endsWith('LAST-STDERR')).toBe(true); + expect(result.stdout).toBe('{"type":"result","result":"done"}\n'); + }); + it('passes metacharacters as one argv value with shell disabled', async () => { const proc = Object.assign(new EventEmitter(), { pid: 123, diff --git a/src/cli/workCommand.test.ts b/src/cli/workCommand.test.ts index b67cbdb0..fb466c40 100644 --- a/src/cli/workCommand.test.ts +++ b/src/cli/workCommand.test.ts @@ -657,6 +657,19 @@ describe('runWorkCommand — SIGTERM takes the same cancellation path (AGT-3423) expect(deps.logs.join('\n')).toContain('Interrupted (SIGTERM)'); }); + it('a real SIGTERM delivered to the process aborts the run through the default wiring', async () => { + const deps = baseDeps(); + delete deps.installSigintHandler; + const observed: boolean[] = []; + deps.exec.mockImplementation(async (_ctx, _task, _path, signal?: AbortSignal) => { + process.emit('SIGTERM', 'SIGTERM'); + observed.push(!!signal?.aborted); + return pipelineResult({ success: false, finalStatus: 'cancelled' }); + }); + expect(await runWorkCommand({ issueIds: ['INT-1'], path: '/repo', yes: true }, deps)).toBe(WORK_EXIT_TERMINATED); + expect(observed).toEqual([true]); + }); + it('default wiring listens for SIGTERM during the run and removes the listener after', async () => { const beforeTerm = process.listenerCount('SIGTERM'); const beforeInt = process.listenerCount('SIGINT');