Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions src/adapters/base.spawn.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,36 @@ describe('argv-safe adapter spawning', () => {
}
});

it('retains a bounded tail of stderr and stdout instead of the whole stream (AGT-3423)', async () => {
const proc = Object.assign(new EventEmitter(), {
pid: 122,
stdout: new PassThrough(), stderr: new PassThrough(),
stdin: Object.assign(new EventEmitter(), { end: vi.fn() }), kill: vi.fn(),
});
const line = `${'e'.repeat(1023)}\n`;
spawnMock.mockImplementationOnce(() => {
queueMicrotask(() => {
for (let i = 0; i < 3 * 1024; i++) proc.stderr.write(line); // 3 MiB of stderr
proc.stderr.end('LAST-STDERR\n');
proc.stdout.end('{"type":"result","result":"done"}\n');
setTimeout(() => proc.emit('close', 0), 20);
});
return proc;
});
const adapter: CliAdapter = {
name: 'claude', capabilities: { supportsStreaming: false, supportsJsonOutput: false, supportsModelSelection: true, managedGit: false, supportedSkills: [] },
isAvailable: async () => true, getDefaultModel: async () => 'sonnet',
buildCommand: () => ({ command: 'fixture-cli', args: [] }),
parseWorkerOutput: () => ({ success: true, summary: '', filesChanged: [], commands: [], output: '' }),
parseReviewerOutput: () => ({ decision: 'approve', feedback: '', issues: [], suggestions: [] }),
};
const result = await spawnCli(adapter, { prompt: 'p', cwd: process.cwd() });
expect(result.stderr.length).toBeLessThanOrEqual(1024 * 1024 + 200);
expect(result.stderr.startsWith('[openswarm: ')).toBe(true);
expect(result.stderr.trimEnd().endsWith('LAST-STDERR')).toBe(true);
expect(result.stdout).toBe('{"type":"result","result":"done"}\n');
});

it('passes metacharacters as one argv value with shell disabled', async () => {
const proc = Object.assign(new EventEmitter(), {
pid: 123,
Expand Down
14 changes: 9 additions & 5 deletions src/adapters/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// Shared spawn logic for all CLI adapters
// ============================================

import { BoundedOutput, MAX_RETAINED_STDERR_CHARS, MAX_RETAINED_STDOUT_CHARS } from './outputRetention.js';
import { spawn } from 'node:child_process';
import fs from 'node:fs/promises';
import { tmpdir } from 'node:os';
Expand Down Expand Up @@ -249,13 +250,14 @@ export async function spawnCli(
}, proc);
}

let stdout = '';
let stderr = '';
// Bounded: keep the tail, which holds the result/cost/error events. (AGT-3423)
const stdoutBuf = new BoundedOutput(MAX_RETAINED_STDOUT_CHARS);
const stderrBuf = new BoundedOutput(MAX_RETAINED_STDERR_CHARS);
let streamBuffer = '';

proc.stdout?.on('data', (data: Buffer) => {
const text = data.toString();
stdout += text;
stdoutBuf.append(text);
if (options.onLog && adapter.capabilities.supportsStreaming) {
streamBuffer = adapter.parseStreamingChunk
? adapter.parseStreamingChunk(text, options.onLog, streamBuffer)
Expand All @@ -264,7 +266,7 @@ export async function spawnCli(
});

proc.stderr?.on('data', (data: Buffer) => {
stderr += data.toString();
stderrBuf.append(data.toString());
});

let exitDrainTimer: NodeJS.Timeout | null = null;
Expand All @@ -280,7 +282,7 @@ export async function spawnCli(
cleanupLifecycle();
terminateCliProcessTree(proc);
const reason = lifecycleController.signal.reason;
session?.record({ type: 'assistant', rawStdout: stdout, rawStderr: stderr });
session?.record({ type: 'assistant', rawStdout: stdoutBuf.toString(), rawStderr: stderrBuf.toString() });
session?.close({
outcome: 'aborted', durationMs: Date.now() - startTime,
error: reason instanceof Error ? `${reason.name}: ${reason.message}` : String(reason),
Expand All @@ -300,6 +302,8 @@ export async function spawnCli(
: parseCliStreamChunk('\n', options.onLog, streamBuffer);
}

const stdout = stdoutBuf.toString();
const stderr = stderrBuf.toString();
session?.record({ type: 'assistant', rawStdout: stdout, rawStderr: stderr });
session?.close({
outcome: code === 0 || code === null ? 'returned' : 'exit_nonzero',
Expand Down
44 changes: 44 additions & 0 deletions src/adapters/outputRetention.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import { describe, expect, it } from 'vitest';
import { BoundedOutput } from './outputRetention.js';
import { extractStreamJsonError } from './base.js';

describe('BoundedOutput (AGT-3423)', () => {
it('keeps everything under the cap', () => {
const out = new BoundedOutput(100);
out.append('a\n');
out.append('b\n');
expect(out.toString()).toBe('a\nb\n');
expect(out.droppedChars).toBe(0);
});

it('never retains more than the cap plus the marker, however much is streamed', () => {
const out = new BoundedOutput(1_000);
for (let i = 0; i < 10_000; i++) out.append(`{"type":"assistant","n":${i}}\n`);
const text = out.toString();
const body = text.slice(text.indexOf('\n') + 1);
expect(body.length).toBeLessThanOrEqual(1_000);
expect(out.droppedChars).toBeGreaterThan(0);
expect(text.startsWith('[openswarm: ')).toBe(true);
// the newest line survives and the body starts on a whole line
expect(body.trimEnd().endsWith('"n":9999}')).toBe(true);
expect(body.startsWith('{"type":"assistant"')).toBe(true);
});

it('keeps the final stream-json result parseable after truncation', () => {
const out = new BoundedOutput(4_096);
for (let i = 0; i < 5_000; i++) out.append(`{"type":"assistant","message":"chunk ${i}"}\n`);
out.append('{"type":"result","is_error":true,"result":"model quota exhausted"}\n');
expect(extractStreamJsonError(out.toString())).toContain('model quota exhausted');
});

it('still bounds a single huge line with no newline', () => {
const out = new BoundedOutput(1_000);
out.append('x'.repeat(50_000));
expect(out.toString().length).toBeLessThan(1_100);
});

it('rejects an invalid cap', () => {
expect(() => new BoundedOutput(0)).toThrow();
expect(() => new BoundedOutput(Number.NaN)).toThrow();
});
});
50 changes: 50 additions & 0 deletions src/adapters/outputRetention.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// ============================================
// OpenSwarm - Bounded CLI output retention (AGT-3423)
// A CLI stage's stdout/stderr used to be concatenated without limit for the
// whole run. Keep the most recent output instead: stream-json results, costs
// and error events are at the end, and parsers read line by line, so the
// retained text starts on a line boundary and is prefixed with a plain-text
// marker that JSON-line parsers skip.
// ============================================

/** Retained stdout per CLI stage. Far above a normal run; a runaway stream is the case it guards. */
export const MAX_RETAINED_STDOUT_CHARS = 32 * 1024 * 1024;
/** Retained stderr per CLI stage. */
export const MAX_RETAINED_STDERR_CHARS = 1024 * 1024;

export class BoundedOutput {
private text = '';
private dropped = 0;

constructor(private readonly maxChars: number) {
if (!Number.isFinite(maxChars) || maxChars < 1) throw new Error(`invalid output cap: ${maxChars}`);
}

append(chunk: string): void {
this.text += chunk;
if (this.text.length > this.maxChars) this.trim();
}

/** Characters dropped from the front so far. */
get droppedChars(): number {
return this.dropped;
}

toString(): string {
return this.dropped > 0
? `[openswarm: ${this.dropped} earlier characters of output were not retained]\n${this.text}`
: this.text;
}

// Trim to three quarters of the cap so a steady stream trims once per
// quarter-cap of input rather than on every chunk.
private trim(): void {
const keep = Math.max(1, Math.floor(this.maxChars * 0.75));
let cut = this.text.length - keep;
// Start on a whole line when one begins reasonably soon after the cut.
const newline = this.text.indexOf('\n', cut);
if (newline !== -1 && newline - cut < keep / 2) cut = newline + 1;
this.dropped += cut;
this.text = this.text.slice(cut);
}
}
54 changes: 54 additions & 0 deletions src/cli/workCommand.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
import { afterEach, describe, it, expect, vi } from 'vitest';
import type { LinearIssueInfo, SwarmConfig } from '../core/types.js';
import type { TaskItem } from '../orchestration/decisionEngine.js';
Expand All @@ -19,6 +19,7 @@
summarizeSettled,
WORK_EXIT_FAILED,
WORK_EXIT_INTERRUPTED,
WORK_EXIT_TERMINATED,
WORK_EXIT_NOT_RUN,
WORK_EXIT_OK,
type WorkCommandDeps,
Expand Down Expand Up @@ -633,6 +634,59 @@
});
});

describe('runWorkCommand — SIGTERM takes the same cancellation path (AGT-3423)', () => {
it('SIGTERM aborts the pipeline signal, still closes the coordinator path, and exits 143', async () => {
let handler: ((signal?: 'SIGINT' | 'SIGTERM') => void) | undefined;
let uninstalled = 0;
const observedAborts: Array<string | undefined> = [];
const deps = baseDeps({
installSigintHandler: (fn) => {
handler = fn;
return () => { uninstalled += 1; };
},
});
deps.exec.mockImplementation(async (_ctx, _task, _path, signal?: AbortSignal) => {
handler?.('SIGTERM');
observedAborts.push(signal?.aborted ? 'aborted' : undefined);
return pipelineResult({ success: false, finalStatus: 'cancelled' });
});
const code = await runWorkCommand({ issueIds: ['INT-1'], path: '/repo', yes: true }, deps);
expect(code).toBe(WORK_EXIT_TERMINATED);
expect(observedAborts).toEqual(['aborted']);
expect(uninstalled).toBe(1);
expect(deps.logs.join('\n')).toContain('Interrupted (SIGTERM)');
});

it('a real SIGTERM delivered to the process aborts the run through the default wiring', async () => {
const deps = baseDeps();
delete deps.installSigintHandler;
const observed: boolean[] = [];
deps.exec.mockImplementation(async (_ctx, _task, _path, signal?: AbortSignal) => {
process.emit('SIGTERM', 'SIGTERM');
observed.push(!!signal?.aborted);
return pipelineResult({ success: false, finalStatus: 'cancelled' });
});
expect(await runWorkCommand({ issueIds: ['INT-1'], path: '/repo', yes: true }, deps)).toBe(WORK_EXIT_TERMINATED);
expect(observed).toEqual([true]);
});

it('default wiring listens for SIGTERM during the run and removes the listener after', async () => {
const beforeTerm = process.listenerCount('SIGTERM');
const beforeInt = process.listenerCount('SIGINT');
let duringTerm = -1;
const deps = baseDeps();
delete deps.installSigintHandler;
deps.exec.mockImplementation(async () => {
duringTerm = process.listenerCount('SIGTERM');
return pipelineResult({ success: true, finalStatus: 'approved' });
});
expect(await runWorkCommand({ issueIds: ['INT-1'], path: '/repo', yes: true }, deps)).toBe(WORK_EXIT_OK);
expect(duringTerm).toBe(beforeTerm + 1);
expect(process.listenerCount('SIGTERM')).toBe(beforeTerm);
expect(process.listenerCount('SIGINT')).toBe(beforeInt);
});
});

describe('runWorkCommand — exit code matrix', () => {
it('all approved → 0', async () => {
const deps = baseDeps();
Expand Down
36 changes: 26 additions & 10 deletions src/cli/workCommand.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ export const WORK_EXIT_OK = 0;
export const WORK_EXIT_FAILED = 1;
export const WORK_EXIT_NOT_RUN = 2;
export const WORK_EXIT_INTERRUPTED = 130;
/** 128 + SIGTERM: a supervisor (systemd, docker stop, CI) stopped the run. */
export const WORK_EXIT_TERMINATED = 143;
export type WorkInterruptSignal = 'SIGINT' | 'SIGTERM';

/** The daemon's fan-out admission (autonomousRunner.executeDurably) with the
* repository's own openswarm.json automation policy layered in. Worktrees
Expand Down Expand Up @@ -136,8 +139,8 @@ export interface WorkCommandDeps {
deliverEffect?: (effect: EffectClaim, source: ITaskSource | null) => Promise<void>;
hasRecoverableWorktree?: (repoPath: string, issueId: string, branchName: string) => Promise<boolean>;
resolveTaskFileScope?: (task: TaskItem, projectPath: string) => Promise<string[]>;
/** Installs the SIGINT handler; returns the uninstaller. */
installSigintHandler?: (onSigint: () => void) => () => void;
/** Installs the SIGINT + SIGTERM handler; returns the uninstaller. */
installSigintHandler?: (onSignal: (signal?: WorkInterruptSignal) => void) => () => void;
isTTY?: boolean;
/** Progress/diagnostics — stderr, so `--json | jq` parses. */
log?: (line: string) => void;
Expand Down Expand Up @@ -559,18 +562,31 @@ async function runWorkCommandInner(

const exec = deps.executePipeline ?? executePipeline;

// SIGINT (Ctrl-C) and SIGTERM (systemd/docker stop/CI cancel) take the same
// path: abort the pipelines, then the finally below releases the uninstaller
// and closes the coordinator. Without a SIGTERM listener the process-tree
// relay re-raised SIGTERM and Node died before that cleanup ran.
const sigint = new AbortController();
let interrupts = 0;
let interruptSignal: WorkInterruptSignal = 'SIGINT';
const forceExit = deps.exit ?? ((code: number) => process.exit(code));
const onSigint = (): void => {
const exitCodeFor = (signal: WorkInterruptSignal): number => (signal === 'SIGTERM' ? WORK_EXIT_TERMINATED : WORK_EXIT_INTERRUPTED);
const onSigint = (signal: WorkInterruptSignal = 'SIGINT'): void => {
interrupts += 1;
if (interrupts > 1) forceExit(WORK_EXIT_INTERRUPTED);
log('Interrupted — aborting running pipelines. Worktrees holding work are preserved; re-run the same command to resume. (Ctrl-C again to force quit)');
sigint.abort(new Error('SIGINT'));
if (interrupts > 1) forceExit(exitCodeFor(signal));
interruptSignal = signal;
log(`Interrupted (${signal}) — aborting running pipelines. Worktrees holding work are preserved; re-run the same command to resume. (Ctrl-C again to force quit)`);
sigint.abort(new Error(signal));
};
const uninstallSigint = (deps.installSigintHandler ?? ((handler: () => void) => {
process.on('SIGINT', handler);
return () => process.removeListener('SIGINT', handler);
const uninstallSigint = (deps.installSigintHandler ?? ((handler: (signal?: WorkInterruptSignal) => void) => {
const onInt = (): void => handler('SIGINT');
const onTerm = (): void => handler('SIGTERM');
process.on('SIGINT', onInt);
process.on('SIGTERM', onTerm);
return () => {
process.removeListener('SIGINT', onInt);
process.removeListener('SIGTERM', onTerm);
};
}))(onSigint);

let settledRows: Array<{ value?: PipelineResult; error?: unknown }>;
Expand Down Expand Up @@ -666,7 +682,7 @@ async function runWorkCommandInner(
const summaries = plan.map((row, index) => summarizeSettled(row, settledRows[index] ?? {}));
const failed = summaries.some((s) => !s.success && s.status !== 'superseded');
const exitCode = interrupts > 0
? WORK_EXIT_INTERRUPTED
? exitCodeFor(interruptSignal)
: failed ? WORK_EXIT_FAILED : WORK_EXIT_OK;

if (opts.json) {
Expand Down
Loading