Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions scripts/lib/runWorkerCommand.cancellation.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/* @vitest-environment node */
import { getEventListeners } from "node:events";
import { mkdtemp, rm, stat } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { expect, it, vi } from "vitest";
import { runWorkerCommand } from "./runWorkerCommand";

it("does not launch work that was cancelled before admission", async () => {
const root = await mkdtemp(join(tmpdir(), "worker-preabort-"));
try {
const marker = join(root, "spawned");
const controller = new AbortController();
controller.abort();
const onTermination = vi.fn();
await expect(
runWorkerCommand(
process.execPath,
["-e", "require('node:fs').writeFileSync(process.argv[1], 'ran')", marker],
{ cwd: root, env: {}, timeoutMs: 1000, signal: controller.signal, onTermination },
),
).rejects.toMatchObject({ message: "Worker command cancelled", failureKind: "cancelled" });
await expect(stat(marker)).rejects.toMatchObject({ code: "ENOENT" });
expect(onTermination).not.toHaveBeenCalled();
expect(getEventListeners(controller.signal, "abort")).toHaveLength(0);
} finally {
await rm(root, { recursive: true, force: true });
}
});

it("removes the cancellation listener when ordinary work finishes", async () => {
const controller = new AbortController();
await expect(
runWorkerCommand(process.execPath, ["-e", "process.stdout.write('finished')"], {
cwd: tmpdir(),
env: {},
timeoutMs: 1000,
signal: controller.signal,
cleanupProcessGroup: true,
}),
).resolves.toEqual({ stdout: "finished", stderr: "" });
expect(getEventListeners(controller.signal, "abort")).toHaveLength(0);
controller.abort();
});
24 changes: 24 additions & 0 deletions scripts/lib/runWorkerCommand.cleanup-timers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,30 @@ it.skipIf(process.platform === "win32").each([-1_000_000, 1_000_000])(
},
);

it.skipIf(process.platform === "win32")(
"classifies unconfirmed cleanup separately from a findings exit",
async () => {
vi.useFakeTimers();
const child = simulatedChild();
vi.spyOn(process, "kill").mockImplementation(() => true);
const pending = runWorkerCommand("generated", [], {
cwd: tmpdir(),
env: {},
timeoutMs: 100,
cleanupProcessGroup: true,
}).catch((error: unknown) => error);
child.emit("close", 1);
await vi.advanceTimersByTimeAsync(60_000);
expect(await pending).toMatchObject({
exitCode: 1,
timedOut: false,
failureKind: "cleanup",
message: "Worker cleanup could not be confirmed",
});
expect(vi.getTimerCount()).toBe(0);
},
);

it("does not claim Windows process-group absence from direct child termination", async () => {
vi.useFakeTimers();
Object.defineProperty(process, "platform", { value: "win32" });
Expand Down
14 changes: 14 additions & 0 deletions scripts/lib/runWorkerCommand.cleanup.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@ it
"nonzero_exit",
"output_limit",
"observer_failure",
"abort",
])("confirms descendant group absence after %s", async (trigger) => {
const shouldTimeout = trigger === "timeout" || trigger === "inherited_pipes";
const controller = new AbortController();
let groupPid: number | undefined;
let termination: WorkerCommandTermination | undefined;
try {
Expand Down Expand Up @@ -48,9 +50,11 @@ it
captureOutput: false,
maxStdoutBytes: 128,
cleanupProcessGroup: true,
signal: controller.signal,
onStderrChunk: (chunk) => {
groupPid = Number(chunk.toString());
if (trigger === "observer_failure") throw new Error("PRIVATE observer failure");
if (trigger === "abort") controller.abort();
},
onTermination: (value) => {
termination = value;
Expand All @@ -61,6 +65,16 @@ it
else {
expect(failure).toBeInstanceOf(CommandFailure);
expect(failure).toMatchObject({ timedOut: shouldTimeout });
const failureKind = shouldTimeout
? "timeout"
: trigger === "nonzero_exit"
? "exit"
: trigger === "observer_failure"
? "observer"
: trigger === "abort"
? "cancelled"
: "output_limit";
expect(failure).toMatchObject({ failureKind });
if (trigger === "nonzero_exit") expect(failure).toMatchObject({ exitCode: 7 });
if (trigger === "output_limit")
expect(failure).toMatchObject({ outputLimitExceeded: "stdout" });
Expand Down
57 changes: 55 additions & 2 deletions scripts/lib/runWorkerCommand.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ export type WorkerCommandTermination = {

export class CommandFailure extends Error {
exitCode: number | null;
failureKind: "exit" | "timeout" | "output_limit" | "observer" | "cleanup" | "cancelled";
outputLimitExceeded?: "stdout" | "stderr";
stderr: string;
stdout: string;
Expand All @@ -26,10 +27,16 @@ export class CommandFailure extends Error {
stderr: string,
timedOut: boolean,
outputLimitExceeded?: "stdout" | "stderr",
failureKind: CommandFailure["failureKind"] = timedOut
? "timeout"
: outputLimitExceeded
? "output_limit"
: "exit",
) {
super(message);
this.name = "CommandFailure";
this.exitCode = exitCode;
this.failureKind = failureKind;
this.stdout = stdout;
this.stderr = stderr;
this.timedOut = timedOut;
Expand All @@ -53,6 +60,7 @@ export async function runWorkerCommand(
cleanupProcessGroup?: boolean;
onStderrChunk?: (chunk: Buffer) => void;
onTermination?: (termination: WorkerCommandTermination) => void;
signal?: AbortSignal;
timeoutMs: number;
},
) {
Expand All @@ -69,6 +77,17 @@ export async function runWorkerCommand(
throw new RangeError(`${stream} output limit must be a non-negative safe integer`);
}
}
if (options.signal?.aborted) {
throw new CommandFailure(
"Worker command cancelled",
null,
"",
"",
false,
undefined,
"cancelled",
);
}
return await new Promise<{ stdout: string; stderr: string }>((resolvePromise, reject) => {
const startedAt = performance.now();
const child = spawn(command, args, {
Expand All @@ -82,6 +101,7 @@ export async function runWorkerCommand(
let outputLimitExceeded: "stdout" | "stderr" | undefined;
let observerFailed = false;
let timedOut = false;
let cancelled = false;
let settled = false;
let forceKillTimeout: NodeJS.Timeout | undefined;
let cleanupTimeout: NodeJS.Timeout | undefined;
Expand Down Expand Up @@ -141,6 +161,7 @@ export async function runWorkerCommand(
output.stderr,
timedOut,
outputLimitExceeded,
"cleanup",
),
);
}, 60_000);
Expand Down Expand Up @@ -174,14 +195,23 @@ export async function runWorkerCommand(
clearTimeout(timeout);
if (forceKillTimeout) clearTimeout(forceKillTimeout);
if (cleanupTimeout) clearTimeout(cleanupTimeout);
options.signal?.removeEventListener("abort", handleAbort);
};
function handleAbort() {
if (settled) return;
cancelled = true;
clearTimeout(timeout);
terminateProcessTree();
}
options.signal?.addEventListener("abort", handleAbort, { once: true });
if (options.signal?.aborted) handleAbort();
child.stdout.on("data", (chunk: Buffer) => appendOutput("stdout", chunk));
child.stderr.on("data", (chunk: Buffer) => appendOutput("stderr", chunk));
child.on("error", (error) => {
if (settled) return;
settled = true;
clearTimers();
if (timedOut) killProcessTree("SIGKILL");
if (timedOut || cancelled) killProcessTree("SIGKILL");
reportTermination(null);
reject(error);
});
Expand All @@ -190,7 +220,11 @@ export async function runWorkerCommand(
settled = true;
clearTimers();
const cleanupRequired =
timedOut || outputLimitExceeded || observerFailed || options.cleanupProcessGroup;
timedOut ||
cancelled ||
outputLimitExceeded ||
observerFailed ||
options.cleanupProcessGroup;
if (cleanupRequired) {
cleanupStartedAt ??= performance.now();
killProcessTree("SIGKILL");
Expand Down Expand Up @@ -219,6 +253,8 @@ export async function runWorkerCommand(
output.stdout,
output.stderr,
timedOut,
undefined,
"observer",
),
);
return;
Expand All @@ -233,6 +269,7 @@ export async function runWorkerCommand(
output.stderr,
timedOut,
outputLimitExceeded,
"output_limit",
),
);
return;
Expand All @@ -245,6 +282,22 @@ export async function runWorkerCommand(
output.stdout,
output.stderr,
timedOut,
undefined,
"cleanup",
),
);
return;
}
if (cancelled && !timedOut) {
reject(
new CommandFailure(
"Worker command cancelled",
code,
output.stdout,
output.stderr,
false,
undefined,
"cancelled",
),
);
return;
Expand Down
1 change: 1 addition & 0 deletions scripts/security/bundled-skillspector-diagnostic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export type BundledSkillSpectorFailureContext = {
/** Index in the worker's sorted, deduplicated declared-root list. */
rootIndex: number;
rootCount: number;
/** Accepted terminal reports; concurrent roots need not finish in index order. */
completedRoots: number;
childBudgetMs: number;
remainingStageBudgetMs: number;
Expand Down
5 changes: 4 additions & 1 deletion scripts/security/run-codex-scan-worker-clawscan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1081,7 +1081,7 @@ JSON`,
false,
);

expect((await readFile(skillSpectorTargets, "utf8")).trim().split("\n")).toEqual([
expect((await readFile(skillSpectorTargets, "utf8")).trim().split("\n").sort()).toEqual([
"artifact/package/skills/alpha",
"artifact/package/skills/beta",
]);
Expand Down Expand Up @@ -1909,6 +1909,9 @@ exit 7`,
const fakeClawScan = join(workspace, "fake-clawscan");
const callLog = join(workspace, "clawscan-called");
const firstRootCompleted = join(workspace, "first-root-completed");
// Keep these exact serial-provider budget and redaction contracts;
// parallel out-of-order accounting is covered separately.
vi.stubEnv("SKILLSPECTOR_PROVIDER", "anthropic");
if (failure === "shared deadline") {
const now = Date.now.bind(Date);
vi.spyOn(Date, "now").mockImplementation(
Expand Down
Loading
Loading