diff --git a/docs/3_TAKEAWAYS.md b/docs/3_TAKEAWAYS.md index b831dbe..0d412db 100644 --- a/docs/3_TAKEAWAYS.md +++ b/docs/3_TAKEAWAYS.md @@ -192,7 +192,9 @@ Implementation: [`src/runner/protocol.ts`](../src/runner/protocol.ts) and A live exec stream can drop while the process continues. The in-Sandbox script therefore writes structured events and exit status to root-owned files. The host can poll for completion and download those files even after losing -the stream. +the stream. Once the script has exited, its output is required: the host +retries the download in the same Sandbox rather than treating a failed +download as an empty run, which would discard a paid investigation. Apply deadlines at several layers: diff --git a/src/sandbox/render/sandbox.ts b/src/sandbox/render/sandbox.ts index d82b8ca..5a3f63a 100644 --- a/src/sandbox/render/sandbox.ts +++ b/src/sandbox/render/sandbox.ts @@ -182,7 +182,11 @@ export async function runInvestigation(client: SandboxClient, id: string, opts: // A dropped exec stream doesn't stop the run, and the API can't reattach to it, so wait for the exit // status run-investigation.sh writes to a file instead. if (exitCode === null && !signal.aborted) exitCode = await awaitExitFile(client, id, runDir, signal); - const output = await readRunFile(client, id, `${runDir}.jsonl`, AbortSignal.timeout(client.t.verifyTimeoutMs)); + // Once the script has exited, its output file exists: a failed download is an infrastructure error, not an empty run. + const outputSignal = AbortSignal.timeout(client.t.verifyTimeoutMs); + const output = exitCode === null + ? await readRunFile(client, id, `${runDir}.jsonl`, outputSignal) + : await readRunOutput(client, id, `${runDir}.jsonl`, outputSignal); return { exitCode, aborted: exitCode === null && signal.aborted, @@ -192,6 +196,22 @@ export async function runInvestigation(client: SandboxClient, id: string, opts: }; } +const OUTPUT_DOWNLOAD_ATTEMPTS = 3; + +/** Retries in the same Sandbox, so a transport error doesn't repeat the model run. */ +async function readRunOutput(client: SandboxClient, id: string, path: string, signal: AbortSignal): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await client.download(id, path, signal); + } catch (err) { + if (attempt === OUTPUT_DOWNLOAD_ATTEMPTS || signal.aborted) { + throw new Error(`could not download ${path} from ${id} after ${attempt} attempt(s)`, { cause: err }); + } + await sleep(client.t.pollMs, undefined, { signal }).catch(() => {}); + } + } +} + /** Polls for the run's exit status until it appears, the sandbox stops running, or `signal` fires. */ async function awaitExitFile(client: SandboxClient, id: string, runDir: string, signal: AbortSignal): Promise { while (!signal.aborted) { diff --git a/test/sandbox-render.test.ts b/test/sandbox-render.test.ts index 350b6c8..ff4d59d 100644 --- a/test/sandbox-render.test.ts +++ b/test/sandbox-render.test.ts @@ -12,6 +12,7 @@ import { openRenderSandbox, runInvestigation } from "../src/sandbox/render/sandb import { buildSnapshot, pruneSnapshots } from "../src/sandbox/render/snapshot.ts"; import { runnerInput } from "./helpers.ts"; import { createLog } from "../src/log.ts"; +import { captureRun, classifyRun } from "../src/tasks.ts"; const GH_TOKEN = "ghp_renderProviderSecret000000000000"; const MODEL_KEY = "sk-ant-api03-renderProviderSecret"; @@ -428,6 +429,79 @@ const runsInvestigation = (events: () => SandboxExecEvent[] | AsyncGenerator (command.includes("run-investigation.sh") ? events() : [exit(0)]); describe("runInvestigation", () => { + it("recovers required output after a transport failure without rerunning the investigation", async () => { + fake.exec_ = runsInvestigation(() => [exit(0)]); + let downloads = 0; + fake.file_ = (path) => { + if (!path.endsWith(".jsonl")) return undefined; + if (++downloads === 1) throw new TypeError("synthetic output transport failure"); + return clarify; + }; + + const res = await runInvestigation(client(), "sbx-run", { input: runnerInput(LAYOUT), modelApiKey: MODEL_KEY, timeoutMs: 60_000 }); + + expect(res.events).toEqual([JSON.parse(clarify)]); + expect(classifyRun(res, captureRun(res.events), { elapsedMs: 1, agentTimeoutMs: 60_000 }).kind).toBe("result"); + expect(downloads).toBe(2); + expect(fake.commands().filter((command) => command.includes("run-investigation.sh"))).toHaveLength(1); + expect(fake.calls.filter((call) => call.op === "upload" && call.path.endsWith("/key"))).toHaveLength(1); + }); + + it("throws an infrastructure error with the last transport cause after three output requests", async () => { + fake.exec_ = runsInvestigation(() => [exit(0)]); + const failures = [1, 2, 3].map((attempt) => new TypeError(`synthetic output failure ${attempt}`)); + let downloads = 0; + fake.file_ = (path) => { + if (!path.endsWith(".jsonl")) return undefined; + throw failures[downloads++]; + }; + + const error = await runInvestigation(client(), "sbx-run", { input: runnerInput(LAYOUT), timeoutMs: 60_000 }).catch((err: unknown) => err); + + expect(error).toMatchObject({ message: expect.stringContaining("after 3 attempt(s)"), cause: failures[2] }); + expect(downloads).toBe(3); + expect(fake.commands().filter((command) => command.includes("run-investigation.sh"))).toHaveLength(1); + }); + + it("keeps a genuinely downloaded empty output as an agent failure, without transport retries", async () => { + fake.exec_ = runsInvestigation(() => [exit(0)]); + let downloads = 0; + fake.file_ = (path) => path.endsWith(".jsonl") ? (downloads++, "") : undefined; + + const res = await runInvestigation(client(), "sbx-run", { input: runnerInput(LAYOUT), timeoutMs: 60_000 }); + + expect(res).toMatchObject({ exitCode: 0, aborted: false, events: [], invalidLines: 0 }); + expect(classifyRun(res, captureRun(res.events), { elapsedMs: 1, agentTimeoutMs: 60_000 })).toMatchObject({ kind: "failed", code: "agent_error" }); + expect(downloads).toBe(1); + }); + + it("keeps an aborted run's output best-effort", async () => { + const controller = new AbortController(); + fake.exec_ = runsInvestigation(async function* () { + controller.abort(); + throw new AbortError(); + }); + + const res = await runInvestigation(client(), "sbx-run", { input: runnerInput(LAYOUT), timeoutMs: 60_000, signal: controller.signal }); + + expect(res).toMatchObject({ exitCode: null, aborted: true, events: [] }); + expect(classifyRun(res, captureRun(res.events), { elapsedMs: 1, agentTimeoutMs: 60_000 })).toMatchObject({ kind: "failed", code: "timeout" }); + }); + + it("recovers a complete result even after an explicit abort", async () => { + const controller = new AbortController(); + fake.exec_ = runsInvestigation(async function* () { + controller.abort(); + throw new AbortError(); + }); + fake.file_ = (path) => path.endsWith(".jsonl") ? clarify : undefined; + + const res = await runInvestigation(client(), "sbx-run", { input: runnerInput(LAYOUT), timeoutMs: 60_000, signal: controller.signal }); + + expect(res.aborted).toBe(true); + expect(classifyRun(res, captureRun(res.events), { elapsedMs: 1, agentTimeoutMs: 60_000 }).kind).toBe("result"); + }); + it("returns the run's events and exit status, streaming stderr and counting invalid lines", async () => { const started = event({ type: "started", pi_version: "0.83.0", model: "m", prompt_version: "v3" }); fake.exec_ = runsInvestigation(() => [out("runner ", "stderr"), out("diagnostics", "stderr"), exit(0)]); @@ -442,6 +516,7 @@ describe("runInvestigation", () => { }); it("uploads the model key as a file beside the input and never puts it in a command", async () => { + fake.file_ = (path) => path.endsWith(".jsonl") ? "" : undefined; await runInvestigation(client(), "sbx-run", { input: runnerInput(LAYOUT), modelApiKey: MODEL_KEY, timeoutMs: 90_000 }); expect(fake.commands().some((c) => c.includes(MODEL_KEY))).toBe(false);