fix(resident): bound drain waits by the fallback deadline

src/core/budgets.ts

@@ -104,11 +104,18 @@ export const HARNESS_PROBE_WAIT_MS = 5 * MINUTE_MS;
  *  `maxMinutes` so one nobody lifted is an hour and a half, not a day. A run
  *  asked during a drain waits at its attach one `pollMs` at a time under its
  *  own lease less `leaseReserveMs` (what the attach and the work after it
- *  need), `waitMaxMs` with no lease to clip it. */
+ *  need), `waitMaxMs` with no lease to clip it — unless a fallback stands
+ *  behind the attach: `fallbackWaitMs` bounds the wait by the FALLBACK's own
+ *  cost, never the deploy's (issue 2101: a run refused by the drain waited
+ *  18 minutes for the whole deploy, then did the job in a seeded sandbox that
+ *  stands up in about two), so a drain refusal on a first attach falls to the
+ *  seeded sandbox after at most a few minutes; a re-attach with no fallback —
+ *  a resumed run, a mid-run recovery — keeps the lease's bound. */
 export const DRAIN = {
   pollMs: 30_000,
   waitMaxMs: 60 * MINUTE_MS,
   leaseReserveMs: 10 * MINUTE_MS,
+  fallbackWaitMs: 3 * MINUTE_MS,
   deployWaitMaxMs: 60 * MINUTE_MS,
   marginMinutes: 5,
   maxMinutes: 90,

src/core/dispatcher.test.ts

@@ -993,6 +993,30 @@ describe("executor provisioning by agent resources", () => {
     expect(quietEvents.some((e) => e.type === "run_note" && e.kind === "drain_wait")).toBe(false);
   });
 
+  // Feature: docs/reference/specs/resident-repos.md item 69 (issue 2101) — a run
+  // the drain sent to the sandbox fallback carries the wait on the SELECTION
+  // (no binding exists), and the dispatcher publishes the same `drain_wait`
+  // note: the incident's run fell cold and `runs friction` counted zero.
+  it("a selection that carries drainWaitMs without a binding — the drain's sandbox fallback — publishes the run's drain_wait note too", async () => {
+    vi.stubEnv("SANDBOX_TOKEN", "tok");
+    vi.stubEnv("GITHUB_APP_ID", "");
+    const registry = new RunRegistry({ genId: () => "run-fell-cold", genToken: () => "tok" });
+    const deps = makeDeps(REMOTE_YAML_FIXTURE, capturingProvider());
+    deps.runRegistry = registry;
+    const fake = { exec: async () => "", readFile: async () => "", writeFile: async () => "" };
+    vi.mocked(makeExecutor).mockImplementationOnce(async () => ({
+      executor: fake,
+      backend: "sandbox" as const,
+      note: "resident attach failed (… drained …) — using fresh sandbox",
+      drainWaitMs: 3 * 60_000,
+    }));
+    await dispatch(deps, msg("agent:coding fix it", "slack:UADMIN"), fakeIO().io);
+    const events = registry.snapshot("run-fell-cold", "tok")!.events;
+    expect(events.find((e) => e.type === "run_note" && e.kind === "drain_wait")).toMatchObject({
+      summary: "waited 3 min at the fleet drain for a deploy to finish",
+    });
+  });
+
   // Feature: docs/reference/specs/tracing.md item 19 — a resident attach that FAILS still
   // grafts the steps it ran under the (failed) attach span; no run exists, so
   // they reach the process sinks.

src/core/dispatcher.ts

@@ -1647,8 +1647,10 @@ export async function dispatch(
     // the note is what `runs friction` reads as the wait's category and the
     // plane's table shows as the run's cause — a half-hour wait with a card
     // that counted "attaching the workspace…" and a friction verdict of
-    // "none" was the incident's shape.
-    const drainWaitMs = round.selection.binding?.drainWaitMs ?? 0;
+    // "none" was the incident's shape. A run the drain sent to the sandbox
+    // fallback carries the wait on the selection instead of a binding (issue
+    // 2101: the incident's run fell cold and `runs friction` counted zero).
+    const drainWaitMs = round.selection.binding?.drainWaitMs ?? round.selection.drainWaitMs ?? 0;
     if (drainWaitMs > 0)
       registry.publish(run.id, {
         type: "run_note",

src/core/trace/displayNames.test.ts

@@ -41,6 +41,14 @@ describe("display names", () => {
     // every streamed prefix family has a rule
     for (const prefix of Object.keys(STREAMED_PREFIXES)) expect(displayNameOf(`${prefix}x`)).not.toBe(`${prefix}x`);
   });
+
+  // The bot client's own attach waits (issue 2101): named beside the Worker's
+  // step vocabulary, never inside it — the step table is scanned against the
+  // Worker's source, and these spans are the bot's.
+  it("the attach's drain and wake waits have display names under the attach prefix", () => {
+    expect(displayNameOf("dispatch.workspace.attach.drain-wait")).toBe("waiting for the deploy to finish");
+    expect(displayNameOf("dispatch.workspace.attach.wake-wait")).toBe("waiting for the resident to wake");
+  });
 });
 
 // The resident Worker names each command it runs (`runOk(argv, "<step>", …)`,

src/core/trace/displayNames.ts

@@ -46,6 +46,15 @@ export const DISPLAY_NAMES = {
 /** The fallback for a prefixed span whose leaf we cannot name. */
 export const GENERIC_STEP_NAME = "a Switchboard step";
 
+/** The bot client's own attach waits (issue 2101): spans under the attach
+ *  span, named here rather than in the resident step table — that table is
+ *  the Worker's vocabulary, scanned against its source, and these waits run
+ *  in the bot. */
+export const ATTACH_WAIT_NAMES = {
+  "drain-wait": "waiting for the deploy to finish",
+  "wake-wait": "waiting for the resident to wake",
+} as const;
+
 /** The display name for any span name: the table for an enumerated name; for a
  *  prefix family the tool's own name (`tool.bash` → `bash`), the MCP tool's own
  *  name (`mcp.<server>.<tool>` → `<tool>`), or the resident step's label, with
@@ -59,7 +68,11 @@ export function displayNameOf(name: string): string {
     return leaf && leaf !== "mcp" ? leaf : GENERIC_STEP_NAME;
   }
   for (const prefix of ["dispatch.workspace.attach.", "run.command."]) {
-    if (name.startsWith(prefix)) return residentStepLabel(name.slice(prefix.length)) ?? GENERIC_STEP_NAME;
+    if (name.startsWith(prefix)) {
+      const leaf = name.slice(prefix.length);
+      if (leaf in ATTACH_WAIT_NAMES) return ATTACH_WAIT_NAMES[leaf as keyof typeof ATTACH_WAIT_NAMES];
+      return residentStepLabel(leaf) ?? GENERIC_STEP_NAME;
+    }
   }
   return GENERIC_STEP_NAME;
 }

src/execution/factory.test.ts

@@ -7,7 +7,7 @@ import { AGENTS, type AgentDef } from "../agents/registry.js";
 import { declaredProfile } from "../config/profile.js";
 import { CloudflareSandboxExecutor } from "./cloudflareSandbox.js";
 import { ExecInfraError, LocalExecutor } from "./executor.js";
-import { ResidentExecutor, ResidentNeedsRefError } from "./resident.js";
+import { DRAIN_FALLBACK_WAIT_MS, DRAIN_POLL_MS, ResidentExecutor, ResidentNeedsRefError } from "./resident.js";
 import {
   gitIdentityEnvs,
   makeExecutor,
@@ -866,6 +866,42 @@ describe("makeExecutor resident selection", () => {
   // /status probe and /attach (503 mirror-busy, 429 pool-exhausted). Any attach
   // failure that is NOT needs-ref must fall back to the per-thread backend with
   // a NAMED note — never a silent stall or a raw ⚠️.
+  // Issue 2101: the first attach has the sandbox fallback below to fall to, so
+  // a drained fleet is waited for under the FALLBACK's own cost — never the
+  // deploy's — and the drain's wait rides the selection so the run's
+  // `drain_wait` note still publishes (the incident's run counted zero).
+  it("a drained fleet's first attach falls to the sandbox after the fallback's own bound, the wait riding the selection as drainWaitMs (issue 2101)", async () => {
+    vi.useFakeTimers();
+    try {
+      stubEnvs();
+      const draining = {
+        status: 503,
+        body: {
+          error: "draining: the resident fleet is closed to new runs for deploy 62e4e9a — the run waits at its attach",
+          status: 503,
+          draining: { since: "2026-09-18T05:00:00.000Z", until: "2026-09-18T06:00:00.000Z", by: "deploy all" },
+        },
+      };
+      const polls = DRAIN_FALLBACK_WAIT_MS / DRAIN_POLL_MS;
+      const { calls } = stubFetch(
+        { body: { state: "warm", reason: "" } },
+        ...Array.from({ length: polls + 1 }, () => draining),
+      );
+      const p = makeExecutor(residentOpts(), repoCtx());
+      await vi.advanceTimersByTimeAsync(DRAIN_FALLBACK_WAIT_MS + 1_000);
+      const { executor, resident, note, drainWaitMs } = await p;
+      expect(executor).toBeInstanceOf(CloudflareSandboxExecutor);
+      expect(resident).toBeFalsy();
+      expect(drainWaitMs).toBe(DRAIN_FALLBACK_WAIT_MS);
+      expect(note).toMatch(
+        /^resident attach failed \(.*did not reopen within the 180s this run could wait.*\) — using fresh sandbox$/,
+      );
+      expect(calls).toEqual(["/status", ...Array.from({ length: polls + 1 }, () => "/attach")]);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
   it("warm probe then a NON-needs-ref attach failure → per-thread executor WITH a named 'resident attach failed' note", async () => {
     stubEnvs();
     const { calls } = stubFetch(

src/execution/factory.ts

@@ -23,6 +23,8 @@ import {
   type SeededSandbox,
 } from "./seedPlan.js";
 import {
+  DRAIN_FALLBACK_WAIT_MS,
+  drainWaitOf,
   ResidentExecutor,
   ResidentLeaseSpentError,
   ResidentNeedsRefError,
@@ -303,6 +305,11 @@ export interface ExecutorSelection {
    *  path — the dispatcher names the path to the model and checks the sha
    *  against the PR head before a review runs. Unset on every other path. */
   binding?: ResidentBinding;
+  /** The drain's share of a failed attach's wait (issue 2101): on the sandbox
+   *  fallback after a drained fleet refused the run, so the dispatcher still
+   *  publishes the run's `drain_wait` note — the resident path carries it on
+   *  the binding instead. */
+  drainWaitMs?: number;
 }
 
 // Resident lifecycle states the bot attaches in — `isServiceable` in
@@ -403,6 +410,8 @@ export async function makeExecutor(
   let reason: string | undefined;
   /** The steps of a resident attach that failed before the sandbox fallback. */
   let failedAttach: ResidentTrace | undefined;
+  /** The drain's share of a failed attach's wait, for the run's `drain_wait` note. */
+  let drainWaitMs: number | undefined;
   if (ctx.repo && opts.execution?.resident) {
     const resident = opts.execution.resident;
     const tokenEnv = resident.tokenEnv ?? "RESIDENT_OPERATOR_TOKEN";
@@ -495,6 +504,10 @@ export async function makeExecutor(
             signal: ctx.stopSignal,
             budgetMs: Math.max(0, FIRST_ATTACH_WAIT_MS - probeWaitMs),
             waitedMs: probeWaitMs,
+            // A drained fleet is waited for under the FALLBACK's own cost, not
+            // the deploy's (issue 2101): this attach has the seeded sandbox
+            // below to fall to, so the drain wait ends in minutes.
+            drainBoundMs: DRAIN_FALLBACK_WAIT_MS,
           },
         );
         // Item 27: the wait is on the card whichever state the restore landed on.
@@ -512,6 +525,7 @@ export async function makeExecutor(
         // is that failure, and falls cold as it always did.
         if (isRunStopError(err)) throw err;
         failedAttach = residentTraceOf(err);
+        drainWaitMs = drainWaitOf(err);
         // Item 27: an attach that fails after a wait names the wait too — the
         // probe's through a typed blip (item 9) and the restore's alike, so a
         // run that started late says why even when it then fell cold.
@@ -561,13 +575,15 @@ export async function makeExecutor(
           backend: perThreadBackend(opts),
           seeded: outcome.seeded,
           ...(failedAttach ? { trace: failedAttach.steps } : {}),
+          ...(drainWaitMs !== undefined ? { drainWaitMs } : {}),
         };
       }
       return {
         executor,
         note: `${reason} — using fresh sandbox${outcome ? ` (${outcome.why})` : ""}`,
         backend: perThreadBackend(opts),
         ...(failedAttach ? { trace: failedAttach.steps } : {}),
+        ...(drainWaitMs !== undefined ? { drainWaitMs } : {}),
       };
     }
   }
@@ -823,8 +839,9 @@ async function openResident(
    *  stop, which ends the wait at once, and the budget past which the attach
    *  fails with the wake's strike; `waitedMs` is a wait the caller already
    *  spent before this attach (the selection probe's), added to the total the
-   *  card names. */
-  wait: { signal?: AbortSignal; budgetMs?: number; waitedMs?: number } = {},
+   *  card names; `drainBoundMs` bounds a drained fleet's wait by the caller's
+   *  fallback cost (issue 2101). */
+  wait: { signal?: AbortSignal; budgetMs?: number; waitedMs?: number; drainBoundMs?: number } = {},
 ): Promise<ExecutorSelection> {
   let executor = new ResidentExecutor(opts);
   let binding: ResidentBinding;

src/execution/resident.test.ts

@@ -18,6 +18,8 @@ import {
   ResidentReuseRefusedError,
   DRAIN_LEASE_RESERVE_MS,
   DRAIN_POLL_MS,
+  DRAIN_FALLBACK_WAIT_MS,
+  drainWaitOf,
   ResidentDrainingError,
   isDrainingRefusal,
 } from "./resident.js";
@@ -2700,4 +2702,146 @@ describe("ResidentExecutor.attach during a fleet drain (item 69)", () => {
     expect(err).toBeInstanceOf(Error);
     expect(calls).toHaveLength(1);
   });
+
+  // Issue 2101: a run refused by the drain waited 18 minutes for the whole
+  // deploy, then did the job in a seeded sandbox that stands up in about two.
+  // Where a fallback stands behind the attach, its cost bounds the wait.
+  it("a fallback bounds the drain wait by its own cost (issue 2101): drainBoundMs ends the wait in the typed error after minutes, with most of the lease left, and the error carries the drain's share", async () => {
+    const polls = DRAIN_FALLBACK_WAIT_MS / DRAIN_POLL_MS;
+    stubFetch(...Array.from({ length: polls + 2 }, () => draining));
+    const ex = new ResidentExecutor({ ...OPTS, remainingMs: () => 45 * 60_000 });
+    const p = ex.attach(undefined, { drainBoundMs: DRAIN_FALLBACK_WAIT_MS }).catch((e: unknown) => e);
+    await vi.advanceTimersByTimeAsync(DRAIN_FALLBACK_WAIT_MS + 1_000);
+    const err = await p;
+    expect(err).toBeInstanceOf(ResidentDrainingError);
+    expect((err as Error).message).toContain(
+      `did not reopen within the ${DRAIN_FALLBACK_WAIT_MS / 1000}s this run could wait`,
+    );
+    // The drain's share rides the error, so the factory's fallback still
+    // publishes the run's `drain_wait` note (the incident counted zero).
+    expect(drainWaitOf(err)).toBe(DRAIN_FALLBACK_WAIT_MS);
+  });
+
+  it("one fallback deadline survives drain/wake hand-offs, so a second drain gets only the first wait's remaining allowance", async () => {
+    const transient = {
+      body: {
+        error: "attach-failed: Durable Object reset because its code was updated.",
+        status: 500,
+        transient: true,
+      },
+    };
+    const firstDrainPolls = DRAIN_FALLBACK_WAIT_MS / DRAIN_POLL_MS - 2;
+    const { calls } = stubFetch(
+      draining,
+      ...Array.from({ length: firstDrainPolls }, () => draining),
+      transient,
+      { body: { state: "warm", reason: "", inFlight: 0 } },
+      draining,
+      draining,
+    );
+    const p = new ResidentExecutor(OPTS)
+      .attach(undefined, { drainBoundMs: DRAIN_FALLBACK_WAIT_MS })
+      .catch((e: unknown) => e);
+
+    await vi.advanceTimersByTimeAsync(DRAIN_FALLBACK_WAIT_MS);
+    const err = await p;
+
+    expect(err).toBeInstanceOf(ResidentDrainingError);
+    expect(drainWaitOf(err)).toBe(DRAIN_FALLBACK_WAIT_MS);
+    expect(calls.map(route)).toEqual([
+      "/attach",
+      ...Array.from({ length: firstDrainPolls + 1 }, () => "/attach"),
+      "/status",
+      "/attach",
+      "/attach",
+    ]);
+  });
+
+  it("the drain wait is one span under the attach span (issue 2101): dispatch.workspace.attach.drain-wait ends ok when the fleet reopens, its duration the wait's own", async () => {
+    const log = recordingSink();
+    const root = createTracer({ clock: () => Date.now() }).start("request", { sinks: [log] });
+    const attachSpan = root.start("dispatch.workspace.attach");
+    stubFetch(draining, draining, { raw: "\n" + JSON.stringify(ATTACH_OK) });
+    const p = new ResidentExecutor(OPTS).attach(attachSpan);
+    await vi.advanceTimersByTimeAsync(2 * DRAIN_POLL_MS);
+    await p;
+    const span = log.ended("dispatch.workspace.attach.drain-wait");
+    expect(span).toMatchObject({ parentSpanId: attachSpan.id, status: "ok", durationMs: 2 * DRAIN_POLL_MS });
+  });
+
+  it("a drain wait that runs out fails its span with the typed classification and no remote words (issue 2101)", async () => {
+    const log = recordingSink();
+    const root = createTracer({ clock: () => Date.now() }).start("request", { sinks: [log] });
+    const attachSpan = root.start("dispatch.workspace.attach");
+    const polls = DRAIN_FALLBACK_WAIT_MS / DRAIN_POLL_MS;
+    stubFetch(...Array.from({ length: polls + 2 }, () => draining));
+    const p = new ResidentExecutor(OPTS)
+      .attach(attachSpan, { drainBoundMs: DRAIN_FALLBACK_WAIT_MS })
+      .catch((e: unknown) => e);
+    await vi.advanceTimersByTimeAsync(DRAIN_FALLBACK_WAIT_MS + 1_000);
+    await p;
+    const span = log.ended("dispatch.workspace.attach.drain-wait");
+    // The typed error's classification, never the refusal's free text: the
+    // words stay on the card note and the attach error (tracing.md item 2).
+    expect(span).toMatchObject({ status: "error", errorKind: "infra", errorCode: "attach" });
+    expect(JSON.stringify(span)).not.toContain("draining:");
+  });
+
+  it("the wake wait after a transient refusal is its own span under the attach span (issue 2101): dispatch.workspace.attach.wake-wait", async () => {
+    const log = recordingSink();
+    const root = createTracer({ clock: () => Date.now() }).start("request", { sinks: [log] });
+    const attachSpan = root.start("dispatch.workspace.attach");
+    stubFetch(
+      { body: { error: "attach-failed: Network connection lost.", status: 500, transient: true } },
+      { body: { state: "warm", reason: "", inFlight: 0 } },
+      { raw: "\n" + JSON.stringify(ATTACH_OK) },
+    );
+    const p = new ResidentExecutor(OPTS).attach(attachSpan);
+    await vi.advanceTimersByTimeAsync(1_000);
+    await p;
+    const span = log.ended("dispatch.workspace.attach.wake-wait");
+    expect(span).toMatchObject({ parentSpanId: attachSpan.id, status: "ok" });
+  });
+
+  it("a stop during the wake wait's initial status probe closes the wake span as failed", async () => {
+    const calls: Array<{ url: string; init: RequestInit }> = [];
+    vi.stubGlobal(
+      "fetch",
+      vi.fn((url: unknown, init?: RequestInit) => {
+        calls.push({ url: String(url), init: init ?? {} });
+        if (calls.length === 1) {
+          return Promise.resolve(
+            new Response(
+              JSON.stringify({ error: "attach-failed: Network connection lost.", status: 500, transient: true }),
+              { status: 200 },
+            ),
+          );
+        }
+        return new Promise<Response>((_resolve, reject) => {
+          const signal = init?.signal;
+          if (signal?.aborted) reject(signal.reason);
+          else signal?.addEventListener("abort", () => reject(signal.reason), { once: true });
+        });
+      }),
+    );
+    const log = recordingSink();
+    const root = createTracer({ clock: () => Date.now() }).start("request", { sinks: [log] });
+    const attachSpan = root.start("dispatch.workspace.attach");
+    const control = new AbortController();
+    const outcome = new ResidentExecutor(OPTS).attach(attachSpan, { signal: control.signal }).catch((e: unknown) => e);
+    await vi.advanceTimersByTimeAsync(0);
+    expect(calls.map(route)).toEqual(["/attach", "/status"]);
+
+    control.abort();
+    await vi.advanceTimersByTimeAsync(0);
+    const err = await outcome;
+
+    expect(err).toBeInstanceOf(ExecInfraError);
+    expect((err as ExecInfraError).reason).toBe("aborted");
+    expect(log.ended("dispatch.workspace.attach.wake-wait")).toMatchObject({
+      parentSpanId: attachSpan.id,
+      status: "error",
+      errorKind: "transport",
+    });
+  });
 });