fix(v2-orchestrator): initialize workspaces through a durable callback

docs/concepts/execution.md

@@ -47,6 +47,7 @@ Agents run on **Amazon Bedrock AgentCore** — a serverless runtime that gives e
 - Each stage spawns a **headless agent CLI** (Kiro, Claude Code, OpenCode, or Codex — selected per project) inside the session.
 - The CLI's only interface to the platform is a stdio **MCP server**. Business reads and writes go to the graph as typed, provenance-stamped artifacts; questions, outputs, and metrics go to the process store and stream live to the browser.
 - Stages run as background jobs with durable callbacks, so a stage can run for hours while the orchestrator is suspended at zero compute.
+- Workspace initialization (clone, intent-branch publish, graph anchor) runs the same way: the orchestrator dispatches `init-ws-start` and waits on a durable callback, so a large clone is not bounded by the orchestrator Lambda's 900-second timeout. The job heartbeats while it runs. The run fails with `init_ws_failed` when the job reports a failure, stops heartbeating for 15 minutes, or has not finished within 2 hours. The intent stays `CREATED` while it runs. Environment revisions built before `init-ws-start` existed, and runs already in flight at deployment, keep the synchronous `init-ws` call.
 - Session storage persists across park/resume, so a stage that asked a question continues its conversation with full context when you answer — even days later.
 
 The CLI authenticates to its model with a bearer token (Bedrock API key) or a Kiro API key configured in [Platform Admin → Agents](../using-the-platform/platform-settings.md#agents). The runtime's IAM role deliberately has no model-invocation permissions — token auth is the only path.

frontend/src/pages/IntentView.test.tsx

@@ -347,6 +347,55 @@ describe('IntentView', () => {
     expect(rewind).not.toHaveBeenCalled();
   });
 
+  it('does not call a claimed run stalled while its workspace is still initializing', async () => {
+    // An orchestrator claimed the run 10 minutes ago and is still cloning a
+    // large repository: CREATED is expected, and a restart would race the clone.
+    const tenMinutesAgo = new Date(Date.now() - 10 * 60_000).toISOString();
+    get.mockResolvedValue(
+      baseDetail({
+        status: 'CREATED',
+        updatedAt: tenMinutesAgo,
+        orchestratorStartedAt: tenMinutesAgo,
+      }),
+    );
+
+    renderAt();
+
+    expect((await screen.findAllByText('My intent')).length).toBeGreaterThan(0);
+    expect(screen.queryByText('Run stalled — never started')).toBeNull();
+    expect(screen.queryByRole('button', { name: 'Restart workflow' })).toBeNull();
+  });
+
+  it('calls an unclaimed CREATED hand-off stalled after two minutes', async () => {
+    get.mockResolvedValue(
+      baseDetail({
+        status: 'CREATED',
+        updatedAt: new Date(Date.now() - 3 * 60_000).toISOString(),
+        orchestratorStartedAt: null,
+      }),
+    );
+
+    renderAt();
+
+    expect(await screen.findByText('Run stalled — never started')).toBeTruthy();
+    expect(screen.getByRole('button', { name: 'Restart workflow' })).toBeTruthy();
+  });
+
+  it('calls a claimed run stalled once the init callback window has passed', async () => {
+    const threeHoursAgo = new Date(Date.now() - 3 * 60 * 60_000).toISOString();
+    get.mockResolvedValue(
+      baseDetail({
+        status: 'CREATED',
+        updatedAt: threeHoursAgo,
+        orchestratorStartedAt: threeHoursAgo,
+      }),
+    );
+
+    renderAt();
+
+    expect(await screen.findByText('Run stalled — never started')).toBeTruthy();
+  });
+
   it('renders one QuestionEditor for the active gate (exclusive expansion)', async () => {
     get.mockResolvedValue({
       ...baseDetail({ status: 'WAITING', pendingHumanTaskId: 'h1' }),

frontend/src/pages/IntentView.tsx

@@ -77,6 +77,13 @@ const CREDENTIAL_FAILURE_CODES = new Set([
   'credential_quota_exhausted',
 ]);
 const NON_EXPORTABLE_STATUSES = new Set(['DRAFT', 'CREATED']);
+// A CREATED run no orchestrator has claimed should leave CREATED within
+// seconds of the hand-off.
+const UNCLAIMED_STALL_AFTER_MS = 2 * 60_000;
+// A claimed run in CREATED is initializing its workspace. The orchestrator's
+// init callback times out after 2 h (v2-orchestrator INIT_WS_CALLBACK_TIMEOUT)
+// and fails the run; the slack covers that failure write.
+const CLAIMED_INIT_STALL_AFTER_MS = 2 * 60 * 60_000 + 10 * 60_000;
 const EXPORT_HARNESSES: Array<{ value: NativeExportHarness; label: string }> = [
   { value: 'claude', label: 'Claude' },
   { value: 'codex', label: 'Codex' },
@@ -391,13 +398,19 @@ export default function IntentView() {
   const fanoutActive = isFanoutActive(detail);
   const reviewGate = reviewGateId ? gates.find((g) => g.humanTaskId === reviewGateId) : null;
   // Stalled detection: a CREATED run whose hand-off never reached a live
-  // orchestrator strands here (init-ws should flip it to RUNNING within
-  // seconds). After >2 min untouched, offer a restart instead of spinning.
+  // orchestrator strands here. After >2 min untouched, offer a restart instead
+  // of spinning. Once an orchestrator has claimed the run, CREATED means the
+  // workspace is still initializing — a large clone legitimately takes many
+  // minutes, and the orchestrator fails the run itself if the init job dies —
+  // so only flag it after the init callback window has clearly passed.
   const lastTouch = intent.updatedAt ?? intent.createdAt;
+  const stallAfterMs = intent.orchestratorStartedAt
+    ? CLAIMED_INIT_STALL_AFTER_MS
+    : UNCLAIMED_STALL_AFTER_MS;
   const isStalled =
     intent.status === 'CREATED' &&
     !!lastTouch &&
-    Date.now() - new Date(lastTouch).getTime() > 120_000;
+    Date.now() - new Date(lastTouch).getTime() > stallAfterMs;
   const isCredentialFailure =
     isFailed && Boolean(intent.failure?.code && CREDENTIAL_FAILURE_CODES.has(intent.failure.code));
   const credentialSettingsPath = isCredentialFailure

frontend/src/services/intents.ts

@@ -167,6 +167,9 @@ export interface Intent {
   createdAt: string | null;
   updatedAt: string | null;
   completedAt: string | null;
+  // When an orchestrator run claimed this execution; null until claimed and
+  // after a relaunch resets it. Absent from older API responses.
+  orchestratorStartedAt?: string | null;
 }
 
 export interface IntentAttachment {

lambda/agentcore/commands/init-ws-start.js

@@ -0,0 +1,224 @@
+import { runCredentialJob } from '../credential-session.js';
+// init-ws-start — the async workspace initialization (the run-stage-start
+// pattern applied to init-ws).
+//
+// The synchronous init-ws path holds the AgentCore request — and the
+// orchestrator's durable step — open for the whole clone. A large repository
+// (field incident: a 1.8 GB / 100k-commit GitLab repo) outlives the
+// orchestrator Lambda's 900 s function timeout; the durable engine then
+// retries init-ws against the half-written checkout. This command accepts the
+// same payload as init-ws PLUS a `callbackId`, runs init-ws as a BACKGROUND
+// job in the container, and returns in milliseconds. The orchestrator suspends
+// on the durable callback at zero compute; the job completes it when init-ws
+// returns.
+//
+// Reliability contract (mirrors run-stage-start):
+//   - the accept response only says "job started"; the init verdict always
+//     travels through the callback, in init-ws's own {ok, reason, detail}
+//     shape — the orchestrator's existing decode applies unchanged;
+//   - EVERY termination of the job sends a callback: success, init-ws's
+//     ok:false failures, and uncaught throws (init_ws_job_crashed);
+//   - the first heartbeat is sent BEFORE the clone starts, so a callback the
+//     container cannot reach fails the dispatch synchronously;
+//   - heartbeats fire while the clone runs so the orchestrator's
+//     heartbeatTimeout separates "large repository" from "dead container";
+//   - one init job per execution: the workspace is reserved before the first
+//     heartbeat is awaited; a duplicate start for the SAME callback shares the
+//     first dispatch's acceptance (durable dispatch retry), a different
+//     callback is refused (init_job_already_running) — two clones must never
+//     race into the same workspace directory, and a job only releases the
+//     reservation it owns;
+//   - a finished job's callback id is remembered for a bounded time, so a
+//     dispatch retry that arrives after the job completed is answered
+//     alreadyCompleted instead of cloning again;
+//   - the busy tracker is held for the job's lifetime so /ping reports
+//     HealthyBusy and AgentCore keeps the session (and its mount) alive.
+
+import { Logger } from '@aws-lambda-powertools/logger';
+
+const logger = new Logger({
+  persistentKeys: { component: 'agentcore', module: 'init-ws-start' },
+});
+
+const jobKey = (p) => `${p.executionId}:init-ws`;
+
+// How long a finished job's callback id is remembered, so a durable dispatch
+// retry (lost response) is answered instead of starting a second clone.
+// Covers the step's retry backoff with a wide margin.
+const COMPLETED_RETENTION_MS = 60 * 60_000;
+
+// init-ws returns {ok:false, reason, detail} on every handled failure and
+// {ok:true, ...} on success. Anything else is a contract break — surface it as
+// a failure rather than letting the orchestrator mark the workspace ready.
+export const normalizeInitResult = (result) => {
+  if (result && typeof result === 'object' && typeof result.ok === 'boolean') return result;
+  return { ok: false, reason: 'init_ws_no_result', detail: String(result) };
+};
+
+export const createInitWsStart = ({
+  initWs,
+  sendCallbackSuccess,
+  sendCallbackHeartbeat,
+  busy = null,
+  heartbeatIntervalMs = 60_000,
+  activeJobs = new Map(),
+  completedJobs = new Map(),
+  completedRetentionMs = COMPLETED_RETENTION_MS,
+  now = Date.now,
+  log = logger,
+}) => {
+  const pruneCompleted = () => {
+    for (const [key, done] of completedJobs) {
+      if (now() - done.completedAt > completedRetentionMs) completedJobs.delete(key);
+    }
+  };
+
+  const start = async (payload) => {
+    const { callbackId, executionId } = payload ?? {};
+    if (!callbackId) return { ok: false, reason: 'missing_callback_id' };
+    if (!executionId) return { ok: false, reason: 'missing_execution_id' };
+
+    const key = jobKey(payload);
+    pruneCompleted();
+    // A dispatch retry that arrives after its job finished (the first
+    // response was lost) must not clone again or heartbeat a completed
+    // callback: the verdict has already been sent.
+    if (completedJobs.get(key)?.callbackId === callbackId) {
+      return { ok: true, accepted: true, alreadyCompleted: true, callbackId };
+    }
+    const inFlight = activeJobs.get(key);
+    if (inFlight) {
+      if (inFlight.callbackId !== callbackId) {
+        return { ok: false, reason: 'init_job_already_running', detail: key };
+      }
+      // A duplicate of the same attempt shares the first dispatch's outcome,
+      // including one whose initial heartbeat is still pending.
+      const first = await inFlight.acceptance;
+      return first.ok ? { ok: true, accepted: true, alreadyRunning: true, callbackId } : first;
+    }
+
+    // Reserve the workspace before the first await, so an overlapping
+    // dispatch sees this attempt instead of starting a second clone.
+    const jobState = { startedAt: Date.now(), callbackId, heartbeatTimer: null, acceptance: null };
+    activeJobs.set(key, jobState);
+    jobState.acceptance = accept(payload, key, jobState);
+    return jobState.acceptance;
+  };
+
+  const release = (key, jobState) => {
+    if (activeJobs.get(key) === jobState) activeJobs.delete(key);
+  };
+
+  const accept = async (payload, key, jobState) => {
+    const { callbackId, executionId } = payload;
+    let initialHeartbeat;
+    try {
+      initialHeartbeat = await sendCallbackHeartbeat(callbackId);
+    } catch (err) {
+      initialHeartbeat = { delivered: false, error: err?.message ?? String(err) };
+    }
+    if (!initialHeartbeat?.delivered) {
+      release(key, jobState);
+      log.error('init-ws callback heartbeat could not be established', {
+        executionId,
+        callbackId,
+        error: initialHeartbeat?.error ?? null,
+      });
+      return {
+        ok: false,
+        reason: 'init_ws_callback_heartbeat_failed',
+        detail: initialHeartbeat?.error ?? null,
+      };
+    }
+
+    busy?.enter();
+    log.info('init-ws job accepted', { executionId, callbackId });
+
+    // The job payload is init-ws's own argument bag; the callback id is
+    // transport, not an init-ws input.
+    const initPayload = { ...payload };
+    delete initPayload.callbackId;
+    delete initPayload.command;
+
+    const job = runCredentialJob(async () => {
+      try {
+        const scheduleHeartbeat = () => {
+          jobState.heartbeatTimer = setTimeout(async () => {
+            let sent;
+            try {
+              sent = await sendCallbackHeartbeat(callbackId);
+            } catch (err) {
+              sent = { delivered: false, error: err?.message ?? String(err) };
+            }
+            if (!sent?.delivered) {
+              log.error('init-ws callback heartbeat failed', {
+                executionId,
+                callbackId,
+                error: sent?.error ?? null,
+              });
+            }
+            if (activeJobs.get(key) === jobState) scheduleHeartbeat();
+          }, heartbeatIntervalMs);
+        };
+        scheduleHeartbeat();
+
+        let result;
+        try {
+          result = normalizeInitResult(await initWs(initPayload));
+        } catch (err) {
+          result = {
+            ok: false,
+            reason: 'init_ws_job_crashed',
+            detail: err?.message ?? String(err),
+          };
+        }
+        // The synchronous path logs failures in dispatchInvocation; a
+        // background job bypasses it, so keep the same diagnosable record.
+        if (result.ok === false) {
+          log.warn('command returned failure', {
+            command: 'init-ws-start',
+            executionId,
+            callbackId,
+            reason: result.reason,
+            detail: result.detail,
+          });
+        } else {
+          log.info('init-ws job completed', {
+            executionId,
+            callbackId,
+            durationMs: Date.now() - jobState.startedAt,
+          });
+        }
+
+        const sent = await sendCallbackSuccess(callbackId, result);
+        if (!sent?.delivered) {
+          // The orchestrator's callback timeout / heartbeatTimeout is the
+          // backstop; log loudly so an operator can complete it manually.
+          log.error('init-ws callback could not be delivered', {
+            executionId,
+            callbackId,
+            error: sent?.error ?? null,
+          });
+        }
+        return result;
+      } finally {
+        if (jobState.heartbeatTimer) clearTimeout(jobState.heartbeatTimer);
+        completedJobs.set(key, { callbackId, completedAt: now() });
+        release(key, jobState);
+        busy?.leave();
+      }
+    });
+    job.catch((err) =>
+      log.error('init-ws job promise rejected unexpectedly', {
+        executionId,
+        callbackId,
+        error: err?.message ?? String(err),
+      }),
+    );
+
+    return { ok: true, accepted: true, callbackId, jobKey: key };
+  };
+
+  start.activeJobs = activeJobs;
+  return start;
+};

lambda/agentcore/http-server.js

@@ -17,6 +17,10 @@ import {
 //
 // Invocation payloads:
 //   { "command": "init-ws",  ...initWs args }
+//   { "command": "init-ws-start", ...initWs args, callbackId }
+//     → accepts in ms, runs init-ws as a background job (clone + branch
+//       publish can outlive the orchestrator Lambda), completes the
+//       orchestrator's durable callback with init-ws's {ok, reason, detail}.
 //   { "command": "run-stage", ...runStage args }
 //   { "command": "run-stage-start", ...runStage args, stageCallbackId }
 //     → accepts in ms, runs the stage as a background job, completes the
@@ -235,6 +239,7 @@ const main = async () => {
   const { initWs } = await import('./commands/init-ws.js');
   const { runStage } = await import('./commands/run-stage.js');
   const { createRunStageStart } = await import('./commands/run-stage-start.js');
+  const { createInitWsStart } = await import('./commands/init-ws-start.js');
   const { createDiscussionAssistStart } = await import('./commands/discussion-assist-start.js');
   const { createComposePlanStart } = await import('./commands/compose-plan-start.js');
   const { createQuorumEditPlanStart } = await import('./commands/quorum-edit-plan-start.js');
@@ -399,6 +404,17 @@ const main = async () => {
       busy,
       activeJobs: stageJobs,
     })(p);
+  // Async workspace initialization: init-ws as a background job so a large
+  // clone is not bounded by the orchestrator Lambda's function timeout.
+  const initJobs = new Map();
+  handlers.initWsStart = (p) =>
+    createInitWsStart({
+      initWs: (q) => handlers.initWs(q),
+      sendCallbackSuccess: sendStageCallbackSuccess,
+      sendCallbackHeartbeat: sendStageCallbackHeartbeat,
+      busy,
+      activeJobs: initJobs,
+    })(p);
   const discussionJobs = new Map();
   handlers.discussionAssistStart = (p, context) =>
     createDiscussionAssistStart({