diff --git a/docs/concepts/data-model.md b/docs/concepts/data-model.md index 3ade437c6..cd05b824f 100644 --- a/docs/concepts/data-model.md +++ b/docs/concepts/data-model.md @@ -311,25 +311,26 @@ graph LR ### Record types -| Sort-key pattern | Record | Purpose | -| ------------------------------------ | ----------------- | ----------------------------------------------------------------------------------------------- | -| `META` | Execution | Captured intent and workflow configuration, overall status, timestamps, and repository context. | -| `STAGE#` | Stage | State and attempts for one concrete stage invocation, including a unit-specific stage. | -| `EVENT##` | Event | Ordered execution timeline and audit event. | -| `HUMAN#` | Human task | A suspended human gate, its callback, answer, and resolution state. | -| `OUTPUT#` | Output | Ordered agent-output chunk; the zero-padded sequence preserves emission order. | -| `METRIC##` | Metric | Token, cost, duration, or other collected execution metric. | -| `SENSOR##` | Sensor run | Deterministic post-stage validation result. | -| `READ##` | Graph read | Audit record of context read from the knowledge graph. | -| `STEER##` | Steering | Pending or consumed human course correction, ordered by creation time. | -| `UNITPLAN` | Unit plan | Current scheduling snapshot for the construction unit DAG. | -| `UNIT#` | Unit | Scheduling state for one construction lane, including dependencies, branch, and session. | -| `UNITPR##` | Unit pull request | Pull-request lifecycle for one unit and repository. | -| `FEEDBACK##` | Feedback batch | A queued or completed batch of review feedback for a unit. | -| `FEEDBACKCOMMENT##` | Feedback comment | Stable per-comment state within unit feedback processing. | -| `QEDIT#` | Quorum edit | Post-hoc artifact edit plan, approval callback, and apply outcome. | -| `COMPOSE#` | Compose request | Adaptive-workflow proposal and its validation result. | -| `TRACKERSYNC` | Tracker sync | Current external-tracker synchronization state and next check. | +| Sort-key pattern | Record | Purpose | +| ------------------------------------ | ----------------- | ------------------------------------------------------------------------------------------------ | +| `META` | Execution | Captured intent and workflow configuration, overall status, timestamps, and repository context. | +| `STAGE#` | Stage | State and attempts for one concrete stage invocation, including a unit-specific stage. | +| `EVENT##` | Event | Ordered execution timeline and audit event. | +| `HUMAN#` | Human task | A suspended human gate, its callback, answer, and resolution state. | +| `OUTPUT#` | Output | Ordered agent-output chunk; the zero-padded sequence preserves emission order. | +| `METRIC##` | Metric | Token, cost, duration, or other collected execution metric. | +| `SENSOR##` | Sensor run | Deterministic post-stage validation result. | +| `READ##` | Graph read | Audit record of context read from the knowledge graph. | +| `STEER##` | Steering | Pending or consumed human course correction, ordered by creation time. | +| `UNITPLAN` | Unit plan | Current scheduling snapshot for the construction unit DAG. | +| `UNIT#` | Unit | Scheduling state for one construction lane, including dependencies, branch, and session. | +| `UNITPR##` | Unit pull request | Pull-request lifecycle for one unit and repository. | +| `FEEDBACK##` | Feedback batch | A queued or completed batch of review feedback for a unit. | +| `FEEDBACKCOMMENT##` | Feedback comment | Stable per-comment state within unit feedback processing. | +| `QEDIT#` | Quorum edit | Post-hoc artifact edit plan, approval callback, and apply outcome. | +| `COMPOSE#` | Compose request | Adaptive-workflow proposal and its validation result. | +| `DELIVERY#` | Final delivery | Per-repository publication results, captured branch identity, and fenced retry attempt metadata. | +| `TRACKERSYNC` | Tracker sync | Current external-tracker synchronization state and next check. | `UNITPLAN` and `UNIT#…` rows are the scheduling source of truth. Their `UnitOfWork` representation in Neptune is a traceability and UI projection; the diff --git a/docs/concepts/execution.md b/docs/concepts/execution.md index 381f9babb..91af53121 100644 --- a/docs/concepts/execution.md +++ b/docs/concepts/execution.md @@ -69,6 +69,24 @@ Git is deterministic and owned by the engine — **the agent never runs git and For PR-per-unit delivery, DynamoDB is scheduling truth. A unit is complete only after every changed repository confirms the exact ready head is on the intent branch. A partial multi-repository merge preserves merged work, blocks dependents, and enters halt-and-ask. +The lane board calls this outcome **Integrated**. Its stored identifier remains +`MERGED`; it does not mean that the final intent-to-base PR has merged. Counts +use the active unit plan; retained rows from earlier plans remain history. + +A walking skeleton must be a dependency-free unit in the current DAG. A missing +choice receives a dependency-free default before approval. Re-promotion retains +valid captured choices. If the regenerated DAG renames that unit or gives it +dependencies, promotion records the change and offers a new dependency-free +default at the normal fan-out approval. Original answers remain in history. +Execution still rejects invalid choices in an already approved plan. A workflow with `skeleton: off` +still runs all units, without the solo-first ceremony. + +Shared stages and successful completion require every active-plan lane to be +integrated or explicitly omitted by a recorded human skip. Blocked-only work +still enters halt-and-ask. Retry preserves integration; skip also omits the +unintegrated dependent closure; abort fails the run. Partial delivery is presented +separately from complete delivery. + Provider comments never launch agents by themselves. A project member selects comments in the intent's review drawer; the API refetches those comments with fresh credentials, rejects bot/system comments, and queues a bounded, audited feedback batch. The lane revisits its last successful stage, runs its normal sensors, commits and pushes the revision, and posts a summary without resolving provider threads. ## Human gates, park and resume diff --git a/docs/development/recovery-ownership.md b/docs/development/recovery-ownership.md index f7e672899..76f45c6eb 100644 --- a/docs/development/recovery-ownership.md +++ b/docs/development/recovery-ownership.md @@ -31,3 +31,43 @@ Storage must accept a write to persist FAILED state. During a complete outage th worker preserves its structured failure, while durable dispatch and callback reconciliation remain the recovery backstop. These guards protect process state; they do not undo provider requests or Git operations already in flight. + +## Completion boundaries + +New histories capture completion policy 1 in the existing `load-meta` result. +The shared unit-completion evaluator validates the pinned, resolved workflow's +expected sections, current-plan units, repository integration evidence and scoped +human omissions. Section fan-in and final success use the same evaluator against +persisted process state. Missing rows, unfinished work and partial integrations +cannot pass; missing provenance, read failures and ambiguous legacy omissions +remain unverified. Serial workflows and sections excluded by approved workflow +scope/composition remain valid. Pre-deploy `MERGED` rows without an +`integrationResults` attribute retain their supported integrated meaning, including +multi-repository retries and rewinds past completed sections. New integrations +always write the attribute; explicit empty, invalid or incomplete evidence never +receives this legacy exemption. PR-per-unit integration still requires its +per-repository PR records. Unpinned historical serial intents without unit work +keep their success presentation without unit-completion diagnostics. Cancelled +executions and recorded aborts from the current run are classified as aborted +before workflow-pin resolution, without implying that their work was integrated. +Missing or truncated execution reads remain unverified. + +Required integration writes are not best-effort. A lost response is reconciled +from persisted state; an unrecorded integration stops the lane before dependent +work can proceed. Final status and its completion receipt commit together under +run, status, META revision and unit-plan conditions. Initial publication rechecks +completion before opening a PR. These checks never rerun models or repair rows. +Marker-absent histories (policy 0) retain their saved operation sequence and +terminal step. The intermediate completion implementation was not released and +has no separate replay path. Saved historical successes are never rewritten. + +A terminal success with unfinished or unverified work is shown honestly in the +intent detail. Accepted omissions remain partial delivery, not execution defects. +This release does not add automatic preservation-and-continuation recovery for +finished incomplete intents. Existing orphan-lane/PR-review repair and ordinary +rewind remain supported; rewind resets affected lanes, including integrated work, +and clears omission receipts for the work explicitly being retried. Original +answers and events remain historical. Preserve runtime and methodology pins. + +Deploy the runtime, orchestrator, intents API and frontend from the same release. +No automatic production audit, status migration or runtime repinning is performed. diff --git a/docs/using-the-platform/git-integration.md b/docs/using-the-platform/git-integration.md index ba24a7b96..932b89723 100644 --- a/docs/using-the-platform/git-integration.md +++ b/docs/using-the-platform/git-integration.md @@ -72,7 +72,7 @@ The external ID is what stops another user (or another tenant) from using your r Repository identity is the **ARN** (region and account are part of it), so CodeCommit repositories appear as `name (region)` and link to the regional console. Commits made by the engine carry the committer configured on the binding (CodeCommit has no user-identity API to discover one). -What CodeCommit cannot do, declared by the provider so the engine never attempts it: no draft pull requests, no reopening a closed pull request (closed is terminal — a new one is opened instead), no issues tracker, no CI check statuses (approval rules are read instead). Because **PR per unit** relies on draft pull requests to keep each unit review unmergeable while its lane reconciles or revises it, that strategy (and its feedback revisions) is not available for a space with CodeCommit repositories: intent creation is refused with `PR_STRATEGY_UNSUPPORTED`; choose one PR per intent for such a space instead. Pull request state is picked up by the same one-minute reconciler as every other provider; no webhook is needed. +What CodeCommit cannot do, declared by the provider so the engine never attempts it: no draft pull requests, no reopening a closed pull request (closed is terminal — a new execution can open a replacement; final delivery retry refuses closed work), no issues tracker, no CI check statuses (approval rules are read instead). Because **PR per unit** relies on draft pull requests to keep each unit review unmergeable while its lane reconciles or revises it, that strategy (and its feedback revisions) is not available for a space with CodeCommit repositories: intent creation is refused with `PR_STRATEGY_UNSUPPORTED`; choose one PR per intent for such a space instead. Pull request state is picked up by the same one-minute reconciler as every other provider; no webhook is needed. ## Branches @@ -154,3 +154,88 @@ The platform supports two delivery strategies: In the intent view, each unit card shows repository-specific review state and links. Project members can open **Address feedback**, select up to 20 current human-authored comments, and queue a targeted revision. The backend refetches selected comments by provider ID, records their versions, and ignores provider comments unless a member explicitly selects them. The agent does not automatically resolve discussion threads. If a unit PR closes without merging, or only part of a multi-repository unit merges, the run enters halt-and-ask with retry, skip, and abort outcomes. Already merged work is preserved. + +## Check and retry final delivery + +Stage completion and final publication are separate outcomes. A completed intent +can still need attention when its code host rejects or cannot confirm a PR/MR. +The intent page shows **Final delivery** with each repository's branch, target, +result and reason. Published PR links also remain available in Code when the +graph projection was unavailable. **No changes to publish** is a verified +no-diff outcome; missing records and failed reads are not treated as success. + +A project owner or admin can choose **Check delivery**. This is read-only: it +checks the pinned workflow's completion, the captured branches and existing PRs. +Review the repository results, then choose **Retry delivery**. This publishes +only unfinished repositories from their existing intent branches. It does not +run a model, repeat stages or gates, regenerate code, integrate units again, or +merge the final PR. Deliberately accepted partial work remains partial. + +An uncertain previous create request is reconciled with the code host before +another request is made. Successfully published repositories are preserved. +A closed or merged existing PR, a changed intent branch or a missing branch is +shown with an actionable refusal; retry does not recreate closed work or choose +a different target. Restore a missing captured branch, fix repository access, +or review a separate execution/recovery plan as the displayed reason directs. +PR-per-unit workflows keep their existing unit review process; this action +covers their final intent-to-base publication only. + +Older completed runs initially show **Delivery not yet verified**. Checking them +does not rewrite their history. **Save delivery assessment** records a reconciled +or refused result when no publication is needed. Missing ownership or unfinished +workflow/unit records require execution recovery first. + +Only one delivery attempt owns an intent at a time. The page refreshes while it +is publishing. If an attempt stops responding, its lease expires after 20 +minutes; check delivery again to reconcile before retrying. Rewind, relaunch and +other recovery actions wait for publication ownership to end. + +## Final publication recovery + +`DELIVERY#` stores final intent-to-base publication separately +from `META.status`. Repository entries preserve captured provider, branch, target, +head SHA and remote PR identity. Current attempt metadata is mutable; prior-run +delivery rows, unit PR rows, events and human decisions are retained. Publication +success means the PR was created or reconciled, not that it was merged. Graph +projection failure does not erase the durable remote identity. + +`POST .../intents/{intentId}/delivery` requires project owner/admin membership and +entity scope. `{"preview":true}` resolves the original methodology/plan, checks +expected sections including missing rows, and reads provider state. A subsequent +`{"previewToken":"..."}` must match the refreshed preview. Unknown legacy runs +are never automatically migrated. Accepted omission receipts and section-scoped +legacy halt decisions are honored; unfinished work requires ordinary execution recovery or a new intent. + +The process store claims publication with one transaction over META, the checked +UNITPLAN snapshot and the run's delivery row. It compares the current success +status, run, META timestamp and delivery revision; only an expired or absent lease can be claimed. Progress +and result writes require the same run, attempt and revision. Status/run changes +in `updateExecution` refuse an active delivery lease, preventing rewind or a new +run from racing publication. A provider mutation requires at least 150 seconds +of lease remaining. The 20-minute lease exceeds the orchestrator invocation's +900-second maximum plus the source-control Lambda's 120-second maximum. Preserve +that bound if either timeout changes. An uncertain launch keeps the lease since +the invocation might already be accepted; a later check recovers abandonment. + +Retry invokes a separate durable `retry-delivery` history through the configured +orchestrator target. It only uses the shared publication logic and the existing +source-control service. It never invokes AgentCore, stages, Git integration or +models. A failed or incomplete provider lookup blocks creation. Strict lookup +includes closed PRs; a changed source SHA, missing branch, retargeted PR or closed +work requires review instead of implicit replacement. External repository edits +remain outside DynamoDB ownership; branch identity is rechecked immediately +before creation. No provider merge-state polling is added. + +Deploy source-control first (strict lookup and branch preconditions), then the +orchestrator and intents API with the delivery route, then the frontend. Install +all parts from the same release before offering retry. New orchestrator histories +capture `deliveryPolicyVersion: 1` in the existing `load-meta` step. Frozen +pre-change results retain the previous `open-pr` sequence, including #534's +completion marker. Changing an alias does not upgrade their durable checkpoints. +The new API can assess a completed old history without changing its execution +pins; publication retry creates its own history on the configured fixed target. +This release requires no status migration, production recovery or model E2E as +part of deployment. The existing tracker handoff is scheduled after complete publication, including +a successful retry; graph projection retains its owner. Publication recovery +exposes durable PR links directly. Existing historical tracker rows are not +rewritten. diff --git a/frontend/src/components/intent/CodeSection.tsx b/frontend/src/components/intent/CodeSection.tsx index ac1757cd5..5842f392a 100644 --- a/frontend/src/components/intent/CodeSection.tsx +++ b/frontend/src/components/intent/CodeSection.tsx @@ -68,6 +68,18 @@ export function buildCodeItems(detail: IntentDetail): CodeItem[] { const repoPushed = (slug: string) => pushedSummaries.some((s) => s.includes(slug)); const prByRepo = new Map((detail.pullRequests ?? []).map((pr) => [pr.repository ?? '', pr])); + for (const repo of detail.delivery?.repositories ?? []) { + if (repo.pr) + prByRepo.set(repo.repoId, { + id: `delivery-${repo.repoId}`, + repository: repo.repoId, + prUrl: repo.pr.prUrl, + prNumber: String(repo.pr.prNumber), + branch: repo.pr.branch, + baseBranch: repo.pr.baseBranch, + createdAt: null, + }); + } return (detail.intent.repos ?? []) .map((repo) => { const slug = gitRepoSlug(repo); diff --git a/frontend/src/components/intent/DeliveryStatus.test.tsx b/frontend/src/components/intent/DeliveryStatus.test.tsx new file mode 100644 index 000000000..10a001260 --- /dev/null +++ b/frontend/src/components/intent/DeliveryStatus.test.tsx @@ -0,0 +1,117 @@ +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { beforeEach, expect, it, vi } from 'vitest'; +import { DeliveryStatus } from './DeliveryStatus'; +import { intentsService } from '@/services/intents'; +vi.mock('@/services/intents', () => ({ + intentsService: { previewDelivery: vi.fn(), retryDelivery: vi.fn() }, +})); +const repo = { + repoId: 'owner/web', + provider: 'github', + branch: 'intent/feature', + baseBranch: 'develop', + status: 'failed', + reason: 'Provider unavailable', +}; +const preview = { + previewToken: 'checked-head', + historical: false, + retryable: true, + canRecord: true, + delivery: { status: 'partial' as const, repositories: [repo] }, +}; +const reload = vi.fn(async () => {}); +const props = { + projectId: 'project', + intentId: 'intent', + canRetry: true, + delivery: preview.delivery, + reload, +}; +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(intentsService.previewDelivery).mockResolvedValue(preview); + vi.mocked(intentsService.retryDelivery).mockResolvedValue({ + delivery: { status: 'in_progress', repositories: [repo] }, + }); +}); +it('shows the affected repository and checks delivery before retrying only publication', async () => { + render(); + const user = userEvent.setup(); + expect(screen.getByText('Provider unavailable')).toBeInTheDocument(); + expect(screen.getByText(/does not regenerate code/)).toBeInTheDocument(); + await user.click(screen.getByRole('button', { name: 'Check delivery' })); + await user.click(await screen.findByRole('button', { name: 'Retry delivery' })); + await waitFor(() => + expect(intentsService.retryDelivery).toHaveBeenCalledWith('project', 'intent', 'checked-head'), + ); + expect(reload).toHaveBeenCalled(); +}); +it('requires a fresh check after a stale confirmation and displays its explanation', async () => { + vi.mocked(intentsService.retryDelivery).mockRejectedValue( + new Error('Delivery state changed. Check again.'), + ); + render(); + const user = userEvent.setup(); + await user.click(screen.getByRole('button', { name: 'Check delivery' })); + await user.click(await screen.findByRole('button', { name: 'Retry delivery' })); + expect(await screen.findByRole('alert')).toHaveTextContent('Delivery state changed'); + expect(screen.queryByRole('button', { name: 'Retry delivery' })).not.toBeInTheDocument(); +}); +it('keeps completed links visible and hides retry for nonowners or active publication', () => { + const { rerender } = render(); + expect(screen.queryByRole('button')).not.toBeInTheDocument(); + rerender( + , + ); + expect(screen.queryByRole('button')).not.toBeInTheDocument(); + rerender( + , + ); + expect(screen.getByRole('link', { name: 'View pull request #7' })).toHaveAttribute( + 'href', + 'https://example.test/pr/7', + ); + expect(screen.queryByRole('button')).not.toBeInTheDocument(); +}); +it('explains historical unknown status and a read-only assessment of closed work', async () => { + vi.mocked(intentsService.previewDelivery).mockResolvedValue({ + ...preview, + historical: true, + retryable: false, + delivery: { + status: 'failed', + repositories: [ + { + ...repo, + retryable: false, + reason: 'The existing pull request is closed. Retry will not recreate it.', + }, + ], + }, + }); + render(); + expect(screen.getByText('Delivery not yet verified')).toBeInTheDocument(); + await userEvent.click(screen.getByRole('button', { name: 'Check delivery' })); + expect(await screen.findByText(/This older run/)).toBeInTheDocument(); + expect(screen.queryByRole('button', { name: 'Retry delivery' })).not.toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Save delivery assessment' })).toBeInTheDocument(); +}); diff --git a/frontend/src/components/intent/DeliveryStatus.tsx b/frontend/src/components/intent/DeliveryStatus.tsx new file mode 100644 index 000000000..e2b0c3e91 --- /dev/null +++ b/frontend/src/components/intent/DeliveryStatus.tsx @@ -0,0 +1,122 @@ +import { useEffect, useRef, useState } from 'react'; +import { intentsService, type DeliveryPreview, type IntentDelivery } from '@/services/intents'; +import { Button } from '@/components/ui/button'; + +const labels: Record = { + unknown: 'Delivery not yet verified', + pending: 'Ready to publish', + in_progress: 'Publishing pull requests…', + succeeded: 'Pull requests published', + failed: 'Delivery needs attention', + partial: 'Some repositories need attention', + no_change: 'No changes to publish', +}; +export function DeliveryStatus({ + projectId, + intentId, + delivery, + canRetry, + reload, +}: { + projectId: string; + intentId: string; + delivery?: IntentDelivery; + canRetry: boolean; + reload: () => Promise; +}) { + const [preview, setPreview] = useState(null); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const alive = useRef(true); + useEffect(() => { + alive.current = true; + return () => { + alive.current = false; + }; + }, []); + const shown = preview?.delivery ?? delivery; + const status = shown?.status ?? 'unknown'; + const check = async () => { + setBusy(true); + setError(null); + try { + const result = await intentsService.previewDelivery(projectId, intentId); + if (alive.current) setPreview(result); + } catch (cause) { + if (alive.current) + setError(cause instanceof Error ? cause.message : 'Delivery could not be checked'); + } finally { + if (alive.current) setBusy(false); + } + }; + const retry = async () => { + if (!preview) return; + setBusy(true); + setError(null); + try { + await intentsService.retryDelivery(projectId, intentId, preview.previewToken); + if (alive.current) { + setPreview(null); + await reload(); + } + } catch (cause) { + if (alive.current) { + setPreview(null); + setError(cause instanceof Error ? cause.message : 'Delivery retry could not start'); + } + } finally { + if (alive.current) setBusy(false); + } + }; + return ( +
+

{labels[status]}

+

+ Publication is separate from stage execution. Retry publishes the existing intent branch; it + does not regenerate code, rerun stages or merge the pull request. +

+ {shown?.message &&

{shown.message}

} + {shown?.repositories.map((repo) => ( +
+

+ {repo.repoId} · {labels[repo.status] ?? repo.status} +

+

+ {repo.branch} → {repo.baseBranch ?? 'repository default'} +

+ {repo.reason &&

{repo.reason}

} + {repo.pr && ( + + View pull request #{repo.pr.prNumber} + + )} +
+ ))} + {error && ( +

+ {error} +

+ )} + {canRetry && + status !== 'in_progress' && + !['succeeded', 'no_change'].includes(delivery?.status ?? '') && ( +
+ + {preview && (preview.retryable || preview.canRecord) && ( + + )} +
+ )} + {preview?.historical && ( +

+ This older run has no delivery record. Review the branch and repository results above + before confirming. +

+ )} +
+ ); +} diff --git a/frontend/src/components/intent/UnitLaneBoard.test.tsx b/frontend/src/components/intent/UnitLaneBoard.test.tsx index 2631c08c6..6b56fc80d 100644 --- a/frontend/src/components/intent/UnitLaneBoard.test.tsx +++ b/frontend/src/components/intent/UnitLaneBoard.test.tsx @@ -95,7 +95,7 @@ describe('UnitLaneBoardView', () => { // (>=1) rather than uniqueness; state labels are unique per fixture. expect(screen.getAllByText('auth').length).toBeGreaterThanOrEqual(1); expect(screen.getByText('reporting')).toBeInTheDocument(); - expect(screen.getByText('Merged')).toBeInTheDocument(); + expect(screen.getByText('Integrated')).toBeInTheDocument(); expect(screen.getByText('Building')).toBeInTheDocument(); expect(screen.getByText('Blocked')).toBeInTheDocument(); expect(screen.getByText('Pending')).toBeInTheDocument(); @@ -115,7 +115,7 @@ describe('UnitLaneBoardView', () => { it('summarizes only the states that are present', () => { renderView(); - expect(screen.getByText('1 merged')).toBeInTheDocument(); + expect(screen.getByText('1 integrated')).toBeInTheDocument(); expect(screen.getByText('1 building')).toBeInTheDocument(); expect(screen.getByText('1 blocked')).toBeInTheDocument(); expect(screen.getByText('1 pending')).toBeInTheDocument(); @@ -137,7 +137,7 @@ describe('UnitLaneBoardView', () => { expect(screen.queryByText(/failed/)).not.toBeInTheDocument(); expect(screen.queryByText('legacy')).not.toBeInTheDocument(); // real units still counted - expect(screen.getByText('1 merged')).toBeInTheDocument(); + expect(screen.getByText('1 integrated')).toBeInTheDocument(); }); it('renders the live stream only when a running stage instance is provided', () => { diff --git a/frontend/src/components/intent/UnitLaneBoard.tsx b/frontend/src/components/intent/UnitLaneBoard.tsx index 6ba6c59f3..cb995206a 100644 --- a/frontend/src/components/intent/UnitLaneBoard.tsx +++ b/frontend/src/components/intent/UnitLaneBoard.tsx @@ -56,7 +56,7 @@ const STATE_LABEL: Record = { PR_READY: 'Ready to merge', ADDRESSING_FEEDBACK: 'Fixing feedback', MERGING: 'Merging', - MERGED: 'Merged', + MERGED: 'Integrated', BLOCKED: 'Blocked', FAILED: 'Failed', WAITING_INPUT: 'Waiting for input', @@ -402,6 +402,7 @@ function UnitCard({ // ── Presentational board (pure — tested in isolation) ──────────────────────── export interface UnitLaneBoardViewProps { + intentBranch?: string | null; unitPlan: IntentUnitPlan; units: IntentUnit[]; /** Live transcript tail per unit slug (only running units need an entry). */ @@ -418,6 +419,7 @@ export interface UnitLaneBoardViewProps { } export function UnitLaneBoardView({ + intentBranch, unitPlan, units, streamsBySlug, @@ -449,10 +451,8 @@ export function UnitLaneBoardView({ // lambda/shared/v2-process-store.js:1000), so counting over all detail.units // would report states for units that aren't on the board. const planSlugs = useMemo(() => { - const fromBatches = (unitPlan.batches ?? []).flat(); - const slugs = fromBatches.length > 0 ? fromBatches : (unitPlan.units ?? []).map((u) => u.slug); - return new Set(slugs); - }, [unitPlan.batches, unitPlan.units]); + return new Set((unitPlan.units ?? []).map((u) => u.slug)); + }, [unitPlan.units]); // State-count chips — only states actually present among plan units, in a // stable order. @@ -480,6 +480,16 @@ export function UnitLaneBoardView({ return ( +

+ Integrated units are on the intent branch + {intentBranch ? ( + <> + {' '} + {intentBranch} + + ) : null} + . The final pull request is reviewed and merged separately. +

Units @@ -708,6 +718,7 @@ export function UnitLaneBoard({ return ( <> { - if (!pollStatus || !['CREATED', 'RUNNING', 'WAITING'].includes(pollStatus)) return; + if ( + !deliveryPending && + (!pollStatus || !['CREATED', 'RUNNING', 'WAITING'].includes(pollStatus)) + ) + return; const id = setInterval(load, 8000); return () => clearInterval(id); - }, [pollStatus, load]); + }, [pollStatus, deliveryPending, load]); // Refetch debouncing (docs/v2-parallel.md WP7): every lifecycle event used // to trigger an immediate full-DTO refetch — N parallel lanes multiply diff --git a/frontend/src/pages/IntentView.test.tsx b/frontend/src/pages/IntentView.test.tsx index 0c9f0ddbc..07f5c9a97 100644 --- a/frontend/src/pages/IntentView.test.tsx +++ b/frontend/src/pages/IntentView.test.tsx @@ -2380,3 +2380,46 @@ describe('IntentView — resume after a failed gate callback', () => { ).not.toBeInTheDocument(); }); }); + +describe('terminal completion diagnostics without terminal recovery', () => { + it.each(['incomplete', 'unknown', 'partial'] as const)( + 'shows %s evidence without offering preservation recovery', + async (state) => { + get.mockResolvedValue({ + ...baseDetail({ status: 'SUCCEEDED' }), + unitCompletion: { + state, + total: 2, + integrated: 1, + skipped: state === 'partial' ? 1 : 0, + unresolved: + state === 'partial' + ? [] + : [ + { + sectionIndex: 1, + slug: 'remaining', + state: 'BLOCKED', + reason: 'unit_unfinished', + }, + ], + legacyPartial: false, + }, + }); + renderAt(); + expect( + await screen.findByText( + state === 'incomplete' + ? 'Incomplete work' + : state === 'unknown' + ? 'Completion unverified' + : 'Partial delivery', + ), + ).toBeInTheDocument(); + expect( + screen.queryByRole('button', { name: /Preview recovery|Confirm and continue/ }), + ).not.toBeInTheDocument(); + expect(repair).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/frontend/src/pages/IntentView.tsx b/frontend/src/pages/IntentView.tsx index 5819c475e..19572bfdd 100644 --- a/frontend/src/pages/IntentView.tsx +++ b/frontend/src/pages/IntentView.tsx @@ -8,6 +8,7 @@ import { import { useIntent } from '@/contexts/IntentContext'; import { useAuth } from '@/contexts/AuthContext'; import { useProjectCache } from '@/hooks/useProjectsCache'; +import { DeliveryStatus } from '@/components/intent/DeliveryStatus'; import { RecomposePanel } from '@/components/intent/RecomposePanel'; import { IntentConfigurationDialog } from '@/components/intent/IntentConfigurationDialog'; import { DiscussButton } from '@/components/discussion/DiscussButton'; @@ -342,6 +343,8 @@ export default function IntentView() { } const intent = detail.intent; + const completion = detail.unitCompletion; + const terminalCompletion = intent.status === 'SUCCEEDED' ? completion?.state : null; const laneWaits = deriveLaneWaits(detail.stages, gates); const recoveryWaits = Object.values(laneWaits).filter((wait) => wait.kind === 'recovery'); const needsLaneRepair = @@ -417,7 +420,13 @@ export default function IntentView() { {TERMINAL_STATUSES.has(intent.status) && ( - {intent.status} + {terminalCompletion === 'incomplete' + ? 'Incomplete work' + : terminalCompletion === 'unknown' + ? 'Completion unverified' + : terminalCompletion === 'partial' + ? 'Partial delivery' + : intent.status} )} @@ -579,6 +588,44 @@ export default function IntentView() {
)} + {terminalCompletion && terminalCompletion !== 'complete' && ( +
+

+ {terminalCompletion === 'partial' + ? 'Partial delivery was explicitly accepted. Integrated work and the recorded omissions are preserved.' + : terminalCompletion === 'unknown' + ? 'Completion could not be verified. Review the recorded work and available evidence before publishing.' + : 'The recorded success has unfinished current-plan work. Review the remaining units before publishing.'} +

+

+ {completion?.integrated} of {completion?.total} units verified integrated;{' '} + {completion?.skipped} explicitly omitted. +

+ {completion?.unresolved.slice(0, 12).map((unit) => ( +

+ Section {unit.sectionIndex}/{unit.slug}: {unit.reason ?? unit.state} +

+ ))} + {terminalCompletion !== 'partial' && ( +

+ Automatic continuation of finished incomplete intents is not supported. Ordinary + rewind reruns the affected work. +

+ )} +
+ )} + + {intent.status === 'SUCCEEDED' && ( + + )} + {needsLaneRepair && (
diff --git a/frontend/src/services/intents.ts b/frontend/src/services/intents.ts index 298c04780..efc52eaa3 100644 --- a/frontend/src/services/intents.ts +++ b/frontend/src/services/intents.ts @@ -792,16 +792,54 @@ export interface IntentDetail { sensorRuns: IntentSensorRun[]; artifacts: IntentArtifact[]; pullRequests?: IntentPullRequest[]; + delivery?: IntentDelivery; /** Quorum-supported artifact edit sessions (post-hoc document editing). */ quorumEdits?: QuorumEdit[]; /** Composer sessions (scope/grid proposals + authoritative validation). */ composes?: ComposeSession[]; unitPlan?: IntentUnitPlan | null; + unitCompletion?: { + state: 'complete' | 'partial' | 'incomplete' | 'unknown' | 'aborted'; + reason?: string | null; + total: number; + integrated: number; + skipped: number; + unresolved: { + sectionIndex: number; + slug: string; + state: UnitState | 'UNKNOWN' | 'MISSING'; + reason?: string; + }[]; + legacyPartial: boolean; + } | null; units?: IntentUnit[]; unitPrs?: IntentUnitPr[]; feedbackBatches?: IntentFeedbackBatch[]; } +export interface IntentDelivery { + status: 'unknown' | 'pending' | 'in_progress' | 'succeeded' | 'failed' | 'partial' | 'no_change'; + message?: string | null; + leaseUntil?: string; + repositories: { + repoId: string; + provider: string; + branch: string; + baseBranch: string | null; + status: string; + reason?: string; + retryable?: boolean; + pr?: { prUrl: string; prNumber: string | number; branch: string; baseBranch: string | null }; + }[]; +} +export interface DeliveryPreview { + delivery: IntentDelivery; + previewToken: string; + historical: boolean; + retryable: boolean; + canRecord: boolean; +} + export interface IntentRepairResult { intent: Intent; repair: { @@ -1186,6 +1224,14 @@ export const intentsService = { `/projects/${projectId}/intents/${intentId}/rewind`, input, ), + previewDelivery: (projectId: string, intentId: string) => + api.post(`/projects/${projectId}/intents/${intentId}/delivery`, { + preview: true, + }), + retryDelivery: (projectId: string, intentId: string, previewToken: string) => + api.post<{ delivery: IntentDelivery }>(`/projects/${projectId}/intents/${intentId}/delivery`, { + previewToken, + }), repair: (projectId: string, intentId: string) => api.post(`/projects/${projectId}/intents/${intentId}/repair`, {}), // Re-attempt the durable callback for an already-recorded gate answer diff --git a/lambda/agentcore/commands/promote-units.js b/lambda/agentcore/commands/promote-units.js index 698dbc0a9..5bf4730c9 100644 --- a/lambda/agentcore/commands/promote-units.js +++ b/lambda/agentcore/commands/promote-units.js @@ -34,6 +34,7 @@ import { createGraphWriter, closeGraphSource } from '../mcp/graph-writer.js'; import { parseBoltDag } from '../../shared/v2-sensor-contract.js'; +import { effectiveSkeleton, skeletonProblem } from '../../shared/unit-completion.js'; const DAG_ARTIFACT_TYPE = 'unit-of-work-dependency'; @@ -122,20 +123,39 @@ export const promoteUnits = async (payload, deps) => { // batch (batches are sorted). The fan-out approval can override. // Consistent: a stale read would drop decisions the fan-out approval saved. const existingPlan = await store.getUnitPlan(executionId, { consistentRead: true }); + const priorSkeleton = existingPlan?.walkingSkeleton; + const priorProblem = + priorSkeleton == null ? null : skeletonProblem({ units, batches }, priorSkeleton); + const walkingSkeleton = priorProblem + ? effectiveSkeleton({ units, batches }) + : effectiveSkeleton({ units, batches, walkingSkeleton: priorSkeleton }); const plan = await store.putUnitPlan({ executionId, units, batches, sourceArtifactId: artifact.id ?? null, producedByStageInstanceId: stageInstanceId, - // Preserve previously captured human decisions across a re-promotion — - // a rewind re-produces the DAG, not the humans' skip/skeleton/autonomy - // answers (the unit-DAG stage's gate re-confirms them on re-approval). + // Keep valid choices across re-promotion. A regenerated DAG can rename + // or add dependencies to the old skeleton; choose a valid default BEFORE + // the normal fan-out approval, preserving the original answer in history. skipMatrix: existingPlan?.skipMatrix ?? {}, - walkingSkeleton: existingPlan?.walkingSkeleton ?? batches[0]?.[0] ?? null, + walkingSkeleton, autonomyMode: existingPlan?.autonomyMode ?? null, }); - const sync = await store.syncUnitRows({ executionId, units, batches, sectionIndexes }); + const sync = await store.syncUnitRows({ + executionId, + units, + batches, + sectionIndexes, + }); + if (priorProblem) + await event( + 'v2.units.skeleton_reselected', + `${priorProblem} Regenerated DAG default: ${walkingSkeleton}; review it at fan-out approval.`, + ); + // Execution still rejects invalid choices in an already approved plan. + const selectionProblem = skeletonProblem(plan); + if (selectionProblem) await event('v2.units.decisions_invalid', selectionProblem); // 4. Traceability mirror (never blocks promotion outcome — the DDB truth // is already written; a mirror failure is recorded and visible). @@ -164,6 +184,7 @@ export const promoteUnits = async (payload, deps) => { unitCount: units.length, batchCount: batches.length, walkingSkeleton: plan.walkingSkeleton, + selectionProblem, sync, mirror, }; diff --git a/lambda/agentcore/test/completion-store.test.js b/lambda/agentcore/test/completion-store.test.js new file mode 100644 index 000000000..bf084c980 --- /dev/null +++ b/lambda/agentcore/test/completion-store.test.js @@ -0,0 +1,207 @@ +import { randomUUID } from 'node:crypto'; +import { afterAll, beforeAll, expect, it } from 'vitest'; +import { TransactWriteCommand } from '@aws-sdk/lib-dynamodb'; +import { createProcessStore } from '../../shared/v2-process-store.js'; +import { finalizeUnitCompletion, unitPlanIdentity } from '../../shared/unit-completion.js'; +import { createV2Table, deleteV2Table, makeDdb } from './helpers/v2-table.js'; + +const tableName = `completion-${randomUUID()}`; +let db; +let store; +beforeAll(async () => { + db = makeDdb(); + await createV2Table(db.client, tableName); + store = createProcessStore({ ddb: db.doc, tableName }); +}); +afterAll(async () => { + await deleteV2Table(db.client, tableName); + db.client.destroy(); +}); +const plan = { stages: [{ stageId: 'build', parallelSection: 1 }] }; +const setup = async ({ + state = 'MERGED', + projectId = randomUUID(), + repos = [], + prStrategy = 'intent-pr', +} = {}) => { + const executionId = randomUUID(); + await store.createExecution({ + executionId, + intentId: executionId, + projectId, + startedAt: '2026-10-01T00:00:00.000Z', + workflowId: 'workflow', + workflowVersion: 1, + repos, + prStrategy, + methodologyPins: { STAGE: { build: { tenantId: 'SYSTEM', version: 1 } } }, + }); + await store.updateExecution({ executionId, status: 'RUNNING', orchestratorRunId: 'run' }); + await store.putUnitPlan({ executionId, units: [{ slug: 'a', dependsOn: [] }], batches: [['a']] }); + await store.syncUnitRows({ + executionId, + units: [{ slug: 'a', dependsOn: [] }], + sectionIndexes: [1], + }); + await store.updateUnitState({ executionId, sectionIndex: 1, slug: 'a', state }); + return executionId; +}; + +it('commits success and returns the same decision when a durable response is lost', async () => { + const executionId = await setup(); + const args = { store, executionId, runId: 'run', plan }; + const result = await finalizeUnitCompletion(args); + expect(result).toMatchObject({ state: 'complete', integrated: 1 }); + const first = await store.getExecution(executionId); + expect(first.status).toBe('SUCCEEDED'); + expect(await finalizeUnitCompletion(args)).toEqual(result); + expect((await store.getExecution(executionId)).updatedAt).toBe(first.updatedAt); +}); + +it('checks the persisted unitSlug repository rows before accepting an integrated unit', async () => { + const executionId = await setup({ repos: ['one', 'two'], prStrategy: 'pr-per-unit' }); + for (const repository of ['one', 'two']) { + await store.createUnitPr({ + executionId, + sectionIndex: 1, + slug: 'a', + repository, + provider: 'github', + sourceBranch: 'unit', + targetBranch: 'intent', + state: repository === 'one' ? 'MERGED' : 'PARTIALLY_MERGED', + }); + } + expect(await finalizeUnitCompletion({ store, executionId, runId: 'run', plan })).toMatchObject({ + state: 'incomplete', + integrated: 0, + unresolvedCount: 1, + }); + expect((await store.getUnit(executionId, 1, 'a')).state).toBe('MERGED'); +}); + +it('permits serial execution without inventing a unit plan', async () => { + const executionId = randomUUID(); + await store.createExecution({ executionId, intentId: executionId, projectId: 'serial' }); + await store.updateExecution({ executionId, status: 'RUNNING', orchestratorRunId: 'run' }); + expect( + await finalizeUnitCompletion({ store, executionId, runId: 'run', plan: { stages: [] } }), + ).toMatchObject({ state: 'complete', total: 0 }); + expect(await store.getUnitPlan(executionId)).toBeNull(); +}); + +it('preserves rows and records an actionable failure instead of false success', async () => { + const executionId = await setup({ state: 'BLOCKED' }); + const before = await store.getUnit(executionId, 1, 'a'); + expect(await finalizeUnitCompletion({ store, executionId, runId: 'run', plan })).toMatchObject({ + state: 'incomplete', + unresolvedCount: 1, + }); + expect(await store.getExecution(executionId)).toMatchObject({ + status: 'FAILED', + failure: { code: 'completion_inconsistent' }, + }); + expect(await store.getUnit(executionId, 1, 'a')).toEqual(before); +}); + +it.each(['run', 'plan', 'meta'])( + 'rejects a concurrent %s change between read and terminal write', + async (change) => { + const executionId = await setup(); + let raced = false; + const racing = createProcessStore({ + tableName, + ddb: { + send: async (command) => { + if (command instanceof TransactWriteCommand && !raced) { + raced = true; + if (change === 'plan') + await store.putUnitPlan({ + executionId, + units: [{ slug: 'replacement', dependsOn: [] }], + batches: [['replacement']], + }); + else + await store.updateExecution({ + executionId, + ...(change === 'run' + ? { orchestratorRunId: 'replacement' } + : { currentStage: 'new-attempt' }), + }); + } + return db.doc.send(command); + }, + }, + }); + expect( + await finalizeUnitCompletion({ store: racing, executionId, runId: 'run', plan }), + ).toEqual({ state: 'retired' }); + expect((await store.getExecution(executionId)).status).toBe('RUNNING'); + }, +); + +it('detects a plan replaced after the section checkpoint without re-running any stage', async () => { + const executionId = await setup(); + expect( + await finalizeUnitCompletion({ + store, + executionId, + runId: 'run', + plan, + expectedUnitPlanIdentity: unitPlanIdentity({ units: [{ slug: 'prior' }] }), + }), + ).toMatchObject({ state: 'unknown', reason: 'unit_plan_changed_after_section' }); +}); + +it('bounds completion reads and never calls a truncated prefix complete', async () => { + const executionId = await setup(); + expect(await store.getCompletionRecords(executionId, { maxRecords: 2 })).toMatchObject({ + truncated: true, + units: [], + }); +}); + +it('reconciles a committed final transaction whose response was lost', async () => { + const executionId = await setup(); + let lost = false; + const responseLost = createProcessStore({ + tableName, + ddb: { + send: async (command) => { + const result = await db.doc.send(command); + if (command instanceof TransactWriteCommand && !lost) { + lost = true; + throw new Error('response lost after commit'); + } + return result; + }, + }, + }); + expect( + await finalizeUnitCompletion({ store: responseLost, executionId, runId: 'run', plan }), + ).toMatchObject({ state: 'complete' }); + expect((await store.getExecution(executionId)).status).toBe('SUCCEEDED'); +}); + +it('does not overwrite a competing success and active publication lease at final commit', async () => { + const executionId = await setup(); + const racingStore = { + ...store, + updateExecution: async (input) => { + const completed = await store.updateExecution({ executionId, status: 'SUCCEEDED' }); + await store.claimDelivery({ + meta: completed, + delivery: { repositories: [] }, + attemptId: 'publisher', + }); + return store.updateExecution(input); + }, + }; + expect( + await finalizeUnitCompletion({ store: racingStore, executionId, runId: 'run', plan }), + ).toEqual({ state: 'retired' }); + const current = await store.getExecution(executionId); + expect(current.status).toBe('SUCCEEDED'); + expect(current.deliveryAttemptId).toBe('publisher'); + expect(current.completionValidation).toBeUndefined(); +}); diff --git a/lambda/agentcore/test/promote-units.test.js b/lambda/agentcore/test/promote-units.test.js index 747ae234c..44cf946a8 100644 --- a/lambda/agentcore/test/promote-units.test.js +++ b/lambda/agentcore/test/promote-units.test.js @@ -62,6 +62,23 @@ beforeEach(() => { }; }); +it.each(['checkout', 'renamed-unit', ''])( + 'replaces unsupported old skeleton %s before approving a regenerated DAG', + async (walkingSkeleton) => { + store.getUnitPlan.mockResolvedValue({ walkingSkeleton, autonomyMode: 'gated' }); + const result = await promoteUnits(basePayload, deps); + expect(result.ok).toBe(true); + expect(result.walkingSkeleton).toBe('auth'); + expect(result.selectionProblem).toBeFalsy(); + expect(store.putUnitPlan).toHaveBeenCalledWith( + expect.objectContaining({ walkingSkeleton: 'auth', autonomyMode: 'gated' }), + ); + expect(store.appendEvent).toHaveBeenCalledWith( + expect.objectContaining({ type: 'v2.units.skeleton_reselected' }), + ); + }, +); + describe('pickCurrentArtifact', () => { it('prefers non-superseded rows and the newest timestamp', () => { const rows = [ diff --git a/lambda/intents/index.js b/lambda/intents/index.js index eb71651a8..e5fa2fa41 100644 --- a/lambda/intents/index.js +++ b/lambda/intents/index.js @@ -35,7 +35,14 @@ import { isEnvironmentResolutionError, resolveEnvironmentSnapshot, } from '../shared/environment-snapshot.js'; +import { previewDelivery, deliveryView, currentDelivery } from '../shared/final-delivery.js'; import { runtimeTargetInput } from '../shared/runtime-target.js'; +import { + unitCompletion, + assessExecutionCompletion, + activePlanUnits, + hasUnitSkipDecision, +} from '../shared/unit-completion.js'; import { SessionReleaseHandoffError, stopSessions } from '../shared/runtime-session.js'; import { composeSessionIdFor, @@ -1784,6 +1791,19 @@ export const handler = async (event, context) => { const auth = await authorize(g, projectId, sub, response); if (auth.res) return auth.res; + if ( + intentId && + (httpMethod === 'DELETE' || + (httpMethod === 'POST' && /\/(rewind|recompose|cancel|repair)$/.test(path ?? ''))) + ) { + const current = await store.getExecution(intentId, { consistentRead: true }); + if (current?.projectId === projectId && Date.parse(current.deliveryLeaseUntil) > Date.now()) { + return response(409, { + error: 'Delivery is in progress. Wait for it to finish before changing this execution.', + code: 'delivery_in_progress', + }); + } + } // Authenticated unit-review selection. Provider comments are data only: // no webhook/comment can launch an agent. GET refreshes selectable @@ -3989,7 +4009,7 @@ export const handler = async (event, context) => { return response(200, mapIntent(updated)); } - if (intentId && httpMethod === 'DELETE') { + if (intentId && httpMethod === 'DELETE' && path?.endsWith(`/intents/${intentId}`)) { const meta = await store.getExecution(intentId); if (!meta || meta.projectId !== projectId) { return response(404, { error: 'Intent not found' }); @@ -4127,6 +4147,86 @@ export const handler = async (event, context) => { } } + // Publication recovery is independent of stage recovery and always scoped + // to the current run. Preview performs reads only, including old executions. + if (intentId && httpMethod === 'POST' && path?.endsWith('/delivery')) { + if (auth.role !== 'owner' && auth.role !== 'admin') + return response(403, { error: 'Only project owners and admins can retry delivery' }); + const records = await store.getExecutionRecords(intentId, { + includeOutputs: false, + consistentRead: true, + }); + const meta = records.meta; + if (!meta || meta.projectId !== projectId) + return response(404, { error: 'Intent not found' }); + const input = body ? JSON.parse(body) : {}; + try { + const resolved = await intentMethodology.loadPlan(meta); + if (!resolved.valid || !resolved.plan) + return response(409, { error: 'The pinned workflow cannot be verified for delivery' }); + const preview = await previewDelivery({ + meta, + records, + plan: resolved.plan, + operation: sourceControlOperation, + }); + if (input.preview) + return response(200, { ...preview, delivery: deliveryView(meta, preview.delivery) }); + if (!input.previewToken || input.previewToken !== preview.previewToken) + return response(409, { + error: 'Delivery state changed. Check delivery again before retrying.', + code: 'delivery_preview_changed', + }); + const claimed = await store.claimDelivery({ + meta, + delivery: { ...preview.delivery, requestedBy: sub }, + attemptId: randomUUID(), + ifUnitPlan: records.unitPlan, + }); + try { + await lambdaClient.send( + new InvokeCommand({ + FunctionName: ORCHESTRATOR_FN(), + InvocationType: 'Event', + DurableExecutionName: `delivery-${randomBytes(16).toString('hex')}`, + Payload: Buffer.from( + JSON.stringify({ + action: 'retry-delivery', + projectId, + intentId, + executionId: intentId, + runId: meta.orchestratorRunId, + attemptId: claimed.attemptId, + }), + ), + }), + ); + } catch { + // An invocation timeout can mean accepted. Keep the bounded lease: + // immediately releasing it could overlap an already running publisher. + return response(202, { + delivery: deliveryView(meta, claimed), + message: + 'Delivery launch is uncertain. Refresh status; the attempt lease expires automatically.', + }); + } + return response(202, { delivery: deliveryView(meta, claimed) }); + } catch (error) { + if ( + error.statusCode === 409 || + ['ConditionalCheckFailedException', 'TransactionCanceledException'].includes(error.name) + ) + return response(409, { + error: + error.statusCode === 409 + ? error.message + : 'Delivery or execution changed. Refresh before retrying.', + code: error.code ?? 'delivery_changed', + }); + throw error; + } + } + // POST /projects/{projectId}/intents/{intentId}/repair // Recover a parallel section whose stage callbacks became orphaned. This is // intentionally narrower than rewind: merged units remain merged, pending @@ -4164,8 +4264,12 @@ export const handler = async (event, context) => { 'ADDRESSING_FEEDBACK', 'MERGING', ]); - const prUnits = (records.units ?? []).filter( + const activeUnits = records.unitPlan?.units?.length + ? activePlanUnits(records.unitPlan, records.units) + : (records.units ?? []); + const prUnits = activeUnits.filter( (unit) => + !hasUnitSkipDecision(unit, records.unitPlan) && activeReviewStates.has(unit.state) && (records.unitPrs ?? []).some( (pr) => @@ -4208,8 +4312,11 @@ export const handler = async (event, context) => { 'FAILED', 'BLOCKED', ]); - const repairUnits = (records.units ?? []).filter( - (unit) => Number(unit.sectionIndex) === sectionIndex && repairableStates.has(unit.state), + const repairUnits = activeUnits.filter( + (unit) => + Number(unit.sectionIndex) === sectionIndex && + !hasUnitSkipDecision(unit, records.unitPlan) && + repairableStates.has(unit.state), ); const repairSlugs = [...new Set(repairUnits.map((unit) => unit.slug))]; if (repairSlugs.length === 0) { @@ -4727,18 +4834,30 @@ export const handler = async (event, context) => { // already superseded by the first cleanup step, with no action capable // of waking it. FAILED remains rewindable, so a repeated request can // safely finish an interrupted reset. - await store.updateExecution({ - executionId: intentId, - projectId, - status: 'FAILED', - fromStatus: priorStatus, - startedAt: meta.startedAt, - pendingHumanTaskId: null, - resumeRequired: null, - failureReason: 'rewind_in_progress', - completedAt: null, - orchestratorRunId: `retired-${randomBytes(8).toString('hex')}`, - }); + try { + await store.updateExecution({ + executionId: intentId, + projectId, + status: 'FAILED', + fromStatus: priorStatus, + startedAt: meta.startedAt, + pendingHumanTaskId: null, + resumeRequired: null, + failureReason: 'rewind_in_progress', + completedAt: null, + ifOrchestratorRunId: meta.orchestratorRunId, + orchestratorRunId: `retired-${randomBytes(8).toString('hex')}`, + }); + } catch (error) { + if (error?.name === 'ConditionalCheckFailedException') { + return response(409, { + error: + 'Execution changed while preparing rewind. Refresh and review the restart point again.', + code: 'execution_changed', + }); + } + throw error; + } // Retire a parked run first so the woken orchestrator exits quietly (its // gate is superseded) instead of racing the relaunch. await retireParkedRun(intentId, `rewound to ${fromStageId}`); @@ -4806,7 +4925,7 @@ export const handler = async (event, context) => { sectionIndex, slug, state: 'PENDING', - fields: { failureReason: null, blockedOn: null }, + fields: { failureReason: null, blockedOn: null, completionSkip: null }, }) .catch((err) => logger.error('Unit lane reset failed', err, { sectionIndex, slug })), ); @@ -5049,11 +5168,23 @@ export const handler = async (event, context) => { }); // Retire only after every affected artifact was durably snapshotted. - await store.updateExecution({ - executionId: intentId, - fromStatus: meta.status, - orchestratorRunId: `retired-${randomBytes(8).toString('hex')}`, - }); + try { + await store.updateExecution({ + executionId: intentId, + fromStatus: meta.status, + ifOrchestratorRunId: meta.orchestratorRunId, + orchestratorRunId: `retired-${randomBytes(8).toString('hex')}`, + }); + } catch (error) { + if (error?.name === 'ConditionalCheckFailedException') { + return response(409, { + error: + 'Execution changed while preparing recompose. Refresh and review the stage selection again.', + code: 'execution_changed', + }); + } + throw error; + } await retireParkedRun(intentId, `recomposed from ${fromStage.stageId}`); await stopRuntimeSessions(intentId, meta); for (const stageInstanceId of resetIds) { @@ -5213,6 +5344,9 @@ export const handler = async (event, context) => { if (!records.meta || records.meta.projectId !== projectId) { return response(404, { error: 'Intent not found' }); } + const completion = ['SUCCEEDED', 'FAILED', 'CANCELLED'].includes(records.meta.status) + ? await assessExecutionCompletion({ records, loadPlan: intentMethodology.loadPlan }) + : unitCompletion(records); const artifacts = await fetchArtifacts(g, intentId); const pullRequests = await fetchPullRequests(g, intentId); const stagesByInstance = new Map( @@ -5284,7 +5418,9 @@ export const handler = async (event, context) => { // Unit lanes (docs/v2-parallel.md WP4): the promoted UNITPLAN snapshot // + the live UNIT lane rows, so the UI can render the lane board and // attribute per-unit stage instances. Both null/empty pre-promotion. + unitCompletion: completion, unitPlan: mapUnitPlan(records.unitPlan), + delivery: deliveryView(records.meta, currentDelivery(records)), units: (records.units ?? []).map(mapUnit), unitPrs: (records.unitPrs ?? []).map(mapUnitPr), feedbackBatches: (records.feedbackBatches ?? []).map(mapFeedbackBatch), diff --git a/lambda/intents/test/delivery.test.js b/lambda/intents/test/delivery.test.js new file mode 100644 index 000000000..e31d35713 --- /dev/null +++ b/lambda/intents/test/delivery.test.js @@ -0,0 +1,197 @@ +import { randomUUID } from 'node:crypto'; +import { beforeAll, afterAll, beforeEach, it, expect, vi } from 'vitest'; +import gremlin from 'gremlin'; +import { PartitionStrategy } from 'gremlin/lib/process/traversal-strategy.js'; +import { mockClient } from 'aws-sdk-client-mock'; +import { LambdaClient, InvokeCommand } from '@aws-sdk/client-lambda'; +import { SSMClient, GetParameterCommand } from '@aws-sdk/client-ssm'; +import { createProcessStore } from '../../shared/v2-process-store.js'; +import { createV2Table, deleteV2Table, makeDdb } from '../../agentcore/test/helpers/v2-table.js'; +const plan = vi.hoisted(() => ({ stages: [] })); +vi.mock('../../shared/v2-workflow-plan.js', async (original) => ({ + ...(await original()), + loadExecutionPlan: async () => ({ valid: true, plan }), +})); +const partition = `delivery-${randomUUID()}`; +const tableName = `delivery-api-${randomUUID()}`; +const lambda = mockClient(LambdaClient); +const ssm = mockClient(SSMClient); +let db, store, handler, conn, g, meta, projectId, userId, remoteHead, providerCalls; +beforeAll(async () => { + vi.stubEnv('AWS_PROFILE', undefined); + vi.stubEnv('AWS_ENDPOINT_URL_DYNAMODB', process.env.DYNAMODB_LOCAL_ENDPOINT); + vi.stubEnv('V2_PROCESS_TABLE', tableName); + vi.stubEnv('GREMLIN_PARTITION', partition); + vi.stubEnv('V2_ORCHESTRATOR_FUNCTION', 'delivery-orchestrator'); + vi.stubEnv('SOURCE_CONTROL_FUNCTION', 'delivery-source-control'); + db = makeDdb(); + await createV2Table(db.client, tableName); + store = createProcessStore({ ddb: db.doc, tableName }); + ({ handler } = await import('../index.js')); + conn = new gremlin.driver.DriverRemoteConnection( + `ws://${process.env.NEPTUNE_ENDPOINT}:${process.env.GREMLIN_PORT}/gremlin`, + ); + g = gremlin.process.AnonymousTraversalSource.traversal() + .withRemote(conn) + .withStrategies( + new PartitionStrategy({ + partitionKey: '_partition', + writePartition: partition, + readPartitions: [partition], + }), + ); +}); +afterAll(async () => { + await conn.close(); + await deleteV2Table(db.client, tableName); + db.client.destroy(); + lambda.restore(); + ssm.restore(); + vi.unstubAllEnvs(); +}); +beforeEach(async () => { + plan.stages = []; + remoteHead = 'head-one'; + providerCalls = []; + lambda.reset(); + ssm.reset(); + ssm.on(GetParameterCommand).rejects(new Error('no pricing fixture')); + lambda.on(InvokeCommand).callsFake((input) => { + if (input.FunctionName !== 'delivery-source-control') return { StatusCode: 202 }; + const call = JSON.parse(Buffer.from(input.Payload).toString()); + providerCalls.push(call); + return { + Payload: Buffer.from( + JSON.stringify({ + ok: true, + result: { headSha: remoteHead, baseSha: 'base', baseBranch: 'develop', pr: null }, + }), + ), + }; + }); + projectId = randomUUID(); + userId = randomUUID(); + await g + .addV('Project') + .property('id', projectId) + .as('p') + .addV('User') + .property('id', userId) + .addE('HAS_MEMBER') + .from_('p') + .property('role', 'owner') + .next(); + const executionId = randomUUID(); + await store.createExecution({ + executionId, + intentId: executionId, + projectId, + workflowId: 'flow', + workflowVersion: 1, + methodologyPins: { STAGE: {} }, + scope: 'feature', + repos: ['owner/repo'], + branch: 'intent/branch', + baseBranch: 'develop', + }); + meta = await store.updateExecution({ + executionId, + status: 'SUCCEEDED', + orchestratorRunId: 'run-one', + }); +}); +const request = (body, over = {}) => + handler({ + httpMethod: 'POST', + path: `/projects/${projectId}/intents/${meta.executionId}/delivery`, + pathParameters: { projectId, intentId: meta.executionId }, + body: JSON.stringify(body), + requestContext: { authorizer: { claims: { sub: userId } } }, + ...over, + }); +const launches = () => + lambda + .commandCalls(InvokeCommand) + .filter((c) => c.args[0].input.FunctionName === 'delivery-orchestrator'); +it('checks historical delivery read-only then admits exactly one repeated/concurrent confirmation', async () => { + const p = await request({ preview: true }); + expect(p.statusCode).toBe(200); + const preview = JSON.parse(p.body); + expect(preview.historical).toBe(true); + expect(await store.getDelivery(meta.executionId, 'run-one')).toBeNull(); + expect(launches()).toHaveLength(0); + const results = await Promise.all([ + request({ previewToken: preview.previewToken }), + request({ previewToken: preview.previewToken }), + ]); + expect(results.map((r) => r.statusCode).sort()).toEqual([202, 409]); + expect(launches()).toHaveLength(1); + expect( + (await request({}, { path: `/projects/${projectId}/intents/${meta.executionId}/rewind` })) + .statusCode, + ).toBe(409); + const payload = JSON.parse(Buffer.from(launches()[0].args[0].input.Payload).toString()); + expect(payload).toMatchObject({ action: 'retry-delivery', runId: 'run-one', projectId }); + expect(providerCalls.every((c) => c.operation === 'delivery-inspect')).toBe(true); + expect((await store.getExecution(meta.executionId)).status).toBe('SUCCEEDED'); +}); +it('rejects a changed branch preview and never starts publication', async () => { + const preview = JSON.parse((await request({ preview: true })).body); + remoteHead = 'head-two'; + const result = await request({ previewToken: preview.previewToken }); + expect(result.statusCode).toBe(409); + expect(launches()).toHaveLength(0); +}); +it('keeps an uncertain accepted launch fenced until the bounded lease expires', async () => { + const preview = JSON.parse((await request({ preview: true })).body); + lambda + .on(InvokeCommand, { FunctionName: 'delivery-orchestrator' }) + .rejects(new Error('response lost')); + const result = await request({ previewToken: preview.previewToken }); + expect(result.statusCode).toBe(202); + expect(JSON.parse(result.body).message).toContain('uncertain'); + expect((await request({ preview: true })).statusCode).toBe(409); +}); +it('enforces authentication, membership, project scope, and recovery roles', async () => { + expect((await request({ preview: true }, { requestContext: {} })).statusCode).toBe(401); + expect( + ( + await request( + { preview: true }, + { requestContext: { authorizer: { claims: { sub: 'outsider' } } } }, + ) + ).statusCode, + ).toBe(403); + await g + .V() + .has('Project', 'id', projectId) + .outE('HAS_MEMBER') + .property('role', 'member') + .toList(); + expect((await request({ preview: true })).statusCode).toBe(403); + await g.V().has('Project', 'id', projectId).outE('HAS_MEMBER').property('role', 'admin').toList(); + expect((await request({ preview: true })).statusCode).toBe(200); + expect( + (await request({ preview: true }, { pathParameters: { projectId, intentId: 'foreign' } })) + .statusCode, + ).toBe(404); + const foreign = randomUUID(); + await store.createExecution({ + executionId: foreign, + intentId: foreign, + projectId: 'another-project', + }); + expect( + (await request({ preview: true }, { pathParameters: { projectId, intentId: foreign } })) + .statusCode, + ).toBe(404); + expect(launches()).toHaveLength(0); +}); +it('refuses missing or unfinished stage/section records without provider requests', async () => { + plan.stages = [{ stageId: 'build' }]; + expect((await request({ preview: true })).statusCode).toBe(409); + plan.stages = [{ stageId: 'code', parallelSection: 1 }]; + expect((await request({ preview: true })).statusCode).toBe(409); + expect(providerCalls).toHaveLength(0); + expect(launches()).toHaveLength(0); +}); diff --git a/lambda/intents/test/intents.test.js b/lambda/intents/test/intents.test.js index 70eec2313..cfe053dcd 100644 --- a/lambda/intents/test/intents.test.js +++ b/lambda/intents/test/intents.test.js @@ -1,3 +1,7 @@ +import { loadMetaBeforeDelivery } from '../../v2-orchestrator/test/fixtures/load-meta-before-delivery.js'; +import { __durableHandler as runOrchestrator } from '../../v2-orchestrator/index.js'; +import { createProcessStore } from '../../shared/v2-process-store.js'; +import { unitPlanIdentity } from '../../shared/unit-completion.js'; import { beforeAll, beforeEach, afterAll, describe, it, expect, vi } from 'vitest'; import { randomUUID } from 'node:crypto'; import { readFileSync } from 'node:fs'; @@ -2970,6 +2974,51 @@ describe('POST /recompose — in-flight reshape', () => { expect(events.map((e) => e.eventType)).toContain('v2.execution.recomposed'); }); + it.each(['ownership race', 'storage failure'])( + 'stops recompose after a retirement %s', + async (failure) => { + const sub = `u-${randomUUID()}`; + const projectId = await seedV2Project(sub); + seedRecomposeFixtures(); + const intent = await seedRun(sub, projectId, { + metaOver: { orchestratorRunId: 'original-run' }, + rows: [ + { stageInstanceId: 'si-analyze', stageId: 'analyze', state: 'SUCCEEDED' }, + { stageInstanceId: 'si-optional', stageId: 'optional', state: 'WAITING_FOR_HUMAN' }, + ], + }); + const pk = `EXEC#${intent.id}`; + seedGate(intent.id, 'keep-pending', { status: 'pending' }); + if (failure === 'ownership race') { + archiveArtifactsSpy.mockImplementationOnce(async () => { + procStore.get(keyOf(pk, 'META')).orchestratorRunId = 'concurrent-run'; + return []; + }); + } else { + ddbMock + .on(UpdateCommand, { + Key: { pk, sk: 'META' }, + ExpressionAttributeValues: { ':ifOrid': 'original-run' }, + }) + .rejects(new Error('storage unavailable')); + } + const result = await recomposeReq(sub, projectId, intent.id, { + composedGrid: { analyze: 'EXECUTE', optional: 'EXECUTE', build: 'SKIP' }, + }); + expect(result.statusCode).toBe(failure === 'ownership race' ? 409 : 500); + if (failure === 'ownership race') + expect(JSON.parse(result.body).code).toBe('execution_changed'); + expect(procStore.get(keyOf(pk, 'HUMAN#keep-pending')).status).toBe('pending'); + expect(procStore.get(keyOf(pk, 'STAGE#si-optional'))).toMatchObject({ + state: 'WAITING_FOR_HUMAN', + attempt: 0, + }); + expect(procStore.get(keyOf(pk, 'META')).composedGrid).toBeFalsy(); + expect(agentcoreMock.commandCalls(StopRuntimeSessionCommand)).toHaveLength(0); + expect(orchestratorInvokes()).toHaveLength(0); + }, + ); + it('freezes the past: a stage that ran cannot flip to SKIP', async () => { const sub = `u-${randomUUID()}`; const projectId = await seedV2Project(sub); @@ -5383,6 +5432,20 @@ describe('DELETE /projects/{id}/intents/{intentId}', () => { .next(); }; + it('does not dispatch unknown nested DELETE routes as intent deletion', async () => { + const sub = `u-${randomUUID()}`; + const projectId = await seedV2Project(sub); + const intent = JSON.parse((await createIntent(sub, projectId)).body); + const result = await handler({ + httpMethod: 'DELETE', + path: `/projects/${projectId}/intents/${intent.id}/unknown`, + pathParameters: { projectId, intentId: intent.id }, + ...claims(sub), + }); + expect(result.statusCode).toBeGreaterThanOrEqual(400); + expect(procStore.get(keyOf(`EXEC#${intent.id}`, 'META'))).toBeTruthy(); + }); + it('deletes a DRAFT intent (no Neptune anchor yet) — 204 and the META row is gone', async () => { const sub = `u-${randomUUID()}`; const projectId = await seedV2Project(sub); @@ -6227,6 +6290,25 @@ describe('POST /rewind', () => { // ── WP4: the unit dimension on the API surface (docs/v2-parallel.md) ───────── describe('WP4 — unit lanes in the detail DTO', () => { + it('keeps an unpinned serial SUCCEEDED intent free of unit completion diagnostics', async () => { + const sub = `u-${randomUUID()}`; + const projectId = await seedV2Project(sub); + const intent = JSON.parse((await createIntent(sub, projectId)).body); + setStatus(intent.id, { status: 'SUCCEEDED', methodologyRelease: null, methodologyPins: null }); + const result = await handler({ + httpMethod: 'GET', + path: `/projects/${projectId}/intents/${intent.id}`, + pathParameters: { projectId, intentId: intent.id }, + ...claims(sub), + }); + expect(result.statusCode).toBe(200); + expect(JSON.parse(result.body)).toMatchObject({ + intent: { status: 'SUCCEEDED' }, + unitCompletion: null, + unitPlan: null, + units: [], + }); + }); it('surfaces unitPlan + units and lane attribution on stages/gates/events', async () => { const sub = `u-${randomUUID()}`; const projectId = await seedV2Project(sub); @@ -6413,13 +6495,235 @@ describe('WP4 — rewind expands per-unit stage instances', () => { .next(); }; - const repair = (sub, projectId, intentId) => + const repair = (sub, projectId, intentId, input = {}) => handler({ httpMethod: 'POST', path: `/projects/${projectId}/intents/${intentId}/repair`, pathParameters: { projectId, intentId }, + body: JSON.stringify(input), + ...claims(sub), + }); + + const falseSuccess = async () => { + const sub = `u-${randomUUID()}`; + const projectId = await seedV2Project(sub); + seedSectionPlan(); + const intent = JSON.parse((await createIntent(sub, projectId)).body); + await seedIntentAnchor(intent.id); + setStatus(intent.id, { + status: 'SUCCEEDED', + orchestratorRunId: 'finished-run', + branch: 'aidlc/feature', + }); + seedStageRow(intent.id, 'units-gen'); + seedStageRow(intent.id, 'cg', 'foundation', 'SUCCEEDED', 1); + seedStageRow(intent.id, 'bt'); + const pk = `EXEC#${intent.id}`; + procStore.set(keyOf(pk, 'UNITPLAN'), { + pk, + sk: 'UNITPLAN', + executionId: intent.id, + units: [ + { slug: 'foundation', dependsOn: [] }, + { slug: 'feature', dependsOn: ['foundation'] }, + ], + batches: [['foundation'], ['feature']], + walkingSkeleton: 'feature', + autonomyMode: 'autonomous', + }); + for (const [slug, state] of [ + ['foundation', 'MERGED'], + ['feature', 'BLOCKED'], + ['old-plan-unit', 'PENDING'], + ]) { + procStore.set(keyOf(pk, `UNIT#S1#${slug}`), { + pk, + sk: `UNIT#S1#${slug}`, + type: 'Unit', + executionId: intent.id, + sectionIndex: 1, + slug, + state, + branch: `aidlc/feature--${slug}`, + }); + } + await g + .addV('PullRequest') + .property('id', `pr-${intent.id}`) + .property('repository', 'owner/repo') + .property('pr_number', 533) + .property('pr_url', 'https://github.com/owner/repo/pull/533') + .property('branch', 'aidlc/feature') + .addE('HAS_PR') + .from_(gremlin.process.statics.V().has('Intent', 'id', intent.id)) + .next(); + sourceControlOperationHandler = (request) => + request.operation === 'pr-status' ? { state: 'open', sourceBranch: 'aidlc/feature' } : null; + return { sub, projectId, intent, pk }; + }; + + it.each(['rewind', 'start'])( + 'completes publication after %s without rebuilding pre-deploy multi-repo integrated units', + async (action) => { + const { sub, projectId, intent, pk } = await falseSuccess(); + const repos = ['owner/repo', 'owner/second']; + setStatus(intent.id, { status: action === 'start' ? 'FAILED' : 'SUCCEEDED', repos }); + const plan = procStore.get(keyOf(pk, 'UNITPLAN')); + plan.walkingSkeleton = 'foundation'; + procStore.get(keyOf(pk, 'UNIT#S1#feature')).state = 'MERGED'; + seedStageRow(intent.id, 'cg', 'feature', 'SUCCEEDED', 1); + const before = ['foundation', 'feature'].map((slug) => + structuredClone(procStore.get(keyOf(pk, `UNIT#S1#${slug}`))), + ); + expect(before.every((row) => !Object.hasOwn(row, 'integrationResults'))).toBe(true); + const result = await handler({ + httpMethod: 'POST', + path: `/projects/${projectId}/intents/${intent.id}/${action}`, + pathParameters: { projectId, intentId: intent.id }, + body: JSON.stringify(action === 'rewind' ? { fromStageId: 'bt' } : {}), + ...claims(sub), + }); + expect(result.statusCode).toBe(202); + const event = JSON.parse( + Buffer.from(orchestratorInvokes().at(-1).args[0].input.Payload).toString(), + ); + const doc = DynamoDBDocumentClient.from( + new (await import('@aws-sdk/client-dynamodb')).DynamoDBClient({}), + ); + const store = createProcessStore({ ddb: doc, tableName: 'v2-proc-test' }); + const runtime = vi.fn(async () => ({ ok: true, state: 'SUCCEEDED' })); + const openPr = vi.fn(async ({ repo }) => ({ + number: 42, + url: `https://example.test/${repo}/pull/42`, + })); + const ctx = { + logger: { info() {}, error() {}, debug() {} }, + // This inherited regression also covers an A-started history after B deploys. + step: async (name, fn) => + name === 'load-meta' ? loadMetaBeforeDelivery(store, intent.id) : fn(), + createCallback: async (name) => { + if (name.startsWith('await-')) { + const humanTaskId = name.slice('await-'.length); + await store.answerHumanTask({ + executionId: intent.id, + humanTaskId, + status: 'answered', + answer: { decision: 'approve' }, + answeredBy: sub, + }); + return [Promise.resolve({ answer: { decision: 'approve' } }), `cb-${name}`]; + } + return [Promise.resolve({ ok: true, state: 'SUCCEEDED' }), `cb-${name}`]; + }, + runInChildContext: async (_name, fn) => fn(ctx), + promise: { + allSettled: async (_name, promises) => Promise.allSettled(promises), + race: async (_name, promises) => Promise.race(promises), + }, + }; + const completed = await runOrchestrator(event, ctx, { + store, + loadPlan: (args) => planSpy.actual({ ddb: doc, tableName: 'v2-blocks-test', ...args }), + invokeRuntime: runtime, + stopSession: async () => ({ stopped: true }), + broadcast: async () => {}, + comparePrBranches: async () => ({ status: 'ahead' }), + openPr, + }); + expect(completed, JSON.stringify(completed)).toMatchObject({ ok: true }); + expect((await store.getExecution(intent.id)).status).toBe('SUCCEEDED'); + expect(openPr).toHaveBeenCalledTimes(2); + expect(runtime.mock.calls.filter(([p]) => p.unitSlug || p.command === 'merge-lane')).toEqual( + [], + ); + expect( + ['foundation', 'feature'].map((slug) => procStore.get(keyOf(pk, `UNIT#S1#${slug}`))), + ).toEqual(before); + doc.destroy(); + }, + ); + it.each(['ownership race', 'storage failure'])( + 'stops rewind after a retirement %s', + async (failure) => { + const { sub, projectId, intent, pk } = await falseSuccess(); + seedGate(intent.id, 'keep-pending', { status: 'pending' }); + if (failure === 'ownership race') { + archiveArtifactsSpy.mockImplementationOnce(async () => { + procStore.get(keyOf(pk, 'META')).orchestratorRunId = 'concurrent-run'; + return []; + }); + } else { + ddbMock + .on(UpdateCommand, { + Key: { pk, sk: 'META' }, + ExpressionAttributeValues: { ':ifOrid': 'finished-run' }, + }) + .rejects(new Error('storage unavailable')); + } + const result = await handler({ + httpMethod: 'POST', + path: `/projects/${projectId}/intents/${intent.id}/rewind`, + pathParameters: { projectId, intentId: intent.id }, + body: JSON.stringify({ fromStageId: 'cg' }), + ...claims(sub), + }); + expect(result.statusCode).toBe(failure === 'ownership race' ? 409 : 500); + if (failure === 'ownership race') + expect(JSON.parse(result.body).code).toBe('execution_changed'); + expect(procStore.get(keyOf(pk, 'HUMAN#keep-pending')).status).toBe('pending'); + expect(procStore.get(keyOf(pk, `STAGE#${siOf('cg', 'foundation', 1)}`))).toMatchObject({ + state: 'SUCCEEDED', + attempt: 0, + }); + expect(procStore.get(keyOf(pk, 'UNIT#S1#foundation')).state).toBe('MERGED'); + expect(agentcoreMock.commandCalls(StopRuntimeSessionCommand)).toHaveLength(0); + expect(orchestratorInvokes()).toHaveLength(0); + }, + ); + + it('ordinary rewind explicitly clears a prior omission without changing the answered gate', async () => { + const { sub, projectId, intent, pk } = await falseSuccess(); + procStore.get(keyOf(pk, 'UNIT#S1#feature')).completionSkip = { + humanTaskId: 'old-halt', + planIdentity: 'old-plan', + }; + seedGate(intent.id, 'old-halt', { status: 'answered' }); + procStore.get(keyOf(pk, 'HUMAN#old-halt')).answer = { decision: 'skip' }; + const result = await handler({ + httpMethod: 'POST', + path: `/projects/${projectId}/intents/${intent.id}/rewind`, + pathParameters: { projectId, intentId: intent.id }, + body: JSON.stringify({ fromStageId: 'cg' }), ...claims(sub), }); + expect(result.statusCode).toBe(202); + expect(procStore.get(keyOf(pk, 'UNIT#S1#feature'))).toMatchObject({ + state: 'PENDING', + completionSkip: null, + }); + expect(procStore.get(keyOf(pk, 'HUMAN#old-halt'))).toMatchObject({ + status: 'answered', + answer: { decision: 'skip' }, + }); + }); + it.each(['complete', 'partial', 'incomplete'])( + 'keeps terminal %s work outside ordinary repair', + async (state) => { + const { sub, projectId, intent, pk } = await falseSuccess(); + const unit = procStore.get(keyOf(pk, 'UNIT#S1#feature')); + if (state === 'complete') unit.state = 'MERGED'; + if (state === 'partial') + unit.completionSkip = { + humanTaskId: 'accepted', + planIdentity: unitPlanIdentity(procStore.get(keyOf(pk, 'UNITPLAN'))), + }; + const before = structuredClone([...procStore.entries()]); + for (const input of [{}, { preview: true }]) + expect((await repair(sub, projectId, intent.id, input)).statusCode).toBe(409); + expect([...procStore.entries()]).toEqual(before); + expect(orchestratorInvokes()).toHaveLength(0); + }, + ); it('repairs orphaned parallel waits while preserving merged units', async () => { const sub = `u-${randomUUID()}`; @@ -6712,6 +7016,7 @@ describe('WP4 — rewind expands per-unit stage instances', () => { status: 'RUNNING', durableExecutionArn: 'arn:aws:lambda:eu-central-1:123:durable-execution:old', }); + seedStageRow(intent.id, 'units-gen'); seedStageRow(intent.id, 'cg', 'asset-containment', 'WAITING_FOR_HUMAN', 1); procStore.set(keyOf(`EXEC#${intent.id}`, 'UNIT#S1#asset-containment'), { pk: `EXEC#${intent.id}`, diff --git a/lambda/shared/delivery-provider.js b/lambda/shared/delivery-provider.js new file mode 100644 index 000000000..c26eaf6e9 --- /dev/null +++ b/lambda/shared/delivery-provider.js @@ -0,0 +1,32 @@ +// Strict publication reads live beside the provider adapters. No credentials or +// provider response bodies cross this boundary. +import { ProviderError } from './git-providers/errors.js'; + +export const inspectDelivery = async (provider, ctx, repo, { branch, baseBranch, number }) => { + const base = baseBranch || (await provider.getDefaultBranch(ctx, repo)); + if (!base) throw new ProviderError(409, 'The delivery base branch is unknown'); + let pr = null; + if (number != null) { + pr = await provider.getPullRequestStatus(ctx, repo, number); + if (!pr) throw new ProviderError(409, 'The recorded pull request could not be verified'); + } else { + const states = + provider.id === 'bitbucket' ? ['OPEN', 'MERGED', 'DECLINED', 'SUPERSEDED'] : ['all']; + for (const state of states) { + const found = await provider.findPullRequest(ctx, repo, { + sourceBranch: branch, + targetBranch: null, + state, + strict: true, + }); + if (!found) continue; + const ref = found.number ?? found.iid ?? found.pullRequestId ?? found.id; + pr = await provider.getPullRequestStatus(ctx, repo, ref); + if (!pr) throw new ProviderError(409, 'The matching pull request could not be verified'); + break; + } + } + const headSha = await provider.getBranchHead(ctx, repo, branch); + const baseSha = await provider.getBranchHead(ctx, repo, base); + return { baseBranch: base, headSha, baseSha, pr }; +}; diff --git a/lambda/shared/final-delivery.js b/lambda/shared/final-delivery.js new file mode 100644 index 000000000..0facf36a2 --- /dev/null +++ b/lambda/shared/final-delivery.js @@ -0,0 +1,487 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { repoProvider } from './repo-provider.js'; +import { + completionMessage, + completionSections, + unitPlanIdentity, + assessExecutionCompletion, + unverifiedCompletion, +} from './unit-completion.js'; +import { openIntentPrs } from './intent-pr.js'; +import { canonicalJson } from './workflow-checkpoint.js'; + +const done = (repo) => ['succeeded', 'no_change'].includes(repo.status); +const fail = (code, message) => Object.assign(new Error(message), { code, statusCode: 409 }); +const safeError = (error) => { + const code = error?.code; + return [ + 'UNBOUND', + 'BINDING_INVALID', + 'BINDING_REQUIRED', + 'CAPABILITY_REQUIRED', + 'PROVIDER_UNAUTHORIZED', + 'SOURCE_CONTROL_NOT_READY', + 'REPOSITORY_NOT_ON_PROJECT', + 'MISSING_SCOPES', + ].includes(code) + ? 'Source control access needs attention in project settings. Check the repository binding, then retry.' + : 'The provider could not be verified. Check source control access and availability, then retry; existing pull requests will be checked first.'; +}; +export const deliveryStatus = (repositories) => { + if (!repositories.length || repositories.every((r) => r.status === 'no_change')) + return 'no_change'; + if (repositories.every(done)) return 'succeeded'; + if (repositories.some(done)) return 'partial'; + return 'failed'; +}; +export const deliveryView = (meta, delivery) => { + if (!delivery) + return { + status: 'unknown', + repositories: [], + message: + 'Delivery has not been assessed for this run. Check delivery to inspect the recorded branches and existing pull requests.', + }; + const expired = + delivery.status === 'in_progress' && Date.parse(delivery.leaseUntil) <= Date.now(); + return { + status: expired ? 'unknown' : delivery.status, + message: expired + ? 'The previous delivery attempt stopped responding. Check delivery before retrying.' + : null, + leaseUntil: delivery.leaseUntil, + repositories: (delivery.repositories ?? []).map( + ({ repoId, provider, branch, baseBranch, status, reason, retryable, pr }) => ({ + repoId, + provider, + branch, + baseBranch, + status, + reason, + retryable, + pr, + }), + ), + updatedAt: delivery.updatedAt, + }; +}; +export const currentDelivery = (records) => + (records.deliveries ?? []).find((d) => d.orchestratorRunId === records.meta?.orchestratorRunId) ?? + null; + +// Resolve the pinned workflow before offering publication of old terminal rows. +// A missing row is unfinished, including when an entire parallel section is absent. +export const assertDeliveryComplete = async (meta, records, plan) => { + if (meta.status !== 'SUCCEEDED' || !meta.orchestratorRunId) + throw fail( + 'delivery_not_complete', + 'Delivery retry requires a completed run with a recorded owner. Recover unfinished execution first.', + ); + if ( + (records.humanTasks ?? []).some( + (gate) => + gate.status === 'pending' && + (!gate.orchestratorRunId || gate.orchestratorRunId === meta.orchestratorRunId), + ) + ) + throw fail( + 'delivery_gate_pending', + 'A human gate still needs an answer. Review execution recovery before delivery.', + ); + const sections = completionSections(plan); + if (sections.length && !records.unitPlan?.units?.length) + throw fail( + 'delivery_missing_plan', + 'The current unit plan is missing. Review execution recovery before delivery.', + ); + const receipt = meta.completionValidation; + if ( + (meta.completionPolicyVersion >= 1 || receipt?.version >= 1) && + (receipt?.runId !== meta.orchestratorRunId || + !['complete', 'partial'].includes(receipt?.state) || + receipt?.unitPlanIdentity !== unitPlanIdentity(records.unitPlan) || + canonicalJson(receipt?.expectedSections) !== canonicalJson(sections)) + ) + throw fail( + 'delivery_completion_changed', + 'The plan or completion decision changed. Review completion diagnostics before delivery.', + ); + // No unit diagnostic on a legacy serial execution is not evidence of its + // historical workflow definition. Publication still requires verified scope. + const completion = + (await assessExecutionCompletion({ + records: { ...records, meta }, + loadPlan: async () => ({ valid: true, plan }), + })) ?? unverifiedCompletion('legacy_workflow_unpinned'); + if (!['complete', 'partial'].includes(completion.state)) + throw fail( + completion.state === 'unknown' ? 'delivery_unverified' : 'delivery_incomplete', + completionMessage(completion), + ); + for (const stage of plan.stages.filter((s) => s.parallelSection == null)) { + const row = records.stages.find((r) => r.stageId === stage.stageId && !r.unitSlug); + if (!row || !['SUCCEEDED', 'SKIPPED'].includes(row.state)) + throw fail( + 'delivery_stage_incomplete', + 'A workflow stage is unfinished or unverified. Review execution recovery before delivery.', + ); + } + return completion; +}; + +const targets = (meta) => { + if (!Array.isArray(meta.repos)) + throw fail( + 'delivery_repositories_unknown', + 'The captured repositories are unknown. Review the execution before delivery.', + ); + return meta.repos.map((repo) => { + const repoId = typeof repo === 'string' ? repo : repo.url; + return { + repoId, + provider: repoProvider(repoId, meta.gitProvider, meta.repoProviders), + branch: meta.branch, + baseBranch: meta.baseBranches?.[repoId] ?? meta.baseBranch ?? null, + status: 'pending', + }; + }); +}; +const callFor = (operation, meta, repo, name, args) => + operation({ + projectId: meta.projectId, + provider: repo.provider, + repo: repo.repoId, + operation: name, + args, + }); + +const inspect = async (meta, repo, operation) => { + if (done(repo)) return repo; // Publication is complete; merge lifecycle is out of scope. + try { + if (!repo.branch) + return { + ...repo, + status: 'failed', + retryable: false, + reason: 'The captured intent branch is missing. Review the execution before publishing.', + }; + const snapshot = await callFor(operation, meta, repo, 'delivery-inspect', { + branch: repo.branch, + baseBranch: repo.baseBranch, + number: repo.pr?.prNumber, + }); + const next = { + ...repo, + baseBranch: repo.baseBranch ?? snapshot.baseBranch, + headSha: repo.headSha ?? snapshot.headSha, + }; + const pr = snapshot.pr; + if (pr) { + const identity = { + repoId: repo.repoId, + provider: repo.provider, + prUrl: pr.url, + prNumber: pr.number, + branch: pr.sourceBranch, + baseBranch: pr.targetBranch, + }; + if ( + !pr.url || + pr.number == null || + pr.sourceBranch !== repo.branch || + pr.targetBranch !== next.baseBranch + ) + return { + ...next, + status: 'failed', + retryable: false, + reason: + 'The existing pull request has a different branch or target. Review it with the repository owner; retry will not retarget it.', + }; + if (pr.state !== 'open') + return { + ...next, + pr: identity, + status: 'failed', + retryable: false, + reason: `The existing pull request is ${pr.state === 'merged' ? 'merged' : 'closed'}. Review it in the provider; retry will not recreate closed work.`, + }; + if ( + !snapshot.headSha || + pr.headSha !== snapshot.headSha || + (repo.headSha && snapshot.headSha !== repo.headSha) + ) + return { + ...next, + pr: identity, + status: 'failed', + retryable: false, + reason: + 'The intent branch changed after the delivery attempt. Review the new commits before starting a new execution.', + }; + return { + ...next, + pr: identity, + status: 'succeeded', + retryable: false, + reason: 'Existing open pull request reconciled.', + }; + } + if (!snapshot.headSha || !snapshot.baseSha) + return { + ...next, + status: 'failed', + retryable: false, + reason: `The ${!snapshot.headSha ? 'intent' : 'base'} branch does not exist. Restore the captured branch before checking delivery again.`, + }; + if (repo.headSha && snapshot.headSha !== repo.headSha) + return { + ...next, + status: 'failed', + retryable: false, + reason: + 'The intent branch changed after the delivery attempt. Review the new commits before starting a new execution.', + }; + return { + ...next, + status: 'pending', + retryable: true, + reason: 'Ready to publish the captured intent branch. Existing integrated code is preserved.', + }; + } catch (error) { + return { ...repo, status: 'failed', retryable: true, reason: safeError(error) }; + } +}; + +export const previewDelivery = async ({ meta, records, plan, operation }) => { + await assertDeliveryComplete(meta, records, plan); + if (meta.deliveryLeaseUntil && Date.parse(meta.deliveryLeaseUntil) > Date.now()) + throw fail( + 'delivery_in_progress', + 'Delivery is already in progress. Refresh its status; an abandoned attempt can be checked after its lease expires.', + ); + const previous = currentDelivery(records); + const repositories = []; + for (const repo of previous?.repositories ?? targets(meta)) + repositories.push(await inspect(meta, repo, operation)); + const delivery = { + ...previous, + repositories, + status: repositories.some((r) => r.status === 'pending') + ? 'pending' + : deliveryStatus(repositories), + }; + const previewToken = createHash('sha256') + .update( + canonicalJson({ + meta, + stages: records.stages, + unitPlan: records.unitPlan, + units: records.units, + unitPrs: records.unitPrs, + humanTasks: records.humanTasks, + previous, + repositories, + }), + ) + .digest('hex'); + return { + delivery, + previewToken, + historical: !previous, + retryable: repositories.some((r) => r.retryable), + canRecord: !previous || canonicalJson(previous.repositories) !== canonicalJson(repositories), + }; +}; + +// Called by the orchestrator in one versioned durable step. Retry uses the same +// code with an already claimed attempt. No runtime/model/Git integration calls. +export const publishFinalDelivery = async ({ + store, + meta, + operation, + applicationUrl, + claimed = null, + loadPlan, +}) => { + // Re-resolve only captured methodology coordinates; this never runs stages or + // reads a mutable graph DAG. Both initial publication and retry enter here. + const verifyCompletion = async () => { + try { + const records = await store.getCompletionRecords(meta.executionId, { includeStages: true }); + const latest = records.meta; + if ( + latest?.orchestratorRunId !== meta.orchestratorRunId || + latest?.projectId !== meta.projectId + ) + throw fail('delivery_owner_changed', 'Execution ownership changed before delivery.'); + const resolved = await loadPlan(latest); + if (!resolved?.valid || !Array.isArray(resolved.plan?.stages)) + throw fail('delivery_unverified', 'The pinned workflow cannot be verified for delivery.'); + await assertDeliveryComplete( + { ...latest, completionPolicyVersion: meta.completionPolicyVersion }, + records, + resolved.plan, + ); + return records; + } catch (error) { + if (error.statusCode === 409) throw error; + throw fail( + 'delivery_unverified', + 'Completion evidence could not be read. Review completion diagnostics before delivery.', + ); + } + }; + let delivery = claimed; + if (!delivery) { + const latest = await store.getExecution(meta.executionId, { consistentRead: true }); + if (latest?.orchestratorRunId !== meta.orchestratorRunId || latest?.status !== 'SUCCEEDED') + return []; + const previous = await store.getDelivery(meta.executionId, meta.orchestratorRunId); + if (previous && ['succeeded', 'no_change'].includes(previous.status)) + return resultsFor(previous); + const completionRecords = await verifyCompletion(); + delivery = await store.claimDelivery({ + meta: completionRecords.meta, + delivery: previous ?? { repositories: targets(meta) }, + attemptId: randomUUID(), + ifUnitPlan: completionRecords.unitPlan, + }); + } + // Safe to repeat after a interrupted step: load saved repository results. A + // different/expired attempt loses before any provider mutation. + const expectedAttemptId = delivery.attemptId; + delivery = await store.getDelivery(meta.executionId, meta.orchestratorRunId); + if (delivery?.attemptId !== expectedAttemptId) return []; + if (delivery.status !== 'in_progress') return resultsFor(delivery); + await store.saveDelivery({ delivery }); + let completionRecords; + try { + completionRecords = await verifyCompletion(); + } catch (error) { + const reason = + error.statusCode === 409 + ? error.message + : 'Completion evidence could not be read. Review completion diagnostics before delivery.'; + delivery.repositories = delivery.repositories.map((repo) => + done(repo) ? repo : { ...repo, status: 'failed', retryable: false, reason }, + ); + delivery.status = deliveryStatus(delivery.repositories); + await store.saveDelivery({ delivery, finished: true }); + return [{ eventType: 'v2.pr.failed', summary: `Final publication withheld: ${reason}` }]; + } + const records = await store.getExecutionRecords(meta.executionId, { + includeOutputs: false, + consistentRead: true, + }); + for (let i = 0; i < delivery.repositories.length; i++) { + let repo = delivery.repositories[i]; + if (done(repo)) continue; + repo = await inspect(meta, repo, operation); + delivery.repositories[i] = repo; + await store.saveDelivery({ delivery }); // pin branch and remote identity BEFORE create + if (repo.status !== 'pending') continue; + try { + const cmp = await callFor(operation, meta, repo, 'compare', { + base: repo.baseBranch, + head: repo.branch, + }); + if (!['ahead', 'diverged', 'identical', 'behind'].includes(cmp?.status)) { + repo = { + ...repo, + status: 'failed', + retryable: true, + reason: + 'The branch comparison could not be verified. Check the captured branches and retry.', + }; + } else { + const [result] = await openIntentPrs({ + store: { + listEvents: async () => records.events, + listUnits: async () => records.units, + getUnitPlan: async () => records.unitPlan, + }, + meta: { + ...meta, + repos: [repo.repoId], + branch: repo.branch, + baseBranches: { [repo.repoId]: repo.baseBranch }, + }, + executionId: meta.executionId, + runId: meta.orchestratorRunId, + applicationUrl, + comparePrBranches: async () => cmp, + openPr: async ({ branch, baseBranch, title, body, attemptKey }) => { + // Provider inspection may take time. Recheck completion and atomically + // fence its plan with the active attempt immediately before creation. + completionRecords = await verifyCompletion(); + await store.saveDelivery({ delivery, ifUnitPlan: completionRecords.unitPlan }); + let providerResult; + try { + providerResult = await callFor(operation, meta, repo, 'create-pr', { + branch, + baseBranch, + title, + body, + attemptKey, + expectedHeadSha: repo.headSha, + allowRetarget: false, + }); + } catch (error) { + return { failed: true, reason: 'provider_unavailable', error: safeError(error) }; + } + if (providerResult?.prUrl && providerResult.prNumber != null) return providerResult; + if (providerResult?.skipped) return { skipped: true, reason: 'no_changes' }; + return { + failed: true, + reason: providerResult?.conflict + ? 'unmerged_branches' + : ['branch_changed', 'missing_base'].includes(providerResult?.reason) + ? providerResult.reason + : 'provider_rejected', + error: + 'Review repository access, branch protection and unintegrated task branches, then retry.', + }; + }, + }); + repo = { + ...repo, + status: result.pr + ? 'succeeded' + : result.eventType === 'v2.pr.skipped' + ? 'no_change' + : 'failed', + ...(result.pr ? { pr: result.pr } : {}), + retryable: result.eventType === 'v2.pr.failed', + reason: result.summary, + }; + } + } catch (error) { + const completionFailure = error.code?.startsWith('delivery_'); + repo = { + ...repo, + status: 'failed', + retryable: !completionFailure, + reason: completionFailure ? error.message : safeError(error), + }; + } + delivery.repositories[i] = repo; + await store.saveDelivery({ delivery }); + } + delivery.status = deliveryStatus(delivery.repositories); + await store.saveDelivery({ delivery, finished: true }); + return resultsFor(delivery); +}; +const resultsFor = (delivery) => + delivery.repositories.length === 0 + ? [{ eventType: 'v2.pr.skipped', summary: 'No repositories on this intent — no PR to open' }] + : delivery.repositories.map((repo) => ({ + eventType: + repo.status === 'succeeded' + ? 'v2.pr.opened' + : repo.status === 'no_change' + ? 'v2.pr.skipped' + : 'v2.pr.failed', + summary: `${repo.repoId}: ${repo.reason ?? repo.status}`, + ...(repo.pr && repo.status === 'succeeded' ? { pr: repo.pr } : {}), + })); diff --git a/lambda/shared/git-providers/bitbucket.js b/lambda/shared/git-providers/bitbucket.js index 52451428a..a20dcd6bd 100644 --- a/lambda/shared/git-providers/bitbucket.js +++ b/lambda/shared/git-providers/bitbucket.js @@ -719,7 +719,7 @@ const cleanupConstructionTaskBranches = async (ctx, repoId, branch) => { const findPullRequest = async ( ctx, repoId, - { sourceBranch, targetBranch = null, state = 'OPEN' }, + { sourceBranch, targetBranch = null, state = 'OPEN', strict = false }, ) => { const { workspace, repoSlug } = splitWorkspaceRepo(repoId); @@ -728,18 +728,23 @@ const findPullRequest = async ( // field explicitly (BCLOUD-23659). Without `(draft=true OR draft=false)` a // draft unit-PR is silently missed at fan-in, so keep the clause on every // lookup regardless of state. - let query = `source.branch.name="${sourceBranch}" AND state="${stateFilter}" AND (draft=true OR draft=false)`; + let query = `source.branch.name=${JSON.stringify(sourceBranch)} AND state="${stateFilter}" AND (draft=true OR draft=false)`; if (targetBranch) { - query += ` AND destination.branch.name="${targetBranch}"`; + query += ` AND destination.branch.name=${JSON.stringify(targetBranch)}`; } const res = await bbFetch( ctx, `${API_BASE}/repositories/${workspace}/${repoSlug}/pullrequests?q=${encodeURIComponent(query)}&pagelen=100`, ); - if (!res.ok) return null; + if (!res.ok) { + if (strict) throw new ProviderError(res.status, 'Pull-request lookup failed'); + return null; + } const data = await res.json(); + if (strict && (!Array.isArray(data.values) || data.next)) + throw new ProviderError(409, 'Pull-request lookup incomplete'); return Array.isArray(data.values) && data.values.length > 0 ? data.values[0] : null; }; @@ -849,6 +854,19 @@ const createPullRequest = async ( }; }; +const getBranchHead = async (ctx, repoId, branch) => { + const { workspace, repoSlug } = splitWorkspaceRepo(repoId); + const res = await bbFetch( + ctx, + `${API_BASE}/repositories/${workspace}/${repoSlug}/refs/branches/${encodeURIComponent(branch)}`, + ); + if (res.status === 404) return null; + if (!res.ok) throw new ProviderError(res.status, 'Branch lookup failed'); + const data = await res.json(); + if (!data.target?.hash) throw new ProviderError(502, 'Invalid branch response'); + return data.target.hash; +}; + const compareBranches = async (ctx, repoId, { base, head }) => { const { workspace, repoSlug } = splitWorkspaceRepo(repoId); const resolvedBase = base || (await getDefaultBranch(ctx, repoId)) || 'main'; @@ -1055,6 +1073,7 @@ export { addPRComment, getUnmergedConstructionTaskBranches, cleanupConstructionTaskBranches, + getBranchHead, findPullRequest, createPullRequest, compareBranches, @@ -1089,6 +1108,7 @@ export default { addPRComment, getUnmergedConstructionTaskBranches, cleanupConstructionTaskBranches, + getBranchHead, findPullRequest, createPullRequest, compareBranches, diff --git a/lambda/shared/git-providers/codecommit.js b/lambda/shared/git-providers/codecommit.js index 1b281daec..02889338d 100644 --- a/lambda/shared/git-providers/codecommit.js +++ b/lambda/shared/git-providers/codecommit.js @@ -477,13 +477,15 @@ const prStatusFilter = (state) => { const findPullRequest = async ( ctx, repoId, - { sourceBranch, targetBranch = null, state = 'open' }, + { sourceBranch, targetBranch = null, state = 'open', strict = false }, ) => { const { repositoryName } = parseRepo(repoId); // Without the session identity the filter would match nothing and a lookup // would silently miss the platform's own pull request: fail instead. - const authorArn = ctx?.token?.assumedRoleArn; - if (!authorArn) { + // Final-delivery reconciliation must also find work created before a + // project's credential binding changed. The API's author filter is optional. + const authorArn = strict ? undefined : ctx?.token?.assumedRoleArn; + if (!strict && !authorArn) { throw new ProviderError(500, 'CodeCommit pull-request lookup needs the session identity', { action: 'ListPullRequests', }); @@ -580,6 +582,18 @@ const INCOMPARABLE_EXCEPTIONS = new Set([ 'MaximumFileContentToLoadExceededException', ]); +const getBranchHead = async (ctx, repoId, branch) => { + const { repositoryName } = parseRepo(repoId); + const res = await callOr( + clientFor(ctx, repoId), + new GetBranchCommand({ repositoryName, branchName: shortRef(branch) }), + 'GetBranch', + 404, + null, + ); + return res?.branch?.commitId ?? null; +}; + const compareBranches = async (ctx, repoId, { base, head }) => { const resolvedBase = base || (await getDefaultBranch(ctx, repoId)) || 'main'; let options; @@ -769,7 +783,7 @@ const draftFields = (pr) => { const createPullRequest = async ( ctx, repoId, - { branch, baseBranch, title, body, draft = false, attemptKey = '' }, + { branch, baseBranch, title, body, draft = false, attemptKey = '', allowRetarget = true }, ) => { const { arn, region, repositoryName } = parseRepo(repoId); const coordinates = { region, repositoryName }; @@ -813,6 +827,7 @@ const createPullRequest = async ( }; } if (comparison.status === 'missing_base') { + if (!allowRetarget) return { failed: true, reason: 'missing_base' }; const defaultBranch = await getDefaultBranch(ctx, repoId); if (!defaultBranch || defaultBranch === resolvedBase) { return { @@ -858,6 +873,7 @@ const createPullRequest = async ( throw new ProviderError(502, 'CodeCommit CreatePullRequest returned no pull request'); } if (pr.pullRequestStatus !== 'CLOSED') break; + if (!allowRetarget) throw new ProviderError(409, 'The previous pull request is closed'); key = `${attemptKey ?? ''}\nreplaces:${pr.pullRequestId}`; } if (pr.pullRequestStatus === 'CLOSED') { @@ -1227,6 +1243,7 @@ export { addPRComment, getUnmergedConstructionTaskBranches, cleanupConstructionTaskBranches, + getBranchHead, findPullRequest, createPullRequest, compareBranches, @@ -1267,6 +1284,7 @@ export default { addPRComment, getUnmergedConstructionTaskBranches, cleanupConstructionTaskBranches, + getBranchHead, findPullRequest, createPullRequest, compareBranches, diff --git a/lambda/shared/git-providers/github.js b/lambda/shared/git-providers/github.js index 34c0c3f90..bebd8b3a2 100644 --- a/lambda/shared/git-providers/github.js +++ b/lambda/shared/git-providers/github.js @@ -634,9 +634,27 @@ const pullRequestProviderId = (pr) => pr?.node_id ?? pr?.id ?? null; const findPullRequest = async ( ctx, repoId, - { sourceBranch, targetBranch = null, state = 'open' }, + { sourceBranch, targetBranch = null, state = 'open', strict = false }, ) => { const { owner, repo } = splitOwnerRepo(repoId); + if (strict) { + for (let page = 1; page <= 10; page++) { + const res = await ghFetch( + ctx, + `${API_BASE}/repos/${owner}/${repo}/pulls?head=${owner}:${encodeURIComponent(sourceBranch)}&state=${state}&per_page=100&page=${page}`, + ); + if (!res.ok) throw new ProviderError(res.status, 'Pull-request lookup failed'); + const rows = await res.json(); + if (!Array.isArray(rows)) throw new ProviderError(502, 'Invalid pull-request lookup'); + const match = rows.find( + (pr) => + pr.head?.ref === sourceBranch && (targetBranch === null || pr.base?.ref === targetBranch), + ); + if (match) return match; + if (rows.length < 100) return null; + } + throw new ProviderError(409, 'Pull-request lookup truncated'); + } const headRes = await ghFetch( ctx, `${API_BASE}/repos/${owner}/${repo}/pulls?head=${owner}:${encodeURIComponent( @@ -680,7 +698,7 @@ const findPullRequest = async ( const createPullRequest = async ( ctx, repoId, - { branch, baseBranch, title, body, draft = false }, + { branch, baseBranch, title, body, draft = false, allowRetarget = true }, ) => { const { owner, repo } = splitOwnerRepo(repoId); @@ -745,7 +763,7 @@ const createPullRequest = async ( // base was mistyped or deleted since). Resolve the repo's real default // branch and retry once against it — the intent branch was cut from // that HEAD at clone time, so it is a reasonable merge target. - if (isBaseInvalid422(errorText)) { + if (allowRetarget && isBaseInvalid422(errorText)) { const defaultBranch = await getDefaultBranch(ctx, repoId); if (defaultBranch && defaultBranch !== resolvedBase) { res = await postPr(defaultBranch); @@ -799,6 +817,19 @@ const createPullRequest = async ( // 'missing_base' — base branch does not exist // 'unknown' — comparison unavailable (caller falls through to // the POST /pulls behavior; never blocks on this) +const getBranchHead = async (ctx, repoId, branch) => { + const { owner, repo } = splitOwnerRepo(repoId); + const res = await ghFetch( + ctx, + `${API_BASE}/repos/${owner}/${repo}/git/ref/heads/${encodeURIComponent(branch)}`, + ); + if (res.status === 404) return null; + if (!res.ok) throw new ProviderError(res.status, 'Branch lookup failed'); + const data = await res.json(); + if (!data.object?.sha) throw new ProviderError(502, 'Invalid branch response'); + return data.object.sha; +}; + const compareBranches = async (ctx, repoId, { base, head }) => { const { owner, repo } = splitOwnerRepo(repoId); const resolvedBase = base || (await getDefaultBranch(ctx, repoId)) || 'main'; @@ -962,6 +993,7 @@ export { addPRComment, getUnmergedConstructionTaskBranches, cleanupConstructionTaskBranches, + getBranchHead, findPullRequest, createPullRequest, compareBranches, @@ -1001,6 +1033,7 @@ export default { addPRComment, getUnmergedConstructionTaskBranches, cleanupConstructionTaskBranches, + getBranchHead, findPullRequest, createPullRequest, compareBranches, diff --git a/lambda/shared/git-providers/gitlab.js b/lambda/shared/git-providers/gitlab.js index b02750fbf..39ceff4fe 100644 --- a/lambda/shared/git-providers/gitlab.js +++ b/lambda/shared/git-providers/gitlab.js @@ -642,9 +642,28 @@ const cleanupConstructionTaskBranches = async (ctx, repoId, branch) => { const findPullRequest = async ( ctx, repoId, - { sourceBranch, targetBranch = null, state = 'opened' }, + { sourceBranch, targetBranch = null, state = 'opened', strict = false }, ) => { const project = encodeProject(repoId); + if (strict) { + for (let page = 1; page <= 10; page++) { + const res = await glFetch( + ctx, + `${API_BASE}/projects/${project}/merge_requests?source_branch=${encodeURIComponent(sourceBranch)}&state=${state}&per_page=100&page=${page}`, + ); + if (!res.ok) throw new ProviderError(res.status, 'Merge-request lookup failed'); + const rows = await res.json(); + if (!Array.isArray(rows)) throw new ProviderError(502, 'Invalid merge-request lookup'); + const match = rows.find( + (mr) => + mr.source_branch === sourceBranch && + (targetBranch === null || mr.target_branch === targetBranch), + ); + if (match) return match; + if (rows.length < 100) return null; + } + throw new ProviderError(409, 'Merge-request lookup truncated'); + } const res = await glFetch( ctx, `${API_BASE}/projects/${project}/merge_requests?source_branch=${encodeURIComponent( @@ -669,7 +688,7 @@ const findPullRequest = async ( const createPullRequest = async ( ctx, repoId, - { branch, baseBranch, title, body, draft = false }, + { branch, baseBranch, title, body, draft = false, allowRetarget = true }, ) => { const project = encodeProject(repoId); @@ -768,7 +787,7 @@ const createPullRequest = async ( // it is a reasonable merge target. if (text.includes('target branch')) { const defaultBranch = await getDefaultBranch(ctx, repoId); - if (defaultBranch && defaultBranch !== resolvedBase) { + if (allowRetarget && defaultBranch && defaultBranch !== resolvedBase) { res = await postMr(defaultBranch); if (res.ok) { await cleanupConstructionTaskBranches(ctx, repoId, branch); @@ -816,6 +835,18 @@ const createPullRequest = async ( // `from` lacks; an empty list means identical-or-behind — either way there is // nothing to merge, which is all the caller needs to know. // Returns { status: 'ahead'|'identical'|'missing_head'|'missing_base'|'unknown', aheadBy?, base }. +const getBranchHead = async (ctx, repoId, branch) => { + const res = await glFetch( + ctx, + `${API_BASE}/projects/${encodeProject(repoId)}/repository/branches/${encodeURIComponent(branch)}`, + ); + if (res.status === 404) return null; + if (!res.ok) throw new ProviderError(res.status, 'Branch lookup failed'); + const data = await res.json(); + if (!data.commit?.id) throw new ProviderError(502, 'Invalid branch response'); + return data.commit.id; +}; + const compareBranches = async (ctx, repoId, { base, head }) => { const project = encodeProject(repoId); const resolvedBase = base || (await getDefaultBranch(ctx, repoId)) || 'main'; @@ -1006,6 +1037,7 @@ export { addPRComment, getUnmergedConstructionTaskBranches, cleanupConstructionTaskBranches, + getBranchHead, findPullRequest, createPullRequest, compareBranches, @@ -1045,6 +1077,7 @@ export default { addPRComment, getUnmergedConstructionTaskBranches, cleanupConstructionTaskBranches, + getBranchHead, findPullRequest, createPullRequest, compareBranches, diff --git a/lambda/shared/intent-pr.js b/lambda/shared/intent-pr.js new file mode 100644 index 000000000..6296bb204 --- /dev/null +++ b/lambda/shared/intent-pr.js @@ -0,0 +1,257 @@ +import { activePlanUnits } from './unit-completion.js'; +import { repoProvider as sharedRepoProvider } from './repo-provider.js'; +import { buildIntentAttribution } from './pr-attribution.js'; +const repoProvider = (meta, repoId) => + sharedRepoProvider(repoId, meta.gitProvider, meta.repoProviders); + +// ── WP6: open the fan-in PR(s) (intent-pr strategy) ───────────────────────── +// One PR per repo from the intent branch onto the base branch. Provider outcomes +// (opened / already-open / no-changes / guard-conflict / error) become timeline +// events the caller records. The PR body links back to the intent and carries +// an HONEST unit summary: lanes that were skipped after failure are named, so +// the reviewer knows what the increment does NOT contain (the human explicitly +// chose to continue without them — the fan-in gate is the decision record, the +// PR body is its mirror). +export const openIntentPrs = async ({ + openPr, + comparePrBranches = null, + store, + meta, + executionId, + runId = null, + applicationUrl, + log, +}) => { + const repos = meta.repos ?? []; + const branch = meta.branch; + const strategy = meta.prStrategy ?? 'intent-pr'; + // Per-repo base-branch override wins; the legacy single string is the + // project-wide fallback; a repo absent from both falls through to null so + // the provider resolves that repo's ACTUAL default branch (never 'main'). + const baseFor = (repoId) => meta.baseBranches?.[repoId] ?? meta.baseBranch ?? null; + if (repos.length === 0) { + return [ + { eventType: 'v2.pr.skipped', summary: 'No repositories on this intent — no PR to open' }, + ]; + } + // Deterministic lost-work signal (2026-07 incident): did the engine record + // repo work during this run? v2.git.pushed names repos whose commits reached + // the remote; v2.git.push_failed names repos whose work did NOT become + // durable. An artifact-only run has neither — for those, a commit-less + // branch is genuinely "no changes". Best-effort: an unreadable timeline + // just means the signal is silent (never blocks the PR). + let gitEvents = []; + try { + gitEvents = ((await store.listEvents?.(executionId)) ?? []).filter( + (e) => e.eventType === 'v2.git.pushed' || e.eventType === 'v2.git.push_failed', + ); + } catch { + /* signal is best-effort */ + } + const repoGitActivity = (repoId) => ({ + pushed: gitEvents.some((e) => e.eventType === 'v2.git.pushed' && e.summary?.includes(repoId)), + pushFailed: gitEvents.some( + (e) => e.eventType === 'v2.git.push_failed' && e.summary?.includes(repoId), + ), + }); + + // Unit transparency for the PR body (best-effort — a store without unit + // rows, or a pre-WP3 run, just omits the section). + let unitLines = []; + try { + const allUnits = (await store.listUnits?.(executionId)) ?? []; + const unitPlan = await store.getUnitPlan?.(executionId); + const units = unitPlan ? activePlanUnits(unitPlan, allUnits) : allUnits; + const unmerged = units.filter((u) => u.state && u.state !== 'MERGED'); + if (units.length) { + unitLines = [ + '', + `Units: ${units.length} total, ${units.length - unmerged.length} integrated into the intent branch.`, + ...unmerged.map( + (u) => + `- ⚠️ unit \`${u.slug}\` NOT integrated (${u.state}${u.failureReason ? `: ${u.failureReason}` : ''}) — its branch is preserved`, + ), + ]; + } + } catch { + /* transparency section is best-effort */ + } + + const title = meta.title || `AI-DLC: ${branch}`; + const aidlcAttribution = buildIntentAttribution({ + applicationUrl, + projectId: meta.projectId, + intentId: meta.intentId ?? executionId, + }); + const trackerRepository = (() => { + try { + const pathname = decodeURIComponent(new URL(meta.source?.resourceUrl).pathname); + if (meta.source?.provider === 'github-issues') { + const parts = pathname.split('/').filter(Boolean); + return parts.length >= 4 && parts[2] === 'issues' ? parts.slice(0, 2).join('/') : null; + } + if (meta.source?.provider === 'gitlab-issues') { + const marker = '/-/issues/'; + const markerIndex = pathname.indexOf(marker); + return markerIndex > 0 ? pathname.slice(1, markerIndex) : null; + } + } catch { + // A missing or malformed source URL only omits the PR tracker reference. + } + return null; + })(); + const trackerReferenceFor = (provider, repoId) => { + const resourceId = String(meta.source?.resourceId || '').trim(); + if (!resourceId) return null; + const matchingTracker = + (meta.source?.provider === 'github-issues' && provider === 'github') || + (meta.source?.provider === 'gitlab-issues' && provider === 'gitlab'); + if (matchingTracker && trackerRepository) { + const issueRef = + trackerRepository.toLowerCase() === repoId.toLowerCase() + ? `#${resourceId}` + : `${trackerRepository}#${resourceId}`; + return `Closes ${issueRef}`; + } + if (meta.source?.provider === 'jira-cloud') { + return meta.source.resourceUrl + ? `Related task: [${resourceId}](${meta.source.resourceUrl})` + : `Related task: ${resourceId}`; + } + return meta.source?.resourceUrl + ? `Related issue: [#${resourceId}](${meta.source.resourceUrl})` + : null; + }; + + const results = []; + for (const repo of repos) { + const repoId = typeof repo === 'string' ? repo : repo.url; + const provider = repoProvider(meta, repoId); + try { + const trackerReference = trackerReferenceFor(provider, repoId); + const body = [ + `Automated ${provider === 'gitlab' ? 'MR' : 'PR'} created by ${aidlcAttribution} (strategy: ${strategy})`, + ...unitLines, + ...(trackerReference ? ['', trackerReference] : []), + ].join('\n'); + const activity = repoGitActivity(repoId); + // Pre-check: does the intent branch exist remotely with commits ahead of + // base? 'unknown' (comparison unavailable) falls through to the PR call + // — the pre-check adds safety, never a new way to block a good PR. + const cmp = comparePrBranches + ? await comparePrBranches({ + projectId: meta.projectId, + gitProvider: provider, + repoId, + base: baseFor(repoId), + head: branch, + }).catch((e) => ({ status: 'unknown', detail: e?.message })) + : { status: 'unknown' }; + if (cmp.status === 'missing_head') { + results.push({ + eventType: 'v2.pr.failed', + summary: `PR for ${repoId} not possible: intent branch "${branch}" does not exist on the remote — the engine never pushed it${ + activity.pushFailed + ? ' (git push failures were recorded during the run; the work may still sit on the session workspace)' + : '' + }`, + }); + continue; + } + if (cmp.status === 'identical' || cmp.status === 'behind') { + // No commits to merge. Benign ONLY for a run that never had repo work; + // a run that recorded engine git activity but ends commit-less has + // LOST WORK and must fail loudly (the 2026-07 incident: two ENOSPC'd + // commits, a clean-looking branch, and a quiet "no changes" skip). + if (activity.pushed || activity.pushFailed) { + results.push({ + eventType: 'v2.pr.failed', + summary: `PR for ${repoId} not possible: "${branch}" has no commits ahead of ${cmp.base ?? 'the base branch'} although the run recorded ${ + activity.pushFailed ? 'engine push FAILURES' : 'engine pushes' + } — likely lost work; check the run's v2.git events and the session workspace`, + }); + continue; + } + results.push({ + eventType: 'v2.pr.skipped', + summary: `No PR for ${repoId}: no changes between ${branch} and ${cmp.base ?? 'the base branch'} (no repo work was recorded this run)`, + }); + continue; + } + const res = await openPr({ + projectId: meta.projectId, + gitProvider: provider, + repoId, + branch, + baseBranch: baseFor(repoId), + title, + body, + // One creation attempt per orchestrator run. The execution id alone + // is the intent id, identical across a rewind or repair relaunch, so + // it would replay a PR a reviewer closed in the meantime (CodeCommit + // cannot reopen it). A durable replay of this step keeps its run id + // and therefore its key. + attemptKey: runId ? `${executionId}:${runId}` : executionId, + }); + if (res?.prUrl) { + results.push({ + eventType: 'v2.pr.opened', + summary: `${res.existing ? 'PR already open' : 'PR opened'} for ${repoId}: ${res.prUrl}`, + // Structured data for record-pr (the graph write happens in the + // VPC-attached runtime; the orchestrator has no Neptune access). + pr: { + repoId, + provider, + prUrl: res.prUrl, + prNumber: res.prNumber ?? null, + branch, + // The provider retargets to the repo's real default branch when the + // requested base was invalid — record the ACTUAL merge target. + baseBranch: res.retargetedBase ?? baseFor(repoId), + }, + }); + } else if (res?.skipped) { + // The provider's own "no changes" verdict (compare was unavailable). + // Same lost-work override as above: recorded git activity means this + // skip is NOT benign. + if (activity.pushed || activity.pushFailed) { + results.push({ + eventType: 'v2.pr.failed', + summary: `PR for ${repoId} reported "${res.reason ?? 'no changes'}" although the run recorded ${ + activity.pushFailed ? 'engine push FAILURES' : 'engine pushes' + } — likely lost work; check the run's v2.git events and the session workspace`, + }); + } else { + results.push({ + eventType: 'v2.pr.skipped', + summary: `No PR for ${repoId}: ${res.reason ?? 'no changes between the branches'}`, + }); + } + } else if (res?.failed) { + results.push({ + eventType: 'v2.pr.failed', + summary: `PR for ${repoId} failed (${res.reason ?? 'error'}): ${res.error ?? 'no detail'}`, + }); + } else if (res?.conflict) { + results.push({ + eventType: 'v2.pr.failed', + summary: `PR blocked for ${repoId}: ${res.error ?? 'unmerged branches'}${ + res.unmergedBranches?.length ? ` (${res.unmergedBranches.join(', ')})` : '' + }`, + }); + } else { + results.push({ + eventType: 'v2.pr.failed', + summary: `PR for ${repoId} returned an unexpected result: ${JSON.stringify(res).slice(0, 300)}`, + }); + } + } catch (e) { + log?.(`PR open failed for ${repoId}: ${e?.message}`); + results.push({ + eventType: 'v2.pr.failed', + summary: `PR open failed for ${repoId}: ${e?.message ?? String(e)}`, + }); + } + } + return results; +}; diff --git a/lambda/v2-orchestrator/pr-attribution.js b/lambda/shared/pr-attribution.js similarity index 77% rename from lambda/v2-orchestrator/pr-attribution.js rename to lambda/shared/pr-attribution.js index e52e03513..7c9e2c916 100644 --- a/lambda/v2-orchestrator/pr-attribution.js +++ b/lambda/shared/pr-attribution.js @@ -1,4 +1,4 @@ -import { buildIntentUrl } from '../shared/intent-url.js'; +import { buildIntentUrl } from './intent-url.js'; export const buildIntentAttribution = ({ applicationUrl, projectId, intentId }) => { const intentUrl = buildIntentUrl({ applicationUrl, projectId, intentId }); diff --git a/lambda/shared/test/delivery-provider.test.js b/lambda/shared/test/delivery-provider.test.js new file mode 100644 index 000000000..a24b92720 --- /dev/null +++ b/lambda/shared/test/delivery-provider.test.js @@ -0,0 +1,205 @@ +import { describe, expect, it, vi } from 'vitest'; +import { mockClient } from 'aws-sdk-client-mock'; +import { + CodeCommitClient, + GetBranchCommand, + ListPullRequestsCommand, + GetPullRequestCommand, +} from '@aws-sdk/client-codecommit'; +import { getProvider } from '../git-providers.js'; +import { inspectDelivery } from '../delivery-provider.js'; + +const fetchResponse = (data, status = 200) => ({ + ok: status < 400, + status, + json: async () => data, + text: async () => JSON.stringify(data), + headers: { get: () => null }, +}); +describe('strict delivery provider reads', () => { + it.each(['github', 'gitlab', 'bitbucket'])( + '%s never treats denied lookup as no existing PR', + async (name) => { + const provider = getProvider(name); + await expect( + inspectDelivery( + provider, + { token: 'test', fetchImpl: async () => fetchResponse({}, 403) }, + 'org/repo', + { branch: 'intent/feature', baseBranch: 'develop' }, + ), + ).rejects.toThrow(); + }, + ); + it.each(['github', 'gitlab', 'bitbucket'])( + '%s reads a branch SHA and distinguishes missing from unreadable', + async (name) => { + const provider = getProvider(name); + const data = { + github: { object: { sha: 'head-sha' } }, + gitlab: { commit: { id: 'head-sha' } }, + bitbucket: { target: { hash: 'head-sha' } }, + }[name]; + const fetchImpl = vi.fn(async () => fetchResponse(data)); + expect( + await provider.getBranchHead({ token: 'test', fetchImpl }, 'org/repo', 'intent/feature'), + ).toBe('head-sha'); + expect(fetchImpl.mock.calls[0][0]).toContain('intent%2Ffeature'); + expect( + await provider.getBranchHead( + { token: 'test', fetchImpl: async () => fetchResponse({}, 404) }, + 'org/repo', + 'intent/feature', + ), + ).toBeNull(); + await expect( + provider.getBranchHead( + { token: 'test', fetchImpl: async () => fetchResponse({}, 500) }, + 'org/repo', + 'intent/feature', + ), + ).rejects.toThrow(); + }, + ); + it.each(['github', 'gitlab'])( + '%s drains strict lookup pages and refuses a truncated response', + async (name) => { + const provider = getProvider(name); + let calls = 0; + const fetchImpl = vi.fn(async () => + fetchResponse( + ++calls === 1 + ? Array.from({ length: 100 }, () => ({ + head: { ref: 'other' }, + source_branch: 'other', + })) + : [], + ), + ); + expect( + await provider.findPullRequest({ token: 'test', fetchImpl }, 'org/repo', { + sourceBranch: 'intent', + state: 'all', + strict: true, + }), + ).toBeNull(); + expect(calls).toBe(2); + await expect( + provider.findPullRequest( + { + token: 'test', + fetchImpl: async () => fetchResponse(Array.from({ length: 100 }, () => ({}))), + }, + 'org/repo', + { sourceBranch: 'intent', state: 'all', strict: true }, + ), + ).rejects.toThrow(/truncated/); + }, + ); + it.each(['github', 'gitlab'])( + '%s retains a closed matching PR for normalized inspection', + async (name) => { + const provider = getProvider(name); + const raw = + name === 'github' + ? { + number: 7, + html_url: 'https://example.test/pr/7', + head: { ref: 'intent', sha: 'sha' }, + base: { ref: 'develop' }, + state: 'closed', + merged_at: null, + } + : { + iid: 7, + web_url: 'https://example.test/pr/7', + source_branch: 'intent', + target_branch: 'develop', + sha: 'sha', + state: 'closed', + }; + const fetchImpl = async (url) => + fetchResponse( + url.includes('/7') + ? raw + : url.includes('pulls?') || url.includes('merge_requests?') + ? [raw] + : name === 'github' + ? { object: { sha: 'sha' } } + : { commit: { id: 'sha' } }, + ); + const result = await inspectDelivery(provider, { token: 'test', fetchImpl }, 'org/repo', { + branch: 'intent', + baseBranch: 'develop', + }); + expect(result.pr).toMatchObject({ + number: 7, + state: 'closed', + sourceBranch: 'intent', + targetBranch: 'develop', + }); + }, + ); + it('Bitbucket explicitly checks terminal states and rejects incomplete negative lookup', async () => { + const provider = getProvider('bitbucket'); + const urls = []; + const fetchImpl = async (url) => { + urls.push(decodeURIComponent(url)); + return fetchResponse(url.includes('/refs/') ? { target: { hash: 'sha' } } : { values: [] }); + }; + const result = await inspectDelivery(provider, { token: 'test', fetchImpl }, 'org/repo', { + branch: 'intent', + baseBranch: 'develop', + }); + expect(result.pr).toBeNull(); + expect(urls.filter((u) => u.includes('/pullrequests?'))).toHaveLength(4); + expect(urls.join('\n')).toContain('state="MERGED"'); + await expect( + provider.findPullRequest( + { token: 'test', fetchImpl: async () => fetchResponse({ values: [], next: 'next-page' }) }, + 'org/repo', + { sourceBranch: 'intent', strict: true }, + ), + ).rejects.toThrow(/incomplete/); + }); + it('CodeCommit uses GetBranch and searches closed as well as open PRs without creating', async () => { + const mock = mockClient(CodeCommitClient); + try { + mock.on(GetBranchCommand).resolves({ branch: { commitId: 'sha' } }); + mock.on(ListPullRequestsCommand).resolves({ pullRequestIds: ['7'] }); + mock.on(GetPullRequestCommand).resolves({ + pullRequest: { + pullRequestId: '7', + pullRequestStatus: 'CLOSED', + pullRequestTargets: [ + { + repositoryName: 'repo', + sourceReference: 'refs/heads/intent', + destinationReference: 'refs/heads/develop', + sourceCommit: 'sha', + }, + ], + }, + }); + const provider = getProvider('codecommit'); + const result = await inspectDelivery( + provider, + { + token: { + assumedRoleArn: 'arn:aws:sts::123456789012:assumed-role/test/session', + credentials: { accessKeyId: 'test', secretAccessKey: 'test' }, + }, + }, + 'arn:aws:codecommit:eu-west-1:123456789012:repo', + { branch: 'intent', baseBranch: 'develop' }, + ); + expect(result.pr.state).toBe('closed'); + expect( + mock.commandCalls(ListPullRequestsCommand)[0].args[0].input.pullRequestStatus, + ).toBeUndefined(); + expect(mock.commandCalls(ListPullRequestsCommand)[0].args[0].input.authorArn).toBeUndefined(); + } finally { + mock.restore(); + } + }); +}); diff --git a/lambda/shared/test/unit-completion.test.js b/lambda/shared/test/unit-completion.test.js new file mode 100644 index 000000000..548164585 --- /dev/null +++ b/lambda/shared/test/unit-completion.test.js @@ -0,0 +1,331 @@ +import { describe, it, expect, vi } from 'vitest'; +import { + effectiveSkeleton, + skeletonProblem, + unitCompletion, + unitPlanIdentity, + completionSections, + finalizeUnitCompletion, + assessExecutionCompletion, +} from '../unit-completion.js'; + +const unitPlan = { + units: [ + { slug: 'foundation', dependsOn: [] }, + { slug: 'feature', dependsOn: ['foundation'] }, + ], + batches: [['foundation'], ['feature']], + walkingSkeleton: 'feature', +}; +const records = () => ({ + meta: { status: 'SUCCEEDED', orchestratorRunId: 'run1' }, + unitPlan, + units: [ + { slug: 'foundation', sectionIndex: 1, state: 'MERGED' }, + { slug: 'feature', sectionIndex: 1, state: 'BLOCKED' }, + { slug: 'old-plan-unit', sectionIndex: 1, state: 'PENDING' }, + ], +}); + +describe('active-plan completion', () => { + it('counts active work and detects false success regardless of the terminal META status', () => { + expect(unitCompletion(records())).toMatchObject({ + state: 'incomplete', + total: 2, + integrated: 1, + skipped: 0, + }); + }); + it('requires an exact-plan human skip receipt for partial delivery', () => { + const input = records(); + input.units[1].completionSkip = { + humanTaskId: 'halt', + planIdentity: unitPlanIdentity(unitPlan), + }; + expect(unitCompletion(input).state).toBe('partial'); + input.unitPlan = { + ...unitPlan, + units: [...unitPlan.units, { slug: 'new-work', dependsOn: [] }], + }; + expect(unitCompletion(input).state).toBe('incomplete'); + }); + it('keeps legacy accepted partial delivery distinct without trusting a prior run decision', () => { + const input = records(); + input.unitPlan = { ...unitPlan, promotedAt: '2026-10-01T00:00:00.000Z' }; + input.humanTasks = [ + { + humanTaskId: 'eg-halt-s1-r0-run1', + orchestratorRunId: 'run1', + status: 'approved', + answer: { decision: 'skip' }, + sectionIndex: 1, + prompt: 'Lane failure in section 1\n- feature: failed', + createdAt: '2026-10-02T00:00:00.000Z', + }, + ]; + expect(unitCompletion(input)).toMatchObject({ state: 'partial', legacyPartial: true }); + input.meta.orchestratorRunId = 'run2'; + expect(unitCompletion(input).state).toBe('incomplete'); + }); + it('does not apply one legacy skip to other sections or unrelated unfinished units', () => { + const input = records(); + input.humanTasks = [ + { + humanTaskId: 'eg-halt-s1-r0-run1', + orchestratorRunId: 'run1', + status: 'answered', + answer: 'skip', + sectionIndex: 1, + }, + ]; + expect(unitCompletion(input, [1]).state).toBe('unknown'); + expect(unitCompletion(input, [2])).toMatchObject({ state: 'incomplete', skipped: 0, total: 2 }); + }); + it('honors an explicit dependent omission if its per-row receipt write was interrupted', () => { + const input = records(); + input.units[0].state = 'FAILED'; + input.unitPlan = { ...unitPlan, promotedAt: '2026-10-01T00:00:00.000Z' }; + input.humanTasks = [ + { + humanTaskId: 'eg-halt-s1-r0-run1', + orchestratorRunId: 'run1', + status: 'answered', + answer: 'skip', + createdAt: '2026-10-02T00:00:00.000Z', + sectionIndex: 1, + prompt: `- foundation: failed +- skip: explicitly omit these lanes and all their unintegrated dependents from delivery`, + }, + ]; + expect(unitCompletion(input, [1])).toMatchObject({ state: 'partial', skipped: 2 }); + input.humanTasks[0].prompt = `- foundation: failed +- skip: continue without them (their work will be missing downstream)`; + expect(unitCompletion(input, [1])).toMatchObject({ state: 'unknown', skipped: 1 }); + }); + it('uses expected sections even when every row or an entire section is missing', () => { + const sections = completionSections({ + stages: [ + { stageId: 'design', parallelSection: 1 }, + { stageId: 'review' }, + { stageId: 'implement', parallelSection: 2 }, + ], + }); + expect(sections).toEqual([1, 2]); + expect(unitCompletion({ ...records(), units: [] }, sections)).toMatchObject({ + state: 'incomplete', + total: 4, + integrated: 0, + }); + const input = records(); + input.units[1].state = 'MERGED'; + expect(unitCompletion(input, sections)).toMatchObject({ + state: 'incomplete', + integrated: 2, + total: 4, + }); + expect(unitCompletion(input, [1])).toMatchObject({ state: 'complete', total: 2 }); + expect(unitCompletion(input, [])).toMatchObject({ state: 'complete', total: 0 }); + }); + it.each(['PENDING', 'BLOCKED', 'RUNNING', 'FAILED', 'PARTIALLY_MERGED'])( + 'does not resolve an active %s row', + (state) => { + const input = records(); + input.units[1].state = state; + expect(unitCompletion(input, [1])).toMatchObject({ state: 'incomplete', integrated: 1 }); + }, + ); + it('allows serial/no-unit plans but never a missing required unit plan', () => { + expect(unitCompletion({}, [])).toMatchObject({ state: 'complete', total: 0 }); + expect(unitCompletion({}, [1])).toMatchObject({ + state: 'unknown', + reason: 'unit_plan_missing', + }); + expect(unitCompletion({ ...records(), units: [] }).state).toBe('unknown'); + }); + it('checks each repository and preserves the distinction between unknown and partial integration', () => { + const input = records(); + input.meta.repos = ['one', 'two']; + input.units[1].state = 'MERGED'; + expect(unitCompletion(input, [1]).state).toBe('complete'); + for (const row of input.units) + row.integrationResults = [ + { repo: 'one', merged: true }, + { repo: 'two', merged: 'up_to_date' }, + ]; + expect(unitCompletion(input, [1]).state).toBe('complete'); + input.units[1].integrationResults[1].merged = false; + expect(unitCompletion(input, [1])).toMatchObject({ state: 'incomplete', integrated: 1 }); + input.meta.prStrategy = 'pr-per-unit'; + input.unitPrs = input.units.flatMap((row) => + ['one', 'two'].map((repository) => ({ + slug: row.slug, + sectionIndex: 1, + repository, + state: repository === 'one' ? 'MERGED' : 'UNCHANGED', + })), + ); + expect(unitCompletion(input, [1]).state).toBe('complete'); + input.unitPrs[3].state = 'PARTIALLY_MERGED'; + expect(unitCompletion(input, [1]).state).toBe('incomplete'); + }); + it.each([ + [[], 'unknown'], + [[{ repo: 'one', merged: true }], 'unknown'], + [ + [ + { repo: 'one', merged: true }, + { repo: 'two', merged: false }, + ], + 'incomplete', + ], + [null, 'unknown'], + [{}, 'unknown'], + ['invalid', 'unknown'], + [[null], 'unknown'], + [[{}], 'unknown'], + ])( + 'does not treat explicit invalid or incomplete evidence %j as a pre-deploy row', + (evidence, state) => { + const input = records(); + input.meta.repos = ['one', 'two']; + input.units[1].state = 'MERGED'; + input.units[1].integrationResults = evidence; + expect(unitCompletion(input, [1])).toMatchObject({ state, integrated: 1 }); + }, + ); + it('omits unit diagnostics only for legacy unpinned executions without unit work', async () => { + const meta = { status: 'SUCCEEDED', orchestratorRunId: 'old-run' }; + const loadPlan = async () => ({ valid: true, plan: { stages: [{ parallelSection: 1 }] } }); + expect(await assessExecutionCompletion({ records: { meta, units: [] }, loadPlan })).toBeNull(); + expect( + await assessExecutionCompletion({ records: { ...records(), meta }, loadPlan }), + ).toMatchObject({ state: 'unknown', reason: 'legacy_workflow_unpinned' }); + for (const pin of [ + { methodologyPins: { STAGE: {} } }, + { completionValidation: { runId: 'old-run', expectedSections: [1] } }, + ]) { + expect( + await assessExecutionCompletion({ + records: { meta: { ...meta, ...pin }, units: [] }, + loadPlan, + }), + ).toMatchObject({ state: 'unknown', reason: 'unit_plan_missing' }); + } + }); + it.each(['CANCELLED', 'recorded-abort'])( + 'classifies unpinned %s work as aborted before loading workflow definitions', + async (kind) => { + const input = records(); + input.meta.status = kind === 'CANCELLED' ? 'CANCELLED' : 'FAILED'; + if (kind === 'recorded-abort') { + input.meta.failure = { code: 'section_aborted' }; + input.humanTasks = [ + { orchestratorRunId: 'run1', status: 'answered', answer: { decision: 'abort' } }, + ]; + } + const before = structuredClone(input); + const loadPlan = vi.fn(async () => { + throw new Error('workflow pin unavailable'); + }); + expect(await assessExecutionCompletion({ records: input, loadPlan })).toMatchObject({ + state: 'aborted', + reason: 'execution_aborted', + integrated: 1, + total: 2, + unresolved: [expect.objectContaining({ slug: 'feature', state: 'BLOCKED' })], + }); + expect(loadPlan).not.toHaveBeenCalled(); + expect(input).toEqual(before); + }, + ); + it.each(['stale-run', 'missing-run', 'unrelated-failure', 'pending-answer'])( + 'keeps legacy evidence unknown for a %s abort decision', + async (kind) => { + const input = records(); + input.meta.status = 'FAILED'; + input.meta.failure = { + code: kind === 'unrelated-failure' ? 'stage_failed' : 'section_aborted', + }; + input.humanTasks = [ + { + orchestratorRunId: kind === 'stale-run' ? 'prior-run' : 'run1', + status: kind === 'pending-answer' ? 'pending' : 'answered', + answer: 'abort', + }, + ]; + if (kind === 'missing-run') { + delete input.meta.orchestratorRunId; + delete input.humanTasks[0].orchestratorRunId; + } + expect(await assessExecutionCompletion({ records: input, loadPlan: vi.fn() })).toMatchObject({ + state: 'unknown', + reason: 'legacy_workflow_unpinned', + }); + }, + ); + it('retains missing/truncated read semantics before cancellation and handles no-unit cancellation', async () => { + const input = records(); + input.meta.status = 'CANCELLED'; + const loadPlan = vi.fn(); + expect( + await assessExecutionCompletion({ records: { ...input, meta: null }, loadPlan }), + ).toMatchObject({ state: 'unknown', reason: 'execution_missing' }); + expect( + await assessExecutionCompletion({ records: { ...input, truncated: true }, loadPlan }), + ).toMatchObject({ state: 'unknown', reason: 'completion_read_limit' }); + expect( + await assessExecutionCompletion({ + records: { ...input, unitPlan: null, units: [] }, + loadPlan, + }), + ).toMatchObject({ state: 'aborted', total: 0, integrated: 0 }); + expect(loadPlan).not.toHaveBeenCalled(); + }); + it('does not guess section mappings for historical slug-only rows', () => { + const input = records(); + input.units[0].sectionIndex = null; + expect(unitCompletion(input, [1]).state).toBe('unknown'); + }); + it('reports a missing state as unknown without inventing an execution status', () => { + const input = records(); + delete input.units[1].state; + expect(unitCompletion(input, [1])).toMatchObject({ + state: 'unknown', + reason: 'unit_state_missing', + unresolved: [{ slug: 'feature', state: 'UNKNOWN' }], + }); + }); + it('fails closed when completion records were bounded or could not be read', async () => { + expect(unitCompletion({ ...records(), truncated: true }, [1]).state).toBe('unknown'); + await expect( + finalizeUnitCompletion({ + store: { + getExecution: async () => ({ status: 'RUNNING', orchestratorRunId: 'r' }), + getCompletionRecords: async () => { + throw new Error('unavailable'); + }, + }, + executionId: 'e', + runId: 'r', + plan: { stages: [] }, + }), + ).rejects.toThrow('unavailable'); + }); + it('counts missing lane rows as unfinished', () => { + const input = records(); + input.units = []; + expect(unitCompletion(input, [1])).toMatchObject({ + state: 'incomplete', + total: 2, + integrated: 0, + }); + }); +}); + +it('defaults only an absent decision and respects skeleton-off ceremonies', () => { + expect(effectiveSkeleton({ ...unitPlan, walkingSkeleton: null })).toBe('foundation'); + expect(effectiveSkeleton(unitPlan)).toBe('feature'); + expect(skeletonProblem(unitPlan)).toContain('depends on foundation'); + expect(skeletonProblem(unitPlan, undefined, [{ policy: { skeleton: 'off' } }])).toBeNull(); + expect(skeletonProblem({ ...unitPlan, walkingSkeleton: 'foundation' })).toBeNull(); +}); diff --git a/lambda/shared/test/v2-process.test.js b/lambda/shared/test/v2-process.test.js index b714be0d7..7614a6d4c 100644 --- a/lambda/shared/test/v2-process.test.js +++ b/lambda/shared/test/v2-process.test.js @@ -312,7 +312,8 @@ describe('stage attempt ownership transactions', () => { orchestratorRunId: 'run1', }); expect(tx.TransactItems[1].Update).toMatchObject({ - ConditionExpression: 'orchestratorRunId = :ifOrid', + ConditionExpression: + '(attribute_not_exists(deliveryLeaseUntil) OR deliveryLeaseUntil <= :deliveryNow) AND orchestratorRunId = :ifOrid', ExpressionAttributeValues: { ':ifOrid': 'run1', ':ph': 'eg-1', ':status': 'WAITING' }, }); expect( @@ -1956,7 +1957,9 @@ describe('steering store methods', () => { ifOrchestratorRunId: 'run-1', }); const input = ddb.commandCalls(UpdateCommand)[0].args[0].input; - expect(input.ConditionExpression).toBe('orchestratorRunId = :ifOrid'); + expect(input.ConditionExpression).toBe( + '(attribute_not_exists(deliveryLeaseUntil) OR deliveryLeaseUntil <= :deliveryNow) AND orchestratorRunId = :ifOrid', + ); expect(input.ExpressionAttributeValues[':ifOrid']).toBe('run-1'); }); @@ -1973,7 +1976,7 @@ describe('steering store methods', () => { }); const input = ddb.commandCalls(UpdateCommand)[0].args[0].input; expect(input.ConditionExpression).toBe( - '#status = :fromStatus AND orchestratorRunId = :ifOrid AND pendingHumanTaskId = :ifPendingHumanTaskId', + '(attribute_not_exists(deliveryLeaseUntil) OR deliveryLeaseUntil <= :deliveryNow) AND #status = :fromStatus AND orchestratorRunId = :ifOrid AND pendingHumanTaskId = :ifPendingHumanTaskId', ); expect(input.ExpressionAttributeValues).toMatchObject({ ':fromStatus': 'WAITING', diff --git a/lambda/shared/unit-completion.js b/lambda/shared/unit-completion.js new file mode 100644 index 000000000..beb294789 --- /dev/null +++ b/lambda/shared/unit-completion.js @@ -0,0 +1,377 @@ +// Execution truth, not provider PR state. MERGED means integrated into the +// intent branch. A deliberately omitted lane needs a recorded human decision. +import { createHash } from 'node:crypto'; +import { repoUrl } from './repo-provider.js'; + +export const unitPlanIdentity = (plan) => + createHash('sha256') + .update( + JSON.stringify({ + units: plan?.units ?? [], + sourceArtifactId: plan?.sourceArtifactId ?? null, + producedByStageInstanceId: plan?.producedByStageInstanceId ?? null, + }), + ) + .digest('hex'); + +export const skeletonCeremonyDisabled = (stages = []) => + stages.find((stage) => stage.policy?.skeleton)?.policy?.skeleton === 'off'; + +export const skeletonCandidates = (plan) => { + const roots = new Set( + (plan?.units ?? []).filter((unit) => !(unit.dependsOn ?? []).length).map((unit) => unit.slug), + ); + return [ + ...new Set([...(plan?.batches ?? []).flat(), ...(plan?.units ?? []).map((u) => u.slug)]), + ].filter((slug) => roots.has(slug)); +}; + +// Preserve a captured decision, including an invalid one, so approval/execution +// can explain and reject it. Only an absent decision receives a safe default. +export const effectiveSkeleton = (plan) => + plan?.walkingSkeleton ?? skeletonCandidates(plan)[0] ?? null; + +export const skeletonProblem = (plan, skeleton = effectiveSkeleton(plan), stages = []) => { + if (skeletonCeremonyDisabled(stages)) return null; + const unit = plan?.units?.find((item) => item.slug === skeleton); + if (!unit) return `Walking skeleton "${skeleton}" is not in the current unit plan.`; + if (unit.dependsOn?.length) { + return `Walking skeleton "${skeleton}" depends on ${unit.dependsOn.join(', ')}. It must run first, alone; select a dependency-free unit (${skeletonCandidates(plan).join(', ')}).`; + } + return null; +}; + +export const dependentClosure = (plan, slugs) => { + const selected = new Set(slugs); + let changed = true; + while (changed) { + changed = false; + for (const unit of plan.units ?? []) { + if (!selected.has(unit.slug) && unit.dependsOn?.some((slug) => selected.has(slug))) { + selected.add(unit.slug); + changed = true; + } + } + } + return [...selected]; +}; + +export const hasUnitSkipDecision = (unit, plan) => + unit?.state !== 'MERGED' && + Boolean(unit?.completionSkip?.humanTaskId) && + unit.completionSkip.planIdentity === unitPlanIdentity(plan); + +export const activePlanUnits = (plan, units = []) => { + const slugs = new Set((plan?.units ?? []).map((unit) => unit.slug)); + return units.filter((unit) => slugs.has(unit.slug)); +}; + +export const completionSections = (plan) => [ + ...new Set((plan?.stages ?? []).map((stage) => stage.parallelSection).filter((s) => s != null)), +]; + +export const unverifiedCompletion = (reason) => ({ + state: 'unknown', + reason, + total: 0, + integrated: 0, + skipped: 0, + unresolved: [], + legacyPartial: false, +}); + +// Legacy halt prompts listed the exact failed lanes. Their old "without them" +// wording did not explicitly authorize every blocked dependent, much less a +// different section. Missing scope or plan provenance is ambiguity, not consent. +const legacyOmission = (records, sectionIndex, slug) => { + let ambiguous = false; + for (const gate of records.humanTasks ?? []) { + if ( + !gate.orchestratorRunId || + gate.orchestratorRunId !== records.meta?.orchestratorRunId || + !/^eg-halt-s\d/.test(gate.humanTaskId ?? '') || + !['approved', 'answered'].includes(gate.status) || + (gate.answer?.decision ?? gate.answer?.choice ?? gate.answer) !== 'skip' + ) + continue; + const section = gate.sectionIndex ?? Number(gate.humanTaskId.match(/^eg-halt-s(\d+)/)?.[1]); + if (section !== sectionIndex) continue; + const named = [ + ...(gate.prompt ?? '').matchAll( + /^- ([^:\n]+): (?:failed|FAILED|BLOCKED|PENDING|RUNNING|undefined)$/gm, + ), + ].map((match) => match[1]); + const planTime = Date.parse(records.unitPlan?.promotedAt); + const gateTime = Date.parse(gate.createdAt); + if (!named.length || !Number.isFinite(planTime) || !Number.isFinite(gateTime)) { + ambiguous = true; + continue; + } + if (gateTime < planTime) continue; + if (named.includes(slug)) return 'skipped'; + if (dependentClosure(records.unitPlan, named).includes(slug)) { + // Policy 1 made the dependent scope explicit in the accepted prompt. + // Its per-row receipt loop can fail partway through; preserve the rest + // of that recorded decision without broadening the older "without them". + if ( + gate.prompt.includes( + '- skip: explicitly omit these lanes and all their unintegrated dependents from delivery', + ) + ) + return 'skipped'; + ambiguous = true; + } + } + return ambiguous ? 'unknown' : null; +}; + +const integrationProblem = (records, row) => { + const repos = (records.meta?.repos ?? []).map(repoUrl); + const prs = (records.unitPrs ?? []).filter( + (pr) => pr.sectionIndex === row.sectionIndex && (pr.unitSlug ?? pr.slug) === row.slug, + ); + if (prs.some((pr) => !['MERGED', 'UNCHANGED'].includes(pr.state))) { + return 'repository_integration_incomplete'; + } + if (records.meta?.prStrategy === 'pr-per-unit') { + if (repos.some((repo) => !prs.some((pr) => pr.repository === repo))) { + return 'repository_evidence_missing'; + } + } else if (Object.hasOwn(row, 'integrationResults')) { + // Only an absent attribute identifies a pre-deploy MERGED row. New writes + // always persist this attribute, including an empty result array. + if ( + !Array.isArray(row.integrationResults) || + row.integrationResults.some((r) => !r || typeof r.repo !== 'string' || !r.repo) + ) + return 'repository_evidence_invalid'; + if (row.integrationResults.some((r) => r.merged !== true && r.merged !== 'up_to_date')) { + return 'repository_integration_incomplete'; + } + if (repos.some((repo) => !row.integrationResults.some((r) => r.repo === repo))) { + return 'repository_evidence_missing'; + } + } + return null; +}; + +export const unitCompletion = (records, sectionIndexes) => { + if (records.truncated) return unverifiedCompletion('completion_read_limit'); + const plan = records.unitPlan; + if (!plan?.units?.length) { + return sectionIndexes?.length + ? unverifiedCompletion('unit_plan_missing') + : sectionIndexes + ? { ...unverifiedCompletion(null), state: 'complete' } + : null; + } + const rows = activePlanUnits(plan, records.units); + const sections = sectionIndexes ?? [...new Set(rows.map((unit) => unit.sectionIndex))]; + if (!sections.length && sectionIndexes === undefined) { + return unverifiedCompletion('expected_sections_unknown'); + } + if ( + sections.some((s) => s == null) || + (sections.length && rows.some((r) => r.sectionIndex == null)) + ) { + return unverifiedCompletion('legacy_section_mapping_unknown'); + } + const outcomes = sections.flatMap((sectionIndex) => + plan.units.map((unit) => { + const row = rows.find( + (item) => item.slug === unit.slug && item.sectionIndex === sectionIndex, + ); + const problem = row?.state === 'MERGED' ? integrationProblem(records, row) : null; + const skipped = hasUnitSkipDecision(row, plan); + const legacy = + !skipped && row?.state !== 'MERGED' + ? legacyOmission(records, sectionIndex, unit.slug) + : null; + return { + sectionIndex, + slug: unit.slug, + state: row?.state ?? (row ? 'UNKNOWN' : 'MISSING'), + integrated: row?.state === 'MERGED' && !problem, + skipped: skipped || legacy === 'skipped', + legacyPartial: legacy === 'skipped', + reason: + problem ?? + (legacy === 'unknown' + ? 'legacy_skip_scope_unknown' + : row + ? row.state + ? 'unit_unfinished' + : 'unit_state_missing' + : 'unit_row_missing'), + unknown: + problem === 'repository_evidence_missing' || + problem === 'repository_evidence_invalid' || + legacy === 'unknown' || + Boolean(row && !row.state), + }; + }), + ); + const unresolved = outcomes.filter((unit) => !unit.integrated && !unit.skipped); + return { + state: unresolved.length + ? unresolved.some((unit) => unit.unknown) + ? 'unknown' + : 'incomplete' + : outcomes.some((unit) => unit.skipped) + ? 'partial' + : 'complete', + total: outcomes.length, + integrated: outcomes.filter((unit) => unit.integrated).length, + skipped: outcomes.filter((unit) => unit.skipped).length, + unresolved, + legacyPartial: outcomes.some((unit) => unit.legacyPartial), + reason: unresolved.find((unit) => unit.unknown)?.reason ?? unresolved[0]?.reason ?? null, + }; +}; + +export const completionMessage = (completion) => { + const work = completion.unresolved + .slice(0, 12) + .map((unit) => `section ${unit.sectionIndex}/${unit.slug}: ${unit.reason ?? unit.state}`) + .join('; '); + return `${completion.integrated}/${completion.total} units verified integrated, ${completion.skipped} explicitly omitted. ${ + work || completion.reason || 'Completion evidence unavailable' + }${completion.unresolved.length > 12 ? '; further units in diagnostics' : ''}. Work is preserved. Review completion diagnostics. Work remains preserved; ordinary rewind reruns affected stages.`; +}; + +// Called inside one versioned durable step. The final decision and its status +// commit together. A lost step response returns that receipt without rewriting +// the historical decision. Run, META revision and promoted plan are fenced by +// the process-store transaction; this function never writes unit/stage rows. +export const finalizeUnitCompletion = async ({ + store, + executionId, + runId, + plan, + expectedUnitPlanIdentity = null, +}) => { + const saved = await store.getExecution(executionId, { consistentRead: true }); + if ( + saved?.orchestratorRunId === runId && + ['SUCCEEDED', 'FAILED'].includes(saved.status) && + saved.completionValidation?.runId === runId + ) + return saved.completionValidation; + const records = await store.getCompletionRecords(executionId); + const meta = records.meta; + if (!meta || meta.orchestratorRunId !== runId) return { state: 'retired' }; + if (['SUCCEEDED', 'FAILED'].includes(meta.status) && meta.completionValidation?.runId === runId) + return meta.completionValidation; + if (meta.status !== 'RUNNING') return { state: 'retired' }; + const sections = completionSections(plan); + const completion = + expectedUnitPlanIdentity && unitPlanIdentity(records.unitPlan) !== expectedUnitPlanIdentity + ? unverifiedCompletion('unit_plan_changed_after_section') + : unitCompletion(records, sections); + const succeeded = ['complete', 'partial'].includes(completion.state); + const validation = { + ...completion, + unresolved: completion.unresolved.slice(0, 50), + unresolvedCount: completion.unresolved.length, + version: 1, + runId, + expectedSections: sections, + unitPlanIdentity: unitPlanIdentity(records.unitPlan), + message: succeeded ? null : completionMessage(completion), + }; + try { + await store.updateExecution({ + executionId, + projectId: meta.projectId, + startedAt: meta.startedAt, + ifOrchestratorRunId: runId, + fromStatus: 'RUNNING', + ifUpdatedAt: meta.updatedAt, + ifUnitPlan: records.unitPlan, + status: succeeded ? 'SUCCEEDED' : 'FAILED', + completedAt: new Date().toISOString(), + pendingHumanTaskId: null, + failureReason: validation.message, + failure: succeeded + ? null + : { + code: + completion.state === 'unknown' ? 'completion_unverified' : 'completion_inconsistent', + message: validation.message, + }, + completionValidation: validation, + }); + return validation; + } catch (error) { + if (error?.name === 'ConditionalCheckFailedException') return { state: 'retired' }; + // The transaction may have committed before its response (or the store's + // follow-up read) was lost. Reconcile the saved decision before reporting + // failure; never overwrite a committed success because its response failed. + const current = await store + .getExecution(executionId, { consistentRead: true }) + .catch(() => null); + if ( + current?.orchestratorRunId === runId && + ['SUCCEEDED', 'FAILED'].includes(current.status) && + current.completionValidation?.runId === runId + ) + return current.completionValidation; + throw error; + } +}; + +export const assessExecutionCompletion = async ({ records, loadPlan }) => { + if (!records.meta) return unverifiedCompletion('execution_missing'); + if (records.truncated) return unverifiedCompletion('completion_read_limit'); + const capturedSections = + records.meta.completionValidation?.runId === records.meta.orchestratorRunId + ? records.meta.completionValidation?.expectedSections + : null; + const aborted = + records.meta.status === 'CANCELLED' || + (records.meta.failure?.code === 'section_aborted' && + Boolean(records.meta.orchestratorRunId) && + (records.humanTasks ?? []).some( + (gate) => + gate.orchestratorRunId === records.meta.orchestratorRunId && + ['answered', 'approved'].includes(gate.status) && + (gate.answer?.decision ?? gate.answer?.choice ?? gate.answer) === 'abort', + )); + if (aborted) { + // Cancellation is an execution decision, not proof of completed work. It + // does not require resolving the historical workflow; preserve the unit + // evidence we do have without treating unfinished units as integrated. + const completion = unitCompletion( + records, + Array.isArray(capturedSections) ? capturedSections : undefined, + ); + return { + ...(completion ?? unverifiedCompletion(null)), + state: 'aborted', + reason: 'execution_aborted', + }; + } + // Old workflow versions can still reference LATEST stage definitions. A + // fresh resolution cannot prove the sections that old execution walked. + if ( + !records.meta.methodologyRelease && + !records.meta.methodologyPins?.STAGE && + !Array.isArray(capturedSections) + ) { + // Legacy serial executions have no unit work to diagnose. This exception + // cannot hide a current receipt's expected sections or a pinned unit plan. + if (!records.unitPlan?.units?.length && !records.units?.length) return null; + return unverifiedCompletion('legacy_workflow_unpinned'); + } + let resolved; + try { + resolved = await loadPlan(records.meta); + } catch { + return unverifiedCompletion('pinned_plan_read_failed'); + } + if (!resolved?.valid || !Array.isArray(resolved.plan?.stages)) { + return unverifiedCompletion('pinned_plan_unverified'); + } + const completion = unitCompletion(records, capturedSections ?? completionSections(resolved.plan)); + return completion; +}; diff --git a/lambda/shared/v2-process-keys.js b/lambda/shared/v2-process-keys.js index a36400d01..d59df758f 100644 --- a/lambda/shared/v2-process-keys.js +++ b/lambda/shared/v2-process-keys.js @@ -1485,3 +1485,9 @@ export default { buildTrackerSyncRow, buildReceiptRow, }; + +// One publication record per run; prior run outcomes remain readable. +export const deliveryKey = (executionId, runId) => ({ + pk: executionPk(executionId), + sk: `DELIVERY#${runId}`, +}); diff --git a/lambda/shared/v2-process-store.js b/lambda/shared/v2-process-store.js index 5bb00d40b..1913e54b8 100644 --- a/lambda/shared/v2-process-store.js +++ b/lambda/shared/v2-process-store.js @@ -22,6 +22,7 @@ import { import { LOOP_BACK_LIMIT } from './stage-loopback.js'; import { META, + deliveryKey, executionMetaKey, workflowCheckpointKey, stageKey, @@ -133,6 +134,23 @@ const retryTransactionConflicts = (client) => ({ }, }); +const snapshotCondition = (row, fields) => { + if (!row) return { ConditionExpression: 'attribute_not_exists(pk)' }; + const names = {}; + const values = {}; + const conditions = fields.map((field, index) => { + names[`#s${index}`] = field; + if (row[field] === undefined) return `attribute_not_exists(#s${index})`; + values[`:s${index}`] = row[field]; + return `#s${index} = :s${index}`; + }); + return { + ConditionExpression: `attribute_exists(pk) AND ${conditions.join(' AND ')}`, + ExpressionAttributeNames: names, + ...(Object.keys(values).length ? { ExpressionAttributeValues: values } : {}), + }; +}; + const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { if (!client) throw new Error('createProcessStore requires a DynamoDB DocumentClient'); const ddb = retryTransactionConflicts(client); @@ -140,6 +158,47 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { const now = () => (clock ? clock() : new Date().toISOString()); const nextId = () => (ids ? ids() : randomUUID()); + // Used by recovery's retired owner as well as active section decisions. + // The token check and row mutation must commit together. + const updateForRun = async (executionId, runId, params, ifUnitPlan) => { + const { ReturnValues: _returnValues, ...update } = params; + await transact([ + { + ConditionCheck: { + TableName: table(), + Key: executionMetaKey(executionId), + ConditionExpression: 'orchestratorRunId = :run', + ExpressionAttributeValues: { ':run': runId }, + }, + }, + ...(ifUnitPlan === undefined + ? [] + : [ + { + ConditionCheck: { + TableName: table(), + Key: unitPlanKey(executionId), + ...snapshotCondition(ifUnitPlan, [ + 'units', + 'sourceArtifactId', + 'producedByStageInstanceId', + 'promotedAt', + ]), + }, + }, + ]), + { Update: update }, + ]); + const { Item } = await ddb.send( + new GetCommand({ + TableName: table(), + Key: params.Key, + ConsistentRead: true, + }), + ); + return Item ?? null; + }; + const transact = async (TransactItems) => { try { await ddb.send(new TransactWriteCommand({ TransactItems })); @@ -316,6 +375,122 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { return Item ?? null; }; + // Delivery owns publication only. Its lease outlives a 900s orchestrator + // invocation plus a 120s source-control call; a crashed attempt is recoverable. + const getDelivery = async (executionId, runId) => { + const { Item } = await ddb.send( + new GetCommand({ + TableName: table(), + Key: deliveryKey(executionId, runId), + ConsistentRead: true, + }), + ); + return Item ?? null; + }; + const deliveryPlanCheck = (executionId, plan) => + plan === undefined + ? [] + : [ + { + ConditionCheck: { + TableName: table(), + Key: unitPlanKey(executionId), + ...snapshotCondition(plan, [ + 'units', + 'sourceArtifactId', + 'producedByStageInstanceId', + 'promotedAt', + ]), + }, + }, + ]; + const claimDelivery = async ({ meta, delivery, attemptId, ifUnitPlan }) => { + const ts = now(); + const leaseUntil = new Date(Date.parse(ts) + 20 * 60_000).toISOString(); + const item = { + ...delivery, + ...deliveryKey(meta.executionId, meta.orchestratorRunId), + type: 'FinalDelivery', + executionId: meta.executionId, + orchestratorRunId: meta.orchestratorRunId, + attemptId, + leaseUntil, + updatedAt: ts, + status: 'in_progress', + revision: (delivery.revision ?? 0) + 1, + }; + await transact([ + { + Update: { + TableName: table(), + Key: executionMetaKey(meta.executionId), + UpdateExpression: 'SET deliveryAttemptId = :attempt, deliveryLeaseUntil = :until', + ConditionExpression: + '#status = :succeeded AND orchestratorRunId = :run AND updatedAt = :updated AND (attribute_not_exists(deliveryLeaseUntil) OR deliveryLeaseUntil <= :now)', + ExpressionAttributeNames: { '#status': 'status' }, + ExpressionAttributeValues: { + ':succeeded': 'SUCCEEDED', + ':run': meta.orchestratorRunId, + ':updated': meta.updatedAt, + ':attempt': attemptId, + ':until': leaseUntil, + ':now': ts, + }, + }, + }, + { + Put: { + TableName: table(), + Item: item, + ConditionExpression: + delivery.revision == null ? 'attribute_not_exists(pk)' : 'revision = :revision', + ...(delivery.revision == null + ? {} + : { ExpressionAttributeValues: { ':revision': delivery.revision } }), + }, + }, + ...deliveryPlanCheck(meta.executionId, ifUnitPlan), + ]); + return item; + }; + const saveDelivery = async ({ delivery, finished = false, ifUnitPlan }) => { + const ts = now(); + const item = { ...delivery, updatedAt: ts }; + await transact([ + { + Update: { + TableName: table(), + Key: executionMetaKey(delivery.executionId), + UpdateExpression: finished + ? 'REMOVE deliveryAttemptId, deliveryLeaseUntil' + : 'SET deliveryAttemptId = :attempt', + ConditionExpression: + '#status = :succeeded AND orchestratorRunId = :run AND deliveryAttemptId = :attempt AND deliveryLeaseUntil > :now', + ExpressionAttributeNames: { '#status': 'status' }, + ExpressionAttributeValues: { + ':succeeded': 'SUCCEEDED', + ':run': delivery.orchestratorRunId, + ':attempt': delivery.attemptId, + ':now': finished ? ts : new Date(Date.parse(ts) + 150_000).toISOString(), + }, + }, + }, + { + Put: { + TableName: table(), + Item: item, + ConditionExpression: 'attemptId = :attempt AND revision = :revision', + ExpressionAttributeValues: { + ':attempt': delivery.attemptId, + ':revision': delivery.revision, + }, + }, + }, + ...deliveryPlanCheck(delivery.executionId, ifUnitPlan), + ]); + return item; + }; + // Publish the latest checkpoint only while the originating orchestrator still // owns META. The condition and put must share one transaction because they // target different items in the execution partition. @@ -358,8 +533,22 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { return Item ?? null; }; - const deleteWorkflowCheckpoint = async (executionId) => { + const deleteWorkflowCheckpoint = async (executionId, { ifOrchestratorRunId } = {}) => { if (!executionId) throw new Error('deleteWorkflowCheckpoint requires executionId'); + if (ifOrchestratorRunId) { + await transact([ + { + ConditionCheck: { + TableName: table(), + Key: executionMetaKey(executionId), + ConditionExpression: 'orchestratorRunId = :run', + ExpressionAttributeValues: { ':run': ifOrchestratorRunId }, + }, + }, + { Delete: { TableName: table(), Key: workflowCheckpointKey(executionId) } }, + ]); + return null; + } const { Attributes } = await ddb.send( new DeleteCommand({ TableName: table(), @@ -381,6 +570,8 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { status, fromStatus = null, ifOrchestratorRunId = null, + ifUpdatedAt, + ifUnitPlan, ifPendingHumanTaskId, ownership = null, additionalWrites = [], @@ -397,6 +588,7 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { startedAt, completedAt, failureReason, + completionValidation, failure, startedBy, starterName, @@ -489,6 +681,10 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { sets.push('completedAt = :ca'); values[':ca'] = completedAt; } + if (completionValidation !== undefined) { + sets.push('completionValidation = :completionValidation'); + values[':completionValidation'] = completionValidation; + } if (failureReason !== undefined) { sets.push('failureReason = :fr'); values[':fr'] = failureReason; @@ -692,6 +888,12 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { }; if (Object.keys(names).length) params.ExpressionAttributeNames = names; const conditions = []; + if (status !== undefined || orchestratorRunId !== undefined) { + conditions.push( + '(attribute_not_exists(deliveryLeaseUntil) OR deliveryLeaseUntil <= :deliveryNow)', + ); + values[':deliveryNow'] = ts; + } if (fromStatus) { conditions.push('#status = :fromStatus'); params.ExpressionAttributeNames = { ...params.ExpressionAttributeNames, '#status': 'status' }; @@ -701,6 +903,10 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { conditions.push('orchestratorRunId = :ifOrid'); params.ExpressionAttributeValues[':ifOrid'] = ifOrchestratorRunId; } + if (ifUpdatedAt !== undefined) { + conditions.push('updatedAt = :ifUpdatedAt'); + params.ExpressionAttributeValues[':ifUpdatedAt'] = ifUpdatedAt; + } if (ifResumeRequiredFor !== null) { conditions.push('resumeRequired.humanTaskId = :ifRrh'); params.ExpressionAttributeValues[':ifRrh'] = ifResumeRequiredFor; @@ -732,18 +938,40 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { ...scopeActivityWrites(table(), [row], { includePlatform: true }), ); } + const planWrites = + ifUnitPlan === undefined + ? [] + : [ + { + ConditionCheck: { + TableName: table(), + Key: unitPlanKey(executionId), + ...snapshotCondition(ifUnitPlan, [ + 'units', + 'batches', + 'sourceArtifactId', + 'producedByStageInstanceId', + 'walkingSkeleton', + 'skipMatrix', + 'autonomyMode', + 'promotedAt', + 'updatedAt', + ]), + }, + }, + ]; if (ownership) { await writeStageAttempt({ executionId, ownership, - writes: [...additionalWrites, ...credentialWrites, { Update: params }], + writes: [...additionalWrites, ...planWrites, ...credentialWrites, { Update: params }], }); // Transactions cannot return ALL_NEW; read the committed row instead. return getExecution(executionId, { consistentRead: true }); } - if (additionalWrites.length) { + if (additionalWrites.length || planWrites.length) { const { ReturnValues: _returnValues, ...update } = params; - await transact([...additionalWrites, ...credentialWrites, { Update: update }]); + await transact([...additionalWrites, ...planWrites, ...credentialWrites, { Update: update }]); return getExecution(executionId, { consistentRead: true }); } if (credentialBinding !== undefined) { @@ -1355,6 +1583,7 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { humanTaskId, supersededBy = null, ownership = null, + ifOrchestratorRunId = null, }) => { const ts = now(); try { @@ -1387,6 +1616,7 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { await writeStageAttempt({ executionId, ownership, writes: [{ Update: input }] }); return getHumanTask(executionId, humanTaskId, { consistentRead: true }); } + if (ifOrchestratorRunId) return updateForRun(executionId, ifOrchestratorRunId, input); const { Attributes } = await ddb.send(new UpdateCommand(input)); return Attributes; } catch (e) { @@ -1590,6 +1820,7 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { preservePendingCodeCommitRefs = false, loopBackId = null, countsAgainstCap = true, + ifOrchestratorRunId = null, }) => { const existing = await getStage(executionId, stageInstanceId, { consistentRead: Boolean(loopBackId), @@ -1664,6 +1895,7 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { : { ReturnValues: 'ALL_NEW' }), }; if (!loopBackId) { + if (ifOrchestratorRunId) return updateForRun(executionId, ifOrchestratorRunId, stageUpdate); const { Attributes } = await ddb.send(new UpdateCommand(stageUpdate)); return Attributes; } @@ -2074,6 +2306,56 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { return items; }; + // Completion does not need outputs, model prompts, metrics or graph state. + // Bound every read, including a single unusually large execution. A limit is + // explicitly unverified; callers must never treat the prefix as a clean run. + const getCompletionRecords = async ( + executionId, + { maxRecords = 2000, includeStages = false } = {}, + ) => { + if (!Number.isInteger(maxRecords) || maxRecords < 2 || maxRecords > 10000) { + throw new Error('Completion record limit must be 2..10000'); + } + const meta = await getExecution(executionId, { consistentRead: true }); + const unitPlan = await getUnitPlan(executionId, { consistentRead: true }); + const records = { + meta, + unitPlan, + units: [], + unitPrs: [], + humanTasks: [], + stages: [], + truncated: false, + }; + let remaining = maxRecords - 2; + for (const [field, prefix] of [ + ['units', 'UNIT#'], + ['unitPrs', 'UNITPR#'], + ['humanTasks', 'HUMAN#'], + ...(includeStages ? [['stages', 'STAGE#']] : []), + ]) { + let cursor; + do { + const page = await ddb.send( + new QueryCommand({ + TableName: table(), + KeyConditionExpression: 'pk = :pk AND begins_with(sk, :prefix)', + ExpressionAttributeValues: { ':pk': executionPk(executionId), ':prefix': prefix }, + ConsistentRead: true, + Limit: Math.min(remaining + 1, 100), + ExclusiveStartKey: cursor, + }), + ); + const items = page.Items ?? []; + records[field].push(...items.slice(0, remaining)); + if (items.length > remaining) return { ...records, truncated: true }; + remaining -= items.length; + cursor = page.LastEvaluatedKey; + } while (cursor); + } + return records; + }; + const listActiveExecutions = async ({ limit = Infinity } = {}) => { const items = []; let ExclusiveStartKey; @@ -2218,11 +2500,12 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { // All unit lane rows for an execution. SK prefix 'UNIT#' (exact — a bare // 'UNIT' would also match UNITPLAN). Paginated: the orchestrator's lane // scheduler reads this — a dropped page would make lanes invisible. - const listUnits = async (executionId) => { + const listUnits = async (executionId, { consistentRead = false } = {}) => { const items = await queryAll(ddb, { TableName: table(), KeyConditionExpression: 'pk = :pk AND begins_with(sk, :p)', ExpressionAttributeValues: { ':pk': executionPk(executionId), ':p': 'UNIT#' }, + ...(consistentRead ? { ConsistentRead: true } : {}), }); return items.toSorted(bySk); }; @@ -2320,6 +2603,8 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { state, fromStates = null, fields = {}, + ifOrchestratorRunId = null, + ifUnitPlan, }) => { if (!UNIT_STATES.includes(state)) throw new Error(`invalid unit state: ${state}`); const ts = now(); @@ -2369,6 +2654,8 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { }); } try { + if (ifOrchestratorRunId) + return await updateForRun(executionId, ifOrchestratorRunId, params, ifUnitPlan); const { Attributes } = await ddb.send(new UpdateCommand(params)); return Attributes; } catch (e) { @@ -2435,6 +2722,7 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { repository, state, fromStates = null, + ifOrchestratorRunId = null, fields = {}, }) => { if (state !== undefined && !UNIT_PR_STATES.includes(state)) { @@ -2485,6 +2773,7 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { params.ExpressionAttributeValues[`:from${index}`] = value; }); } + if (ifOrchestratorRunId) return updateForRun(executionId, ifOrchestratorRunId, params); try { const { Attributes } = await ddb.send(new UpdateCommand(params)); return Attributes; @@ -2813,6 +3102,7 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { skipMatrix, walkingSkeleton, autonomyMode, + ifOrchestratorRunId = null, }) => { const sets = ['updatedAt = :ts']; const values = { ':ts': now() }; @@ -2828,16 +3118,16 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { sets.push('autonomyMode = :am'); values[':am'] = autonomyMode; } - const { Attributes } = await ddb.send( - new UpdateCommand({ - TableName: table(), - Key: unitPlanKey(executionId), - UpdateExpression: `SET ${sets.join(', ')}`, - ExpressionAttributeValues: values, - ConditionExpression: 'attribute_exists(pk)', - ReturnValues: 'ALL_NEW', - }), - ); + const params = { + TableName: table(), + Key: unitPlanKey(executionId), + UpdateExpression: `SET ${sets.join(', ')}`, + ExpressionAttributeValues: values, + ConditionExpression: 'attribute_exists(pk)', + ReturnValues: 'ALL_NEW', + }; + if (ifOrchestratorRunId) return updateForRun(executionId, ifOrchestratorRunId, params); + const { Attributes } = await ddb.send(new UpdateCommand(params)); return Attributes; }; @@ -3149,9 +3439,13 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { // seconds and outputs are the bulk of a long run's partition. '#' is 0x23; // '$' (0x24) is the next code point, so sk < 'OUTPUT#' and sk >= 'OUTPUT$' // partition the SK space exactly around the OUTPUT# prefix. - const getExecutionRecords = async (executionId, { includeOutputs = true } = {}) => { + const getExecutionRecords = async ( + executionId, + { includeOutputs = true, consistentRead = false } = {}, + ) => { const base = { TableName: table(), + ...(consistentRead ? { ConsistentRead: true } : {}), ExpressionAttributeValues: { ':pk': executionPk(executionId) }, }; const Items = includeOutputs @@ -3186,6 +3480,7 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { feedbackBatches: records.filter((r) => r.sk.startsWith('FEEDBACK#')), quorumEdits: records.filter((r) => r.sk.startsWith('QEDIT#')), composes: records.filter((r) => r.sk.startsWith('COMPOSE#')), + deliveries: records.filter((r) => r.sk.startsWith('DELIVERY#')), trackerSync: records.find((r) => r.sk === 'TRACKERSYNC') ?? null, }; }; @@ -3254,6 +3549,9 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { return { createExecution, getExecution, + getDelivery, + claimDelivery, + saveDelivery, putWorkflowCheckpoint, getWorkflowCheckpoint, deleteWorkflowCheckpoint, @@ -3294,6 +3592,7 @@ const createProcessStore = ({ ddb: client, tableName, clock, ids } = {}) => { appendOutput, getOutputs, listProjectExecutions, + getCompletionRecords, listActiveExecutions, listStaleActiveExecutions, patchExecutionConfig, diff --git a/lambda/source-control/index.js b/lambda/source-control/index.js index 1bc739fc2..3febec8db 100644 --- a/lambda/source-control/index.js +++ b/lambda/source-control/index.js @@ -1,4 +1,5 @@ import gremlin from 'gremlin'; +import { inspectDelivery } from '../shared/delivery-provider.js'; import { fromNodeProviderChain } from '@aws-sdk/credential-providers'; import { getUrlAndHeaders } from 'gremlin-aws-sigv4/lib/utils.js'; import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; @@ -42,6 +43,7 @@ const SOURCE_CONTROL_OPERATIONS = Object.freeze({ compare: 'read', 'find-pr': 'read', 'create-pr': 'write', + 'delivery-inspect': 'read', 'pr-status': 'read', 'set-pr-draft': 'write', 'reopen-pr': 'write', @@ -317,7 +319,15 @@ const callProvider = async (provider, operation, ctx, repo, args = {}) => { targetBranch: args.targetBranch, state: args.state, }); + case 'delivery-inspect': + return inspectDelivery(provider, ctx, repo, args); case 'create-pr': + if (args.expectedHeadSha) { + const head = await provider.getBranchHead(ctx, repo, args.branch); + if (head !== args.expectedHeadSha) return { failed: true, reason: 'branch_changed' }; + if (!args.baseBranch || !(await provider.getBranchHead(ctx, repo, args.baseBranch))) + return { failed: true, reason: 'missing_base' }; + } return provider.createPullRequest(ctx, repo, args); case 'pr-status': return provider.getPullRequestStatus(ctx, repo, args.number); diff --git a/lambda/v2-orchestrator/index.js b/lambda/v2-orchestrator/index.js index 695033988..c08b5341b 100644 --- a/lambda/v2-orchestrator/index.js +++ b/lambda/v2-orchestrator/index.js @@ -39,7 +39,6 @@ import { executionCredentialBinding, } from '../shared/agent-credential-service.js'; import { commandDefinition } from '../shared/agent-command-registry.js'; -import { repoProvider as sharedRepoProvider } from '../shared/repo-provider.js'; import { createProcessStore } from '../shared/v2-process-store.js'; import { eventTypeOf } from '../shared/v2-process-keys.js'; import { loadExecutionPlan } from '../shared/v2-workflow-plan.js'; @@ -65,6 +64,14 @@ import { } from '../shared/stage-loopback.js'; import { broadcastToIntentChannel } from '../shared/ws-fanout.js'; import { resolveRuntimeTarget } from '../shared/runtime-target.js'; +import { + skeletonProblem, + finalizeUnitCompletion, + completionSections, + unitCompletion, + completionMessage, + unitPlanIdentity, +} from '../shared/unit-completion.js'; import { sdkTarget, stopSession as stopSharedSession } from '../shared/runtime-session.js'; import { awaitEngineGate, @@ -75,7 +82,9 @@ import { fanoutGateAddendum, } from './section.js'; import { runQuorumEdit } from './quorum-edit.js'; -import { buildIntentAttribution } from './pr-attribution.js'; +import { openIntentPrs } from '../shared/intent-pr.js'; +import { publishFinalDelivery } from '../shared/final-delivery.js'; +import { createIntentMethodologyLoader } from '../shared/intent-methodology.js'; import { bindGateCallback, ownsAnsweredGate, unparkGate } from './gate-callback.js'; import { OVERRIDE_REASON_MAX } from '../shared/gate-answer.js'; import { intentSessionIdFor } from '../shared/runtime-session-ids.js'; @@ -213,9 +222,6 @@ const defaultSourceControlOperation = async ({ return body.result; }; -const repoProvider = (meta, repoId) => - sharedRepoProvider(repoId, meta.gitProvider, meta.repoProviders); - // Map a timeline event type to the live broadcast payload the UI routes on // (useIntentEvents → refetch on agent.workspace / agent.execution). The // orchestrator is NOT VPC-attached and reaches the connections table over the @@ -246,6 +252,17 @@ const livePayloadFor = (type, summary) => { // cross the source-control service boundary; credentials never enter this process. const defaultDeps = () => ({ store: defaultStore, + publishDelivery: (args) => + publishFinalDelivery({ + ...args, + operation: defaultSourceControlOperation, + loadPlan: createIntentMethodologyLoader({ + ddb, + s3, + tableName: BLOCKS_TABLE, + bucket: ARTIFACTS_BUCKET, + }).loadPlan, + }), loadPlan: (args) => loadExecutionPlan({ ddb, tableName: BLOCKS_TABLE(), ...args }), invokeRuntime: defaultInvokeRuntime, issueAgentCredentialGrant: (claims) => issueAgentCredentialGrant(ssm, claims), @@ -360,6 +377,22 @@ const defaultDeps = () => ({ }, }); +// The existing tracker handoff consumes the complete set of published PRs. +// Publication retry does not add another tracker/provider polling loop. +const publishTrackerDelivery = (store, meta, pullRequests) => + store.putTrackerSync({ + executionId: meta.executionId, + projectId: meta.projectId, + intentId: meta.intentId ?? meta.executionId, + source: meta.source, + pullRequests, + intentTitle: meta.title ?? null, + workflowId: meta.workflowId, + workflowVersion: meta.workflowVersion, + scope: meta.scope, + branch: meta.branch ?? null, + }); + const handler = async (event, ctx, deps = defaultDeps()) => { const { store, @@ -384,6 +417,50 @@ const handler = async (event, ctx, deps = defaultDeps()) => { if (event.action === 'quorum-edit') { return runQuorumEdit(event, ctx, deps); } + if (event.action === 'retry-delivery') { + const retryMeta = await ctx.step('delivery-load', () => + store.getExecution(executionId, { consistentRead: true }), + ); + if ( + !retryMeta || + retryMeta.projectId !== event.projectId || + retryMeta.orchestratorRunId !== event.runId + ) + return { ok: false, reason: 'retired' }; + const results = await ctx.step('delivery-publish', () => + deps.publishDelivery({ + store, + meta: retryMeta, + applicationUrl, + claimed: { attemptId: event.attemptId }, + }), + ); + const publishedPrs = results.map((result) => result.pr).filter(Boolean); + if ( + publishedPrs.length && + !results.some((result) => result.eventType === 'v2.pr.failed') && + retryMeta.source?.bindingId && + retryMeta.source?.resourceId + ) { + await ctx.step('delivery-tracker', async () => { + try { + await publishTrackerDelivery(store, retryMeta, publishedPrs); + } catch { + await store.appendEvent({ + executionId, + type: 'v2.tracker.failed', + summary: 'Tracker synchronization could not be scheduled after delivery retry', + }); + } + }); + } + await ctx.step('delivery-notify', async () => { + for (const result of results) + await store.appendEvent({ executionId, type: result.eventType, summary: result.summary }); + await broadcast(intentId, { action: 'agent.pr', summary: 'Delivery status updated' }); + }); + return { ok: true }; + } if (event.action !== 'start') { // Resume is handled out-of-band via SendDurableExecutionCallbackSuccess // against the suspended callback — there is no separate resume invocation. @@ -391,9 +468,14 @@ const handler = async (event, ctx, deps = defaultDeps()) => { return { ok: false, reason: 'not_a_start' }; } - const meta = await ctx.step('load-meta', () => - store.getExecution(executionId, { consistentRead: true }), - ); + const meta = await ctx.step('load-meta', async () => { + const loaded = await store.getExecution(executionId, { + consistentRead: true, + }); + // Version behavior in the existing first checkpoint. Old histories have no + // marker and retain their operation sequence; repair creates a new history. + return loaded ? { ...loaded, completionPolicyVersion: 1, deliveryPolicyVersion: 1 } : null; + }); if (!meta) return { ok: false, reason: 'execution_not_found' }; const runtimeTarget = resolveRuntimeTarget(meta, RUNTIME_ARN()); const stopIntentSession = (sessionId) => stopSession(sessionId, runtimeTarget); @@ -1114,6 +1196,7 @@ const handler = async (event, ctx, deps = defaultDeps()) => { publishCheckpoint, ids: { projectId, intentId, executionId }, runId: null, // stamped below once minted + completionPolicyVersion: meta.completionPolicyVersion ?? 0, intentBranch: meta.branch, cloneBase: { repos: meta.repos ?? [], @@ -1510,7 +1593,9 @@ const handler = async (event, ctx, deps = defaultDeps()) => { sectionIndex: fanoutSection.index, unitPlan: unitPlanForGate, sectionStages: fanoutSection.stages, - skeleton: defaultSkeletonFor(unitPlanForGate), + skeleton: defaultSkeletonFor(unitPlanForGate, { + legacy: !meta.completionPolicyVersion, + }), }), ] : []), @@ -1883,12 +1968,26 @@ const handler = async (event, ctx, deps = defaultDeps()) => { }); const effective = { walkingSkeleton: - overrides.walkingSkeleton ?? defaultSkeletonFor(unitPlanForGate), + overrides.walkingSkeleton ?? + defaultSkeletonFor(unitPlanForGate, { + legacy: !meta.completionPolicyVersion, + }), skipMatrix: overrides.skipMatrix ?? unitPlanForGate.skipMatrix ?? {}, }; + const problem = skeletonProblem( + unitPlanForGate, + effective.walkingSkeleton, + fanoutSection.stages, + ); + if (meta.completionPolicyVersion && problem) return { ...effective, problem }; try { - await store.updateUnitPlanDecisions({ executionId, ...effective }); - } catch { + await store.updateUnitPlanDecisions({ + executionId, + ...effective, + ...(meta.completionPolicyVersion ? { ifOrchestratorRunId: runId } : {}), + }); + } catch (error) { + if (meta.completionPolicyVersion) throw error; /* promotion defaults stand when the patch write is lost */ } if (overrides.rejected.length) { @@ -1906,6 +2005,8 @@ const handler = async (event, ctx, deps = defaultDeps()) => { return effective; }, ); + if (decisions.problem) + return await fail('invalid_walking_skeleton', decisions.problem); await emitEvent( ctx, `fanout-approved-${stage.stageId}-${round}`, @@ -2078,24 +2179,64 @@ const handler = async (event, ctx, deps = defaultDeps()) => { // Terminal success — CAS on the ownership token: a retired run (relaunched // under a new runId while this one was still unwinding) exits quietly. The // verdict is the step result so a durable replay sees it too. - const ownedFinish = await ctx.step('finish-succeeded', async () => { + let ownedFinish; + if (meta.completionPolicyVersion >= 1) { + let completion; try { - await store.updateExecution({ - executionId, - projectId, - status: 'SUCCEEDED', - startedAt: meta.startedAt, - completedAt: nowIso(), - failureReason: null, - failure: null, - ...(runId ? { ifOrchestratorRunId: runId } : {}), - }); - return true; - } catch (e) { - if (e?.name === 'ConditionalCheckFailedException') return false; - throw e; + completion = await ctx.step('finish-completion-v1', () => + finalizeUnitCompletion({ + store, + executionId, + runId, + plan: planResult.plan, + expectedUnitPlanIdentity: sectionToolkit.completionUnitPlan + ? unitPlanIdentity(sectionToolkit.completionUnitPlan) + : null, + }), + ); + } catch { + return fail( + 'completion_unverified', + 'Completion state could not be read or committed. Work is preserved; review diagnostics before retrying.', + null, + { fromStatus: 'RUNNING' }, + ); } - }); + if (completion.state === 'retired') return { ok: false, reason: 'retired', intentId }; + if (!['complete', 'partial'].includes(completion.state)) { + await emitEvent( + ctx, + 'completion-failed-event-v2', + 'v2.execution.failed', + completion.message, + ); + return { + ok: false, + reason: + completion.state === 'unknown' ? 'completion_unverified' : 'completion_inconsistent', + intentId, + }; + } + ownedFinish = true; + } else + ownedFinish = await ctx.step('finish-succeeded', async () => { + try { + await store.updateExecution({ + executionId, + projectId, + status: 'SUCCEEDED', + startedAt: meta.startedAt, + completedAt: nowIso(), + failureReason: null, + failure: null, + ...(runId ? { ifOrchestratorRunId: runId } : {}), + }); + return true; + } catch (e) { + if (e?.name === 'ConditionalCheckFailedException') return false; + throw e; + } + }); if (!ownedFinish) { logger.info('retired run skipped terminal success write'); return { ok: false, reason: 'retired', intentId }; @@ -2107,9 +2248,64 @@ const handler = async (event, ctx, deps = defaultDeps()) => { // intent branch onto the base branch, via the source-control service (the // v1 unmerged-branch guard lives inside createPullRequest). PR problems // never un-succeed the run — every outcome is a loud timeline event. A - const prResults = await ctx.step('open-pr', async () => - openIntentPrs({ - openPr, + const prResults = await ctx.step('open-pr', async () => { + if (meta.deliveryPolicyVersion) { + try { + return await deps.publishDelivery({ + store, + meta: { ...meta, orchestratorRunId: runId }, + applicationUrl, + }); + } catch { + // Store/provider uncertainty is delivery uncertainty, never a stage failure. + return [ + { + eventType: 'v2.pr.failed', + summary: + 'Delivery could not be verified. Check delivery to reconcile provider state before retrying.', + }, + ]; + } + } + + const verifyCompletion = async () => { + const records = await store.getCompletionRecords(executionId); + const receipt = records.meta?.completionValidation; + if ( + records.meta?.orchestratorRunId !== runId || + records.meta?.status !== 'SUCCEEDED' || + receipt?.runId !== runId || + !['complete', 'partial'].includes(receipt?.state) || + receipt?.unitPlanIdentity !== unitPlanIdentity(records.unitPlan) + ) + throw new Error( + 'Execution ownership, plan or completion decision changed before publication', + ); + const completion = unitCompletion(records, completionSections(planResult.plan)); + if (!['complete', 'partial'].includes(completion.state)) + throw new Error(completionMessage(completion)); + }; + if (meta.completionPolicyVersion >= 1) { + try { + await verifyCompletion(); + } catch { + return [ + { + eventType: 'v2.pr.failed', + summary: + 'Final publication withheld: completion evidence is incomplete or unavailable. Work is preserved; review completion diagnostics.', + }, + ]; + } + } + return openIntentPrs({ + openPr: + meta.completionPolicyVersion >= 1 + ? async (input) => { + await verifyCompletion(); + return openPr(input); + } + : openPr, comparePrBranches, store, meta, @@ -2117,8 +2313,8 @@ const handler = async (event, ctx, deps = defaultDeps()) => { runId, applicationUrl, log: (m) => logger.info(m), - }), - ); + }); + }); for (let i = 0; i < prResults.length; i++) { const r = prResults[i]; await emitEvent(ctx, `pr-event-${i}`, r.eventType, r.summary); @@ -2128,21 +2324,15 @@ const handler = async (event, ctx, deps = defaultDeps()) => { // un-succeeds a run whose PR already exists on the remote. const openedPrs = prResults.map((r) => r.pr).filter(Boolean); if (openedPrs.length > 0) { - if (meta.source?.bindingId && meta.source?.resourceId) { + if ( + meta.source?.bindingId && + meta.source?.resourceId && + (!meta.deliveryPolicyVersion || + !prResults.some((result) => result.eventType === 'v2.pr.failed')) + ) { const published = await ctx.step('publish-tracker-sync', async () => { try { - await store.putTrackerSync({ - executionId, - projectId, - intentId, - source: meta.source, - pullRequests: openedPrs, - intentTitle: meta.title ?? null, - workflowId, - workflowVersion, - scope, - branch: meta.branch ?? null, - }); + await publishTrackerDelivery(store, meta, openedPrs); return true; } catch (error) { logger.error('tracker sync publication failed', error); @@ -2535,256 +2725,6 @@ const validationPrompt = ({ ].join('\n'); }; -// ── WP6: open the fan-in PR(s) (intent-pr strategy) ───────────────────────── -// One PR per repo from the intent branch onto the base branch. Provider outcomes -// (opened / already-open / no-changes / guard-conflict / error) become timeline -// events the caller records. The PR body links back to the intent and carries -// an HONEST unit summary: lanes that were skipped after failure are named, so -// the reviewer knows what the increment does NOT contain (the human explicitly -// chose to continue without them — the fan-in gate is the decision record, the -// PR body is its mirror). -const openIntentPrs = async ({ - openPr, - comparePrBranches = null, - store, - meta, - executionId, - runId = null, - applicationUrl, - log, -}) => { - const repos = meta.repos ?? []; - const branch = meta.branch; - const strategy = meta.prStrategy ?? 'intent-pr'; - // Per-repo base-branch override wins; the legacy single string is the - // project-wide fallback; a repo absent from both falls through to null so - // the provider resolves that repo's ACTUAL default branch (never 'main'). - const baseFor = (repoId) => meta.baseBranches?.[repoId] ?? meta.baseBranch ?? null; - if (repos.length === 0) { - return [ - { eventType: 'v2.pr.skipped', summary: 'No repositories on this intent — no PR to open' }, - ]; - } - // Deterministic lost-work signal (2026-07 incident): did the engine record - // repo work during this run? v2.git.pushed names repos whose commits reached - // the remote; v2.git.push_failed names repos whose work did NOT become - // durable. An artifact-only run has neither — for those, a commit-less - // branch is genuinely "no changes". Best-effort: an unreadable timeline - // just means the signal is silent (never blocks the PR). - let gitEvents = []; - try { - gitEvents = ((await store.listEvents?.(executionId)) ?? []).filter( - (e) => e.eventType === 'v2.git.pushed' || e.eventType === 'v2.git.push_failed', - ); - } catch { - /* signal is best-effort */ - } - const repoGitActivity = (repoId) => ({ - pushed: gitEvents.some((e) => e.eventType === 'v2.git.pushed' && e.summary?.includes(repoId)), - pushFailed: gitEvents.some( - (e) => e.eventType === 'v2.git.push_failed' && e.summary?.includes(repoId), - ), - }); - - // Unit transparency for the PR body (best-effort — a store without unit - // rows, or a pre-WP3 run, just omits the section). - let unitLines = []; - try { - const units = (await store.listUnits?.(executionId)) ?? []; - const unmerged = units.filter((u) => u.state && u.state !== 'MERGED'); - if (units.length) { - unitLines = [ - '', - `Units: ${units.length} total, ${units.length - unmerged.length} merged.`, - ...unmerged.map( - (u) => - `- ⚠️ unit \`${u.slug}\` NOT merged (${u.state}${u.failureReason ? `: ${u.failureReason}` : ''}) — its branch is preserved`, - ), - ]; - } - } catch { - /* transparency section is best-effort */ - } - - const title = meta.title || `AI-DLC: ${branch}`; - const aidlcAttribution = buildIntentAttribution({ - applicationUrl, - projectId: meta.projectId, - intentId: meta.intentId ?? executionId, - }); - const trackerRepository = (() => { - try { - const pathname = decodeURIComponent(new URL(meta.source?.resourceUrl).pathname); - if (meta.source?.provider === 'github-issues') { - const parts = pathname.split('/').filter(Boolean); - return parts.length >= 4 && parts[2] === 'issues' ? parts.slice(0, 2).join('/') : null; - } - if (meta.source?.provider === 'gitlab-issues') { - const marker = '/-/issues/'; - const markerIndex = pathname.indexOf(marker); - return markerIndex > 0 ? pathname.slice(1, markerIndex) : null; - } - } catch { - // A missing or malformed source URL only omits the PR tracker reference. - } - return null; - })(); - const trackerReferenceFor = (provider, repoId) => { - const resourceId = String(meta.source?.resourceId || '').trim(); - if (!resourceId) return null; - const matchingTracker = - (meta.source?.provider === 'github-issues' && provider === 'github') || - (meta.source?.provider === 'gitlab-issues' && provider === 'gitlab'); - if (matchingTracker && trackerRepository) { - const issueRef = - trackerRepository.toLowerCase() === repoId.toLowerCase() - ? `#${resourceId}` - : `${trackerRepository}#${resourceId}`; - return `Closes ${issueRef}`; - } - if (meta.source?.provider === 'jira-cloud') { - return meta.source.resourceUrl - ? `Related task: [${resourceId}](${meta.source.resourceUrl})` - : `Related task: ${resourceId}`; - } - return meta.source?.resourceUrl - ? `Related issue: [#${resourceId}](${meta.source.resourceUrl})` - : null; - }; - - const results = []; - for (const repo of repos) { - const repoId = typeof repo === 'string' ? repo : repo.url; - const provider = repoProvider(meta, repoId); - try { - const trackerReference = trackerReferenceFor(provider, repoId); - const body = [ - `Automated ${provider === 'gitlab' ? 'MR' : 'PR'} created by ${aidlcAttribution} (strategy: ${strategy})`, - ...unitLines, - ...(trackerReference ? ['', trackerReference] : []), - ].join('\n'); - const activity = repoGitActivity(repoId); - // Pre-check: does the intent branch exist remotely with commits ahead of - // base? 'unknown' (comparison unavailable) falls through to the PR call - // — the pre-check adds safety, never a new way to block a good PR. - const cmp = comparePrBranches - ? await comparePrBranches({ - projectId: meta.projectId, - gitProvider: provider, - repoId, - base: baseFor(repoId), - head: branch, - }).catch((e) => ({ status: 'unknown', detail: e?.message })) - : { status: 'unknown' }; - if (cmp.status === 'missing_head') { - results.push({ - eventType: 'v2.pr.failed', - summary: `PR for ${repoId} not possible: intent branch "${branch}" does not exist on the remote — the engine never pushed it${ - activity.pushFailed - ? ' (git push failures were recorded during the run; the work may still sit on the session workspace)' - : '' - }`, - }); - continue; - } - if (cmp.status === 'identical' || cmp.status === 'behind') { - // No commits to merge. Benign ONLY for a run that never had repo work; - // a run that recorded engine git activity but ends commit-less has - // LOST WORK and must fail loudly (the 2026-07 incident: two ENOSPC'd - // commits, a clean-looking branch, and a quiet "no changes" skip). - if (activity.pushed || activity.pushFailed) { - results.push({ - eventType: 'v2.pr.failed', - summary: `PR for ${repoId} not possible: "${branch}" has no commits ahead of ${cmp.base ?? 'the base branch'} although the run recorded ${ - activity.pushFailed ? 'engine push FAILURES' : 'engine pushes' - } — likely lost work; check the run's v2.git events and the session workspace`, - }); - continue; - } - results.push({ - eventType: 'v2.pr.skipped', - summary: `No PR for ${repoId}: no changes between ${branch} and ${cmp.base ?? 'the base branch'} (no repo work was recorded this run)`, - }); - continue; - } - const res = await openPr({ - projectId: meta.projectId, - gitProvider: provider, - repoId, - branch, - baseBranch: baseFor(repoId), - title, - body, - // One creation attempt per orchestrator run. The execution id alone - // is the intent id, identical across a rewind or repair relaunch, so - // it would replay a PR a reviewer closed in the meantime (CodeCommit - // cannot reopen it). A durable replay of this step keeps its run id - // and therefore its key. - attemptKey: runId ? `${executionId}:${runId}` : executionId, - }); - if (res?.prUrl) { - results.push({ - eventType: 'v2.pr.opened', - summary: `${res.existing ? 'PR already open' : 'PR opened'} for ${repoId}: ${res.prUrl}`, - // Structured data for record-pr (the graph write happens in the - // VPC-attached runtime; the orchestrator has no Neptune access). - pr: { - repoId, - provider, - prUrl: res.prUrl, - prNumber: res.prNumber ?? null, - branch, - // The provider retargets to the repo's real default branch when the - // requested base was invalid — record the ACTUAL merge target. - baseBranch: res.retargetedBase ?? baseFor(repoId), - }, - }); - } else if (res?.skipped) { - // The provider's own "no changes" verdict (compare was unavailable). - // Same lost-work override as above: recorded git activity means this - // skip is NOT benign. - if (activity.pushed || activity.pushFailed) { - results.push({ - eventType: 'v2.pr.failed', - summary: `PR for ${repoId} reported "${res.reason ?? 'no changes'}" although the run recorded ${ - activity.pushFailed ? 'engine push FAILURES' : 'engine pushes' - } — likely lost work; check the run's v2.git events and the session workspace`, - }); - } else { - results.push({ - eventType: 'v2.pr.skipped', - summary: `No PR for ${repoId}: ${res.reason ?? 'no changes between the branches'}`, - }); - } - } else if (res?.failed) { - results.push({ - eventType: 'v2.pr.failed', - summary: `PR for ${repoId} failed (${res.reason ?? 'error'}): ${res.error ?? 'no detail'}`, - }); - } else if (res?.conflict) { - results.push({ - eventType: 'v2.pr.failed', - summary: `PR blocked for ${repoId}: ${res.error ?? 'unmerged branches'}${ - res.unmergedBranches?.length ? ` (${res.unmergedBranches.join(', ')})` : '' - }`, - }); - } else { - results.push({ - eventType: 'v2.pr.failed', - summary: `PR for ${repoId} returned an unexpected result: ${JSON.stringify(res).slice(0, 300)}`, - }); - } - } catch (e) { - log?.(`PR open failed for ${repoId}: ${e?.message}`); - results.push({ - eventType: 'v2.pr.failed', - summary: `PR open failed for ${repoId}: ${e?.message ?? String(e)}`, - }); - } - } - return results; -}; - export const lambdaHandler = withDurableExecution(handler); // Exported for unit tests that drive the control flow with a fake DurableContext. export const __durableHandler = handler; diff --git a/lambda/v2-orchestrator/section.js b/lambda/v2-orchestrator/section.js index f7bea1ff8..e67a30702 100644 --- a/lambda/v2-orchestrator/section.js +++ b/lambda/v2-orchestrator/section.js @@ -50,8 +50,18 @@ // wakes: a superseded gate (cancel/rewind) retires the run with NO writes. import processKeysPkg from '../shared/v2-process-keys.js'; +import { + dependentClosure, + effectiveSkeleton, + hasUnitSkipDecision, + skeletonCeremonyDisabled, + skeletonProblem, + unitPlanIdentity, + unitCompletion, + completionMessage, +} from '../shared/unit-completion.js'; import { stageIsNoopForUnit } from '../shared/unit-kind-pruning.js'; -import { buildIntentAttribution } from './pr-attribution.js'; +import { buildIntentAttribution } from '../shared/pr-attribution.js'; import { bindGateCallback, unparkGate } from './gate-callback.js'; import { assertPrStrategySupported } from '../shared/pr-strategy.js'; import { repoProvider as sharedRepoProvider } from '../shared/repo-provider.js'; @@ -375,14 +385,14 @@ export const validateFanoutOverrides = (answer, { slugs, sectionStages, bySlug } // it is (still) a known unit, else the first slug of the first topological // wave. Shared by the fan-out approval (index.js) and the section runner so // the gate presents exactly what the section would run. -export const defaultSkeletonFor = (unitPlan) => { - const bySlug = new Map((unitPlan?.units ?? []).map((u) => [u.slug, u])); - const waves = ( - unitPlan?.batches?.length ? unitPlan.batches : [(unitPlan?.units ?? []).map((u) => u.slug)] - ).map((w) => w.filter((slug) => bySlug.has(slug))); - return unitPlan?.walkingSkeleton && bySlug.has(unitPlan.walkingSkeleton) +export const defaultSkeletonFor = (unitPlan, { legacy = false } = {}) => { + if (!legacy) return effectiveSkeleton(unitPlan); + const slugs = new Set((unitPlan?.units ?? []).map((unit) => unit.slug)); + return slugs.has(unitPlan?.walkingSkeleton) ? unitPlan.walkingSkeleton - : (waves.flat()[0] ?? null); + : ((unitPlan?.batches?.length ? unitPlan.batches.flat() : [...slugs]).find((slug) => + slugs.has(slug), + ) ?? null); }; // The fan-out summary appended to the unit-DAG stage's validation gate (A2 @@ -400,7 +410,12 @@ export const fanoutGateAddendum = ({ sectionIndex, unitPlan, sectionStages, skel `- ${u.slug}${(u.dependsOn ?? []).length ? ` (depends on ${(u.dependsOn ?? []).join(', ')})` : ''}`, ), '', - `Walking skeleton: ${skeleton} (runs first, solo, with a mandatory review gate).`, + skeletonCeremonyDisabled(sectionStages) + ? 'Walking-skeleton ceremony is disabled: all units run with the normal stage and batch gates.' + : `Walking skeleton: ${skeleton} (runs first, solo, with a mandatory review gate).`, + ...(skeletonProblem(unitPlan, skeleton, sectionStages) + ? [skeletonProblem(unitPlan, skeleton, sectionStages)] + : []), `Per-unit stages: ${sectionStages.map((s) => s.stageId).join(', ')}.`, `Skip matrix (CONDITIONAL stages skipped per unit): ${ Object.keys(unitPlan.skipMatrix ?? {}).length @@ -438,18 +453,21 @@ export const runParallelSection = async (segment, toolkit) => { unitPrProvider = null, applicationUrl, runId = null, + completionPolicyVersion = 1, // Derive-time enrichment mode ('off'|'llm'), snapshotted on META — rides // the lane derive-artifacts dispatches like the once-per-workflow hook. deriveEnrichment = 'off', attachments = [], } = toolkit; const { executionId, intentId, projectId } = ids; + const checkedCompletion = completionPolicyVersion >= 1; const sk = `s${segment.index}`; // Consistent: the fan-out approval patched this plan's decisions just before. const unitPlan = await ctx.step(`load-unit-plan-${sk}`, () => store.getUnitPlan(executionId, { consistentRead: true }), ); + if (checkedCompletion) toolkit.completionUnitPlan = unitPlan; if (!unitPlan || (unitPlan.units ?? []).length === 0) { return await fail( 'unit_plan_missing', @@ -461,6 +479,16 @@ export const runParallelSection = async (segment, toolkit) => { unitPlan.batches?.length ? unitPlan.batches : [unitPlan.units.map((u) => u.slug)] ).map((w) => w.filter((slug) => bySlug.has(slug))); const laneOrder = waves.flat(); + if (checkedCompletion) { + const problem = skeletonProblem(unitPlan, undefined, segment.stages); + if (problem) return fail('invalid_walking_skeleton', problem); + if (laneOrder.length !== bySlug.size || new Set(laneOrder).size !== bySlug.size) { + return fail( + 'invalid_unit_waves', + 'Every current-plan unit must occur exactly once in the scheduling waves.', + ); + } + } // ── fan-out decisions (A2 rules 2/7/8) ──────────────────────────────────── // The human approved the fan-out at the unit-DAG stage's validation gate @@ -468,7 +496,7 @@ export const runParallelSection = async (segment, toolkit) => { // UNITPLAN via updateUnitPlanDecisions. The plan row read above (a durable // step) is therefore the section's scheduling truth — no second gate here. const decisions = { - walkingSkeleton: defaultSkeletonFor(unitPlan), + walkingSkeleton: defaultSkeletonFor(unitPlan, { legacy: !checkedCompletion }), skipMatrix: unitPlan.skipMatrix ?? {}, }; await emitEvent( @@ -485,6 +513,7 @@ export const runParallelSection = async (segment, toolkit) => { const withIntegrationLock = makeMergeLock(); // slug → 'MERGED' | 'FAILED' | 'BLOCKED' — settled lane states (this run). const laneState = new Map(); + const skipped = new Set(); // slug → in-flight lane DurablePromise (wavefront cross-lane coordination). const lanePromises = new Map(); const usesUnitPrs = prStrategy === 'pr-per-unit'; @@ -504,8 +533,28 @@ export const runParallelSection = async (segment, toolkit) => { // gate are not replayed and dependency waits can continue from durable DDB // truth rather than the empty in-memory map. const persistedUnits = await ctx.step(`load-unit-states-${sk}`, () => - store.listUnits(executionId), + store.listUnits(executionId, { consistentRead: checkedCompletion }), ); + if (checkedCompletion) { + let initial; + try { + initial = await ctx.step(`load-completion-v1-${sk}`, async () => { + const records = await store.getCompletionRecords(executionId); + if (records.meta?.orchestratorRunId !== runId) return { state: 'retired' }; + return unitCompletion(records, [segment.index]); + }); + } catch { + return fail( + 'completion_unverified', + 'Existing unit integration evidence could not be read. Work is preserved.', + ); + } + if (initial.state === 'retired') return { ok: false, reason: 'retired', intentId }; + // Validate recorded repository evidence before releasing dependencies. + // Unfinished lanes may execute; uncertain or contradictory integration may not. + if (initial.state === 'unknown' || initial.unresolved.some((unit) => unit.state === 'MERGED')) + return fail('completion_unverified', completionMessage(initial)); + } const persistedBySlug = new Map( (persistedUnits ?? []) .filter((row) => Number(row.sectionIndex) === Number(segment.index)) @@ -519,6 +568,14 @@ export const runParallelSection = async (segment, toolkit) => { ) { laneState.set(row.slug, 'MERGED'); } + if ( + checkedCompletion && + Number(row.sectionIndex) === Number(segment.index) && + bySlug.has(row.slug) && + hasUnitSkipDecision(row, unitPlan) + ) { + skipped.add(row.slug); + } } const callProvider = async (method, args) => { @@ -815,7 +872,10 @@ export const runParallelSection = async (segment, toolkit) => { sectionIndex: segment.index, slug, state: 'MERGED', - fields: { mergedAt: true }, + fields: { + mergedAt: true, + ...(checkedCompletion ? { integrationResults: [] } : {}), + }, }); }); return { slug, state: 'MERGED' }; @@ -1508,6 +1568,7 @@ export const runParallelSection = async (segment, toolkit) => { state: 'MERGED', fields: { mergedAt: true, + ...(checkedCompletion ? { integrationResults: [] } : {}), integrationOwner: false, blockedReason: null, }, @@ -1638,6 +1699,8 @@ export const runParallelSection = async (segment, toolkit) => { let depState = laneState.get(dep); if (lanePromises.has(dep)) { const settled = await lanePromises.get(dep).catch(() => ({ state: 'FAILED' })); + if (checkedCompletion && settled?.state === 'TERMINAL') + return { slug, state: 'TERMINAL', value: settled.value }; depState = settled?.state; } if (depState !== 'MERGED') { @@ -1983,6 +2046,7 @@ export const runParallelSection = async (segment, toolkit) => { intentId, executionId, unitSlug: slug, + ...(checkedCompletion ? { sectionIndex: segment.index } : {}), repos: cloneBase.repos, unitBranch, intentBranch, @@ -2051,22 +2115,59 @@ export const runParallelSection = async (segment, toolkit) => { stageId: 'merge-lane', reason: merge?.reason ?? 'merge_failed', detail: merge?.detail ?? null, + integrationResults: merge?.results, }); } - await laneCtx.step(`unit-merged-${sk}-${slug}${rTag}`, async () => { - try { - await store.updateUnitState({ - executionId, - sectionIndex: segment.index, - slug, - state: 'MERGED', - fromStates: ['MERGING', 'RUNNING'], - fields: { mergedAt: true }, - }); - } catch { - /* tolerated */ - } - }); + const integrationRecorded = await laneCtx.step( + `unit-merged-${sk}-${slug}${rTag}`, + async () => { + try { + const saved = await store.updateUnitState({ + executionId, + sectionIndex: segment.index, + slug, + state: 'MERGED', + fromStates: ['MERGING', 'RUNNING'], + ...(checkedCompletion ? { ifOrchestratorRunId: runId, ifUnitPlan: unitPlan } : {}), + fields: { + mergedAt: true, + ...(checkedCompletion + ? { + integrationResults: (merge.results ?? []).map(({ repo, merged, reason }) => ({ + repo, + merged: merged ?? false, + ...(reason ? { reason } : {}), + })), + } + : {}), + }, + }); + if (checkedCompletion && !saved) + throw new Error('Unit integration write was not accepted'); + return true; + } catch (error) { + if (!checkedCompletion) return true; // frozen legacy behavior + const records = await store.getCompletionRecords(executionId); + if ( + records.meta?.orchestratorRunId !== runId || + unitPlanIdentity(records.unitPlan) !== unitPlanIdentity(unitPlan) + ) + return false; + const row = records.units.find( + (unit) => unit.sectionIndex === segment.index && unit.slug === slug, + ); + const completion = unitCompletion(records, [segment.index]); + if ( + row?.state === 'MERGED' && + !completion.unresolved.some((unit) => unit.slug === slug) + ) + return true; + throw error; + } + }, + ); + if (integrationRecorded === false) + return { slug, state: 'TERMINAL', value: { ok: false, reason: 'retired', intentId } }; await emitEvent( laneCtx, `unit-merged-event-${sk}-${slug}${rTag}`, @@ -2092,7 +2193,12 @@ export const runParallelSection = async (segment, toolkit) => { } }; - const laneFailed = async (laneCtx, slug, round, { stageId, reason, detail }) => { + const laneFailed = async ( + laneCtx, + slug, + round, + { stageId, reason, detail, integrationResults }, + ) => { const rTag = round > 0 ? `-r${round}` : ''; await laneCtx.step(`unit-failed-${sk}-${slug}${rTag}`, async () => { try { @@ -2101,7 +2207,21 @@ export const runParallelSection = async (segment, toolkit) => { sectionIndex: segment.index, slug, state: 'FAILED', - fields: { failureReason: `${stageId}: ${reason}${detail ? ` (${detail})` : ''}` }, + ...(checkedCompletion ? { ifOrchestratorRunId: runId, ifUnitPlan: unitPlan } : {}), + fields: { + failureReason: `${stageId}: ${reason}${detail ? ` (${detail})` : ''}`, + ...(checkedCompletion && integrationResults + ? { + integrationResults: integrationResults.map( + ({ repo, merged, reason: repoReason }) => ({ + repo, + merged: merged ?? false, + ...(repoReason ? { reason: repoReason } : {}), + }), + ), + } + : {}), + }, }); } catch { /* lane bookkeeping must never mask the failure */ @@ -2146,9 +2266,6 @@ export const runParallelSection = async (segment, toolkit) => { return { terminal }; }; - const blocked = () => - [...laneState.entries()].filter(([, s]) => s === 'BLOCKED').map(([slug]) => slug); - // Halt-and-ask (stage-protocol.md): wait for all lanes (done by the caller), // preserve successes (pushed work + MERGED lanes), ask retry / skip / abort. // An unparseable answer RE-ASKS (fresh gate, one human answer per attempt) @@ -2157,6 +2274,12 @@ export const runParallelSection = async (segment, toolkit) => { // deterministic fallback is abort (audited), never an unbounded hot loop. // Returns 'retry' | 'skip' | a terminal value (abort/retired). const haltAndAsk = async (round, failedSlugs, idSuffix = '') => { + const unresolvedStates = [...laneState.entries()] + .filter(([slug, state]) => state !== 'MERGED' && !skipped.has(slug)) + .map(([, state]) => state); + const failedCount = unresolvedStates.filter((state) => state === 'FAILED').length; + const blockedCount = unresolvedStates.filter((state) => state === 'BLOCKED').length; + const otherCount = unresolvedStates.length - failedCount - blockedCount; const detailLines = failedSlugs.map((slug) => { const row = laneState.get(slug); return `- ${slug}: ${row === 'FAILED' ? 'failed' : row}`; @@ -2166,14 +2289,14 @@ export const runParallelSection = async (segment, toolkit) => { name: `halt-${sk}${idSuffix}-r${round}${attempt ? `-a${attempt}` : ''}`, sectionIndex: segment.index, prompt: [ - `Lane failure in section ${segment.index} (${failedSlugs.length} unit(s) failed, ${blocked().length} blocked, ${ + `Unresolved work in section ${segment.index} (${failedCount} failed, ${blockedCount} blocked${otherCount ? `, ${otherCount} other unresolved` : ''}, ${ [...laneState.values()].filter((s) => s === 'MERGED').length - } merged).`, + } integrated).`, ...detailLines, '', 'All pushed work is preserved on the unit branches. Choose:', - '- retry: re-run the failed lanes (and their blocked dependents) in place', - '- skip: continue without them (their work will be missing downstream)', + '- retry: re-run unresolved lanes (and their blocked dependents) in place; integrated work is preserved', + '- skip: explicitly omit these lanes and all their unintegrated dependents from delivery', '- abort: fail the run', ...(attempt ? [ @@ -2210,11 +2333,11 @@ export const runParallelSection = async (segment, toolkit) => { return { terminal: await fail( 'section_aborted', - `section ${segment.index}: units [${failedSlugs.join(', ')}] failed; human aborted`, + `section ${segment.index}: units [${failedSlugs.join(', ')}] unresolved; human aborted`, ), }; } - return { choice }; + return { choice, humanTaskId: halt.gate.humanTaskId }; } }; @@ -2250,7 +2373,7 @@ export const runParallelSection = async (segment, toolkit) => { initialSlugs, { tag, idSuffix = '', feedbackTaskId = null, revive = false }, ) => { - let toRun = initialSlugs; + let toRun = initialSlugs.filter((slug) => !skipped.has(slug)); let round = 0; // A retry round re-runs the previous round's FAILED lanes and revives // their BLOCKED dependents. Unbounded by design — each round is one @@ -2262,16 +2385,46 @@ export const runParallelSection = async (segment, toolkit) => { laneOpts: { idSuffix, feedbackTaskId, revive }, }); if (terminal) return terminal; - const failed = toRun.filter((s) => laneState.get(s) === 'FAILED'); + const failed = toRun.filter((s) => + checkedCompletion + ? laneState.get(s) !== 'MERGED' && !skipped.has(s) + : laneState.get(s) === 'FAILED', + ); if (failed.length === 0) return null; - const decision = await haltAndAsk(round, failed, idSuffix); + const decisionSuffix = checkedCompletion ? `${tag}${idSuffix}` : idSuffix; + const decision = await haltAndAsk(round, failed, decisionSuffix); if (decision.terminal) return decision.terminal; if (decision.choice === 'skip') { + if (checkedCompletion) { + const omitted = dependentClosure(unitPlan, failed).filter( + (slug) => laneState.get(slug) !== 'MERGED' && !skipped.has(slug), + ); + await ctx.step(`record-lane-skips-${sk}${decisionSuffix}-r${round}`, async () => { + for (const slug of omitted) { + const saved = await store.updateUnitState({ + executionId, + sectionIndex: segment.index, + slug, + state: laneState.get(slug) ?? 'BLOCKED', + ifOrchestratorRunId: runId, + fields: { + completionSkip: { + humanTaskId: decision.humanTaskId, + planIdentity: unitPlanIdentity(unitPlan), + }, + }, + }); + if (!saved) throw new Error('Unit skip decision lost execution ownership'); + } + return omitted; + }); + omitted.forEach((slug) => skipped.add(slug)); + } await emitEvent( ctx, - `skip-lanes-${sk}${idSuffix}-r${round}`, + `skip-lanes-${sk}${checkedCompletion ? decisionSuffix : idSuffix}-r${round}`, 'v2.units.lanes_skipped', - `Continuing section ${segment.index} without [${failed.join(', ')}] (human skip); dependents stay blocked`, + `Continuing section ${segment.index} without [${failed.join(', ')}] and their unintegrated dependents (human skip); their work is omitted from delivery`, { sectionIndex: segment.index }, ); return null; @@ -2292,7 +2445,7 @@ export const runParallelSection = async (segment, toolkit) => { }); } round += 1; - toRun = [...failed, ...toRun.filter((s) => laneState.get(s) === 'BLOCKED')]; + toRun = [...new Set([...failed, ...toRun.filter((s) => laneState.get(s) === 'BLOCKED')])]; } }; @@ -2305,8 +2458,7 @@ export const runParallelSection = async (segment, toolkit) => { // policy, which only exists in release mode — so an unpinned run takes the // pre-Phase-6 path byte for byte. const skeleton = decisions.walkingSkeleton; - const skeletonCeremonyOff = - (segment.stages.find((stage) => stage.policy?.skeleton)?.policy?.skeleton ?? null) === 'off'; + const skeletonCeremonyOff = skeletonCeremonyDisabled(segment.stages); const skeletonAlreadyApproved = laneState.get(skeleton) === 'MERGED' && CONSTRUCTION_AUTONOMY_MODES.includes(unitPlan.autonomyMode); @@ -2392,14 +2544,51 @@ export const runParallelSection = async (segment, toolkit) => { // ── phase 2: autonomy ladder (A2 rule 9), then the remaining lanes ─────── const remaining = laneOrder.filter( - (s) => (skeletonCeremonyOff || s !== skeleton) && laneState.get(s) !== 'MERGED', + (s) => + (skeletonCeremonyOff || s !== skeleton) && laneState.get(s) !== 'MERGED' && !skipped.has(s), ); + const validateCompletion = async () => { + if (checkedCompletion) { + let completion; + try { + completion = await ctx.step(`section-completion-v1-${sk}`, async () => { + const records = await store.getCompletionRecords(executionId); + if (records.meta?.orchestratorRunId !== runId) return { state: 'retired' }; + if (unitPlanIdentity(records.unitPlan) !== unitPlanIdentity(unitPlan)) + return { + state: 'unknown', + reason: 'unit_plan_changed_after_section', + integrated: 0, + total: unitPlan.units.length, + skipped: 0, + unresolved: [], + }; + return unitCompletion(records, [segment.index]); + }); + } catch { + return fail( + 'completion_unverified', + `Section ${segment.index} completion evidence could not be read. Work is preserved.`, + ); + } + if (completion.state === 'retired') return { ok: false, reason: 'retired', intentId }; + return ['complete', 'partial'].includes(completion.state) + ? null + : fail( + completion.state === 'unknown' ? 'completion_unverified' : 'section_incomplete', + completionMessage(completion), + ); + } + return null; // Marker-absent histories retain the pre-check operation sequence. + }; if (remaining.length === 0) { + const incomplete = await validateCompletion(); + if (incomplete) return incomplete; await emitEvent( ctx, `fan-in-${sk}`, 'v2.units.fan_in', - `Section ${segment.index} complete: ${[...laneState.values()].filter((s) => s === 'MERGED').length}/${laneOrder.length} unit(s) merged`, + `Section ${segment.index} resolved: ${[...laneState.values()].filter((s) => s === 'MERGED').length}/${laneOrder.length} unit(s) integrated${skipped.size ? `; ${skipped.size} explicitly omitted by human decision` : ''}`, { sectionIndex: segment.index }, ); return null; @@ -2433,8 +2622,13 @@ export const runParallelSection = async (segment, toolkit) => { /* META mirror is best-effort; the UNITPLAN write below is the truth */ } try { - await store.updateUnitPlanDecisions({ executionId, autonomyMode: parsed }); - } catch { + await store.updateUnitPlanDecisions({ + executionId, + autonomyMode: parsed, + ...(checkedCompletion ? { ifOrchestratorRunId: runId } : {}), + }); + } catch (error) { + if (checkedCompletion) throw error; /* tolerated — the step result governs this run */ } return parsed; @@ -2522,14 +2716,14 @@ export const runParallelSection = async (segment, toolkit) => { } // ── fan-in bookkeeping (A2 rule 5) ──────────────────────────────────────── + const incomplete = await validateCompletion(); + if (incomplete) return incomplete; const merged = [...laneState.values()].filter((s) => s === 'MERGED').length; await emitEvent( ctx, `fan-in-${sk}`, 'v2.units.fan_in', - `Section ${segment.index} complete: ${merged}/${laneOrder.length} unit(s) merged${ - merged < laneOrder.length ? ' (some lanes skipped after failure — see halt decisions)' : '' - }`, + `Section ${segment.index} resolved: ${merged}/${laneOrder.length} unit(s) integrated${skipped.size ? `; ${skipped.size} explicitly omitted by human decision` : ''}`, { sectionIndex: segment.index }, ); return null; diff --git a/lambda/v2-orchestrator/test/completion-section.test.js b/lambda/v2-orchestrator/test/completion-section.test.js new file mode 100644 index 000000000..9b3748733 --- /dev/null +++ b/lambda/v2-orchestrator/test/completion-section.test.js @@ -0,0 +1,206 @@ +import { randomUUID } from 'node:crypto'; +import { afterAll, beforeAll, expect, it, vi } from 'vitest'; +import { createProcessStore } from '../../shared/v2-process-store.js'; +import { createV2Table, deleteV2Table, makeDdb } from '../../agentcore/test/helpers/v2-table.js'; +import { runParallelSection } from '../section.js'; + +const tableName = `completion-section-${randomUUID()}`; +let db; +let store; +beforeAll(async () => { + db = makeDdb(); + await createV2Table(db.client, tableName); + store = createProcessStore({ ddb: db.doc, tableName }); +}); +afterAll(async () => { + await deleteV2Table(db.client, tableName); + db.client.destroy(); +}); +const setup = async (fault) => { + const executionId = randomUUID(); + const runId = 'run'; + const units = [ + { slug: 'foundation', dependsOn: [] }, + { slug: 'dependent', dependsOn: ['foundation'] }, + ]; + await store.createExecution({ + executionId, + intentId: executionId, + projectId: 'project', + repos: ['one', 'two'], + }); + await store.updateExecution({ executionId, status: 'RUNNING', orchestratorRunId: runId }); + await store.putUnitPlan({ + executionId, + units, + batches: [['foundation'], ['dependent']], + autonomyMode: 'autonomous', + }); + await store.syncUnitRows({ executionId, units, sectionIndexes: [1] }); + if (fault === 'legacy') + for (const { slug } of units) + await store.updateUnitState({ + executionId, + sectionIndex: 1, + slug, + state: 'MERGED', + fields: { mergedAt: true }, + }); + if (fault === 'contradictory') + await store.updateUnitState({ + executionId, + sectionIndex: 1, + slug: 'foundation', + state: 'MERGED', + fields: { + integrationResults: [ + { repo: 'one', merged: true }, + { repo: 'two', merged: false }, + ], + }, + }); + const events = []; + const executeStage = vi.fn(async () => ({ state: 'SUCCEEDED' })); + const invokeRuntime = vi.fn(async (payload) => + payload.command === 'merge-lane' + ? { + ok: true, + results: + fault === 'empty-results' + ? [] + : [ + { repo: 'one', merged: true }, + { repo: 'two', merged: true }, + ], + } + : { ok: true }, + ); + const ctx = { + logger: { error() {} }, + step: async (_name, fn) => fn(), + runInChildContext: async (_name, fn) => fn(ctx), + promise: { allSettled: async (_name, promises) => Promise.allSettled(promises) }, + createCallback: async (name) => { + const humanTaskId = name.replace(/^await-/, ''); + await store.answerHumanTask({ + executionId, + humanTaskId, + status: 'answered', + answer: 'abort', + answeredBy: 'operator', + ifOrchestratorRunId: runId, + }); + return [Promise.resolve({ answer: 'abort' }), `cb-${name}`]; + }, + }; + const guarded = { + ...store, + updateUnitState: async (input) => { + if (input.slug === 'foundation' && input.state === 'MERGED') { + if (fault === 'write-failed') throw new Error('required integration write failed'); + if (fault === 'lost-response') { + await store.updateUnitState(input); + throw new Error('response lost'); + } + if (fault === 'retired') { + await store.updateExecution({ executionId, orchestratorRunId: 'successor' }); + return store.updateUnitState(input); + } + } + return store.updateUnitState(input); + }, + }; + const toolkit = { + ctx, + store: guarded, + runId, + completionPolicyVersion: 1, + ids: { executionId, intentId: executionId, projectId: 'project' }, + intentBranch: 'intent', + cloneBase: { repos: ['one', 'two'] }, + intentSessionId: 'session', + maxParallelUnits: 2, + invokeRuntime, + executeStage, + stopSession: async () => {}, + broadcast: async () => {}, + emitEvent: async (_ctx, _name, type, summary) => events.push({ type, summary }), + fail: async (reason, detail) => ({ ok: false, reason, detail }), + stageInstanceIdFor: (stage, slug) => `${stage}-${slug}`, + }; + return { + executionId, + events, + executeStage, + invokeRuntime, + run: () => + runParallelSection( + { index: 1, stages: [{ stageId: 'build', policy: { skeleton: 'off' } }] }, + toolkit, + ), + }; +}; + +it('does not release dependent work when the required MERGED write fails', async () => { + const world = await setup('write-failed'); + expect(await world.run()).toMatchObject({ ok: false, reason: 'section_aborted' }); + expect(world.executeStage.mock.calls.map(([, , options]) => options.unitSlug)).toEqual([ + 'foundation', + ]); + expect(world.invokeRuntime.mock.calls.filter(([p]) => p.command === 'merge-lane')).toHaveLength( + 1, + ); + expect((await store.getUnit(world.executionId, 1, 'foundation')).state).toBe('FAILED'); + expect((await store.getUnit(world.executionId, 1, 'dependent')).state).toBe('BLOCKED'); + expect(world.events.some((event) => event.type === 'v2.units.fan_in')).toBe(false); +}); +it('reconciles a lost integration response without rerunning stage or merge', async () => { + const world = await setup('lost-response'); + expect(await world.run()).toBeNull(); + expect(world.executeStage).toHaveBeenCalledTimes(2); + expect(world.invokeRuntime.mock.calls.filter(([p]) => p.command === 'merge-lane')).toHaveLength( + 2, + ); + for (const row of await store.listUnits(world.executionId)) { + expect(row).toMatchObject({ + state: 'MERGED', + integrationResults: [ + { repo: 'one', merged: true }, + { repo: 'two', merged: true }, + ], + }); + } +}); +it('does not let contradictory persisted repository evidence unblock dependencies', async () => { + const world = await setup('contradictory'); + expect(await world.run()).toMatchObject({ ok: false, reason: 'completion_unverified' }); + expect(world.executeStage).not.toHaveBeenCalled(); + expect(world.invokeRuntime).not.toHaveBeenCalled(); +}); +it('does not mark successor work failed when the integration owner is retired', async () => { + const world = await setup('retired'); + expect(await world.run()).toMatchObject({ ok: false, reason: 'retired' }); + expect((await store.getExecution(world.executionId)).orchestratorRunId).toBe('successor'); + expect((await store.getUnit(world.executionId, 1, 'foundation')).state).toBe('MERGING'); + expect((await store.getUnit(world.executionId, 1, 'dependent')).state).toBe('PENDING'); +}); + +it('persists explicit empty integration evidence instead of producing another legacy row', async () => { + const world = await setup('empty-results'); + expect(await world.run()).toMatchObject({ ok: false }); + expect(await store.getUnit(world.executionId, 1, 'foundation')).toMatchObject({ + integrationResults: [], + }); +}); + +it('hydrates pre-deploy multi-repo MERGED rows from the real store without rebuilding units', async () => { + const world = await setup('legacy'); + expect(await world.run()).toBeNull(); + expect(world.executeStage).not.toHaveBeenCalled(); + expect(world.invokeRuntime).not.toHaveBeenCalled(); + expect( + (await store.listUnits(world.executionId)).every( + (row) => row.state === 'MERGED' && !Object.hasOwn(row, 'integrationResults'), + ), + ).toBe(true); +}); diff --git a/lambda/v2-orchestrator/test/final-completion.test.js b/lambda/v2-orchestrator/test/final-completion.test.js new file mode 100644 index 000000000..0f7192c1f --- /dev/null +++ b/lambda/v2-orchestrator/test/final-completion.test.js @@ -0,0 +1,137 @@ +import { expect, it, vi } from 'vitest'; +import { __durableHandler } from '../index.js'; +import { loadMetaBeforeDelivery } from './fixtures/load-meta-before-delivery.js'; +import { unitPlanIdentity } from '../../shared/unit-completion.js'; + +const setup = () => { + const meta = { + executionId: 'intent', + intentId: 'intent', + projectId: 'project', + status: 'RUNNING', + workflowId: 'workflow', + workflowVersion: 1, + scope: 'feature', + repos: ['owner/repo'], + branch: 'intent-branch', + startedAt: '2026-10-01T00:00:00.000Z', + }; + const unitPlan = { + units: [ + { slug: 'a', dependsOn: [] }, + { slug: 'b', dependsOn: ['a'] }, + ], + batches: [['a'], ['b']], + autonomyMode: 'autonomous', + }; + const units = unitPlan.units.map(({ slug }) => ({ sectionIndex: 1, slug, state: 'MERGED' })); + const records = { meta, unitPlan, units: structuredClone(units) }; + const events = []; + const store = { + getExecution: async () => meta, + updateExecution: vi.fn(async (input) => { + Object.assign(meta, input); + return { ...meta }; + }), + getUnitPlan: async () => unitPlan, + listUnits: async () => units, // scheduler's earlier state, independent of final read + getCompletionRecords: vi.fn(async () => records), + appendEvent: async (event) => { + events.push(event.type); + }, + }; + const deps = { + store, + loadPlan: async () => ({ + valid: true, + plan: { stages: [{ stageId: 'build', parallelSection: 1 }] }, + }), + invokeRuntime: vi.fn(async () => ({ ok: true })), + stopSession: async () => ({ stopped: true }), + broadcast: async () => {}, + comparePrBranches: async () => ({ status: 'ahead' }), + openPr: vi.fn(async () => ({ number: 1, url: 'https://example.test/pr/1' })), + }; + const ctx = { + // A-started histories retain their initial publication path after B deploys. + // Modern publication is exercised with the real store in final-delivery.test.js. + step: async (name, fn) => + name === 'load-meta' + ? loadMetaBeforeDelivery(store, meta.executionId) + : name.startsWith('section-completion-v1-') || name.startsWith('load-completion-v1-') + ? { state: 'complete', unresolved: [] } + : fn(), + logger: { info() {}, error() {} }, + }; + return { + deps, + records, + events, + run: () => + __durableHandler( + { + action: 'start', + intentId: 'intent', + executionId: 'intent', + }, + ctx, + deps, + ), + }; +}; + +it.each(['BLOCKED', 'RUNNING', 'FAILED', 'PENDING', 'missing'])( + 'withholds both success and final PR for an independent final %s observation', + async (state) => { + const world = setup(); + if (state === 'missing') world.records.units = []; + else world.records.units[1].state = state; + const result = await world.run(); + expect(result).toMatchObject({ ok: false, reason: 'completion_inconsistent' }); + expect(world.records.meta.status).toBe('FAILED'); + expect(world.events).not.toContain('v2.execution.succeeded'); + expect(world.deps.openPr).not.toHaveBeenCalled(); + expect(world.deps.invokeRuntime.mock.calls.some(([p]) => p.command === 'run-stage-start')).toBe( + false, + ); + }, +); + +it('publishes complete work and plan-bound approved omissions', async () => { + for (const skipped of [false, true]) { + const world = setup(); + if (skipped) { + world.records.units[1].state = 'BLOCKED'; + world.records.units[1].completionSkip = { + humanTaskId: 'approved-skip', + planIdentity: unitPlanIdentity(world.records.unitPlan), + }; + } + expect(await world.run()).toMatchObject({ ok: true }); + expect(world.records.meta.completionValidation.state).toBe(skipped ? 'partial' : 'complete'); + expect(world.deps.openPr).toHaveBeenCalledOnce(); + } +}); + +it('fails visibly without publication when final reads fail', async () => { + const world = setup(); + world.deps.store.getCompletionRecords.mockRejectedValue(new Error('DynamoDB unavailable')); + expect(await world.run()).toMatchObject({ ok: false, reason: 'completion_unverified' }); + expect(world.records.meta.status).toBe('FAILED'); + expect(world.deps.openPr).not.toHaveBeenCalled(); + expect(world.events).not.toContain('v2.execution.succeeded'); +}); + +it('rechecks publication after saved success without rewriting that decision', async () => { + const world = setup(); + let reads = 0; + world.deps.store.getCompletionRecords.mockImplementation(async () => { + if (++reads === 2) world.records.units[1].state = 'BLOCKED'; + return world.records; + }); + await world.run(); + expect(world.deps.openPr).not.toHaveBeenCalled(); + expect(world.events).toContain('v2.pr.failed'); + expect(world.records.meta.completionValidation.state).toBe('complete'); + expect(world.records.meta.status).toBe('SUCCEEDED'); +}); diff --git a/lambda/v2-orchestrator/test/final-delivery.test.js b/lambda/v2-orchestrator/test/final-delivery.test.js new file mode 100644 index 000000000..25ebab449 --- /dev/null +++ b/lambda/v2-orchestrator/test/final-delivery.test.js @@ -0,0 +1,793 @@ +import { randomUUID } from 'node:crypto'; +import { PutCommand } from '@aws-sdk/lib-dynamodb'; +import { beforeAll, afterAll, beforeEach, describe, it, expect, vi } from 'vitest'; +import { createV2Table, deleteV2Table, makeDdb } from '../../agentcore/test/helpers/v2-table.js'; +import { createProcessStore } from '../../shared/v2-process-store.js'; +import { + previewDelivery, + publishFinalDelivery, + deliveryView, + currentDelivery, +} from '../../shared/final-delivery.js'; +import { __durableHandler } from '../index.js'; +import { finalizeUnitCompletion, unitPlanIdentity } from '../../shared/unit-completion.js'; + +const tableName = `delivery-${randomUUID()}`; +let db, store, meta, remote, operation, timestamp, workflowPlan; +const loadPlan = async () => ({ valid: true, plan: workflowPlan }); +beforeAll(async () => { + db = makeDdb(); + await createV2Table(db.client, tableName); +}); +afterAll(async () => { + await deleteV2Table(db.client, tableName); + db.client.destroy(); +}); +beforeEach(async () => { + workflowPlan = { stages: [] }; + timestamp = Date.now(); + store = createProcessStore({ + ddb: db.doc, + tableName, + clock: () => new Date(timestamp).toISOString(), + }); + const executionId = randomUUID(); + await store.createExecution({ + executionId, + intentId: executionId, + projectId: 'project', + repos: ['owner/api', 'owner/web'], + branch: 'intent/feature', + baseBranch: 'develop', + methodologyPins: { STAGE: {} }, + source: { bindingId: 'tracker-binding', resourceId: '42', provider: 'github-issues' }, + }); + meta = await store.updateExecution({ + executionId, + status: 'SUCCEEDED', + orchestratorRunId: 'run1', + }); + remote = new Map(); + operation = vi.fn(async ({ repo, operation: name, args }) => { + if (name === 'delivery-inspect') + return { + headSha: 'head1', + baseSha: 'base1', + baseBranch: args.baseBranch, + pr: remote.get(repo) ?? null, + }; + if (name === 'compare') return { status: 'ahead', base: args.base }; + if (name === 'create-pr') { + const pr = { + number: remote.size + 1, + url: `https://example.test/${repo}/pull/1`, + sourceBranch: args.branch, + targetBranch: args.baseBranch, + headSha: 'head1', + state: 'open', + }; + remote.set(repo, pr); + return { prUrl: pr.url, prNumber: pr.number }; + } + throw new Error(`unexpected operation ${name}`); + }); +}); +const publish = (over = {}) => + publishFinalDelivery({ + store, + meta, + operation, + loadPlan, + applicationUrl: 'https://example.test', + ...over, + }); +const row = () => store.getDelivery(meta.executionId, meta.orchestratorRunId); +const preview = async () => + previewDelivery({ + meta: await store.getExecution(meta.executionId, { consistentRead: true }), + records: await store.getExecutionRecords(meta.executionId), + plan: workflowPlan, + operation, + }); +const createCalls = () => + operation.mock.calls.map(([call]) => call).filter((call) => call.operation === 'create-pr'); +const retry = async () => { + const p = await preview(); + const claimed = await store.claimDelivery({ + meta: await store.getExecution(meta.executionId, { consistentRead: true }), + delivery: p.delivery, + attemptId: randomUUID(), + }); + return publish({ claimed }); +}; + +const completeParallelWork = async ({ partial = false } = {}) => { + workflowPlan = { stages: [{ stageId: 'code', parallelSection: 1 }] }; + const executionId = meta.executionId; + await store.updateExecution({ executionId, status: 'RUNNING' }); + const units = [ + { slug: 'api', dependsOn: [] }, + { slug: 'web', dependsOn: ['api'] }, + ]; + await store.putUnitPlan({ executionId, units, batches: [['api'], ['web']] }); + await store.syncUnitRows({ executionId, units, sectionIndexes: [1] }); + const unitPlan = await store.getUnitPlan(executionId); + for (const unit of units) + await store.updateUnitState({ + executionId, + sectionIndex: 1, + slug: unit.slug, + state: partial && unit.slug === 'web' ? 'BLOCKED' : 'MERGED', + fields: + partial && unit.slug === 'web' + ? { + completionSkip: { humanTaskId: 'approved', planIdentity: unitPlanIdentity(unitPlan) }, + } + : { integrationResults: meta.repos.map((repo) => ({ repo, merged: true })) }, + }); + expect( + await finalizeUnitCompletion({ store, executionId, runId: 'run1', plan: workflowPlan }), + ).toMatchObject({ state: partial ? 'partial' : 'complete' }); + meta = await store.getExecution(executionId); +}; + +describe('final delivery with real DynamoDB ownership', () => { + it.each(['start', 'retry-delivery'])( + 'publishes pre-deploy multi-repo MERGED rows through %s without rebuilding units', + async (action) => { + await completeParallelWork(); + await store.updateUnitPlanDecisions({ + executionId: meta.executionId, + autonomyMode: 'autonomous', + }); + const before = await store.listUnits(meta.executionId); + for (const unit of before) { + delete unit.integrationResults; + await db.doc.send(new PutCommand({ TableName: tableName, Item: unit })); + } + let attemptId; + if (action === 'retry-delivery') { + const p = await preview(); + const claimed = await store.claimDelivery({ + meta, + delivery: p.delivery, + attemptId: 'legacy-retry', + }); + attemptId = claimed.attemptId; + } else { + meta = await store.updateExecution({ executionId: meta.executionId, status: 'CREATED' }); + } + const invokeRuntime = vi.fn(async () => ({ ok: true })); + const result = await __durableHandler( + { + action, + executionId: meta.executionId, + intentId: meta.intentId, + projectId: meta.projectId, + runId: meta.orchestratorRunId, + attemptId, + }, + { step: (_name, fn) => fn() }, + { + store, + loadPlan, + publishDelivery: (args) => publishFinalDelivery({ ...args, operation, loadPlan }), + invokeRuntime, + broadcast: vi.fn(), + }, + ); + expect(result).toMatchObject({ ok: true }); + meta = await store.getExecution(meta.executionId); + expect(meta.status).toBe('SUCCEEDED'); + expect((await row()).status).toBe('succeeded'); + expect(createCalls()).toHaveLength(2); + expect( + invokeRuntime.mock.calls.some( + ([p]) => p.unitSlug || p.command === 'run-stage-start' || p.command === 'merge-lane', + ), + ).toBe(false); + expect(await store.listUnits(meta.executionId)).toEqual(before); + }, + ); + + it.each( + ['initial', 'retry'].flatMap((path) => + ['blocked', 'missing-section', 'multi-repo', 'read-failure', 'changed-plan'].map((reason) => [ + path, + reason, + ]), + ), + )('withholds %s publication after %s completion evidence changes', async (path, reason) => { + await completeParallelWork(); + let claimed; + if (path === 'retry') { + const p = await preview(); + claimed = await store.claimDelivery({ meta, delivery: p.delivery, attemptId: 'retry' }); + } + operation.mockClear(); + let checkedStore = store; + if (reason === 'read-failure') + checkedStore = { + ...store, + getCompletionRecords: async () => { + throw new Error('read unavailable'); + }, + }; + else if (reason === 'missing-section') + workflowPlan = { + stages: [...workflowPlan.stages, { stageId: 'verify', parallelSection: 2 }], + }; + else if (reason === 'changed-plan') + await store.putUnitPlan({ + executionId: meta.executionId, + units: [{ slug: 'replacement' }], + batches: [['replacement']], + }); + else + await store.updateUnitState({ + executionId: meta.executionId, + sectionIndex: 1, + slug: 'web', + state: reason === 'blocked' ? 'BLOCKED' : 'MERGED', + ...(reason === 'multi-repo' + ? { fields: { integrationResults: [{ repo: 'owner/api', merged: true }] } } + : {}), + }); + const before = await store.getExecution(meta.executionId); + const invokeRuntime = vi.fn(); + if (claimed) { + await __durableHandler( + { + action: 'retry-delivery', + executionId: meta.executionId, + intentId: meta.intentId, + projectId: meta.projectId, + runId: meta.orchestratorRunId, + attemptId: claimed.attemptId, + }, + { step: (_name, fn) => fn() }, + { + store: checkedStore, + publishDelivery: (args) => publishFinalDelivery({ ...args, operation, loadPlan }), + invokeRuntime, + broadcast: vi.fn(), + }, + ); + expect((await row()).status).toBe('failed'); + expect((await store.getExecution(meta.executionId)).deliveryLeaseUntil).toBeUndefined(); + } else await expect(publish({ store: checkedStore })).rejects.toThrow(); + expect(operation).not.toHaveBeenCalled(); + expect(invokeRuntime).not.toHaveBeenCalled(); + const after = await store.getExecution(meta.executionId); + expect(after.completionValidation).toEqual(before.completionValidation); + expect(after.status).toBe('SUCCEEDED'); + }); + + it.each([false, true])( + 'retries failed publication of verified work without rerunning stages (partial=%s)', + async (partial) => { + await completeParallelWork({ partial }); + const original = operation.getMockImplementation(); + operation.mockImplementation((call) => + call.operation === 'create-pr' ? Promise.reject(new Error('offline')) : original(call), + ); + await publish(); + expect((await row()).status).toBe('failed'); + const before = await store.getCompletionRecords(meta.executionId); + operation.mockImplementation(original); + await retry(); + expect((await row()).status).toBe('succeeded'); + const after = await store.getCompletionRecords(meta.executionId); + expect(after.units).toEqual(before.units); + expect(after.meta.completionValidation).toEqual(before.meta.completionValidation); + expect(after.meta.completionValidation.state).toBe(partial ? 'partial' : 'complete'); + }, + ); + + it('withholds a provider mutation when evidence changes during provider inspection', async () => { + await completeParallelWork(); + const original = operation.getMockImplementation(); + operation.mockImplementation(async (call) => { + if (call.operation === 'compare') + await store.updateUnitState({ + executionId: meta.executionId, + sectionIndex: 1, + slug: 'web', + state: 'BLOCKED', + }); + return original(call); + }); + await publish(); + expect(createCalls()).toHaveLength(0); + expect((await row()).status).toBe('failed'); + }); + + it('does not assess an unpinned legacy workflow as verified', async () => { + const legacy = { ...meta }; + delete legacy.methodologyPins; + await expect( + previewDelivery({ + meta: legacy, + records: await store.getExecutionRecords(meta.executionId), + plan: workflowPlan, + operation, + }), + ).rejects.toMatchObject({ code: 'delivery_unverified' }); + expect(operation).not.toHaveBeenCalled(); + }); + + it('fences delivery plan changes and keeps lease exclusion on completion/recovery transactions', async () => { + await completeParallelWork(); + const records = await store.getCompletionRecords(meta.executionId); + const p = await preview(); + const claimed = await store.claimDelivery({ + meta, + delivery: p.delivery, + attemptId: 'delivery', + ifUnitPlan: records.unitPlan, + }); + await expect( + store.updateExecution({ + executionId: meta.executionId, + status: 'RUNNING', + orchestratorRunId: 'recovery', + ifOrchestratorRunId: meta.orchestratorRunId, + ifUpdatedAt: meta.updatedAt, + ifUnitPlan: records.unitPlan, + completionValidation: null, + }), + ).rejects.toMatchObject({ name: 'ConditionalCheckFailedException' }); + expect((await store.getExecution(meta.executionId)).completionValidation).toEqual( + meta.completionValidation, + ); + await store.putUnitPlan({ + executionId: meta.executionId, + units: [{ slug: 'new' }], + batches: [['new']], + }); + await expect( + store.saveDelivery({ delivery: claimed, ifUnitPlan: records.unitPlan }), + ).rejects.toMatchObject({ name: 'ConditionalCheckFailedException' }); + await store.saveDelivery({ delivery: { ...claimed, status: 'failed' }, finished: true }); + await expect( + store.claimDelivery({ + meta: await store.getExecution(meta.executionId), + delivery: await row(), + attemptId: 'stale-preview', + ifUnitPlan: records.unitPlan, + }), + ).rejects.toMatchObject({ name: 'ConditionalCheckFailedException' }); + }); + + it('preserves the no-repository event and does not treat missing repository metadata as no change', async () => { + const unknownMeta = { ...meta }; + delete unknownMeta.repos; + await expect( + previewDelivery({ + meta: unknownMeta, + records: await store.getExecutionRecords(meta.executionId), + plan: { stages: [] }, + operation, + }), + ).rejects.toMatchObject({ code: 'delivery_repositories_unknown' }); + meta = { ...meta, repos: [] }; + await db.doc.send(new PutCommand({ TableName: tableName, Item: meta })); + expect(await publish()).toEqual([ + { eventType: 'v2.pr.skipped', summary: 'No repositories on this intent — no PR to open' }, + ]); + expect((await row()).status).toBe('no_change'); + expect(operation).not.toHaveBeenCalled(); + }); + it('keeps preview tokens stable when stored object attributes are reordered', async () => { + const records = await store.getExecutionRecords(meta.executionId); + const args = { meta, records, plan: { stages: [] }, operation }; + const first = await previewDelivery(args); + const reordered = Object.fromEntries(Object.entries(meta).reverse()); + expect((await previewDelivery({ ...args, meta: reordered })).previewToken).toBe( + first.previewToken, + ); + }); + it('publishes each repository once and leaves stage success independent', async () => { + await publish(); + await publish(); + expect(createCalls()).toHaveLength(2); + expect(createCalls()[0].args).toMatchObject({ + expectedHeadSha: 'head1', + baseBranch: 'develop', + allowRetarget: false, + attemptKey: `${meta.executionId}:run1`, + }); + expect((await row()).status).toBe('succeeded'); + expect((await store.getExecution(meta.executionId)).status).toBe('SUCCEEDED'); + expect(currentDelivery(await store.getExecutionRecords(meta.executionId)).status).toBe( + 'succeeded', + ); + }); + it('preserves success in a partial delivery and retries only the failed repository', async () => { + const original = operation.getMockImplementation(); + let reject = true; + operation.mockImplementation((call) => + call.operation === 'create-pr' && call.repo === 'owner/web' && reject + ? Promise.reject(new Error('private provider detail')) + : original(call), + ); + await publish(); + expect((await row()).status).toBe('partial'); + expect(JSON.stringify(await row())).not.toContain('private provider detail'); + reject = false; + await retry(); + expect(createCalls().map((c) => c.repo)).toEqual(['owner/api', 'owner/web', 'owner/web']); + expect((await row()).status).toBe('succeeded'); + }); + it('reconciles provider success with a lost response without creating again', async () => { + const original = operation.getMockImplementation(); + let lost = true; + operation.mockImplementation(async (call) => { + const result = await original(call); + if (call.operation === 'create-pr' && lost) { + lost = false; + throw new Error('socket closed'); + } + return result; + }); + await publish(); + expect((await row()).status).toBe('partial'); + await retry(); + expect((await row()).status).toBe('succeeded'); + expect(createCalls()).toHaveLength(2); + }); + it.each(['closed', 'merged'])( + 'refuses to recreate a previously %s PR after a lost response', + async (state) => { + const original = operation.getMockImplementation(); + operation.mockImplementation((call) => + call.operation === 'create-pr' ? Promise.reject(new Error('timeout')) : original(call), + ); + await publish(); + remote.set('owner/api', { + number: 8, + url: 'https://example.test/pr/8', + sourceBranch: meta.branch, + targetBranch: 'develop', + headSha: 'head1', + state, + }); + const p = await preview(); + expect(p.delivery.repositories[0]).toMatchObject({ status: 'failed', retryable: false }); + expect(p.delivery.repositories[0].reason).toContain(state); + await retry(); + expect(createCalls().filter((c) => c.repo === 'owner/api')).toHaveLength(1); + }, + ); + it.each(['missing', 'changed'])('refuses %s head branches on retry', async (kind) => { + const original = operation.getMockImplementation(); + operation.mockImplementation((call) => + call.operation === 'create-pr' ? Promise.reject(new Error('denied')) : original(call), + ); + await publish(); + operation.mockImplementation((call) => + call.operation === 'delivery-inspect' + ? { + headSha: kind === 'missing' ? null : 'new-head', + baseSha: 'base1', + baseBranch: 'develop', + } + : original(call), + ); + await retry(); + expect(createCalls()).toHaveLength(2); + expect((await row()).repositories.every((r) => r.retryable === false)).toBe(true); + }); + it('records ordinary no-diff as no change but never masks a failed timeline read', async () => { + const original = operation.getMockImplementation(); + operation.mockImplementation((call) => + call.operation === 'compare' ? { status: 'identical', base: 'develop' } : original(call), + ); + await publish(); + expect((await row()).status).toBe('no_change'); + expect(createCalls()).toHaveLength(0); + // Different run; the old no-change record remains historical. + meta = await store.updateExecution({ + executionId: meta.executionId, + orchestratorRunId: 'run2', + }); + const badStore = { + ...store, + getExecutionRecords: async () => { + throw new Error('unreadable'); + }, + }; + await expect(publish({ store: badStore })).rejects.toThrow('unreadable'); + expect((await row()).status).toBe('in_progress'); + expect(createCalls()).toHaveLength(0); + }); + it('refuses no-diff after engine push failure and failed strict provider reads', async () => { + await store.appendEvent({ + executionId: meta.executionId, + type: 'v2.git.push_failed', + summary: 'owner/api failed push', + }); + const original = operation.getMockImplementation(); + operation.mockImplementation((call) => + call.operation === 'compare' ? { status: 'identical' } : original(call), + ); + await publish(); + expect((await row()).repositories[0].status).toBe('failed'); + operation.mockImplementation(() => { + throw new Error('provider read denied'); + }); + await retry(); + expect(createCalls()).toHaveLength(0); + expect((await row()).repositories[0].status).toBe('failed'); + }); + it('admits one concurrent retry and fences a competing rewind/new run', async () => { + const p = await preview(); + const claims = await Promise.allSettled( + ['first', 'second'].map((attemptId) => + store.claimDelivery({ meta, delivery: p.delivery, attemptId }), + ), + ); + expect(claims.filter((r) => r.status === 'fulfilled')).toHaveLength(1); + await expect( + store.updateExecution({ + executionId: meta.executionId, + status: 'FAILED', + orchestratorRunId: 'rewind', + }), + ).rejects.toMatchObject({ name: 'ConditionalCheckFailedException' }); + const claimed = claims.find((r) => r.status === 'fulfilled').value; + await publish({ claimed }); + expect(createCalls()).toHaveLength(2); + await expect(store.saveDelivery({ delivery: claimed })).rejects.toMatchObject({ + name: 'ConditionalCheckFailedException', + }); + }); + it('recovers an abandoned attempt and rejects its late write and provider call', async () => { + const first = await store.claimDelivery({ + meta, + delivery: { repositories: [] }, + attemptId: 'abandoned', + }); + timestamp += 21 * 60_000; + const second = await store.claimDelivery({ + meta, + delivery: await row(), + attemptId: 'replacement', + }); + await expect(store.saveDelivery({ delivery: first })).rejects.toMatchObject({ + name: 'ConditionalCheckFailedException', + }); + expect(await publish({ claimed: first })).toEqual([]); + expect(createCalls()).toHaveLength(0); + await store.saveDelivery({ delivery: { ...second, status: 'no_change' }, finished: true }); + }); + it('rejects a retry preview when a new run already owns the execution', async () => { + const p = await preview(); + await store.updateExecution({ + executionId: meta.executionId, + status: 'RUNNING', + orchestratorRunId: 'replacement', + }); + await expect( + store.claimDelivery({ meta, delivery: p.delivery, attemptId: 'stale' }), + ).rejects.toMatchObject({ name: 'ConditionalCheckFailedException' }); + expect(await publish()).toEqual([]); + expect(createCalls()).toHaveLength(0); + }); + it.each(['CANCELLED', 'FAILED'])( + 'does not publish or offer retry for an aborted %s execution', + async (status) => { + meta = await store.updateExecution({ + executionId: meta.executionId, + status, + methodologyPins: null, + failure: + status === 'FAILED' + ? { code: 'section_aborted', message: 'Execution aborted by its owner.' } + : null, + }); + await expect(preview()).rejects.toMatchObject({ code: 'delivery_not_complete' }); + expect(await publish()).toEqual([]); + expect(operation).not.toHaveBeenCalled(); + expect(await row()).toBeNull(); + }, + ); + it('old terminal records remain unknown until read-only assessment; missing sections block it', async () => { + expect(deliveryView(meta, await row()).status).toBe('unknown'); + expect((await preview()).historical).toBe(true); + expect(await row()).toBeNull(); + expect(createCalls()).toHaveLength(0); + const records = await store.getExecutionRecords(meta.executionId); + await expect( + previewDelivery({ + meta, + records, + plan: { stages: [{ stageId: 'build', parallelSection: 1 }] }, + operation, + }), + ).rejects.toMatchObject({ code: 'delivery_missing_plan' }); + records.unitPlan = { units: [{ slug: 'api' }] }; + await expect( + previewDelivery({ + meta, + records, + plan: { stages: [{ stageId: 'build', parallelSection: 1 }] }, + operation, + }), + ).rejects.toMatchObject({ code: 'delivery_incomplete' }); + }); + it('honors plan-bound partial approvals but never a pending gate or a skip for another section', async () => { + const records = await store.getExecutionRecords(meta.executionId); + records.unitPlan = { + units: [{ slug: 'api' }, { slug: 'web' }], + promotedAt: '2026-10-01T00:00:00.000Z', + }; + records.units = [ + { + slug: 'api', + sectionIndex: 1, + state: 'MERGED', + integrationResults: meta.repos.map((repo) => ({ repo, merged: true })), + }, + { + slug: 'web', + sectionIndex: 1, + state: 'SKIPPED', + completionSkip: { humanTaskId: 'skip', planIdentity: unitPlanIdentity(records.unitPlan) }, + }, + ]; + const args = { + meta, + records, + plan: { stages: [{ stageId: 'code', parallelSection: 1 }] }, + operation, + }; + expect((await previewDelivery(args)).historical).toBe(true); + records.humanTasks = [{ status: 'pending', orchestratorRunId: meta.orchestratorRunId }]; + await expect(previewDelivery(args)).rejects.toMatchObject({ code: 'delivery_gate_pending' }); + delete records.units[1].completionSkip; + records.humanTasks = [ + { + humanTaskId: 'eg-halt-s2-r0', + status: 'answered', + orchestratorRunId: meta.orchestratorRunId, + answer: { decision: 'skip' }, + createdAt: '2026-10-02T00:00:00.000Z', + prompt: 'Lane failure in section 2\n- web: failed', + }, + ]; + await expect(previewDelivery(args)).rejects.toMatchObject({ code: 'delivery_incomplete' }); + records.humanTasks[0].humanTaskId = 'eg-halt-s1-r0'; + records.humanTasks[0].prompt = 'Lane failure in section 1\n- web: failed'; + expect((await previewDelivery(args)).historical).toBe(true); + }); + it('retry dispatch runs publication only, without models, stages or Git integration', async () => { + const p = await preview(); + const claimed = await store.claimDelivery({ meta, delivery: p.delivery, attemptId: 'retry' }); + const invokeRuntime = vi.fn(); + const result = await __durableHandler( + { + action: 'retry-delivery', + executionId: meta.executionId, + intentId: meta.intentId, + projectId: meta.projectId, + runId: meta.orchestratorRunId, + attemptId: claimed.attemptId, + }, + { step: (_name, fn) => fn() }, + { + store, + publishDelivery: (args) => publishFinalDelivery({ ...args, operation, loadPlan }), + invokeRuntime, + broadcast: vi.fn(), + }, + ); + expect(result.ok).toBe(true); + expect(invokeRuntime).not.toHaveBeenCalled(); + expect((await store.getTrackerSync(meta.executionId)).pullRequests).toHaveLength(2); + expect((await row()).status).toBe('succeeded'); + }); + it.each(['unavailable', 'replaced-plan'])( + 'withholds initial publication at the actual start entrypoint when evidence becomes %s', + async (fault) => { + meta = await store.updateExecution({ executionId: meta.executionId, status: 'CREATED' }); + let publishing = false; + let capturedMeta; + const guardedStore = { + ...store, + getCompletionRecords: async (...args) => { + if (publishing && fault === 'unavailable') + throw new Error('completion store unavailable'); + return store.getCompletionRecords(...args); + }, + }; + const invokeRuntime = vi.fn(async () => ({ ok: true })); + const result = await __durableHandler( + { action: 'start', executionId: meta.executionId, intentId: meta.intentId }, + { + step: async (name, fn) => { + if (name === 'open-pr') { + publishing = true; + if (fault === 'replaced-plan') + await store.putUnitPlan({ + executionId: meta.executionId, + units: [{ slug: 'replacement' }], + batches: [['replacement']], + }); + } + const value = await fn(); + if (name === 'load-meta') capturedMeta = value; + return value; + }, + }, + { + store: guardedStore, + loadPlan, + publishDelivery: (args) => publishFinalDelivery({ ...args, operation, loadPlan }), + invokeRuntime, + broadcast: vi.fn(), + }, + ); + expect(result.ok).toBe(true); // Execution completed; publication was withheld separately. + expect(capturedMeta).toMatchObject({ completionPolicyVersion: 1, deliveryPolicyVersion: 1 }); + expect(capturedMeta.recoveryRuntime).toBeUndefined(); + meta = await store.getExecution(meta.executionId); + expect(meta.status).toBe('SUCCEEDED'); + expect(operation).not.toHaveBeenCalled(); + expect(await row()).toBeNull(); + expect( + (await store.listEvents(meta.executionId)).some((e) => e.eventType === 'v2.pr.failed'), + ).toBe(true); + expect( + invokeRuntime.mock.calls.some(([payload]) => payload.command === 'run-stage-start'), + ).toBe(false); + }, + ); + + it.each([false, true])( + 'persists publication when graph projection fails (captured runtime=%s)', + async (capturedRuntime) => { + meta = await store.updateExecution({ executionId: meta.executionId, status: 'CREATED' }); + const runtimeTarget = { + agentRuntimeArn: 'arn:aws:bedrock-agentcore:eu-west-1:123456789012:runtime/captured', + qualifier: 'original', + capacityProviderArn: null, + }; + if (capturedRuntime) { + meta.environment = { + runtimeArn: runtimeTarget.agentRuntimeArn, + runtimeEndpoint: runtimeTarget.qualifier, + }; + await db.doc.send(new PutCommand({ TableName: tableName, Item: meta })); + } + const commands = []; + const invokeRuntime = vi.fn(async (payload, _session, target) => { + if (capturedRuntime) expect(target).toEqual(runtimeTarget); + commands.push(payload.command); + return payload.command === 'record-pr' ? { ok: false } : { ok: true }; + }); + const result = await __durableHandler( + { + action: 'start', + executionId: meta.executionId, + intentId: meta.intentId, + }, + { step: (_name, fn) => fn() }, + { + store, + loadPlan: async () => ({ valid: true, plan: { stages: [] } }), + publishDelivery: (args) => publishFinalDelivery({ ...args, operation, loadPlan }), + invokeRuntime, + broadcast: vi.fn(), + }, + ); + expect(result.ok).toBe(true); + meta = await store.getExecution(meta.executionId); + expect(meta.status).toBe('SUCCEEDED'); + expect(meta.completionValidation.version).toBe(1); + expect((await row()).status).toBe('succeeded'); + expect(commands).toContain('record-pr'); + expect(deliveryView(meta, await row()).repositories[0].pr.prUrl).toBeTruthy(); + }, + ); +}); diff --git a/lambda/v2-orchestrator/test/fixtures/completion-handler.js b/lambda/v2-orchestrator/test/fixtures/completion-handler.js new file mode 100644 index 000000000..0aa03434a --- /dev/null +++ b/lambda/v2-orchestrator/test/fixtures/completion-handler.js @@ -0,0 +1,36 @@ +import { __durableHandler as handler } from '../../index.js'; +import { repoUrl } from '../../../shared/repo-provider.js'; +import { openIntentPrs } from '../../../shared/intent-pr.js'; +import { withCompletionReads } from './completion-store.js'; + +// Upgrade the existing control-flow fakes to the persisted completion read +// contract. Production logic still runs; real conditional stores and deliberately +// incomplete snapshots are covered by the completion tests. +export const __durableHandler = (event, ctx, deps) => + handler(event, ctx, { + ...deps, + publishDelivery: + deps.publishDelivery ?? + (({ store, meta, applicationUrl }) => + openIntentPrs({ + store, + meta, + applicationUrl, + executionId: meta.executionId, + runId: meta.orchestratorRunId, + openPr: deps.openPr, + comparePrBranches: deps.comparePrBranches, + })), + store: withCompletionReads(deps.store), + invokeRuntime: async (...args) => { + const result = await deps.invokeRuntime(...args); + const [payload] = args; + if (payload.command === 'merge-lane' && result?.ok && result.results === undefined) { + return { + ...result, + results: (payload.repos ?? []).map((repo) => ({ repo: repoUrl(repo), merged: true })), + }; + } + return result; + }, + }); diff --git a/lambda/v2-orchestrator/test/fixtures/completion-store.js b/lambda/v2-orchestrator/test/fixtures/completion-store.js new file mode 100644 index 000000000..bcb285df3 --- /dev/null +++ b/lambda/v2-orchestrator/test/fixtures/completion-store.js @@ -0,0 +1,56 @@ +// Stateful completion reads for the orchestration fakes. Existing tests own +// their store methods and assertions; these reads assemble the writes those +// fakes accepted. Conditional-write races use DynamoDB Local separately. +import { vi } from 'vitest'; + +export const withCompletionReads = (store) => { + const calls = {}; + for (const name of ['updateExecution', 'updateUnitState', 'createUnitPr', 'updateUnitPr']) { + if (!store[name]) continue; + if (!store[name].mock) { + const original = store[name]; + store[name] = vi.fn((...args) => original(...args)); + } + calls[name] = () => + store[name].mock.calls + .filter((_, index) => store[name].mock.settledResults?.[index]?.type !== 'rejected') + .map(([input]) => input); + } + store.getCompletionRecords = async (executionId) => { + const loaded = await store.getExecution(executionId); + const meta = { ...loaded }; + for (const input of calls.updateExecution?.() ?? []) { + for (const field of ['status', 'orchestratorRunId', 'completionValidation']) { + if (input[field] !== undefined) meta[field] = input[field]; + } + } + const units = new Map( + ((await store.listUnits?.(executionId)) ?? []).map((row) => [ + `${row.sectionIndex}:${row.slug}`, + { ...row }, + ]), + ); + for (const input of calls.updateUnitState?.() ?? []) { + const key = `${input.sectionIndex}:${input.slug}`; + units.set(key, { ...units.get(key), ...input, ...input.fields }); + } + const unitPrs = new Map( + ((await store.listUnitPrs?.(executionId)) ?? []).map((row) => [ + `${row.sectionIndex}:${row.unitSlug ?? row.slug}:${row.repository}`, + { ...row }, + ]), + ); + for (const input of [...(calls.createUnitPr?.() ?? []), ...(calls.updateUnitPr?.() ?? [])]) { + const key = `${input.sectionIndex}:${input.slug}:${input.repository}`; + unitPrs.set(key, { ...unitPrs.get(key), ...input, ...input.fields }); + } + return { + meta, + unitPlan: (await store.getUnitPlan?.(executionId)) ?? null, + units: [...units.values()], + unitPrs: [...unitPrs.values()], + humanTasks: [], + }; + }; + return store; +}; diff --git a/lambda/v2-orchestrator/test/fixtures/dependent-skeleton-plan.json b/lambda/v2-orchestrator/test/fixtures/dependent-skeleton-plan.json new file mode 100644 index 000000000..4c42d3665 --- /dev/null +++ b/lambda/v2-orchestrator/test/fixtures/dependent-skeleton-plan.json @@ -0,0 +1,83 @@ +{ + "units": [ + { + "dependsOn": [], + "slug": "notifications-data-infra", + "kind": null + }, + { + "dependsOn": ["notifications-data-infra"], + "slug": "notification-core", + "kind": "service" + }, + { + "dependsOn": ["notification-core", "notifications-data-infra"], + "slug": "execution-event-capture", + "kind": "service" + }, + { + "kind": "service", + "dependsOn": ["notification-core", "notifications-data-infra"], + "slug": "notifications-api" + }, + { + "dependsOn": ["notifications-api"], + "kind": "ui", + "slug": "fe-inbox" + }, + { + "dependsOn": ["notification-core", "notifications-api", "fe-inbox"], + "kind": "service", + "slug": "realtime-user-channel" + }, + { + "dependsOn": ["realtime-user-channel", "fe-inbox"], + "kind": "ui", + "slug": "fe-realtime-alerts" + }, + { + "dependsOn": [ + "notification-core", + "notifications-data-infra", + "notifications-api", + "fe-inbox" + ], + "kind": "service", + "slug": "email-setup" + }, + { + "dependsOn": [ + "email-setup", + "execution-event-capture", + "realtime-user-channel", + "fe-realtime-alerts" + ], + "kind": "service", + "slug": "email-immediate" + }, + { + "dependsOn": ["email-immediate", "execution-event-capture"], + "kind": "service", + "slug": "email-digest" + }, + { + "slug": "notification-observability", + "dependsOn": ["email-digest", "execution-event-capture", "notification-core"], + "kind": null + } + ], + "batches": [ + ["notifications-data-infra"], + ["notification-core"], + ["execution-event-capture", "notifications-api"], + ["fe-inbox"], + ["email-setup", "realtime-user-channel"], + ["fe-realtime-alerts"], + ["email-immediate"], + ["email-digest"], + ["notification-observability"] + ], + "walkingSkeleton": "notification-core", + "autonomyMode": "autonomous", + "skipMatrix": {} +} diff --git a/lambda/v2-orchestrator/test/fixtures/load-meta-before-completion.js b/lambda/v2-orchestrator/test/fixtures/load-meta-before-completion.js new file mode 100644 index 000000000..d34d6ea06 --- /dev/null +++ b/lambda/v2-orchestrator/test/fixtures/load-meta-before-completion.js @@ -0,0 +1,30 @@ +// Frozen load-meta body from main 4fd4558ae8b064d8d99efcbd905d38b0b853c27a, +// also used by the deployed pre-fix orchestrator. Do not modernize this fixture. +export const loadMetaBeforeCompletion = (store, executionId) => + store.getExecution(executionId, { consistentRead: true }); + +// Frozen finish-succeeded body from main c4b6afff3147be23c567492b3a5ea9e692af7e0f. +export const finishSucceededBeforeCompletion = async ({ + store, + executionId, + projectId, + meta, + runId, +}) => { + try { + await store.updateExecution({ + executionId, + projectId, + status: 'SUCCEEDED', + startedAt: meta.startedAt, + completedAt: new Date().toISOString(), + failureReason: null, + failure: null, + ...(runId ? { ifOrchestratorRunId: runId } : {}), + }); + return true; + } catch (e) { + if (e?.name === 'ConditionalCheckFailedException') return false; + throw e; + } +}; diff --git a/lambda/v2-orchestrator/test/fixtures/load-meta-before-delivery.js b/lambda/v2-orchestrator/test/fixtures/load-meta-before-delivery.js new file mode 100644 index 000000000..bb3db95bf --- /dev/null +++ b/lambda/v2-orchestrator/test/fixtures/load-meta-before-delivery.js @@ -0,0 +1,5 @@ +// Frozen load-meta boundary from the corrected preventive completion policy (PR #534). +export const loadMetaBeforeDelivery = async (store, executionId) => { + const loaded = await store.getExecution(executionId, { consistentRead: true }); + return loaded ? { ...loaded, completionPolicyVersion: 1 } : null; +}; diff --git a/lambda/v2-orchestrator/test/gate-findings.test.js b/lambda/v2-orchestrator/test/gate-findings.test.js index 7782a3d4c..7e464bac1 100644 --- a/lambda/v2-orchestrator/test/gate-findings.test.js +++ b/lambda/v2-orchestrator/test/gate-findings.test.js @@ -8,7 +8,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { Logger } from '@aws-lambda-powertools/logger'; -import { __durableHandler } from '../index.js'; +import { __durableHandler } from './fixtures/completion-handler.js'; const makeCtx = (over = {}) => { const stageCallbacks = new Map(); diff --git a/lambda/v2-orchestrator/test/lane-gate-resume.test.js b/lambda/v2-orchestrator/test/lane-gate-resume.test.js index cb61bf3ef..25fe1b9d7 100644 --- a/lambda/v2-orchestrator/test/lane-gate-resume.test.js +++ b/lambda/v2-orchestrator/test/lane-gate-resume.test.js @@ -1,5 +1,5 @@ import { describe, it, expect, vi } from 'vitest'; -import { __durableHandler } from '../index.js'; +import { __durableHandler } from './fixtures/completion-handler.js'; const makeCtx = () => { const stageCallbacks = new Map(); diff --git a/lambda/v2-orchestrator/test/learnings-ritual.test.js b/lambda/v2-orchestrator/test/learnings-ritual.test.js index d158fb8a5..dcf2405a7 100644 --- a/lambda/v2-orchestrator/test/learnings-ritual.test.js +++ b/lambda/v2-orchestrator/test/learnings-ritual.test.js @@ -7,7 +7,7 @@ // and a failure to record a learning never costs the run its approval. import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { __durableHandler } from '../index.js'; +import { __durableHandler } from './fixtures/completion-handler.js'; import { buildEventRow } from '../../shared/v2-process-keys.js'; // Timeline fixtures go through the SAME row builder the process store persists diff --git a/lambda/v2-orchestrator/test/orchestrator-replay.test.js b/lambda/v2-orchestrator/test/orchestrator-replay.test.js index f873e2cd0..7734c4ecb 100644 --- a/lambda/v2-orchestrator/test/orchestrator-replay.test.js +++ b/lambda/v2-orchestrator/test/orchestrator-replay.test.js @@ -4,9 +4,14 @@ import { LocalDurableTestRunner, WaitingOperationStatus, } from '@aws/durable-execution-sdk-js-testing'; -import { __durableHandler } from '../index.js'; +import { __durableHandler } from './fixtures/completion-handler.js'; import { awaitEngineGate } from '../section.js'; import { awaitEngineGate as previousEngineGate } from './fixtures/engine-gate-before-recovery.js'; +import { loadMetaBeforeDelivery } from './fixtures/load-meta-before-delivery.js'; +import { + loadMetaBeforeCompletion, + finishSucceededBeforeCompletion, +} from './fixtures/load-meta-before-completion.js'; // --------------------------------------------------------------------------- // REAL replay coverage for the orchestrator's async stage flow (WP1). @@ -292,6 +297,207 @@ const completeStage = async (runner, opName, result) => { }; describe('orchestrator on the real durable runner (replay semantics)', () => { + it.each([ + [loadMetaBeforeCompletion, 0], + [loadMetaBeforeDelivery, 1], + ])( + 'resumes a frozen pre-delivery load-meta checkpoint (%s) without upgrading publication', + async (loadFrozenMeta, completionVersion) => { + const world = makeWorld({ stages: [{ stageId: 'a' }] }); + let upgraded = false; + const loadedVersions = []; + let deliveryCalls = 0; + world.deps.publishDelivery = async () => { + deliveryCalls++; + return []; + }; + let firstLoadCalls = 0; + let markSuspended; + const suspended = new Promise((resolve) => { + markSuspended = resolve; + }); + const handler = withDurableExecution((event, ctx) => { + const wrapped = new Proxy(ctx, { + get: (target, key) => + key === 'step' + ? async (name, fn, options) => { + const result = await target.step( + name, + !upgraded && name === 'load-meta' + ? async () => { + firstLoadCalls++; + return loadFrozenMeta(world.deps.store, event.executionId); + } + : fn, + options, + ); + if (name === 'load-meta') + loadedVersions.push(result.completionPolicyVersion ?? 0); + return result; + } + : typeof target[key] === 'function' + ? target[key].bind(target) + : target[key], + }); + return __durableHandler(event, wrapped, world.deps); + }); + const runner = new LocalDurableTestRunner({ + handlerFunction: async (...input) => { + const result = await handler(...input); + if (result.Status === 'PENDING') { + upgraded = true; + markSuspended(); + } + return result; + }, + }); + const completion = runner.run({ + payload: { action: 'start', intentId: 'i1', executionId: 'i1' }, + }); + await suspended; + await completeStage(runner, 'stage-cb-a', { ok: true, state: 'SUCCEEDED' }); + const execution = await completion; + expect(execution.getResult()).toMatchObject({ ok: true }); + expect(execution.getInvocations().length).toBeGreaterThan(1); + expect(firstLoadCalls).toBe(1); + expect(new Set(loadedVersions)).toEqual(new Set([completionVersion])); + // New delivery code would call this collaborator; frozen histories never do. + expect(deliveryCalls).toBe(0); + expect(world.invokes.filter((payload) => payload.command === 'run-stage-start')).toHaveLength( + 1, + ); + }, + ); + + it.each(['before', 'after'])( + 'replays a frozen marker-absent checkpoint %s terminal success', + async (boundary) => { + const world = makeWorld({ stages: [{ stageId: 'a' }] }); + let upgraded = false; + let finishCalls = 0; + const names = []; + const handler = withDurableExecution((event, ctx) => { + let loadedMeta; + let runId; + const wrapped = new Proxy(ctx, { + get: (target, key) => + key === 'step' + ? async (name, fn, options) => { + names.push(name); + if (name === 'open-pr' && boundary === 'after') { + const [pause] = await target.createCallback('test-after-legacy-success'); + await pause; + } + const result = await target.step( + name, + !upgraded && name === 'load-meta' + ? () => loadMetaBeforeCompletion(world.deps.store, event.executionId) + : !upgraded && name === 'finish-succeeded' + ? () => { + finishCalls++; + return finishSucceededBeforeCompletion({ + store: world.deps.store, + executionId: event.executionId, + projectId: loadedMeta.projectId, + meta: loadedMeta, + runId, + }); + } + : fn, + options, + ); + if (name === 'load-meta') loadedMeta = result; + if (name === 'mint-run-id') runId = result.runId; + return result; + } + : typeof target[key] === 'function' + ? target[key].bind(target) + : target[key], + }); + return __durableHandler(event, wrapped, world.deps); + }); + const runner = new LocalDurableTestRunner({ handlerFunction: handler }); + const completion = runner.run({ + payload: { action: 'start', intentId: 'i1', executionId: 'i1' }, + }); + const stage = await runner + .getOperation('stage-cb-a') + .waitForData(WaitingOperationStatus.STARTED); + if (boundary === 'before') upgraded = true; + await stage.sendCallbackSuccess(JSON.stringify({ ok: true, state: 'SUCCEEDED' })); + if (boundary === 'after') { + const pause = await runner + .getOperation('test-after-legacy-success') + .waitForData(WaitingOperationStatus.STARTED); + upgraded = true; + await pause.sendCallbackSuccess(JSON.stringify({ ok: true })); + } + const result = await completion; + expect(result.getResult()).toMatchObject({ ok: true }); + expect(names).not.toContain('finish-completion-v1'); + expect(world.statusWrites.filter((status) => status === 'SUCCEEDED')).toHaveLength(1); + expect(finishCalls).toBe(boundary === 'after' ? 1 : 0); + expect(world.statusWrites).not.toContain('FAILED'); + }, + ); + it('resumes a frozen pre-fix load-meta checkpoint without silently upgrading completion policy', async () => { + const world = makeWorld({ stages: [{ stageId: 'a' }] }); + let upgraded = false; + const loadedVersions = []; + let firstLoadCalls = 0; + let markSuspended; + const suspended = new Promise((resolve) => { + markSuspended = resolve; + }); + const handler = withDurableExecution((event, ctx) => { + const wrapped = new Proxy(ctx, { + get: (target, key) => + key === 'step' + ? async (name, fn, options) => { + const result = await target.step( + name, + !upgraded && name === 'load-meta' + ? async () => { + firstLoadCalls++; + return loadMetaBeforeCompletion(world.deps.store, event.executionId); + } + : fn, + options, + ); + if (name === 'load-meta') loadedVersions.push(result.completionPolicyVersion ?? 0); + return result; + } + : typeof target[key] === 'function' + ? target[key].bind(target) + : target[key], + }); + return __durableHandler(event, wrapped, world.deps); + }); + const runner = new LocalDurableTestRunner({ + handlerFunction: async (...input) => { + const result = await handler(...input); + if (result.Status === 'PENDING') { + upgraded = true; + markSuspended(); + } + return result; + }, + }); + const completion = runner.run({ + payload: { action: 'start', intentId: 'i1', executionId: 'i1' }, + }); + await suspended; + await completeStage(runner, 'stage-cb-a', { ok: true, state: 'SUCCEEDED' }); + const execution = await completion; + expect(execution.getResult()).toMatchObject({ ok: true }); + expect(execution.getInvocations().length).toBeGreaterThan(1); + expect(firstLoadCalls).toBe(1); + expect(new Set(loadedVersions)).toEqual(new Set([0])); + expect(world.invokes.filter((payload) => payload.command === 'run-stage-start')).toHaveLength( + 1, + ); + }); + it('resumes a stage gate after its unpark committed without a checkpoint', async () => { const world = makeWorld({ stages: [{ stageId: 'a', stageInstanceId: 'si-a' }] }); const meta = { ...META }; diff --git a/lambda/v2-orchestrator/test/orchestrator.test.js b/lambda/v2-orchestrator/test/orchestrator.test.js index ba30d148a..e60c71d2b 100644 --- a/lambda/v2-orchestrator/test/orchestrator.test.js +++ b/lambda/v2-orchestrator/test/orchestrator.test.js @@ -1,5 +1,7 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { __durableHandler } from '../index.js'; +import { __durableHandler } from './fixtures/completion-handler.js'; +import { promoteUnits } from '../../agentcore/commands/promote-units.js'; +import dependentSkeletonPlan from './fixtures/dependent-skeleton-plan.json'; // The orchestrator's control flow is driven through an injected `deps` bag and a // fake DurableContext — no real AWS/Neptune. This isolates the sequencing logic @@ -134,6 +136,7 @@ beforeEach(() => { putTrackerSync: vi.fn(async (args) => args), failRunningStageAttempt: vi.fn(async () => null), listUnits: vi.fn(async () => []), + updateUnitPlanDecisions: vi.fn(async () => ({})), getUnit: vi.fn(async () => null), }, loadPlan: vi.fn(async () => ({ @@ -1879,6 +1882,10 @@ describe('WP5 — parallel sections: lanes, skeleton, ladder, halt-and-ask', () // Complete the missing lane instance → rewind proceeds straight to bt. rows[planStageInstanceId('aidlc-v2@1', 'cg', 'billing', 1)] = doneRow; + deps.store.listUnits = vi.fn(async () => [ + { sectionIndex: 1, slug: 'auth', state: 'MERGED' }, + { sectionIndex: 1, slug: 'billing', state: 'MERGED' }, + ]); invokes.length = 0; ctx = makeCtx(); deps.invokeRuntime = makeRuntime(ctx, sectionScript); @@ -2090,6 +2097,9 @@ describe('PR per unit delivery', () => { ); expect(invokes.map((payload) => payload.command)).toContain('reconcile-lane'); expect(invokes.map((payload) => payload.command)).toContain('refresh-intent'); + expect(unitStates.find((row) => row.state === 'MERGED').fields).toMatchObject({ + integrationResults: [], + }); expect(invokes).toContainEqual( expect.objectContaining({ command: 'record-unit-pr', @@ -2116,6 +2126,16 @@ describe('PR per unit delivery', () => { ); }); + it('records the evidence attribute for an unchanged unit PR integration', async () => { + configure({ statusFor: async () => null }); + deps.unitPrProvider.compare.mockResolvedValue({ status: 'identical' }); + expect(await start()).toMatchObject({ ok: true }); + expect(unitStates.find((row) => row.state === 'MERGED').fields).toMatchObject({ + integrationResults: [], + }); + expect(deps.unitPrProvider.createDraft).not.toHaveBeenCalled(); + }); + it('parks one callback for a long unchanged PR wait without growing durable operations', async () => { const originalCreateCallback = ctx.createCallback; let resolvePrWait; @@ -2492,6 +2512,161 @@ describe('WP5 — engine-gate decisions', () => { const start = () => __durableHandler({ action: 'start', intentId: 'i1', executionId: 'i1' }, ctx, deps); + it('rejects the observed dependent skeleton before building, shared checks or final PR creation', async () => { + unitPlanRow = structuredClone(dependentSkeletonPlan); + const result = await start(); + expect(result).toMatchObject({ ok: false, reason: 'invalid_walking_skeleton' }); + expect(stageStarts().filter((payload) => payload.unitSlug)).toEqual([]); + expect(stageStarts().some((payload) => payload.stageId === 'bt')).toBe(false); + expect(deps.openPr).not.toHaveBeenCalled(); + expect( + deps.store.updateExecution.mock.calls.some(([input]) => input.status === 'SUCCEEDED'), + ).toBe(false); + expect(unitPlanRow.walkingSkeleton).toBe('notification-core'); + }); + + it.each([ + ['renamed-unit', false], + ['billing', false], + ['billing', true], + ])( + 'plain approval executes a regenerated DAG after old skeleton %s becomes invalid (ceremony off: %s)', + async (oldSkeleton, ceremonyOff) => { + unitPlanRow.walkingSkeleton = oldSkeleton; + if (ceremonyOff) { + const plan = SECTION_PLAN(); + plan.plan.stages + .filter((stage) => stage.parallelSection === 1) + .forEach((stage) => { + stage.policy = { skeleton: 'off' }; + }); + deps.loadPlan.mockResolvedValue(plan); + } + const graph = { + lookupArtifacts: async () => [ + { + id: 'regenerated', + content: + '```yaml\nunits:\n - name: auth\n depends_on: []\n - name: billing\n depends_on: [auth]\n```', + }, + ], + mirrorUnitDag: async () => ({}), + }; + deps.invokeRuntime = makeRuntime(ctx, async (payload) => { + if (payload.command !== 'promote-units') return sectionScript(payload); + return promoteUnits(payload, { + store: { + ...deps.store, + putUnitPlan: async (plan) => { + unitPlanRow = plan; + return plan; + }, + syncUnitRows: async () => ({ created: [], preserved: [], orphaned: [] }), + }, + openGraph: async () => ({}), + createWriter: () => graph, + }); + }); + const result = await start(); + expect(result.ok).toBe(true); + expect(unitPlanRow.walkingSkeleton).toBe('auth'); + if (ceremonyOff) + expect( + deps.store.getHumanTask.mock.calls.some(([, id]) => id.startsWith('eg-skeleton')), + ).toBe(false); + expect( + invokes + .filter((payload) => payload.command === 'merge-lane') + .map((payload) => payload.unitSlug), + ).toEqual(['auth', 'billing']); + expect( + deps.store.updateExecution.mock.calls.some(([input]) => input.status === 'SUCCEEDED'), + ).toBe(true); + }, + ); + + it('does not execute a different default when an approved fan-out decision cannot be saved', async () => { + deps.store.updateUnitPlanDecisions.mockRejectedValue(new Error('decision store unavailable')); + const result = await start(); + expect(result.ok).toBe(false); + expect(stageStarts().filter((payload) => payload.unitSlug)).toEqual([]); + expect(deps.openPr).not.toHaveBeenCalled(); + }); + + it('integrates all eleven units with the dependency-free control, preserving the recovered foundation', async () => { + unitPlanRow = { + ...structuredClone(dependentSkeletonPlan), + walkingSkeleton: 'notifications-data-infra', + }; + deps.store.listUnits.mockResolvedValue([ + { slug: 'notifications-data-infra', sectionIndex: 1, state: 'MERGED' }, + { slug: 'old-plan-unit', sectionIndex: 1, state: 'PENDING' }, + ]); + const result = await start(); + expect(result.ok).toBe(true); + const integrations = invokes.filter((payload) => payload.command === 'merge-lane'); + expect(integrations).toHaveLength(10); + expect(integrations.some((payload) => payload.unitSlug === 'notifications-data-infra')).toBe( + false, + ); + expect(stageStarts().some((payload) => payload.stageId === 'bt')).toBe(true); + expect( + deps.store.appendEvent.mock.calls + .map(([event]) => event) + .find((event) => event.type === 'v2.units.fan_in').summary, + ).toContain('11/11'); + }); + + it.each([false, true])( + 'counts disjoint halt outcomes and prevents unresolved completion (failed prerequisite: %s)', + async (failedPrerequisite) => { + unitPlanRow = { + units: [ + ...(failedPrerequisite ? [{ slug: 'prerequisite', dependsOn: [] }] : []), + { + slug: 'blocked', + dependsOn: [failedPrerequisite ? 'prerequisite' : 'missing-prerequisite'], + }, + ], + batches: [failedPrerequisite ? ['prerequisite', 'blocked'] : ['blocked']], + walkingSkeleton: 'blocked', + autonomyMode: 'autonomous', + }; + const plan = SECTION_PLAN(); + plan.plan.stages + .filter((stage) => stage.parallelSection === 1) + .forEach((stage) => { + stage.policy = { skeleton: 'off' }; + }); + deps.loadPlan.mockResolvedValue(plan); + const answered = gateReads({ 'eg-halt': { answer: { decision: 'abort' } } }); + deps.store.getHumanTask = vi.fn(async (executionId, id) => { + if ( + id.startsWith('eg-halt') && + !deps.store.createHumanTask.mock.calls.some(([gate]) => gate.humanTaskId === id) + ) + return null; + return answered(executionId, id); + }); + if (failedPrerequisite) + deps.invokeRuntime = makeRuntime(ctx, (payload) => + payload.stageId === 'cg' && payload.unitSlug === 'prerequisite' + ? { ok: false, state: 'FAILED', reason: 'sensor_blocked' } + : sectionScript(payload), + ); + const result = await start(); + expect(result).toMatchObject({ ok: false, reason: 'section_aborted' }); + expect(stageStarts().some((payload) => payload.stageId === 'bt')).toBe(false); + expect(deps.openPr).not.toHaveBeenCalled(); + const halt = deps.store.createHumanTask.mock.calls + .map(([gate]) => gate) + .find((gate) => gate.humanTaskId.startsWith('eg-halt')); + expect(halt.prompt).toContain( + `${failedPrerequisite ? 1 : 0} failed, 1 blocked, 0 integrated`, + ); + }, + ); + it('halt-and-ask RETRY re-runs the failed lane under fresh round identities and completes the run', async () => { let cgBillingAttempts = 0; deps.store.getHumanTask = gateReads({ 'eg-halt': { answer: { decision: 'retry' } } }); @@ -2604,6 +2779,19 @@ describe('WP5 — engine-gate decisions', () => { const eventCalls = deps.store.appendEvent.mock.calls.map((c) => c[0]); expect(eventCalls.some((e) => e.type === 'v2.units.lanes_skipped')).toBe(true); expect(eventCalls.find((e) => e.type === 'v2.units.fan_in')?.summary).toContain('1/2'); + expect( + unitStates.find((unit) => unit.slug === 'billing' && unit.fields?.completionSkip), + ).toMatchObject({ + fields: { + completionSkip: { + humanTaskId: expect.stringMatching(/^eg-halt/), + planIdentity: expect.any(String), + }, + }, + }); + expect(eventCalls.find((e) => e.type === 'v2.units.fan_in')?.summary).toContain( + 'explicitly omitted', + ); }); it('AUTONOMOUS wavefront: a failed lane BLOCKS its dependents while independents finish', async () => { @@ -2634,7 +2822,7 @@ describe('WP5 — engine-gate decisions', () => { // c never dispatched a stage — it blocked on b. expect(stageStarts().some((p) => p.unitSlug === 'c')).toBe(false); const cStates = unitStates.filter((u) => u.slug === 'c').map((u) => u.state); - expect(cStates).toEqual(['BLOCKED']); + expect([...new Set(cStates)]).toEqual(['BLOCKED']); expect(unitStates.find((u) => u.slug === 'c' && u.state === 'BLOCKED').fields).toMatchObject({ blockedOn: 'b', }); @@ -2836,8 +3024,8 @@ describe('WP5 — engine-gate decisions', () => { it('an uninterpretable halt-and-ask answer RE-ASKS; the follow-up choice is honored', async () => { deps.store.getHumanTask = gateReads({ - 'eg-halt-s1-r0-a1': { answer: { decision: 'skip' } }, - 'eg-halt-s1-r0': { answer: { decision: 'garbage' } }, + 'eg-halt-s1-w1-r0-a1': { answer: { decision: 'skip' } }, + 'eg-halt-s1-w1-r0': { answer: { decision: 'garbage' } }, }); deps.invokeRuntime = makeRuntime(ctx, (payload) => { if (payload.command === 'init-ws') return { ok: true }; @@ -3000,6 +3188,21 @@ describe('WP6 — PR opened on SUCCEEDED (intent-pr)', () => { __durableHandler({ action: 'start', intentId: 'i1', executionId: 'i1' }, ctx, deps); const events = () => deps.store.appendEvent.mock.calls.map((c) => c[0]); + it('counts only current-plan units in the final PR summary, retaining older rows', async () => { + const rows = [ + { slug: 'auth', state: 'MERGED' }, + { slug: 'billing', state: 'MERGED' }, + { slug: 'retired-plan-unit', state: 'PENDING' }, + ]; + deps.store.listUnits.mockResolvedValue(rows); + deps.store.getUnitPlan = vi.fn(async () => UNIT_PLAN()); + expect((await start()).ok).toBe(true); + const body = deps.openPr.mock.calls[0][0].body; + expect(body).toContain('2 total, 2 integrated into the intent branch'); + expect(body).not.toContain('retired-plan-unit'); + expect(rows).toHaveLength(3); + }); + it('opens one PR per repo from the intent branch with execution provenance in the body', async () => { deps.store.getExecution = vi.fn(async () => ({ ...META, @@ -3037,7 +3240,7 @@ describe('WP6 — PR opened on SUCCEEDED (intent-pr)', () => { 'created by [AI-DLC](https://aidlc.example.test/space/p1/intent/i1)', ); expect(deps.openPr.mock.calls[0][0].body).toContain('strategy: intent-pr'); - expect(deps.openPr.mock.calls[0][0].body).toContain('2 total, 2 merged'); + expect(deps.openPr.mock.calls[0][0].body).toContain('2 total, 2 integrated'); const opened = events().filter((e) => e.type === 'v2.pr.opened'); expect(opened).toHaveLength(2); expect(opened[0].summary).toContain('https://github.com/o/r/pull/7'); @@ -3239,9 +3442,9 @@ describe('WP6 — PR opened on SUCCEEDED (intent-pr)', () => { deps.openPr = vi.fn(async () => ({ prUrl: 'https://x/pr/1', prNumber: 1 })); await start(); const body = deps.openPr.mock.calls[0][0].body; - expect(body).toContain('3 total, 1 merged'); - expect(body).toContain('unit `billing` NOT merged (FAILED: merge-lane: markers_remain)'); - expect(body).toContain('unit `checkout` NOT merged (BLOCKED)'); + expect(body).toContain('3 total, 1 integrated'); + expect(body).toContain('unit `billing` NOT integrated (FAILED: merge-lane: markers_remain)'); + expect(body).toContain('unit `checkout` NOT integrated (BLOCKED)'); }); it('an identical branch records v2.pr.skipped without calling the provider create API', async () => { diff --git a/lambda/v2-orchestrator/test/release-coexistence-simulation.test.js b/lambda/v2-orchestrator/test/release-coexistence-simulation.test.js index 239927928..3e30714c9 100644 --- a/lambda/v2-orchestrator/test/release-coexistence-simulation.test.js +++ b/lambda/v2-orchestrator/test/release-coexistence-simulation.test.js @@ -19,7 +19,7 @@ import { methodologyReleasePinFromManifest, } from '../../shared/release-resolver.js'; import { loadExecutionPlan } from '../../shared/v2-workflow-plan.js'; -import { __durableHandler } from '../index.js'; +import { __durableHandler } from './fixtures/completion-handler.js'; vi.spyOn(Logger.prototype, 'info').mockImplementation(() => {}); diff --git a/lambda/v2-orchestrator/test/section-integration.test.js b/lambda/v2-orchestrator/test/section-integration.test.js index c130e249e..9833e49b5 100644 --- a/lambda/v2-orchestrator/test/section-integration.test.js +++ b/lambda/v2-orchestrator/test/section-integration.test.js @@ -7,7 +7,7 @@ import { LocalDurableTestRunner, WaitingOperationStatus, } from '@aws/durable-execution-sdk-js-testing'; -import { __durableHandler } from '../index.js'; +import { __durableHandler } from './fixtures/completion-handler.js'; import { initLane, mergeLane } from '../../agentcore/commands/lane.js'; import { resolveConflict } from '../../agentcore/commands/resolve-conflict.js'; import { runGit, commitAndPushAll, fetchOrigin } from '../../agentcore/git-engine.js'; @@ -295,7 +295,9 @@ const makeWorld = ({ remote, unitPlan, fileFor, beforeStage = null, conflictAgen // wait for the row to appear, flip it, complete its durable callback. const answerEngineGate = async (world, prefix, patch = { status: 'answered' }) => { let id; - for (let i = 0; i < 400 && !id; i++) { + // Real Git and durable callbacks can exceed 10s under concurrent test load. + // Keep the wait bounded within the existing 120s test deadline. + for (let i = 0; i < 1200 && !id; i++) { id = [...world.gates.keys()].find((k) => k.startsWith(prefix)); if (!id) await new Promise((r) => setTimeout(r, 25)); } @@ -455,7 +457,7 @@ describe('WP8 fixture 2 — forced merge conflict → halt-and-ask SKIP (real gi : { name: 'shared.txt', content: `${slug} version\n` }, beforeStage: async (payload, w) => { if (payload.unitSlug !== 'c') return; - for (let i = 0; i < 400 && !w.unitStates.includes('b:MERGED'); i++) { + for (let i = 0; i < 1200 && !w.unitStates.includes('b:MERGED'); i++) { await new Promise((r) => setTimeout(r, 25)); } }, @@ -492,6 +494,7 @@ describe('WP8 fixture 2 — forced merge conflict → halt-and-ask SKIP (real gi 'c:RUNNING', 'c:MERGING', 'c:FAILED', + 'c:FAILED', // Human omission receipt retains the original failure state. ]); const cFailed = world.events.find((e) => e.type === 'v2.unit.failed' && e.unitSlug === 'c'); expect(cFailed?.summary).toContain('markers_remain'); @@ -558,7 +561,7 @@ describe('WP6 fixture 3 — conflict RESOLVED by the resolution stage (real git) : { name: 'shared.txt', content: `${slug} version\n` }, beforeStage: async (payload, w) => { if (payload.unitSlug !== 'c') return; - for (let i = 0; i < 400 && !w.unitStates.includes('b:MERGED'); i++) { + for (let i = 0; i < 1200 && !w.unitStates.includes('b:MERGED'); i++) { await new Promise((r) => setTimeout(r, 25)); } }, diff --git a/lambda/v2-orchestrator/test/skeleton-switch.test.js b/lambda/v2-orchestrator/test/skeleton-switch.test.js index e7c1e14ed..0cc0daaf5 100644 --- a/lambda/v2-orchestrator/test/skeleton-switch.test.js +++ b/lambda/v2-orchestrator/test/skeleton-switch.test.js @@ -17,7 +17,7 @@ import { LocalDurableTestRunner, WaitingOperationStatus, } from '@aws/durable-execution-sdk-js-testing'; -import { __durableHandler } from '../index.js'; +import { __durableHandler } from './fixtures/completion-handler.js'; vi.setConfig({ testTimeout: 60_000 }); diff --git a/lambda/v2-orchestrator/test/stage-loopback.test.js b/lambda/v2-orchestrator/test/stage-loopback.test.js index ff86c68ef..215b24d5e 100644 --- a/lambda/v2-orchestrator/test/stage-loopback.test.js +++ b/lambda/v2-orchestrator/test/stage-loopback.test.js @@ -6,7 +6,7 @@ // outside release mode, and the recommendation behavior at the cap. import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { __durableHandler } from '../index.js'; +import { __durableHandler } from './fixtures/completion-handler.js'; import { LOOP_BACK_RECORDED_EVENT } from '../../shared/stage-loopback.js'; import { buildStageRow } from '../../shared/v2-process-keys.js'; diff --git a/lambda/v2-orchestrator/test/stage-skip.test.js b/lambda/v2-orchestrator/test/stage-skip.test.js index ee90dd4b1..710c87090 100644 --- a/lambda/v2-orchestrator/test/stage-skip.test.js +++ b/lambda/v2-orchestrator/test/stage-skip.test.js @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { __durableHandler } from '../index.js'; +import { __durableHandler } from './fixtures/completion-handler.js'; // Stage-skipping behaviors of the durable orchestrator (shared/stage-skip.js): // - intent-level skips (META.skipStageIds) ride load-plan, get SKIPPED audit diff --git a/terraform/modules/api/main.tf b/terraform/modules/api/main.tf index 4058c054f..872b971cd 100644 --- a/terraform/modules/api/main.tf +++ b/terraform/modules/api/main.tf @@ -263,6 +263,7 @@ resource "aws_api_gateway_deployment" "main" { module.cors_intent_rewind, module.cors_intent_resume, module.cors_intent_repair, + module.cors_intent_delivery, module.cors_intent_compose, module.cors_intent_compose_report_upload, module.cors_intent_composes, @@ -447,6 +448,7 @@ resource "aws_api_gateway_deployment" "main" { aws_api_gateway_resource.intent_rewind.id, aws_api_gateway_resource.intent_resume.id, aws_api_gateway_resource.intent_repair.id, + aws_api_gateway_resource.intent_delivery.id, aws_api_gateway_resource.intent_realtime_token.id, aws_api_gateway_resource.intent_gates.id, aws_api_gateway_resource.intent_gate.id, diff --git a/terraform/modules/api/routes.tf b/terraform/modules/api/routes.tf index a15f2aa0f..300086fb9 100644 --- a/terraform/modules/api/routes.tf +++ b/terraform/modules/api/routes.tf @@ -1348,6 +1348,12 @@ resource "aws_api_gateway_resource" "intent_repair" { path_part = "repair" } +resource "aws_api_gateway_resource" "intent_delivery" { + rest_api_id = aws_api_gateway_rest_api.main.id + parent_id = aws_api_gateway_resource.intent.id + path_part = "delivery" +} + # Composer sessions (Adaptive Workflows): start a compose (front/report), # presign the report upload, and list this intent's sessions. resource "aws_api_gateway_resource" "intent_compose" { @@ -1541,6 +1547,7 @@ locals { cancel_post = { resource = aws_api_gateway_resource.intent_cancel.id, method = "POST" } rewind_post = { resource = aws_api_gateway_resource.intent_rewind.id, method = "POST" } resume_post = { resource = aws_api_gateway_resource.intent_resume.id, method = "POST" } + delivery_post = { resource = aws_api_gateway_resource.intent_delivery.id, method = "POST" } repair_post = { resource = aws_api_gateway_resource.intent_repair.id, method = "POST" } compose_post = { resource = aws_api_gateway_resource.intent_compose.id, method = "POST" } compose_upload = { resource = aws_api_gateway_resource.intent_compose_report_upload.id, method = "POST" } @@ -1680,6 +1687,12 @@ module "cors_intent_repair" { resource_id = aws_api_gateway_resource.intent_repair.id } +module "cors_intent_delivery" { + source = "./cors" + rest_api_id = aws_api_gateway_rest_api.main.id + resource_id = aws_api_gateway_resource.intent_delivery.id +} + module "cors_intent_compose" { source = "./cors" rest_api_id = aws_api_gateway_rest_api.main.id