diff --git a/.agent/execplans/013_operator_validation_command_center.md b/.agent/execplans/013_operator_validation_command_center.md new file mode 100644 index 0000000..9c6a1f4 --- /dev/null +++ b/.agent/execplans/013_operator_validation_command_center.md @@ -0,0 +1,769 @@ +# Ship the operator validation command center + +This ExecPlan is a living document. The sections `Progress`, `Surprises & Discoveries`, `Decision Log`, and `Outcomes & Retrospective` must be kept up to date as work proceeds. + +This repository implements the Switchboard service. This plan must be maintained in accordance with `.agent/PLANS.md`. + +Primary issue: #136. Parent epic: #111. Cheapest-capable local routing was squash-merged through PR #135 at `223df7752716dd6ad35e75ba7613eeb03cfb2887`. + +## Purpose / Big Picture + +Turn the execution broker from a collection of trusted APIs into a usable operator product. + +After this slice, an operator should be able to open the existing Switchboard dashboard and complete one whole local-first validation workflow without stitching together curl commands: + +```text +configure worker routing + -> request validation for a GitHub PR + -> inspect the exact resolved head + -> explicitly approve and queue + -> observe cheapest-capable execution or exact reuse + -> inspect compact evidence + -> explicitly publish current or stale evidence + -> review bounded history and avoided-work metrics +``` + +The dashboard must preserve existing task, maintenance, configuration, diagnostics, analytics, and live-document behavior. This is a large vertical slice, not a replacement of the existing domains. + +The user-visible value is proof that Switchboard can make deterministic local validation genuinely convenient: it resolves exact source identity, chooses a trusted local worker, reuses exact retained evidence when safe, and shows what execution was avoided without claiming unmeasured money or paid-agent credits. + +The product remains: + +```text +PUBLIC DEVELOPER PREVIEW — NOT PRODUCTION READY +``` + +## Progress + +- [x] PR #135 squash-merged and issue #134 completed. +- [x] Exact base selected: `223df7752716dd6ad35e75ba7613eeb03cfb2887`. +- [x] Issue #136 created as one large end-to-end product slice. +- [x] Canonical branch `feat/operator-validation-command-center` created from the exact base. +- [x] Initial living ExecPlan created. +- [x] Verify local/remote preflight and establish a clean isolated worktree. +- [x] Run the pre-change focused backend, adapter, worker, and strict-browser baselines. +- [x] Audit current dashboard state, execution APIs, GitHub adapter identity, and persistence upgrade path. +- [x] Lock bounded operator projection and adapter-policy contracts. +- [x] Implement additive adapter policy handling and restart compatibility without adapter schema changes. +- [x] Implement bounded worker, request, overview, and history projection APIs. +- [x] Implement truthful avoided-work aggregation. +- [x] Implement the Validation Broker dashboard workspace and profile-management UX. +- [x] Implement explicit lifecycle and publication controls. +- [x] Add server-backed fresh-then-reused routed GitHub validation proof. +- [x] Add strict browser and accessibility acceptance. +- [x] Add public-safe screenshot and operator documentation. +- [x] Run the complete protected local matrix and public-hygiene audit. +- [x] Push the initial implementation commits and require its complete hosted matrix. +- [x] Record final connector review `4890406546` and isolate its three correction clusters. +- [x] Bound unknown preferred-executor failures with rollback and no created records. +- [x] Add a real `ExecutionClient`/`LocalWorker` fresh-then-reuse worker-trust acceptance. +- [x] Complete queued route, terminal quota/timing/source, and worker-state visibility. +- [x] Run the corrected complete local matrix and repeat public-hygiene validation. +- [x] Record final connector review `4890873592` and its activity, request-identity, capability-summary, and quota-reset corrections. +- [x] Implement deterministic worker-activity precedence and direct plus API projection coverage. +- [x] Complete selected-request identity/policy and bounded worker capability/quota-reset visibility. +- [x] Run the final corrected complete local matrix and repeat public-hygiene validation. +- [ ] Push the ultimate head and require the corrected complete hosted matrix. +- [ ] Complete connector re-review while keeping PR #137 draft and unmerged. + +## Surprises & Discoveries + +- Observation: the current dashboard is one long server-served HTML page backed by vanilla JavaScript, HTMX, Tailwind CDN classes, and a small local stylesheet rather than a component framework. + Evidence: `web/index.html` and `web/static/app.js`. The implementation should extend the established architecture unless a framework migration is independently justified, which is outside this slice. + +- Observation: the browser already stores the admin token only in local storage and sends it for protected mutations. That pattern can serve the validation workspace, but the token must never render or enter application state returned by the server. + Evidence: `ADMIN_TOKEN_STORAGE_KEY`, `loadAdminToken`, and `persistAdminToken` in `web/static/app.js`. + +- Observation: existing execution APIs expose work orders, runs, evidence, route provenance, routing-profile CRUD, and lifecycle actions, but their list endpoints are broad primitives rather than a bounded operator projection. + Evidence: `server/api/routers/execution.py`. A projection layer is preferable to making the browser join several unbounded domain lists. + +- Observation: the GitHub adapter creates and reads one request by ID but does not expose a bounded list endpoint, and its create contract currently accepts only repository, PR number, and manifest identity. + Evidence: `server/api/routers/github_execution.py` and `server/github_adapter/schemas.py`. Optional reuse/routing inputs must be additive, strict, and included in adapter idempotency. + +- Observation: the GitHub adapter response already exposes tested head, base SHA, linked work-order status, terminal run, evidence fingerprint, and publication state. The command center can build on this instead of inventing a second lifecycle. + Evidence: `GitHubValidationRequestOut` in `server/github_adapter/schemas.py`. + +- Observation: exact reuse and routed execution already persist source-run provenance, route cost snapshots, quota state, timestamps, and terminal run state. Avoided-work metrics should be derived from those immutable records rather than written into a mutable savings ledger. + Evidence: `ExecutionRun` route and reuse fields in `server/models.py`. + +- Observation: existing strict Playwright tests spin up a real FastAPI application with a file-backed SQLite database and use browser route interception only for deliberately simulated failures. + Evidence: `web/tests/test_ui.py`. New browser tests should continue exercising the real application wherever possible and use bounded mocks only for GitHub transport-dependent surfaces. + +- Observation: connector planning review `4888524384` established the canonical branch, draft PR #137, and this vertical-slice contract at the planning head. + Evidence: the branch began at `e905a3d7020c76d940c0938b2617d55d6d57de7e`; planning-head Commitlint workflow `31249483907` and CI workflow `31249483899` succeeded. + +- Observation: the pre-change shared Python 3.14 environment was dependency-dirty even though the focused suites were green. + Evidence: `python -m pip check` reported unrelated global conflicts for OpenCV versus NumPy and Streamlit versus Pillow, plus a stale invalid `~andit` installation warning. The pre-change backend/adapter matrix passed 169 tests with 8 warnings, the worker matrix passed 36 tests, and strict Playwright passed 2 tests with zero skips. + +- Observation: additive GitHub adapter policy does not require a persisted adapter-table column. + Evidence: the linked `ExecutionWorkOrder` already owns reuse, routing, integer ceiling, quota, and preferred-executor policy. Keeping that record authoritative avoids duplicating mutable state and avoids any `github_validation_requests` schema migration or `schema_version = 1` constraint change. + +- Observation: the exact pre-#136 adapter key must remain a separate calculation rather than a partially populated form of the new key. + Evidence: `_legacy_idempotency_key` preserves the prior canonical input byte-for-byte. Only an all-default request checks it after a new-key miss; a recovered legacy row is returned without changing its immutable key or linked work order. + +- Observation: the current-main manifest restart regression remains the relevant prior-schema startup proof because this slice changes no table shape. + Evidence: `test_main_manifest_schema_survives_repeated_startup` creates the manifest table in its merged-main shape without `updated_at`, runs lifespan twice, resolves/lists the trusted manifest both times, proves one identity, and verifies routing tables and columns remain present. The adapter compatibility regression independently proves legacy default identity recovery without duplication. + +- Observation: the first implementation-head Linux matrix ran fast enough for the new hard-bounds ASGI requests to consume the shared default-client rate-limit bucket before a later live-file authentication test. + Evidence: CI workflow `31279411403` returned `429` instead of the later test's expected `401`, while slower Windows runs remained green. Giving the endpoint test a distinct synthetic ASGI client identity isolates its request accounting without changing production limits, bypass rules, or authentication behavior. The same workflow also confirmed that `list_workers` needs the focused `PLR0913` annotation already used for other typed projection boundaries. + +- Observation: final connector review `4890406546` found that the direct-completion projection fixture and synthetic browser completion exercised lifecycle math and UI states but bypassed the worker trust boundary. + Evidence: neither fixture invoked `LocalWorker.poll_once()`, the trusted step runner, `EvidenceStore`, nor worker-local retained-evidence verification. The correction adds a separate file-backed real-worker acceptance and keeps the direct fixtures explicitly scoped to projection and UI behavior. + +- Observation: `preferred_executor_not_found` crossed from `ExecutionService.create_work_order` through the GitHub adapter route as an `ExecutionDomainError`, while that route caught only adapter errors. + Evidence: the corrected boundary maps expected execution not-found failures to a bounded 404, rolls back, and proves zero `GitHubValidationRequest` and `ExecutionWorkOrder` rows plus no open transaction. + +- Observation: final connector review `4890873592` found that the worker activity labels did not state or prove precedence when persisted status, freshness, and capacity disagreed. + Evidence: `_worker_activity_state` now classifies draining/offline or malformed records as unavailable, stale heartbeat or checkout polling before considering capacity, busy/full workers as capacity constrained, and only the remaining healthy workers as active. A six-worker persisted matrix proves active, genuinely busy, stale, inconsistent, draining, and offline cases in both insertion orders through direct projection and real HTTP reads, with overview buckets summing exactly to total workers. + +- Observation: the redacted worker projection already had enough declared, allowlisted capability scalars to support safe operator diagnosis, but returning the arbitrary `capabilities` document would have violated the bounded surface. + Evidence: worker cards receive only Python/Node version, Docker, a maximum of eight bounded browser names, GPU, Unity, desktop automation, network posture, and the fixed read-only repository capability. Tests prove arbitrary capability keys and values never enter the projection or rendered page. + +- Observation: a persisted quota-reset timestamp is stored without timezone metadata in SQLite and therefore serialized without an offset by the projection response. + Evidence: preserving that raw string during profile replacement caused the strict API schema to reject the edit. The browser now normalizes an existing reset timestamp to an explicit UTC ISO value before replacement; the strict browser profile-edit and revision-conflict workflow passes without changing quota semantics. + +## Decision Log + +- Decision: deliver backend projections, operator UX, history, avoided-work metrics, documentation, and end-to-end acceptance in one large vertical PR. + Rationale: the owner explicitly prefers fewer, larger coherent slices; the value of the execution broker is not obvious until the complete operator loop is visible. + Date/Author: 2026-08-08 / owner and connector planning + +- Decision: retain the existing vanilla HTML/JavaScript dashboard architecture for this slice. + Rationale: a framework migration would add unrelated build, dependency, and review risk without being required to ship the operator workflow. + Date/Author: 2026-08-08 / connector planning + +- Decision: add bounded operator projections rather than making the browser join raw work-order, run, request, worker, and profile lists. + Rationale: one server-owned projection can enforce pagination, redaction, stable ordering, and no-double-counting rules consistently. + Date/Author: 2026-08-08 / connector planning + +- Decision: extend the GitHub adapter request with optional reuse and routing policy inputs using compatibility defaults. + Rationale: the dashboard must be able to request the broker behavior already implemented without creating or mutating work orders through a separate hidden workflow. + Date/Author: 2026-08-08 / owner and connector planning + +- Decision: adapter idempotency must bind every accepted execution-policy input. + Rationale: two requests with materially different reuse, routing, quota, ceiling, or preferred-worker policy must not collapse into one adapter request. + Date/Author: 2026-08-08 / connector planning + +- Decision: derive avoided-work metrics only from successful reused runs and immutable linked source runs. + Rationale: this produces an auditable exact count and historical duration estimate without pretending to know actual paid-agent cost. + Date/Author: 2026-08-08 / owner and connector planning + +- Decision: comparison units remain operator-defined routing values, never currency or credits. + Rationale: the system does not integrate a provider or authoritative billing source. + Date/Author: 2026-08-08 / owner and connector planning + +- Decision: approval and publication remain explicit button actions. + Rationale: this slice makes the workflow convenient but must not weaken the accepted operator-control boundary. + Date/Author: 2026-08-08 / owner and connector planning + +- Decision: use bounded polling for execution state unless implementation proves a focused execution-event WebSocket materially simplifies the system. + Rationale: the current APIs are sufficient; a new event transport is not a requirement for the user outcome. + Date/Author: 2026-08-08 / connector planning + +- Decision: keep execution policy authoritative on the linked work order and add no adapter persistence. + Rationale: every new request field already maps to an accepted work-order field, so duplicating it in `github_validation_requests` would introduce drift, a needless prior-schema migration, and a schema-version constraint change without adding recoverability. + Date/Author: 2026-08-08 / implementation + +- Decision: expose one bounded projection module with a 100-row maximum, 10,000 maximum offset, 365-day maximum window, stable request ordering, and redacted worker summaries. + Rationale: these hard bounds give the browser the joined state it needs without N+1 queries or raw-domain overexposure. + Date/Author: 2026-08-08 / implementation + +- Decision: preserve the existing page shell while rendering the new command center as a visually distinct dark operator workspace. + Rationale: the final implementation carries the concept's dense scan-friendly metrics and two-column control surface without replacing the existing task, maintenance, configuration, diagnostics, analytics, live-document, or WebSocket UI. + Date/Author: 2026-08-08 / implementation + +- Decision: use existing bounded route-assessment and run-detail point reads for the selected request instead of widening the history list. + Rationale: queued assessment and exact run responses already contain the required scalar route, quota, timing, cleanup, and reuse-source state without exposing candidate lists, logs, commands, paths, or environments. + Date/Author: 2026-08-09 / connector correction + +- Decision: treat the direct-completion adapter test as a focused projection fixture and the synthetic browser app as UI acceptance only. + Rationale: only the new real `ExecutionClient`/`LocalWorker` test executes the trusted manifest, persists worker-owned evidence, performs local cryptographic verification, and proves the reused run skips the step runner. + Date/Author: 2026-08-09 / connector correction + +- Decision: worker activity uses one deterministic precedence: unavailable, stale, capacity constrained, then active. + Rationale: lifecycle and malformed-state safety must win first; stale trust signals must never be hidden behind a busy/full label; only fresh, valid workers may be described as capacity constrained or active. + Date/Author: 2026-08-09 / final connector correction + +- Decision: expose only small typed capability fields already accepted by worker registration, never the arbitrary capability document. + Rationale: operators need bounded runtime compatibility context, while internal markers, paths, host identity, commands, environments, and future capability extensions must remain server-redacted by default. + Date/Author: 2026-08-09 / final connector correction + +## Outcomes & Retrospective + +The operator can now complete the complete validation workflow in the existing +dashboard without assembling curl calls: inspect workers and profiles, create or +revision-safely replace a profile, reset quota, request an exact GitHub PR head, +approve/queue or apply another valid lifecycle action, monitor route/run/reuse/ +evidence state, explicitly publish current or stale evidence, and review bounded +history and avoided-work metrics. Existing task and operational surfaces remain +in the same page and their strict browser regressions remain green. + +The server boundary is one focused read-only projection module plus four bounded +authenticated routes: overview (1-365 UTC days), history and GitHub request lists +(1-100 rows and offsets through 10,000), and worker/profile summaries. Request +history is newest-first by creation time then numeric ID, joins only the latest +run, accepts typed repository/PR/work-order/run/reuse/routing/publication/time +filters, and omits executable, local, private, credential, and unbounded fields. + +Every new adapter policy input participates in the new identity. Existing +all-default requests remain recoverable through the exact pre-#136 key and are +returned without mutating identity; non-default requests never use legacy +fallback. Policy is authoritative on the linked work order, so this slice adds +no table column, schema-version change, or migration. The existing merged-main +manifest-shape regression ran lifespan startup twice and remained green. + +The focused direct-completion fixture still proves projection formulas and +lifecycle joins, but is no longer represented as the worker trust proof. The +file-backed real-worker acceptance registers and actively polls two real outbound +workers, rejects the higher-cost claimant twice, runs the trusted manifest on the +lower-cost worker, retains marker/result/log/artifact evidence, publishes current, +then performs same-worker exact retained-evidence verification for a distinct +reused run without a second step-runner call. Moving the mocked head produces a +stale second publication while preserving the tested SHA. Overview/history prove +one fresh success, one reused success, one avoided execution, nonzero source time, +3 comparison units, 50% reuse, and one current plus one stale publication. + +Strict Playwright passed all three tests with zero skips. The command-center test +proved native malformed-input rejection, server-loaded selectors, UI profile +creation/replacement/quota reset, visible revision conflict recovery, explicit +lifecycle/publication transitions, fresh/reused and current/stale rendering, +disabled-action reasons, keyboard order, zero unexpected console errors, and +page containment at 1440 and 390 pixels. The approved screenshot was generated +from that no-network file-backed application with synthetic public data and was +visually inspected against the generated concept. + +The final connector pass makes activity classification deterministic under +conflicting persisted signals and verifies the same buckets through direct and +HTTP projection reads. The selected request now keeps repository, pull-request, +reuse-policy, and routing-policy identity adjacent to route/run evidence. Worker +cards add only typed allowlisted runtime capabilities plus the next quota-reset +state; scheduled reset values survive profile replacement as timezone-aware API +inputs. The synthetic screenshot was recaptured from the corrected bounded UI. + +The product remains a public developer preview for trusted local networks. MCP, +paid-agent/provider execution, billing, browser/desktop/RPA workers, automatic +approval/publication, webhooks, repository writes, full-log transfer, and +production multi-tenancy remain deliberately out of scope. + +## Context and Orientation + +### Existing operator dashboard + +- `web/index.html` contains the complete dashboard markup. +- `web/static/app.js` owns browser state, API helpers, rendering, actions, polling, and WebSocket behavior. +- `web/static/styles.css` contains local styles layered over Tailwind CDN utilities. +- `web/tests/test_ui.py` launches the real FastAPI service and runs strict Playwright acceptance. +- `server/api/routers/ui.py` serves the dashboard document. + +The current dashboard already presents: + +- system maintenance state; +- configuration; +- diagnostics; +- task analytics; +- task creation and lifecycle; +- live documents. + +The new workspace must preserve those surfaces. + +### Execution domain + +- `server/models.py` defines `ExecutionWorkOrder`, `ExecutionWorker`, `WorkerRoutingProfile`, `ExecutionRun`, and `ExecutionLease`. +- `server/execution/enums.py` defines work-order, run, reuse, routing, quota, worker, and approval states. +- `server/execution/schemas.py` defines strict input/output contracts. +- `server/execution/repository.py` owns database queries and transactional mutations. +- `server/execution/service.py` owns lifecycle, routing, reuse, quota, and evidence validation. +- `server/api/routers/execution.py` exposes authenticated operator and worker routes. +- `server/api/lifecycle.py` provides additive repeated-startup compatibility. + +### GitHub adapter + +- `server/github_adapter/schemas.py` defines strict create and response contracts. +- `server/github_adapter/service.py` resolves exact PR identity, creates the linked work order, and publishes managed evidence. +- `server/github_adapter/repository.py` owns request persistence. +- `server/github_adapter/transport.py` enforces bounded GitHub transport behavior. +- `server/api/routers/github_execution.py` exposes authenticated request, status, and publish routes. +- `server/models.py` defines `GitHubValidationRequest`. + +### Existing test foundations + +- `server/tests/test_github_adapter_service.py` +- `server/tests/test_github_adapter_transport.py` +- `server/tests/test_execution_routing.py` +- `server/tests/test_execution_reuse.py` +- `server/tests/test_execution_concurrency.py` +- `server/tests/test_execution_startup.py` +- `client/python/tests/test_execution_worker_server_smoke.py` +- `client/python/tests/test_execution_worker_reuse.py` +- `web/tests/test_ui.py` + +## Plan of Work + +### 1. Establish exact baseline + +Verify the branch and merge base, create or reuse a clean isolated worktree, and run focused baselines for: + +- GitHub adapter service/transport/startup; +- execution contracts/routing/reuse/concurrency/startup; +- worker server-smoke/reuse/strict work-order parsing; +- current strict Playwright dashboard tests. + +Record exact counts and environment limitations before implementation. + +### 2. Extend GitHub adapter policy identity + +Add optional strict fields to the adapter request: + +```text +reuse_policy +routing_policy +maximum_cost_units +required_quota_units +preferred_executor +``` + +Compatibility defaults: + +```text +never +first_available +null +0 +null +``` + +Update: + +- request schema; +- internal request identity; +- idempotency hash; +- persistence or authoritative linked-order recovery; +- work-order creation; +- response projection where useful; +- prior-schema repeated-startup compatibility; +- tests for every policy dimension. + +Do not accept manifest parameters, commands, paths, environment values, or credentials. + +### 3. Add bounded read models + +Create a focused operator projection module, likely under `server/execution/` or `server/application/`, rather than expanding already-large lifecycle methods without structure. + +Define typed outputs for: + +- overview metrics; +- worker/profile summaries; +- GitHub request list items; +- history rows; +- paginated metadata; +- current request detail when a joined projection materially reduces client calls. + +Use stable ordering and explicit hard bounds. + +Suggested history row fields: + +```text +request ID +repository and PR number +tested head SHA +manifest identity +work-order ID/status +terminal/latest run ID/status +routing policy and selected worker +estimated cost units +reuse decision and source run +run duration +evidence fingerprint +publication state/decision +created/updated timestamps +``` + +No full logs, artifact bytes, commands, argv, local paths, environments, private URLs, or credentials may enter the projection. + +### 4. Implement overview aggregation + +Derive metrics from authoritative persisted records within a caller-supplied bounded window. + +At minimum calculate: + +- request counts; +- work-order status counts; +- run status counts; +- fresh successful runs; +- reused successful runs; +- unavailable exact reuse; +- deterministic executions avoided; +- reused source-duration sum where source timing is complete; +- comparison units avoided where the reused run has a route cost snapshot; +- reuse rate; +- current/stale publication counts; +- active/stale/capacity-constrained worker counts. + +Do not double-count: + +- multiple attempts for one work order in work-order counts; +- publication updates; +- source runs themselves as avoided executions; +- reused runs without successful terminal state; +- missing source timing or missing cost snapshots. + +### 5. Add operator APIs + +Add authenticated typed endpoints equivalent to: + +```text +GET /api/execution/operator/overview +GET /api/execution/operator/history +GET /api/execution/workers +GET /api/execution/github/requests +``` + +Use explicit bounds for limit, offset/cursor, time window, and filters. + +The request-list endpoint should return bounded joined lifecycle state without forcing N+1 client calls. + +Preserve existing routes and response compatibility. + +### 6. Build the Validation Broker workspace + +Extend `web/index.html` with accessible sections or tabs for: + +- broker overview cards; +- new GitHub PR validation form; +- request detail and lifecycle actions; +- worker/routing-profile management; +- execution history and filters. + +Extend `web/static/app.js` with focused state and render functions. Avoid one monolithic function where small modules or grouped functions improve reviewability without a framework migration. + +The form should load manifest and worker choices from the server. + +Actions must include: + +- request validation; +- approve; +- approve and queue; +- queue; +- cancel; +- expire; +- refresh; +- publish evidence; +- create/replace routing profile; +- reset quota. + +Only render actions allowed by the current lifecycle state. Confirm destructive transitions. + +### 7. Monitor and explain state + +Show exact head, route, quota, run, reuse, evidence, and publication state with concise badges and a timeline. + +Use bounded polling with cleanup when the page unloads. Do not create uncontrolled request loops. + +Display conflict and stale responses through existing toast patterns plus persistent inline status where the operator needs recovery guidance. + +### 8. Implement history and avoided-work UX + +Add cards for: + +- deterministic executions avoided; +- reference execution time avoided; +- comparison units avoided; +- fresh runs; +- reused runs; +- reuse rate; +- current/stale publications. + +Use explicit labels and help text that comparison units are not money or measured paid-agent credits. + +Add a filterable table with exact SHA copy controls and fresh/reused/current/stale badges. + +### 9. Add end-to-end server proof + +Use mocked GitHub transport and existing server-backed worker helpers to prove: + +1. two workers and profiles; +2. cheapest-capable selection; +3. fresh successful evidence; +4. current publication; +5. second exact policy request; +6. local exact reuse without validation steps; +7. correct overview/history metrics; +8. moved-head stale publication. + +The proof must remain offline and use no real credentials. + +### 10. Add strict browser acceptance + +Extend Playwright tests for the principal workflow, responsiveness, keyboard navigation, disabled action reasons, optimistic conflicts, fresh/reused history, and current/stale publication. + +Use strict mode and require zero skips. + +Test at least one desktop and one narrow viewport for horizontal containment. + +### 11. Documentation and screenshot + +Update README, API, architecture, configuration, local-worker operations, public status, and a focused command-center guide. + +Capture a public-safe screenshot only from synthetic local data. Inspect it for: + +- no token; +- no local path; +- no machine identity; +- no private URL; +- no real credential-shaped value; +- no misleading actual-savings claim. + +### 12. Complete validation and delivery + +Run focused tests first, then the full protected matrix. Remove generated artifacts. Audit public changes. Commit in intentional functional groups. Push normally to the existing branch. Keep the PR draft until connector review is complete. + +## Concrete Steps + +### Preflight + +From the primary checkout, inspect only: + +```powershell +git fetch origin --prune --tags +git status --short --branch --untracked-files=all +git worktree list --porcelain +git rev-parse origin/main +git rev-parse origin/feat/operator-validation-command-center +git merge-base origin/main origin/feat/operator-validation-command-center +``` + +Require: + +```text +origin/main: 223df7752716dd6ad35e75ba7613eeb03cfb2887 +starting branch head: +merge base: 223df7752716dd6ad35e75ba7613eeb03cfb2887 +``` + +Use a clean isolated worktree. Do not switch, reset, clean, or alter an unrelated primary checkout. + +### Focused baselines + +Run at minimum: + +```powershell +python -m pip check + +python -m pytest -q -p no:cacheprovider ` + server/tests/test_github_adapter_service.py ` + server/tests/test_github_adapter_transport.py ` + server/tests/test_execution_startup.py ` + server/tests/test_execution_contracts.py ` + server/tests/test_execution_routing.py ` + server/tests/test_execution_reuse.py ` + server/tests/test_execution_concurrency.py + +python -m pytest -q -p no:cacheprovider ` + client/python/tests/test_execution_worker_server_smoke.py ` + client/python/tests/test_execution_worker_reuse.py ` + client/python/tests/test_execution_worker_strict_work_order.py + +$env:SWITCHBOARD_STRICT_PLAYWRIGHT = "1" +python -m pytest web/tests/test_ui.py -rA +``` + +### Expected implementation files + +The actual design determines the final set, but likely files include: + +```text +server/models.py +server/api/lifecycle.py +server/api/routers/execution.py +server/api/routers/github_execution.py +server/api/__init__.py +server/execution/entities.py +server/execution/schemas.py +server/execution/repository.py +server/execution/service.py +server/execution/operator_projection.py +server/github_adapter/schemas.py +server/github_adapter/repository.py +server/github_adapter/service.py +web/index.html +web/static/app.js +web/static/styles.css +web/tests/test_ui.py +server/tests/test_execution_operator_projection.py +server/tests/test_github_adapter_service.py +server/tests/test_execution_startup.py +client/python/tests/test_execution_worker_server_smoke.py +docs/API.md +docs/configuration.md +docs/architecture/local-execution-broker.md +docs/operations/local-worker.md +docs/operations/validation-command-center.md +docs/reports/status.md +README.md +``` + +Do not treat this as permission for unrelated edits. + +### Complete local matrix + +Run the repository's current protected commands, including: + +```powershell +python -m pip check +python -m pre_commit run --all-files --show-diff-on-failure +python scripts/dev.py check-todos --root . +python -m ruff check server client scripts tests web switchboard_cli.py switchboard_client.py +python -m black --check server client scripts tests web switchboard_cli.py switchboard_client.py +python -m mypy --config-file mypy.ini server client scripts + +New-Item -ItemType Directory -Force reports | Out-Null +python -m pytest --maxfail=1 --disable-warnings --junitxml=reports/pytest.xml + +$env:SWITCHBOARD_STRICT_PLAYWRIGHT = "1" +python -m pytest web/tests/test_ui.py -rA +``` + +Run the configured coverage suite and all existing module thresholds exactly as defined by the current workflow and prior ExecPlans. + +Also run: + +```powershell +python -m bandit -q -r server -x server/tests +python -m pip_audit --progress-spinner=off -r server/requirements-dev.txt +gitleaks detect --verbose + +New-Item -ItemType Directory -Force lychee | Out-Null +lychee --config lychee.toml --no-progress ` + README.md CHANGELOG.md SECURITY.md CONTRIBUTING.md CODE_OF_CONDUCT.md ` + "docs/**/*.md" ` + --exclude-path docs/history ` + --exclude-path archive + +git diff --check +git status --short --branch --untracked-files=all +``` + +Use the established compatible interpreter for pinned tools when the host interpreter has a documented tool incompatibility. Do not silently accept a scanner that skipped analysis. + +## Validation and Acceptance + +The slice is accepted only when all of the following are proven: + +1. Existing callers omit new adapter fields and retain current defaults. +2. Every policy input changes adapter idempotency when it should. +3. Existing adapter databases upgrade and start twice without duplication or data loss. +4. Worker/profile APIs are bounded and do not allow worker-owned profile writes. +5. Overview and history are paginated, stably ordered, and redacted. +6. Avoided execution count equals successful reused runs only. +7. Source-duration estimate uses the linked immutable source run only. +8. Missing source duration or route cost is excluded rather than guessed. +9. The dashboard can request, approve, queue, monitor, and publish a validation. +10. The dashboard can create/update a profile and handle revision conflicts. +11. Fresh and reused runs display distinctly. +12. Current and stale publication display distinctly. +13. A real server-backed offline fresh-then-reused flow passes. +14. Existing task dashboard behavior remains green. +15. Strict browser tests execute with zero skips. +16. Desktop and narrow viewport containment pass. +17. A public-safe screenshot contains only synthetic data. +18. Full local and hosted matrices pass. +19. Public-hygiene review finds no credential, path, machine, private URL, log, database, report, or cache leakage. +20. PR #137 remains draft and unmerged pending connector review. + +## Idempotence and Recovery + +- Adapter schema changes must use idempotent repeated-startup compatibility. +- Projection endpoints are read-only and safe to retry. +- List/history pagination must not mutate last-seen state. +- Route assessment must not refresh worker poll timestamps unless the accepted implementation explicitly documents and tests a change; the current preference is no mutation. +- UI polling must cancel prior timers and tolerate repeated refresh. +- Optimistic profile conflicts must refresh state rather than overwrite a newer revision. +- Repeated request submission with identical full identity should return the existing adapter request. +- Publication remains idempotent under the managed-comment contract. +- The screenshot must be regenerated only after the final UI state is stable. +- If implementation stops, preserve the isolated worktree and report local/remote SHAs plus every dirty path. Do not reset, clean, stash, or recreate the branch. + +## Artifacts and Notes + +Record during implementation: + +- exact preflight SHAs; +- baseline counts; +- adapter schema before/after and prior-schema test; +- example bounded overview/history payloads using synthetic values; +- fresh and reused run IDs from the server-backed proof; +- source duration and comparison-unit calculations; +- strict browser counts and viewport evidence; +- screenshot path and exact tested SHA; +- full pytest count; +- aggregate and module coverage; +- security, dependency, secret, and link results; +- public-hygiene result; +- final local/remote SHA equality; +- final hosted Commitlint and CI IDs in PR #137's external review record. + +Current corrected-head local evidence: + +- preflight: `origin/main` and merge base were `223df7752716dd6ad35e75ba7613eeb03cfb2887`; the local and remote implementation branch began at `e905a3d7020c76d940c0938b2617d55d6d57de7e`; the isolated worktree was clean and exclusively owned the branch; +- correction preflight: local and remote `feat/operator-validation-command-center` both began at `5bb0ec7a696cb512708e3f60e05c2e281b6255b0`; `origin/main` and the merge base remained `223df7752716dd6ad35e75ba7613eeb03cfb2887`; PR #137 remained open, draft, unmerged, and exclusively owned by the clean isolated worktree; +- final-correction preflight: local and remote `feat/operator-validation-command-center` both began at `4f3613e29cc4aca133b815ec935bbf76f6e89ad9`; `origin/main` and the merge base remained `223df7752716dd6ad35e75ba7613eeb03cfb2887`; PR #137 remained open, draft, unmerged, and exclusively owned by the clean isolated worktree. Connector review `4890873592` and checkpoint comments `5230659404`/`5230660442` define this pass; +- pre-change backend/adapter baseline: 169 passed, 8 warnings; +- pre-change worker baseline: 36 passed; +- pre-change strict browser baseline: 2 passed, zero skips; +- implemented server-backed proof: request/work-order/run IDs are database-local synthetic IDs; request 1 completed fresh on the lower-cost worker, request 2 completed as a distinct reused run on the same worker with an empty step list, reference duration 7 seconds, route comparison units 3, one avoided execution, and a 50% reuse rate; publication was current before the mocked head moved and stale afterward; +- focused implementation checks before documentation: the new policy/operator tests passed 4 tests, the adapter/startup target passed 44 tests, and the new strict command-center browser test passed 1 test; +- corrected public screenshot: `docs/assets/switchboard-validation-command-center.png` is a 1232-by-2276 RGB capture from the offline file-backed UI acceptance application with synthetic public-repository data. It shows queued/terminal route, quota, duration, reuse-source, publication, and bounded worker state and was visually inspected; no token, authorization value, path, machine identity, private URL, credential-shaped value, environment value, or financial-savings claim is present; +- final focused backend/adapter/startup matrix: 173 passed with 8 existing SQLAlchemy deprecation warnings; +- final focused worker matrix: 36 passed; +- final strict browser matrix: 3 passed with zero skips in 21.55 seconds; +- complete pytest: 576 passed, 5 documented platform skips, and 344 warnings in 404.09 seconds; +- coverage pytest: the same 576 passed and 5 skipped; aggregate configured coverage was 93%; all 16 gates passed at 95.40%, 94.17%, 100%, 100%, 92.59%, 95%, 87.76%, 100%, 100%, 97.73%, 90.16%, 97.59%, 94.83%, 100%, 90.71%, and 91.30% in workflow order; +- correction-focused backend matrix: 95 passed; the unknown preferred executor returned bounded HTTP 404 `preferred_executor_not_found`, created zero adapter requests and zero work orders, left no open transaction, and returned no internal or transport data; +- correction-focused worker matrix: 37 passed. The dedicated real-worker acceptance ran all 7 trusted `validate-switchboard@1` steps for fresh run 1, retained `ownership.json`, `result.json`, step logs, and all 14 declared artifacts with verified hashes, then created reused run 2 from run 1 with local retained-evidence verification, the identical immutable source fingerprint, zero runner calls, zero steps, and zero copied artifacts. The source evidence tree remained byte-for-byte unchanged, the mocked publication decisions were current then stale after the head moved, overview/history totals were 1 fresh, 1 reused, 1 avoided execution, 3 comparison units, 50% reuse, 1 current, 1 stale, and 2 history rows, and both the canonical repository and worker worktree roots were clean; +- corrected strict browser matrix: 3 passed with zero skips in 26.40 seconds. Queued route assessment; fresh route/quota/duration; reused source run/fingerprint/zero steps; current/stale publication; worker status/freshness/capacity/platform; revision-conflict recovery; keyboard order; desktop and 390-pixel containment; exact `Reference execution time avoided` copy; token non-rendering; and zero unexpected console errors all passed; +- final-correction focused activity projection: 9 passed. Both insertion orders produced 6 total workers partitioned into 1 active, 1 stale, 2 capacity constrained, and 2 unavailable; the genuinely checked-out busy worker persisted `busy` with `1/1` capacity, and direct plus HTTP projections agreed; +- final-correction in-app browser QA: the live file-backed synthetic workspace rendered repository/PR and configured reuse/routing policy, the allowlisted capability summary, scheduled and unscheduled quota resets, and no arbitrary capability marker. Strict Playwright then passed 3 tests with zero skips after the timezone-aware reset-preservation correction; +- final-correction public screenshot: `docs/assets/switchboard-validation-command-center.png` was recaptured as a 1232-by-2266 RGB image from the disposable local acceptance app using synthetic public repository, request, worker, routing, and quota data, visually inspected, and contains no token, path, workstation identity, private URL, credential, or financial claim; +- final-correction focused matrices: backend/adapter/startup/routing/reuse/activity passed 104 tests; worker trust/reuse/strict work-order passed 37 tests; the dedicated real `ExecutionClient`/`LocalWorker` fresh-then-reuse test passed separately. The trust proof retained the previously recorded 7 fresh steps, 0 reused steps, distinct source/reused run IDs, immutable source evidence fingerprint, current/stale publication decisions, 1 fresh + 1 reused + 1 avoided totals, 50% reuse, and clean canonical/worker repositories; +- final-correction complete pytest: 587 passed, 5 documented platform skips, and 344 warnings in 516.45 seconds; +- final-correction coverage pytest: the same 587 passed and 5 skipped in 445.40 seconds; aggregate configured coverage was 93%; all 16 gates passed at 95.40%, 94.17%, 100%, 100%, 92.59%, 95%, 87.76%, 100%, 100%, 97.73%, 90.16%, 97.59%, 94.83%, 100%, 90.71%, and 91.30% in workflow order; +- final-correction strict browser: 3 passed with zero skips in 19.14 seconds after the final formatter-neutral test refactor and timezone-preserving profile-edit correction; +- final-correction quality/security matrix: clean Python 3.11 `pip check`, pinned pre-commit, TODO policy, Ruff, Black, Mypy over 178 source files, Bandit 1.8.6 under Python 3.11.14, pip-audit, Gitleaks over 259 commits, Lychee, and `git diff --check` passed. The shared Python 3.14 dependency conflicts remain the documented unrelated baseline and no tool, rule, threshold, or scan was weakened; +- corrected complete pytest: 578 passed, 5 documented platform skips, and 344 warnings in 724.68 seconds; +- corrected coverage pytest: the same 578 passed and 5 skipped; aggregate configured coverage was 93%; all 16 gates passed at 95.40%, 94.17%, 100%, 100%, 92.59%, 95%, 87.76%, 100%, 100%, 97.73%, 90.16%, 97.59%, 94.83%, 100%, 90.71%, and 91.30% in workflow order; +- corrected quality/security matrix: clean Python 3.11 `pip check`, pre-commit, TODO policy, repository-pinned Ruff, Black, Mypy over 177 source files, Bandit 1.8.6 on Python 3.11.14, pip-audit, Gitleaks over 255 commits, Lychee, strict Playwright, and `git diff --check` passed. The shared host retains its documented unrelated dependency and newer-Ruff drift, so no repository rule or scanner was weakened; +- clean Python 3.11 `pip check`, pre-commit, TODO policy, Ruff, Black, Mypy, compatible-interpreter Bandit, pip-audit, Gitleaks over 250 commits, Lychee, and `git diff --check` passed; +- the shared Python 3.14 environment retains the unrelated baseline OpenCV/NumPy and Streamlit/Pillow conflicts. A clean external Python 3.11 environment proved dependency consistency and direct gates. The fixed worker argv `python` resolves the base Astral interpreter ahead of a Windows venv under its intentionally sanitized environment, so the established host interpreter was retained for worker/full pytest while direct Python 3.11 gates used the clean environment; no worker command or trust boundary was weakened; +- generated reports, coverage data, databases, caches, bytecode, link output, and temporary environments are removed before commit. The final public-hygiene audit, local/remote equality, and hosted ultimate-head workflow IDs are recorded during delivery; ultimate-head IDs belong in PR #137's external review record rather than a self-referential plan commit. + +## Interfaces and Dependencies + +Expected new or extended interfaces include equivalents of: + +```python +class GitHubValidationCreateIn: + reuse_policy: ReusePolicy = ReusePolicy.NEVER + routing_policy: RoutingPolicy = RoutingPolicy.FIRST_AVAILABLE + maximum_cost_units: int | None = None + required_quota_units: int = 0 + preferred_executor: str | None = None + +class ExecutionOperatorOverviewOut: + window: TimeWindowOut + requests: RequestMetricsOut + work_orders: WorkOrderMetricsOut + runs: RunMetricsOut + avoided_work: AvoidedWorkMetricsOut + publications: PublicationMetricsOut + workers: WorkerMetricsOut + +class ExecutionHistoryPageOut: + items: list[ExecutionHistoryItemOut] + limit: int + offset: int + total: int +``` + +Names may change to fit repository conventions, but contracts must remain typed, bounded, and strict. + +No new runtime dependency is expected. Any dependency addition requires explicit justification, pinning, license review, audit evidence, and documentation. diff --git a/README.md b/README.md index 40cf058..c725677 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Switchboard is a reference implementation for coordinating multiple agents again - **Lease-based ownership** — agents claim work with expiry and heartbeat semantics that reduce duplicate execution. - **Trusted local execution** — approved exact-SHA work orders are claimed by an outbound local worker using fixed reviewed argv, read-only disposable worktrees, bounded output, cancellation, and cleanup. - **Compact validation evidence** — `validate-switchboard@1` records strict step outcomes, parsed test/coverage/security summaries, dependency-lock hashes, retained artifact hashes, and a deterministic fingerprint without returning full local logs. +- **Validation Broker workspace** — operators can configure local-worker routing, resolve a GitHub pull request to an exact head, approve and queue it, distinguish fresh execution from exact reuse, publish current or stale evidence, and inspect bounded history without assembling API calls by hand. - **Live state synchronization** — plan changes are broadcast to the dashboard and clients over WebSockets. - **Live-file hosting** — agents can fetch mutable documents by URL; mutation endpoints can be protected with an admin token. - **Operational visibility** — health, readiness, diagnostics, metrics hooks, structured logs, and rate limiting. @@ -59,6 +60,12 @@ python scripts/run_uvicorn.py Open [http://localhost:8000/](http://localhost:8000/) to view the operator dashboard. +The **Validation Broker** workspace uses the same optional admin-token boundary as +the execution APIs. Its comparison units are operator-authored routing values, +not currency, provider credits, or measured savings. See the +[command-center operations guide](docs/operations/validation-command-center.md) +for the end-to-end workflow and trust boundaries. + ### 3. Create a task ```bash @@ -152,6 +159,7 @@ See [SECURITY.md](SECURITY.md) and [docs/configuration.md](docs/configuration.md - **[Configuration](docs/configuration.md)** — environment variables and runtime settings. - **[Agent Integration](docs/ai-interface.md)** — how agents interact with Switchboard. - **[Local Worker Operations](docs/operations/local-worker.md)** — trusted repository mapping, worker configuration, execution, evidence retention, and limitations. +- **[Validation Command Center](docs/operations/validation-command-center.md)** — browser workflow, bounded projections, exact-reuse metrics, and publication controls. - **[Public Status](docs/reports/status.md)** — current developer-preview posture and release boundaries. - **[Two-Agent Workflow](docs/visuals/TWO_AGENT_WORKFLOW.md)** — dependency-unlock sequence. - **[Documentation Index](docs/index.md)** — full navigation. diff --git a/client/python/tests/test_execution_worker_server_smoke.py b/client/python/tests/test_execution_worker_server_smoke.py index 8686b00..499aaa7 100644 --- a/client/python/tests/test_execution_worker_server_smoke.py +++ b/client/python/tests/test_execution_worker_server_smoke.py @@ -18,6 +18,7 @@ from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from client.python.execution_worker import worker as worker_module from client.python.execution_worker.client import ExecutionClient from client.python.execution_worker.config import WorkerConfig from client.python.execution_worker.evidence import EvidenceStore @@ -331,6 +332,16 @@ def _request( return decoded +def _retained_hashes(directory: Path) -> dict[str, str]: + hashes: dict[str, str] = {} + for path in directory.rglob("*"): + if path.is_file(): + hashes[path.relative_to(directory).as_posix()] = hashlib.sha256( + path.read_bytes() + ).hexdigest() + return hashes + + def test_server_backed_worker_smoke_executes_exact_sha_and_releases_lease( tmp_path: Path, ) -> None: @@ -802,7 +813,7 @@ async def prepare() -> None: published = _request( app, "POST", - ("/api/execution/github/requests/" f"{created['request_id']}/publish"), + (f"/api/execution/github/requests/{created['request_id']}/publish"), {}, ) assert published["publication_state"] == "published_current" @@ -816,7 +827,7 @@ async def prepare() -> None: stale = _request( app, "POST", - ("/api/execution/github/requests/" f"{created['request_id']}/publish"), + (f"/api/execution/github/requests/{created['request_id']}/publish"), {}, ) assert stale["publication_state"] == "published_stale" @@ -843,10 +854,7 @@ async def prepare() -> None: _request( app, "POST", - ( - "/api/execution/work-orders/" - f"{fork_request['work_order_id']}/approve" - ), + (f"/api/execution/work-orders/{fork_request['work_order_id']}/approve"), {}, ) assert worker.poll_once() is True @@ -854,7 +862,7 @@ async def prepare() -> None: fork_runs = _request( app, "GET", - ("/api/execution/runs?work_order_id=" f"{fork_request['work_order_id']}"), + (f"/api/execution/runs?work_order_id={fork_request['work_order_id']}"), ) assert len(fork_runs) == 1 assert fork_runs[0]["status"] == "failed" @@ -899,3 +907,329 @@ async def prepare() -> None: finally: app.dependency_overrides.clear() asyncio.run(engine.dispose()) + + +def test_routed_github_validation_executes_then_reuses_real_local_worker( # noqa: PLR0915 - complete issue #136 worker trust proof + tmp_path: Path, +) -> None: + canonical, tested_shas, status_before = _validation_repository(tmp_path) + tested_sha, moved_sha = tested_shas + engine = create_async_engine( + f"sqlite+aiosqlite:///{tmp_path / 'operator-worker-acceptance.db'}" + ) + sessions = async_sessionmaker(engine, expire_on_commit=False) + app = create_app(AppConfig(include_ui=False)) + github = _MockGitHubAcceptanceTransport(_resolved_pull_request(tested_sha)) + + async def isolated_session() -> AsyncGenerator[AsyncSession, None]: + async with sessions() as session: + yield session + + def github_service(session: SessionDependency) -> GitHubAdapterService: + return GitHubAdapterService( + dependencies=GitHubAdapterDependencies( + repository=GitHubAdapterRepository(session), + execution=build_execution_service(session), + transport=github, + ), + settings=GitHubSettings( + api_url="https://api.github.com", + operator_id="operator-worker-acceptance", + token=_GITHUB_TEST_TOKEN, + ), + clock=utcnow_naive, + ) + + app.dependency_overrides[get_session] = isolated_session + app.dependency_overrides[get_github_adapter_service] = github_service + + async def prepare() -> None: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + + asyncio.run(prepare()) + cheap_config = WorkerConfig( + base_url="http://switchboard.test", + worker_id="operator-worker-cheap", + display_name="Operator worker cheap", + admin_token=_TOKEN, + worker_root=tmp_path / "operator-cheap-worktrees", + evidence_root=tmp_path / "operator-cheap-evidence", + repositories={"Nobodyworld/dev-agent-switchboard": canonical}, + execution_timeout_seconds=3600, + heartbeat_interval_seconds=5, + ) + expensive_config = WorkerConfig( + base_url="http://switchboard.test", + worker_id="operator-worker-expensive", + display_name="Operator worker expensive", + admin_token=_TOKEN, + worker_root=tmp_path / "operator-expensive-worktrees", + evidence_root=tmp_path / "operator-expensive-evidence", + repositories={"Nobodyworld/dev-agent-switchboard": canonical}, + execution_timeout_seconds=3600, + heartbeat_interval_seconds=5, + ) + try: + with ( + ExecutionClient( + cheap_config.base_url, + cheap_config.worker_id, + cheap_config.admin_token, + session=_AsgiSession(app), # type: ignore[arg-type] + ) as cheap_client, + ExecutionClient( + expensive_config.base_url, + expensive_config.worker_id, + expensive_config.admin_token, + session=_AsgiSession(app), # type: ignore[arg-type] + ) as expensive_client, + ): + cheap_worker = LocalWorker(cheap_config, cheap_client) + expensive_worker = LocalWorker(expensive_config, expensive_client) + cheap_worker.start() + expensive_worker.start() + for worker_id, cost in ( + (cheap_config.worker_id, 3), + (expensive_config.worker_id, 9), + ): + _request( + app, + "POST", + "/api/execution/routing-profiles", + { + "schema_version": 1, + "worker_id": worker_id, + "enabled": True, + "estimated_cost_units_per_run": cost, + "quota_capacity_units": 20, + "quota_remaining_units": 20, + "quota_reset_at": None, + "routing_priority": 0, + }, + ) + cheap_client.heartbeat_worker(status="online") + expensive_client.heartbeat_worker(status="online") + assert cheap_worker.poll_once() is False + assert expensive_worker.poll_once() is False + + request_payload = { + "repository_full_name": "Nobodyworld/dev-agent-switchboard", + "pull_request_number": 125, + "manifest": {"name": "validate-switchboard", "version": "1"}, + "routing_policy": "cheapest_capable", + "reuse_policy": "never", + "maximum_cost_units": 10, + "required_quota_units": 2, + } + fresh_request = _request( + app, + "POST", + "/api/execution/github/pull-requests/validate", + request_payload, + ) + _request( + app, + "POST", + f"/api/execution/work-orders/{fresh_request['work_order_id']}/approve", + {"queue": True}, + ) + expensive_client.heartbeat_worker(status="online") + assert expensive_worker.poll_once() is False + + original_runner = worker_module.run_step + original_verifier = worker_module.verify_reuse_candidate + with ( + patch.object( + worker_module, "run_step", wraps=original_runner + ) as runner, + patch.object( + worker_module, + "verify_reuse_candidate", + wraps=original_verifier, + ) as verifier, + ): + cheap_client.heartbeat_worker(status="online") + assert cheap_worker.poll_once() is True + manifest = get_trusted_manifest("validate-switchboard", "1") + assert manifest is not None + fresh_step_count = len(manifest.execution_steps) + assert runner.call_count == fresh_step_count + assert verifier.call_count == 0 + + fresh_runs = _request( + app, + "GET", + f"/api/execution/runs?work_order_id={fresh_request['work_order_id']}", + ) + assert len(fresh_runs) == 1 + fresh_run = fresh_runs[0] + fresh_evidence = ExecutionEvidence.model_validate( + _request( + app, + "GET", + f"/api/execution/runs/{fresh_run['id']}/evidence", + ) + ) + assert fresh_run["status"] == "succeeded" + assert fresh_run["worker_id"] == cheap_config.worker_id + assert fresh_run["route_provenance"]["estimated_cost_units"] == 3 + assert fresh_run["route_provenance"]["required_quota_units"] == 2 + assert fresh_evidence.tested_sha == tested_sha + assert fresh_evidence.reuse_provenance.decision == "fresh" + assert len(fresh_evidence.steps) == fresh_step_count + assert len(fresh_evidence.artifacts) == len( + manifest.artifact_declarations + ) + fresh_directory = cheap_config.evidence_root / f"run-{fresh_run['id']}" + assert (fresh_directory / "ownership.json").is_file() + assert (fresh_directory / "result.json").is_file() + assert list((fresh_directory / "logs").glob("*.log")) + for artifact in fresh_evidence.artifacts: + retained = fresh_directory.joinpath( + *artifact.relative_path.split("/") + ) + assert retained.is_file() + assert ( + hashlib.sha256(retained.read_bytes()).hexdigest() + == artifact.sha256 + ) + source_hashes = _retained_hashes(fresh_directory) + assert source_hashes + + published_current = _request( + app, + "POST", + f"/api/execution/github/requests/{fresh_request['request_id']}/publish", + {}, + ) + assert published_current["publication_state"] == "published_current" + assert published_current["publication_decision"] == "current" + + reused_request = _request( + app, + "POST", + "/api/execution/github/pull-requests/validate", + {**request_payload, "reuse_policy": "allow_exact"}, + ) + assert reused_request["request_id"] != fresh_request["request_id"] + assert reused_request["work_order_id"] != fresh_request["work_order_id"] + _request( + app, + "POST", + f"/api/execution/work-orders/{reused_request['work_order_id']}/approve", + {"queue": True}, + ) + expensive_client.heartbeat_worker(status="online") + assert expensive_worker.poll_once() is False + cheap_client.heartbeat_worker(status="online") + assert cheap_worker.poll_once() is True + assert runner.call_count == fresh_step_count + assert verifier.call_count == 1 + + reused_runs = _request( + app, + "GET", + f"/api/execution/runs?work_order_id={reused_request['work_order_id']}", + ) + assert len(reused_runs) == 1 + reused_run = reused_runs[0] + reused_evidence = ExecutionEvidence.model_validate( + _request( + app, + "GET", + f"/api/execution/runs/{reused_run['id']}/evidence", + ) + ) + assert reused_run["id"] != fresh_run["id"] + assert reused_run["worker_id"] == cheap_config.worker_id + assert reused_run["reuse_decision"] == "reused" + assert reused_run["reused_from_run_id"] == fresh_run["id"] + assert ( + reused_run["source_evidence_fingerprint"] == fresh_evidence.fingerprint + ) + assert reused_evidence.steps == [] + assert reused_evidence.artifacts == [] + assert reused_evidence.reuse_provenance.source_run_id == fresh_run["id"] + assert ( + reused_evidence.reuse_provenance.source_evidence_fingerprint + == fresh_evidence.fingerprint + ) + assert _retained_hashes(fresh_directory) == source_hashes + reused_directory = cheap_config.evidence_root / f"run-{reused_run['id']}" + assert (reused_directory / "ownership.json").is_file() + assert (reused_directory / "result.json").is_file() + + github.resolved = _resolved_pull_request(moved_sha) + published_stale = _request( + app, + "POST", + f"/api/execution/github/requests/{reused_request['request_id']}/publish", + {}, + ) + assert published_stale["publication_state"] == "published_stale" + assert published_stale["publication_decision"] == "stale" + assert published_stale["tested_head_sha"] == tested_sha + assert published_stale["publication_head_sha"] == moved_sha + + overview = _request( + app, + "GET", + "/api/execution/operator/overview?window_days=1", + ) + history = _request( + app, + "GET", + "/api/execution/operator/history?limit=25&offset=0", + ) + assert overview["runs"]["fresh_successful"] == 1 + assert overview["runs"]["reused_successful"] == 1 + assert overview["avoided_work"]["deterministic_executions_avoided"] == 1 + assert overview["avoided_work"]["reference_seconds_avoided"] > 0 + assert overview["avoided_work"]["comparison_units_avoided"] == 3 + assert overview["avoided_work"]["reuse_rate"] == 0.5 + assert overview["publications"] == {"current": 1, "stale": 1} + assert history["total"] == 2 + history_by_request = {item["request_id"]: item for item in history["items"]} + assert ( + history_by_request[fresh_request["request_id"]]["reuse_decision"] + == "fresh" + ) + assert ( + history_by_request[reused_request["request_id"]]["reuse_decision"] + == "reused" + ) + assert ( + history_by_request[reused_request["request_id"]]["reused_from_run_id"] + == fresh_run["id"] + ) + + async def capacity_proof() -> tuple[int, list[int]]: + async with sessions() as database: + leases = await database.scalar( + select(func.count()).select_from(ExecutionLease) + ) + active_counts = ( + ( + await database.execute( + select(ExecutionWorker.active_run_count).order_by( + ExecutionWorker.worker_id + ) + ) + ) + .scalars() + .all() + ) + return int(leases or 0), [int(value) for value in active_counts] + + leases, active_counts = asyncio.run(capacity_proof()) + assert leases == 0 + assert active_counts == [0, 0] + assert list(cheap_config.worker_root.glob("run-*")) == [] + assert list(expensive_config.worker_root.glob("run-*")) == [] + assert _git(canonical, "status", "--porcelain=v1") == status_before + assert _git(canonical, "rev-parse", "HEAD") == moved_sha + assert tested_sha != moved_sha + finally: + app.dependency_overrides.clear() + asyncio.run(engine.dispose()) diff --git a/docs/API.md b/docs/API.md index 8d62199..0ea1420 100644 --- a/docs/API.md +++ b/docs/API.md @@ -32,7 +32,7 @@ Use this page as the concise endpoint index; use [ai-interface.md](ai-interface. | `/api/execution/routing-profiles` | `GET`, `POST` | List or create privileged operator-owned local-worker cost, quota, and priority profiles. | | `/api/execution/routing-profiles/{worker_id}` | `GET`, `PUT` | Read or revision-protected replace one worker routing profile. | | `/api/execution/routing-profiles/{worker_id}/quota-reset` | `POST` | Apply an explicit monotonic, revision-protected quota replacement. | -| `/api/execution/workers` | `POST` | Register or refresh a read-only worker capability declaration. | +| `/api/execution/workers` | `GET`, `POST` | Read a bounded, redacted operator worker/profile projection or register/refresh a read-only worker capability declaration. | | `/api/execution/workers/{worker_id}/heartbeat` | `POST` | Refresh a registered worker heartbeat and availability state. | | `/api/execution/checkout` | `POST` | Atomically assign one capability-compatible queued work order to one worker. | | `/api/execution/runs` | `GET` | List historical execution attempts; filter with `work_order_id`. | @@ -44,8 +44,11 @@ Use this page as the concise endpoint index; use [ai-interface.md](ai-interface. | `/api/execution/runs/{id}/complete` | `POST` | Record `succeeded`, `failed`, `timed_out`, or `cancelled` after ownership validation. | | `/api/execution/leases/expire` | `POST` | Timeout stale runs, release worker capacity, and safely requeue their work orders. | | `/api/execution/github/pull-requests/validate` | `POST` | Resolve an allowlisted GitHub PR to one exact head and create or return one normal pending work order. | +| `/api/execution/github/requests` | `GET` | List a bounded, stably ordered adapter projection with repository, lifecycle, reuse, and publication filters. | | `/api/execution/github/requests/{request_id}` | `GET` | Return bounded adapter identity and execution/publication lifecycle. | | `/api/execution/github/requests/{request_id}/publish` | `POST` | Recheck the PR head and synchronously create or update one bounded managed comment as current or stale. | +| `/api/execution/operator/overview` | `GET` | Return database-derived request, run, reuse, publication, avoided-work, and worker counts for a bounded day window. | +| `/api/execution/operator/history` | `GET` | Return bounded, paginated, newest-first request/work-order/latest-run history with redacted route and evidence fields. | | `/api/plan` | `GET` | Return current plan snapshot used by agents and dashboard. | | `/api/execplans/index` | `GET` | Return ExecPlan registry index in JSON (default) or YAML based on query/header negotiation. | | `/health/live` | `GET` | Liveness probe returning process and probe observations. | @@ -116,7 +119,10 @@ operator `routing_priority`, and finally lexical worker ID. An explicit `preferred_executor` is a hard pin to a known worker. It overrides ranking but never approval, liveness, polling, status, capacity, capability, network, read-only, cost, quota, or profile-enabled checks, and it never falls back to a -different worker when unavailable. +different worker when unavailable. An unknown hard pin submitted through the +GitHub validation route returns bounded `404 preferred_executor_not_found`; the +request transaction is rolled back before either an adapter request or work +order exists. Every checkout by a known authenticated worker records only that requester's `last_checkout_poll_at` using server time. Poll freshness and heartbeat @@ -170,18 +176,51 @@ retention is not changed. ## GitHub exact-PR adapter The manual outbound adapter reuses `SWITCHBOARD_ADMIN_TOKEN` authentication. -Its create request accepts only `repository_full_name`, `pull_request_number`, -and a trusted manifest `name`/`version`. Stable GitHub identities, exact head -SHA, base provenance, manifest digest, work-order identity, terminal evidence, -comment identity, and publication state are server-owned. Unknown fields, -including commands, URLs, paths, status, hashes, worker IDs, and comment IDs, -return `422`. - -An identical authenticated actor + stable PR + exact head + trusted manifest -request returns the same adapter and work-order identities. A new head or -credential actor creates a distinct request. Stable actor ownership identifiers -remain server-owned and are not returned. The work order remains -`pending_approval` until the normal explicit approval route is called. +Its create request accepts `repository_full_name`, `pull_request_number`, a +trusted manifest `name`/`version`, and optional strict `reuse_policy`, +`routing_policy`, `maximum_cost_units`, `required_quota_units`, and +`preferred_executor` fields. Defaults preserve the original `never` plus +`first_available` behavior. Stable GitHub identities, exact head SHA, base +provenance, manifest digest, work-order identity, terminal evidence, comment +identity, and publication state are server-owned. Unknown or executable-shaped +fields still return `422`. + +An identical authenticated actor + stable PR + exact head + trusted manifest + +complete execution-policy request returns the same adapter and work-order +identities. Every accepted execution-policy field participates in the new +idempotency identity. An all-default request also recognizes the exact legacy +pre-command-center identity, so an existing default request is returned without +mutation or duplication; a non-default request never falls back to that legacy +identity. Policy remains authoritative on the linked work order rather than on +adapter-owned schema columns. A new head, credential actor, or material policy +creates a distinct request. The work order remains `pending_approval` until the +normal explicit approval route is called. + +The operator projections accept bounded pagination (`limit` at most `100`, +`offset` at most `10000`) and the overview accepts a day window at most `365`. +History has stable newest-first ordering and joins only the latest run per +request. It never returns commands, argv, logs, environment dumps, local paths, +credentials, candidate lists, or complete worker capabilities. Avoided-work +counts include successful reused runs only; reference seconds come from each +linked successful source run's persisted start/finish interval, and comparison +units come from the reused run's persisted route estimate. Missing values are +excluded rather than guessed. + +`GET /api/execution/workers` derives one activity label using safety-first +precedence: unavailable, stale, capacity constrained, then active. Its worker +summary contains only safe typed declarations (OS/architecture, Python/Node, +Docker, up to eight bounded browser names, GPU, Unity, desktop automation, +network posture, and the false repository-write capability) plus bounded profile +and freshness state. It never returns the arbitrary capability document. Profile +summaries distinguish a missing quota reset from a scheduled timestamp. + +The command center combines the exact request status with existing bounded +`GET /api/execution/work-orders/{id}/route-assessment` and +`GET /api/execution/runs/{id}` reads. The former supplies a non-mutating queued +candidate decision; the latter supplies persisted route/quota provenance, +timestamps, cleanup, reuse source run/fingerprint, and compact evidence after a +run exists. The browser never requests full logs, commands, argv, environment +values, local paths, or unbounded candidate data. Publication requires terminal compact evidence and re-resolves the PR immediately before its managed comment is written. A moved or unavailable head @@ -201,6 +240,9 @@ See for credential permissions, marker recovery, local commit availability, and transport limits. +See [Validation command center](operations/validation-command-center.md) for +the browser workflow and projection semantics. + ## Related Docs - [ai-interface.md](ai-interface.md) diff --git a/docs/architecture/local-execution-broker.md b/docs/architecture/local-execution-broker.md index b589a70..5ef1bd3 100644 --- a/docs/architecture/local-execution-broker.md +++ b/docs/architecture/local-execution-broker.md @@ -249,6 +249,63 @@ Provider budgets, external remaining-rate-limit ingestion, and paid-agent routing remain later concerns. Deterministic local validation must not invoke a paid coding agent. +### Operator validation command center (#136) + +The dashboard adds a browser workspace over the accepted execution and GitHub +adapter boundaries; it does not create a second scheduler or execution path. +Lifecycle actions still call the explicit approve, queue, cancel, expire, and +publish routes. Only the selected active request is polled, with one replaceable +timer that is cleared when selection changes or the page unloads. + +Server-owned projections assemble the operator view with bounded, stably ordered +queries. The browser does not join unbounded request, work-order, run, worker, +profile, evidence, and publication lists. History selects only the latest run +per adapter request and returns compact scalar provenance. Worker summaries +combine the declared worker with its operator-owned profile without exposing +capability dumps, local roots, or private connectivity data. + +The selected-request surface makes two bounded point reads in addition to its +request/history projection: current route assessment while queued and the exact +run after assignment. This exposes candidate count, reason, hard-pin state, +profile revision, quota reservation state, timestamps, duration, cleanup, and +reuse source provenance without broadening list responses or returning worker +logs. Worker cards render explicit declared status, derived activity, +OS/architecture, capacity, heartbeat/poll freshness, and profile state. Derived +activity has deterministic precedence: draining/offline and malformed records +are unavailable; expired heartbeat or poll freshness is stale even when the +worker is busy/full; a remaining busy/full worker is capacity constrained; and +only a fresh valid worker with spare capacity is active. The projection adds +only typed allowlisted Python/Node, Docker, bounded browser, GPU, Unity, desktop, +network, and read-only repository declarations. Arbitrary capability documents +remain redacted. Worker profiles expose the next quota-reset timestamp or an +explicit unscheduled state. + +The selected-request view keeps repository, pull-request number, and configured +reuse/routing policy beside the exact tested SHA and bounded route/run evidence. +These are identity fields from the persisted request/work order, not browser-side +guesses or joins. + +The browser acceptance uses synthetic completion only to drive visible lifecycle +states. It is not a worker trust proof. A separate file-backed acceptance uses +the real outbound `ExecutionClient` and `LocalWorker`, executes the trusted +manifest on the deterministically selected lower-cost worker, verifies retained +evidence locally on reuse, and proves the step runner is not called a second +time. + +Adapter request identity binds every accepted result-affecting execution policy. +For compatibility, an all-default request may resolve the exact pre-#136 legacy +identity; that row and linked work order are returned unchanged. A non-default +request never uses legacy fallback. The linked work order remains the single +authoritative persistence location for reuse, routing, cost, quota, and preferred +executor policy, so no adapter schema migration is required. + +Avoided-work metrics are projections, not a mutable savings ledger. One +successful reused run counts as one deterministic execution avoided. Reference +seconds are the non-negative persisted duration of its linked successful source +run, when available. Comparison units are the reused run's persisted route cost, +when available. These units are local routing comparisons only and are not money, +credits, provider usage, or verified spend reduction. + ## Execution-Plane Domain ### WorkOrder diff --git a/docs/assets/switchboard-validation-command-center.png b/docs/assets/switchboard-validation-command-center.png new file mode 100644 index 0000000..2665fcd Binary files /dev/null and b/docs/assets/switchboard-validation-command-center.png differ diff --git a/docs/configuration.md b/docs/configuration.md index e7e8519..d75572a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -49,6 +49,14 @@ These values are local operator comparison units only: Switchboard does not interpret them as currency, credits, spend, savings, billing, or a provider rate limit. No provider credential or paid-agent configuration is introduced. +The Validation Broker workspace edits the same persisted profiles through the +revision-protected APIs and reads active/stale worker state through a bounded +projection. It does not introduce browser-owned routing configuration. The +browser reads the existing admin token from local storage only when sending an +authenticated request; the token is never returned by a projection or rendered +into the workspace. Overview windows are request parameters bounded from 1 +through 365 days and do not mutate worker heartbeat or checkout-poll freshness. + ## Outbound GitHub adapter The manual exact-PR adapter uses server-only environment configuration: @@ -69,6 +77,13 @@ configuration surface. See [GitHub exact pull-request validation](operations/github-exact-pr-validation.md) for the complete operator and security contract. +Validation requests may add strict per-request reuse and routing policy. These +values are persisted on the linked execution work order, whose schema already +owns the authoritative policy fields. The adapter table therefore needs no new +policy columns or startup migration. Existing all-default adapter identities +remain discoverable through the exact prior idempotency calculation; non-default +requests use the complete policy-bound identity. + ## Exact evidence reuse Evidence reuse has no global environment-variable switch and no diff --git a/docs/operations/local-worker.md b/docs/operations/local-worker.md index 627fa65..63b59af 100644 --- a/docs/operations/local-worker.md +++ b/docs/operations/local-worker.md @@ -78,6 +78,12 @@ See [GitHub exact pull-request validation](github-exact-pr-validation.md) for the operator workflow. +The dashboard's [Validation command center](validation-command-center.md) +provides the same request, approval, routing-profile, lifecycle, evidence, and +publication operations through bounded server projections. It does not change +the worker's outbound-only pull loop, trust model, repository mapping, or local +evidence retention rules. + ## Runtime behavior and limits Phase 1 is deliberately single-concurrency. Configuration must set diff --git a/docs/operations/validation-command-center.md b/docs/operations/validation-command-center.md new file mode 100644 index 0000000..1ef1d8c --- /dev/null +++ b/docs/operations/validation-command-center.md @@ -0,0 +1,158 @@ +# Validation command center + +The **Validation Broker** workspace turns the existing trusted execution APIs +into one operator workflow. It remains a public developer preview for localhost +or controlled trusted networks; it is not a hosted execution service, remote +shell, provider router, or billing dashboard. + +![Validation Broker command center with synthetic data](../assets/switchboard-validation-command-center.png) + +The screenshot was captured from the offline browser acceptance application +using synthetic repository, SHA, worker, route, quota, evidence, and publication +data. It contains no admin token, credential-shaped value, local path, machine +identity, private URL, or claim of actual financial savings. + +## Operator workflow + +1. Start Switchboard and at least one configured outbound local worker. Keep the + worker's heartbeat and checkout-poll intervals inside the server freshness + windows. +2. Open the dashboard and, when configured, enter the existing admin token in + the dashboard's settings. The browser keeps it in local storage and attaches + it to protected requests using the accepted `Authorization: Bearer` form; + the server never returns it to the workspace. +3. Create or edit an operator-owned worker routing profile. Profile replacement + and quota reset require the latest revision. A `409` refreshes visible state + and leaves a persistent recovery message instead of overwriting newer data. +4. Enter an allowlisted `owner/repository`, pull-request number, trusted manifest, + reuse policy, routing policy, optional maximum comparison units, required + quota, and optional hard worker pin. +5. Request validation. Switchboard resolves the authenticated actor and the + exact current PR head, then creates or returns one pending work order whose + complete execution policy participates in adapter idempotency. +6. Inspect the exact SHA and explicitly approve and queue the request. Approval + is never implied by request creation. +7. Allow the outbound worker to claim the job. The selected request is polled + while active; selecting another request replaces the timer. +8. Inspect the queued route assessment or persisted terminal route, quota state, + candidate count, profile revision, run timestamps and duration, fresh/reused + decision, reused source provenance, cleanup, compact evidence fingerprint, + and publication decision. +9. Explicitly publish evidence. Switchboard re-resolves the PR head immediately; + a moved head is labelled stale without rewriting the tested SHA. +10. Use bounded history and filters to review prior requests. Copy actions expose + only stable IDs, exact SHAs, and compact fingerprints already present in the + projection. + +Disabled actions remain visible and provide a reason. The workspace supports +keyboard navigation and collapses to one column at narrow widths without page +horizontal overflow. + +## Manual acceptance + +On an offline development instance, register two synthetic trusted workers and +give them enabled profiles with different comparison-unit values. Submit a +`never` plus `cheapest_capable` request, approve and queue it, and confirm the +lower-cost eligible worker is selected. Complete and publish it, then submit the +same exact validation identity with `allow_exact`; confirm a distinct request +produces a distinct reused run with no executed validation steps. Move the mocked +PR head and publish again; the second record must become stale while the tested +SHA remains unchanged. Finally, filter history to reused rows, force one stale +profile revision, tab through the request form, and verify both a desktop and a +390-pixel viewport have no page-level horizontal overflow. The automated strict +browser regression performs this procedure with a file-backed database and no +network or real GitHub credential. Its completion endpoint is deliberately a +synthetic UI fixture; it does not claim to prove worker-local execution or +cryptographic reuse verification. That trust path is separately exercised by +the file-backed `ExecutionClient`/`LocalWorker` acceptance in +`client/python/tests/test_execution_worker_server_smoke.py`, which runs the +trusted manifest once, retains evidence, verifies the exact retained source on +the worker, and skips the step runner only for the reused run. + +## Projection and identity contract + +The workspace reads four authenticated projections: + +- `GET /api/execution/operator/overview?window_days=30` +- `GET /api/execution/operator/history?limit=25&offset=0` +- `GET /api/execution/workers?limit=100&offset=0` +- `GET /api/execution/github/requests?limit=25&offset=0` + +Limits are at most `100`, offsets at most `10000`, and metric windows at most +`365` days. History is newest-first with stable ID tie-breaking and one latest +run per request. Filters cover repository, lifecycle, fresh/reused decision, and +publication state. Projection responses exclude commands, argv, full logs, +environment dumps, credentials, local paths, candidate lists, and complete +worker capabilities. + +The selected request uses the existing bounded route-assessment and run-detail +APIs. Before assignment it displays the selected candidate, abstract comparison +units, eligible count, bounded reason, and hard-pin decision. After assignment it +displays persisted worker/profile provenance, required and reserved quota, +reservation state, timestamps, measured duration, reuse source identity, +evidence fingerprint, cleanup, and terminal state. Missing values are displayed +as unavailable rather than inferred. Repository, pull-request number, and the +configured reuse and routing policies remain visible beside that evidence so the +operator does not have to infer request identity from history. + +Worker cards separately show declared status, server-derived activity, safe +OS/architecture, active/max capacity, heartbeat and checkout-poll timestamps, +and operator-owned profile state including scheduled or unscheduled quota reset. +Activity precedence is unavailable for draining/offline or malformed records, +then stale for expired heartbeat or checkout polling, then capacity constrained +for a fresh busy/full worker, and active otherwise. The capability summary is a +strict allowlist: Python/Node version, Docker, at most eight bounded browser +names, GPU, Unity, desktop automation, network posture, and read-only repository +support. The arbitrary capability document, host identity, paths, commands, +argv, environments, and credentials are never returned to this surface. + +The request idempotency identity includes repository, stable PR identity, exact +head, trusted manifest, authenticated actor, reuse policy, routing policy, +maximum comparison units, required quota, and preferred executor. Requests with +different policy do not collapse. For an all-default request only, Switchboard +also checks the exact pre-command-center key so existing adapter rows keep their +request and work-order identities without mutation. Policy is stored +authoritatively on the linked work order; the adapter schema needs no new policy +columns or migration. + +## Metrics + +The overview is derived from persisted database records for the selected window: + +- **Deterministic executions avoided** counts successful reused runs only. +- **Reference execution time avoided** sums the persisted `finished_at - started_at` + duration of each linked successful source run when both timestamps are valid. +- **Comparison units avoided** sums the reused run's persisted route estimate + when present. +- **Fresh successful runs**, **reused successful runs**, and **reuse rate** use + terminal successful runs only; the rate denominator is their sum. +- **Current** and **stale publications** use persisted publication state. + +Missing source duration or route cost contributes nothing rather than an +estimate. A source run is not itself counted as avoided. Comparison units are +operator-defined local routing values, not currency, paid-agent credits, +provider usage, actual spend, or measured savings. + +## Recovery and limitations + +- An empty projection is a valid state; configure a worker/profile and create a + request rather than treating it as a server failure. +- A stale profile revision returns `409`. Reload the profile and deliberately + reapply the intended values. +- Cheapest-capable routing still requires fresh heartbeat, active polling, + capacity, capabilities, network/read-only compatibility, an enabled profile, + cost within the optional ceiling, and sufficient quota. +- Exact reuse remains same-worker and requires local retained evidence proof. + `allow_exact` falls back to fresh execution; `require_exact` never validates + when proof is unavailable. +- The GitHub adapter resolves identity and publishes a managed comment but does + not fetch source. The exact commit must already exist in the worker's approved + canonical checkout. +- Full logs and artifact bytes remain under the worker-owned evidence root. The + browser receives compact evidence only. +- There is no MCP, paid-agent, provider, browser-worker, desktop/RPA, webhook, + auto-approval, auto-publication, or production multi-tenant scope in this slice. + +See [Local worker operations](local-worker.md), +[GitHub exact pull-request validation](github-exact-pr-validation.md), and the +[execution API reference](../API.md) for the underlying contracts. diff --git a/docs/reports/status.md b/docs/reports/status.md index d85c793..2be3fa9 100644 --- a/docs/reports/status.md +++ b/docs/reports/status.md @@ -1,6 +1,6 @@ # Public Developer Preview Status -_Last reviewed: 2026-07-23_ +_Last reviewed: 2026-08-08_ ## Classification @@ -27,7 +27,30 @@ The merged foundation includes: - the merged `validate-switchboard@1` workflow with strict compact evidence, worker-owned retained artifacts, dependency-lock hashes, parsed validation results, deterministic fingerprints, server-side local-path rejection, and the read-only - `GET /api/execution/runs/{run_id}/evidence` endpoint. + `GET /api/execution/runs/{run_id}/evidence` endpoint; +- exact GitHub pull-request resolution and explicit current/stale compact-evidence + publication; +- exact same-worker evidence reuse and deterministic cheapest-capable local routing + with operator-owned cost/quota profiles. + +Draft PR #137 adds the Validation Broker command center over that merged control +plane: bounded operator projections, policy-aware request identity, explicit +lifecycle controls, routing-profile management, fresh/reused history, and +database-derived avoided-work metrics. It remains draft and is not part of the +merged baseline until review and the protected matrix complete. + +Connector review `4890406546` required final correction evidence before that +review can complete: a real `ExecutionClient`/`LocalWorker` fresh-then-reuse +acceptance, bounded rollback for unknown preferred workers, and complete +route/quota/source/timing plus worker-state visibility. The browser fixture +remains UI-only; it is not cited as cryptographic retained-evidence proof. + +Connector review `4890873592` required the final presentation and projection +corrections: deterministic safety-first activity precedence with persisted direct +and HTTP proof, visible repository/PR plus configured reuse/routing policy, a +strictly allowlisted worker capability summary, and explicit scheduled versus +unscheduled quota-reset state. The correction preserves routing, reservation, +reuse, evidence, execution, and GitHub adapter behavior. Operator setup, repository allowlisting, evidence retention, and trust limitations are documented in the [local worker operations guide](../operations/local-worker.md). @@ -89,12 +112,8 @@ not by itself authorize a release, production deployment, or public hosted servi The execution-broker roadmap is tracked in issue [#111](https://github.com/Nobodyworld/dev-agent-switchboard/issues/111). -The next bounded slices are now unblocked from the Phase 1B merge: - -- [#122](https://github.com/Nobodyworld/dev-agent-switchboard/issues/122) — resolve - an exact GitHub pull-request head and publish compact validation evidence; -- [#121](https://github.com/Nobodyworld/dev-agent-switchboard/issues/121) — exact - evidence reuse with worker-local availability proof. - -Fresh execution remains the baseline. Do not infer that roadmap work is implemented, -merged, or release-authorized merely because its issue is fully specified. +Issue [#136](https://github.com/Nobodyworld/dev-agent-switchboard/issues/136) +tracks the current command-center product slice. Fresh execution remains the +default and explicit approval/publication remain mandatory. Do not infer that a +draft pull request is merged or release-authorized merely because its workflow is +documented. diff --git a/server/api/routers/execution.py b/server/api/routers/execution.py index c979a2e..747136a 100644 --- a/server/api/routers/execution.py +++ b/server/api/routers/execution.py @@ -6,7 +6,8 @@ from __future__ import annotations -from typing import NoReturn +import datetime as dt +from typing import Literal, NoReturn from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.exc import IntegrityError @@ -24,7 +25,12 @@ WorkerRegistration, WorkOrderDraft, ) -from server.execution.enums import ExecutionRunStatus +from server.execution.enums import ( + ExecutionRunStatus, + ReuseDecision, + RoutingPolicy, + WorkOrderStatus, +) from server.execution.evidence import ExecutionEvidence from server.execution.exceptions import ( ApprovalDeniedError, @@ -38,6 +44,15 @@ RepositoryWritePolicyError, UnknownManifestError, ) +from server.execution.operator_projection import ( + MAX_OPERATOR_LIMIT, + MAX_OPERATOR_OFFSET, + MAX_OPERATOR_WINDOW_DAYS, + ExecutionHistoryPageOut, + ExecutionOperatorOverviewOut, + ExecutionOperatorProjection, + ExecutionWorkerPageOut, +) from server.execution.schemas import ( ApproveWorkOrderIn, CheckoutIn, @@ -62,10 +77,92 @@ WorkOrderCreateIn, WorkOrderOut, ) +from server.settings import get_execution_routing_settings router = APIRouter(dependencies=[Depends(require_admin_token)]) +@router.get( + "/api/execution/operator/overview", + response_model=ExecutionOperatorOverviewOut, +) +async def get_operator_overview( + session: SessionDependency, + window_days: int = Query(default=30, ge=1, le=MAX_OPERATOR_WINDOW_DAYS), +) -> ExecutionOperatorOverviewOut: + """Return bounded validation, reuse, publication, and worker metrics.""" + + freshness = get_execution_routing_settings() + return await ExecutionOperatorProjection(session).overview( + window_days=window_days, + heartbeat_freshness_seconds=freshness.heartbeat_freshness_seconds, + active_poll_freshness_seconds=freshness.active_poll_freshness_seconds, + ) + + +@router.get( + "/api/execution/operator/history", + response_model=ExecutionHistoryPageOut, +) +async def list_operator_history( # noqa: PLR0913 + session: SessionDependency, + limit: int = Query(default=25, ge=1, le=MAX_OPERATOR_LIMIT), + offset: int = Query(default=0, ge=0, le=MAX_OPERATOR_OFFSET), + repository_full_name: str | None = Query( + default=None, min_length=3, max_length=255 + ), + pull_request_number: int | None = Query(default=None, ge=1), + work_order_status: WorkOrderStatus | None = None, + run_status: ExecutionRunStatus | None = None, + reuse_decision: ReuseDecision | None = None, + routing_policy: RoutingPolicy | None = None, + publication_state: ( + Literal[ + "not_published", + "published_current", + "published_stale", + "retryable_failure", + "failed", + ] + | None + ) = None, + created_after: dt.datetime | None = None, + created_before: dt.datetime | None = None, +) -> ExecutionHistoryPageOut: + """List redacted validation history in stable newest-first order.""" + + return await ExecutionOperatorProjection(session).list_history( + limit=limit, + offset=offset, + repository_full_name=repository_full_name, + pull_request_number=pull_request_number, + work_order_status=(work_order_status.value if work_order_status else None), + run_status=(run_status.value if run_status else None), + reuse_decision=(reuse_decision.value if reuse_decision else None), + routing_policy=(routing_policy.value if routing_policy else None), + publication_state=publication_state, + created_after=created_after, + created_before=created_before, + ) + + +@router.get("/api/execution/workers", response_model=ExecutionWorkerPageOut) +async def list_execution_workers( + session: SessionDependency, + limit: int = Query(default=25, ge=1, le=MAX_OPERATOR_LIMIT), + offset: int = Query(default=0, ge=0, le=MAX_OPERATOR_OFFSET), +) -> ExecutionWorkerPageOut: + """List bounded worker and operator-owned routing profile summaries.""" + + freshness = get_execution_routing_settings() + return await ExecutionOperatorProjection(session).list_workers( + limit=limit, + offset=offset, + heartbeat_freshness_seconds=freshness.heartbeat_freshness_seconds, + active_poll_freshness_seconds=freshness.active_poll_freshness_seconds, + ) + + def _raise_domain_error(error: ExecutionDomainError) -> NoReturn: """Map expected execution-domain errors to documented HTTP responses.""" diff --git a/server/api/routers/github_execution.py b/server/api/routers/github_execution.py index cecb957..910d8c1 100644 --- a/server/api/routers/github_execution.py +++ b/server/api/routers/github_execution.py @@ -2,9 +2,10 @@ from __future__ import annotations -from typing import NoReturn +import datetime as dt +from typing import Literal, NoReturn -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.exc import IntegrityError from server.api.dependencies import ( @@ -12,6 +13,19 @@ SessionDependency, require_admin_token, ) +from server.execution.enums import ( + ExecutionRunStatus, + ReuseDecision, + RoutingPolicy, + WorkOrderStatus, +) +from server.execution.exceptions import ExecutionDomainError, ExecutionNotFoundError +from server.execution.operator_projection import ( + MAX_OPERATOR_LIMIT, + MAX_OPERATOR_OFFSET, + ExecutionHistoryPageOut, + ExecutionOperatorProjection, +) from server.github_adapter.errors import ( GitHubAdapterError, GitHubManifestError, @@ -28,6 +42,52 @@ router = APIRouter(dependencies=[Depends(require_admin_token)]) +@router.get( + "/api/execution/github/requests", + response_model=ExecutionHistoryPageOut, +) +async def list_github_validation_requests( # noqa: PLR0913 + session: SessionDependency, + limit: int = Query(default=25, ge=1, le=MAX_OPERATOR_LIMIT), + offset: int = Query(default=0, ge=0, le=MAX_OPERATOR_OFFSET), + repository_full_name: str | None = Query( + default=None, min_length=3, max_length=255 + ), + pull_request_number: int | None = Query(default=None, ge=1), + work_order_status: WorkOrderStatus | None = None, + run_status: ExecutionRunStatus | None = None, + reuse_decision: ReuseDecision | None = None, + routing_policy: RoutingPolicy | None = None, + publication_state: ( + Literal[ + "not_published", + "published_current", + "published_stale", + "retryable_failure", + "failed", + ] + | None + ) = None, + created_after: dt.datetime | None = None, + created_before: dt.datetime | None = None, +) -> ExecutionHistoryPageOut: + """List bounded GitHub request and linked lifecycle projections.""" + + return await ExecutionOperatorProjection(session).list_history( + limit=limit, + offset=offset, + repository_full_name=repository_full_name, + pull_request_number=pull_request_number, + work_order_status=(work_order_status.value if work_order_status else None), + run_status=(run_status.value if run_status else None), + reuse_decision=(reuse_decision.value if reuse_decision else None), + routing_policy=(routing_policy.value if routing_policy else None), + publication_state=publication_state, + created_after=created_after, + created_before=created_before, + ) + + def _raise_adapter_error(error: GitHubAdapterError) -> NoReturn: reason = str(error) if isinstance(error, GitHubRequestNotFoundError): @@ -44,6 +104,16 @@ def _raise_adapter_error(error: GitHubAdapterError) -> NoReturn: raise HTTPException(status_code=409, detail="github_adapter_conflict") from error +def _raise_execution_error(error: ExecutionDomainError) -> NoReturn: + """Keep expected execution validation failures bounded at the adapter route.""" + + if isinstance(error, ExecutionNotFoundError): + raise HTTPException(status_code=404, detail=str(error)) from error + raise HTTPException( + status_code=409, detail="execution_lifecycle_conflict" + ) from error + + async def _commit(session: SessionDependency) -> None: try: await session.commit() @@ -71,10 +141,18 @@ async def request_pull_request_validation( pull_request_number=body.pull_request_number, manifest_name=body.manifest.name, manifest_version=body.manifest.version, + reuse_policy=body.reuse_policy, + routing_policy=body.routing_policy, + maximum_cost_units=body.maximum_cost_units, + required_quota_units=body.required_quota_units, + preferred_executor=body.preferred_executor, ) except GitHubAdapterError as error: await session.rollback() _raise_adapter_error(error) + except ExecutionDomainError as error: + await session.rollback() + _raise_execution_error(error) await _commit(session) return GitHubValidationRequestOut.from_status(status) diff --git a/server/execution/operator_projection.py b/server/execution/operator_projection.py new file mode 100644 index 0000000..98f064b --- /dev/null +++ b/server/execution/operator_projection.py @@ -0,0 +1,686 @@ +"""Bounded, redacted read models for the validation broker dashboard.""" + +from __future__ import annotations + +import datetime as dt +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints +from sqlalchemy import Select, func, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import InstrumentedAttribute, aliased +from sqlalchemy.sql.elements import ColumnElement + +from server.models import ( + ExecutionRun, + ExecutionWorker, + ExecutionWorkOrder, + GitHubValidationRequest, + WorkerRoutingProfile, +) +from server.time_utils import utcnow_naive + +from .enums import ExecutionRunStatus, NetworkPolicy, ReuseDecision, WorkerStatus + +MAX_OPERATOR_LIMIT = 100 +MAX_OPERATOR_OFFSET = 10_000 +MAX_OPERATOR_WINDOW_DAYS = 365 +MAX_OPERATOR_BROWSER_COUNT = 8 +MAX_OPERATOR_BROWSER_LENGTH = 64 +SHA256_HEX_LENGTH = 64 +BrowserName = Annotated[ + str, + StringConstraints( + strip_whitespace=True, + min_length=1, + max_length=MAX_OPERATOR_BROWSER_LENGTH, + ), +] + + +class OperatorProjectionModel(BaseModel): + """Strict base for browser-safe operator projections.""" + + model_config = ConfigDict(extra="forbid") + + +class OperatorWindowOut(OperatorProjectionModel): + """Inclusive UTC window applied to persisted record timestamps.""" + + days: int = Field(ge=1, le=MAX_OPERATOR_WINDOW_DAYS) + starts_at: dt.datetime + ends_at: dt.datetime + + +class OperatorRequestMetricsOut(OperatorProjectionModel): + total: int = Field(ge=0) + + +class OperatorWorkOrderMetricsOut(OperatorProjectionModel): + total: int = Field(ge=0) + by_status: dict[str, int] + + +class OperatorRunMetricsOut(OperatorProjectionModel): + total: int = Field(ge=0) + by_status: dict[str, int] + fresh_successful: int = Field(ge=0) + reused_successful: int = Field(ge=0) + unavailable_exact_reuse: int = Field(ge=0) + + +class AvoidedWorkMetricsOut(OperatorProjectionModel): + deterministic_executions_avoided: int = Field(ge=0) + reference_seconds_avoided: float = Field(ge=0) + comparison_units_avoided: int = Field(ge=0) + reuse_rate: float = Field(ge=0, le=1) + + +class OperatorPublicationMetricsOut(OperatorProjectionModel): + current: int = Field(ge=0) + stale: int = Field(ge=0) + + +class OperatorWorkerMetricsOut(OperatorProjectionModel): + total: int = Field(ge=0) + active: int = Field(ge=0) + stale: int = Field(ge=0) + capacity_constrained: int = Field(ge=0) + unavailable: int = Field(ge=0) + + +class ExecutionOperatorOverviewOut(OperatorProjectionModel): + window: OperatorWindowOut + requests: OperatorRequestMetricsOut + work_orders: OperatorWorkOrderMetricsOut + runs: OperatorRunMetricsOut + avoided_work: AvoidedWorkMetricsOut + publications: OperatorPublicationMetricsOut + workers: OperatorWorkerMetricsOut + + +class RoutingProfileSummaryOut(OperatorProjectionModel): + enabled: bool + estimated_cost_units_per_run: int = Field(ge=0) + quota_capacity_units: int = Field(ge=0) + quota_remaining_units: int = Field(ge=0) + quota_reset_at: dt.datetime | None + routing_priority: int = Field(ge=0) + revision: int = Field(ge=1) + updated_at: dt.datetime + + +class ExecutionWorkerSummaryOut(OperatorProjectionModel): + worker_id: str = Field(min_length=1, max_length=128) + display_name: str = Field(min_length=1, max_length=255) + operating_system: str = Field(min_length=1, max_length=64) + architecture: str = Field(min_length=1, max_length=64) + python_version: str | None = Field(default=None, max_length=64) + node_version: str | None = Field(default=None, max_length=64) + docker_available: bool + browsers: list[BrowserName] = Field(max_length=MAX_OPERATOR_BROWSER_COUNT) + gpu_available: bool + unity_available: bool + desktop_available: bool + network_policy_capability: Literal["disabled", "worker_restricted"] + repository_write_capability: Literal[False] + status: str = Field(min_length=1, max_length=32) + activity_state: Literal["active", "stale", "capacity_constrained", "unavailable"] + active_run_count: int = Field(ge=0) + max_concurrency: int = Field(ge=1) + last_heartbeat_at: dt.datetime + last_checkout_poll_at: dt.datetime | None + profile: RoutingProfileSummaryOut | None + + +class ExecutionWorkerPageOut(OperatorProjectionModel): + items: list[ExecutionWorkerSummaryOut] + limit: int = Field(ge=1, le=MAX_OPERATOR_LIMIT) + offset: int = Field(ge=0, le=MAX_OPERATOR_OFFSET) + total: int = Field(ge=0) + + +class ExecutionHistoryItemOut(OperatorProjectionModel): + request_id: int = Field(ge=1) + repository_full_name: str = Field(min_length=3, max_length=255) + pull_request_number: int = Field(ge=1) + tested_head_sha: str = Field(pattern=r"^[0-9a-f]{40}$") + manifest_name: str = Field(min_length=1, max_length=128) + manifest_version: str = Field(min_length=1, max_length=64) + work_order_id: int = Field(ge=1) + work_order_status: str = Field(min_length=1, max_length=32) + routing_policy: str = Field(min_length=1, max_length=32) + preferred_executor: str | None = Field(default=None, max_length=128) + maximum_cost_units: int | None = Field(default=None, ge=0) + required_quota_units: int = Field(ge=0) + run_id: int | None = Field(default=None, ge=1) + run_status: str | None = Field(default=None, max_length=32) + selected_worker_id: str | None = Field(default=None, max_length=128) + estimated_cost_units: int | None = Field(default=None, ge=0) + reuse_decision: str | None = Field(default=None, max_length=32) + reused_from_run_id: int | None = Field(default=None, ge=1) + run_duration_seconds: float | None = Field(default=None, ge=0) + evidence_fingerprint: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$") + publication_state: str = Field(min_length=1, max_length=32) + publication_decision: str = Field(min_length=1, max_length=16) + created_at: dt.datetime + updated_at: dt.datetime + + +class ExecutionHistoryPageOut(OperatorProjectionModel): + items: list[ExecutionHistoryItemOut] + limit: int = Field(ge=1, le=MAX_OPERATOR_LIMIT) + offset: int = Field(ge=0, le=MAX_OPERATOR_OFFSET) + total: int = Field(ge=0) + + +def _enum_value(value: object) -> str: + return str(getattr(value, "value", value)) + + +def _utc_naive(value: dt.datetime) -> dt.datetime: + if value.tzinfo is None or value.utcoffset() is None: + return value + return value.astimezone(dt.UTC).replace(tzinfo=None) + + +def _fresh(value: dt.datetime | None, *, now: dt.datetime, seconds: int) -> bool: + return value is not None and _utc_naive(value) >= now - dt.timedelta( + seconds=seconds + ) + + +class ExecutionOperatorProjection: + """Build stable operator views without exposing executable or sensitive data.""" + + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def overview( + self, + *, + window_days: int, + heartbeat_freshness_seconds: int, + active_poll_freshness_seconds: int, + now: dt.datetime | None = None, + ) -> ExecutionOperatorOverviewOut: + now = _utc_naive(now or utcnow_naive()) + starts_at = now - dt.timedelta(days=window_days) + + request_total = int( + await self._session.scalar( + select(func.count(GitHubValidationRequest.id)).where( + GitHubValidationRequest.created_at >= starts_at + ) + ) + or 0 + ) + work_order_counts = await self._grouped_counts( + ExecutionWorkOrder.status, + ExecutionWorkOrder.created_at >= starts_at, + ) + run_counts = await self._grouped_counts( + ExecutionRun.status, + ExecutionRun.created_at >= starts_at, + ) + reuse_rows = ( + await self._session.execute( + select( + ExecutionRun.reuse_decision, + ExecutionRun.status, + ExecutionRun.route_estimated_cost_units, + ExecutionRun.reused_from_run_id, + ).where(ExecutionRun.created_at >= starts_at) + ) + ).all() + fresh = sum( + status == ExecutionRunStatus.SUCCEEDED and decision == ReuseDecision.FRESH + for decision, status, _cost, _source in reuse_rows + ) + reused_rows = [ + row + for row in reuse_rows + if row.status == ExecutionRunStatus.SUCCEEDED + and row.reuse_decision == ReuseDecision.REUSED + ] + reused = len(reused_rows) + unavailable = sum( + decision == ReuseDecision.UNAVAILABLE + for decision, _status, _cost, _source in reuse_rows + ) + source_ids = { + row.reused_from_run_id for row in reused_rows if row.reused_from_run_id + } + source_durations: dict[int, float] = {} + if source_ids: + source_runs = ( + await self._session.execute( + select( + ExecutionRun.id, + ExecutionRun.started_at, + ExecutionRun.finished_at, + ).where(ExecutionRun.id.in_(source_ids)) + ) + ).all() + source_durations = { + run_id: max(0.0, (finished - started).total_seconds()) + for run_id, started, finished in source_runs + if started is not None and finished is not None and finished >= started + } + seconds_avoided = sum( + source_durations.get(row.reused_from_run_id, 0.0) for row in reused_rows + ) + units_avoided = sum( + row.route_estimated_cost_units + for row in reused_rows + if row.route_estimated_cost_units is not None + ) + + publication_counts = await self._grouped_counts( + GitHubValidationRequest.publication_state, + GitHubValidationRequest.created_at >= starts_at, + ) + worker_rows = ( + await self._session.execute( + select( + ExecutionWorker.status, + ExecutionWorker.active_run_count, + ExecutionWorker.max_concurrency, + ExecutionWorker.last_heartbeat_at, + ExecutionWorker.last_checkout_poll_at, + ) + ) + ).all() + worker_states = [ + _worker_activity_state( + status=status, + active_run_count=active_runs, + max_concurrency=max_concurrency, + last_heartbeat_at=heartbeat, + last_checkout_poll_at=poll, + now=now, + heartbeat_freshness_seconds=heartbeat_freshness_seconds, + active_poll_freshness_seconds=active_poll_freshness_seconds, + ) + for status, active_runs, max_concurrency, heartbeat, poll in worker_rows + ] + denominator = fresh + reused + return ExecutionOperatorOverviewOut( + window=OperatorWindowOut( + days=window_days, + starts_at=starts_at.replace(tzinfo=dt.UTC), + ends_at=now.replace(tzinfo=dt.UTC), + ), + requests=OperatorRequestMetricsOut(total=request_total), + work_orders=OperatorWorkOrderMetricsOut( + total=sum(work_order_counts.values()), by_status=work_order_counts + ), + runs=OperatorRunMetricsOut( + total=sum(run_counts.values()), + by_status=run_counts, + fresh_successful=fresh, + reused_successful=reused, + unavailable_exact_reuse=unavailable, + ), + avoided_work=AvoidedWorkMetricsOut( + deterministic_executions_avoided=reused, + reference_seconds_avoided=seconds_avoided, + comparison_units_avoided=units_avoided, + reuse_rate=(reused / denominator if denominator else 0.0), + ), + publications=OperatorPublicationMetricsOut( + current=publication_counts.get("published_current", 0), + stale=publication_counts.get("published_stale", 0), + ), + workers=OperatorWorkerMetricsOut( + total=len(worker_states), + active=worker_states.count("active"), + stale=worker_states.count("stale"), + capacity_constrained=worker_states.count("capacity_constrained"), + unavailable=worker_states.count("unavailable"), + ), + ) + + async def list_workers( + self, + *, + limit: int, + offset: int, + heartbeat_freshness_seconds: int, + active_poll_freshness_seconds: int, + ) -> ExecutionWorkerPageOut: + now = utcnow_naive() + total = int( + await self._session.scalar(select(func.count(ExecutionWorker.id))) or 0 + ) + rows = ( + await self._session.execute( + select(ExecutionWorker, WorkerRoutingProfile) + .outerjoin( + WorkerRoutingProfile, + WorkerRoutingProfile.worker_id == ExecutionWorker.worker_id, + ) + .order_by(ExecutionWorker.worker_id) + .limit(limit) + .offset(offset) + ) + ).all() + items = [] + for worker, profile in rows: + profile_out = None + if profile is not None: + profile_out = RoutingProfileSummaryOut.model_validate( + { + "enabled": profile.enabled, + "estimated_cost_units_per_run": ( + profile.estimated_cost_units_per_run + ), + "quota_capacity_units": profile.quota_capacity_units, + "quota_remaining_units": profile.quota_remaining_units, + "quota_reset_at": profile.quota_reset_at, + "routing_priority": profile.routing_priority, + "revision": profile.revision, + "updated_at": profile.updated_at, + } + ) + items.append( + ExecutionWorkerSummaryOut( + worker_id=worker.worker_id, + display_name=worker.display_name, + operating_system=worker.operating_system, + architecture=worker.architecture, + python_version=worker.python_version, + node_version=worker.node_version, + docker_available=worker.docker_available, + browsers=_bounded_browsers(worker.browsers), + gpu_available=worker.gpu_available, + unity_available=worker.unity_available, + desktop_available=worker.desktop_available, + network_policy_capability=_network_policy_capability( + worker.network_policy_capability + ), + repository_write_capability=worker.repository_write_capability, + status=_enum_value(worker.status), + activity_state=_worker_activity_state( + status=worker.status, + active_run_count=worker.active_run_count, + max_concurrency=worker.max_concurrency, + last_heartbeat_at=worker.last_heartbeat_at, + last_checkout_poll_at=worker.last_checkout_poll_at, + now=now, + heartbeat_freshness_seconds=heartbeat_freshness_seconds, + active_poll_freshness_seconds=active_poll_freshness_seconds, + ), + active_run_count=worker.active_run_count, + max_concurrency=worker.max_concurrency, + last_heartbeat_at=worker.last_heartbeat_at, + last_checkout_poll_at=worker.last_checkout_poll_at, + profile=profile_out, + ) + ) + return ExecutionWorkerPageOut( + items=items, limit=limit, offset=offset, total=total + ) + + async def list_history( # noqa: PLR0913 + self, + *, + limit: int, + offset: int, + repository_full_name: str | None = None, + pull_request_number: int | None = None, + work_order_status: str | None = None, + run_status: str | None = None, + reuse_decision: str | None = None, + routing_policy: str | None = None, + publication_state: str | None = None, + created_after: dt.datetime | None = None, + created_before: dt.datetime | None = None, + ) -> ExecutionHistoryPageOut: + latest_run = aliased(ExecutionRun) + latest_run_id = ( + select(func.max(ExecutionRun.id)) + .where(ExecutionRun.work_order_id == ExecutionWorkOrder.id) + .correlate(ExecutionWorkOrder) + .scalar_subquery() + ) + statement = ( + select(GitHubValidationRequest, ExecutionWorkOrder, latest_run) + .join( + ExecutionWorkOrder, + ExecutionWorkOrder.id == GitHubValidationRequest.work_order_id, + ) + .outerjoin(latest_run, latest_run.id == latest_run_id) + ) + statement = self._history_filters( + statement, + latest_run=latest_run, + repository_full_name=repository_full_name, + pull_request_number=pull_request_number, + work_order_status=work_order_status, + run_status=run_status, + reuse_decision=reuse_decision, + routing_policy=routing_policy, + publication_state=publication_state, + created_after=created_after, + created_before=created_before, + ) + count_statement = select(func.count(GitHubValidationRequest.id)).join( + ExecutionWorkOrder, + ExecutionWorkOrder.id == GitHubValidationRequest.work_order_id, + ) + if reuse_decision is not None or run_status is not None: + count_statement = count_statement.outerjoin( + latest_run, latest_run.id == latest_run_id + ) + count_statement = self._history_filters( + count_statement, + latest_run=latest_run, + repository_full_name=repository_full_name, + pull_request_number=pull_request_number, + work_order_status=work_order_status, + run_status=run_status, + reuse_decision=reuse_decision, + routing_policy=routing_policy, + publication_state=publication_state, + created_after=created_after, + created_before=created_before, + ) + total = int(await self._session.scalar(count_statement) or 0) + rows = ( + await self._session.execute( + statement.order_by( + GitHubValidationRequest.created_at.desc(), + GitHubValidationRequest.id.desc(), + ) + .limit(limit) + .offset(offset) + ) + ).all() + return ExecutionHistoryPageOut( + items=[_history_item(request, order, run) for request, order, run in rows], + limit=limit, + offset=offset, + total=total, + ) + + async def _grouped_counts( + self, column: InstrumentedAttribute[Any], predicate: ColumnElement[bool] + ) -> dict[str, int]: + rows = ( + await self._session.execute( + select(column, func.count()).where(predicate).group_by(column) + ) + ).all() + return {_enum_value(value): int(count) for value, count in rows} + + @staticmethod + def _history_filters( # noqa: PLR0913 + statement: Select[Any], + *, + latest_run: Any, + repository_full_name: str | None, + pull_request_number: int | None, + work_order_status: str | None, + run_status: str | None, + reuse_decision: str | None, + routing_policy: str | None, + publication_state: str | None, + created_after: dt.datetime | None, + created_before: dt.datetime | None, + ) -> Select[Any]: + if repository_full_name is not None: + statement = statement.where( + GitHubValidationRequest.repository_full_name == repository_full_name + ) + if pull_request_number is not None: + statement = statement.where( + GitHubValidationRequest.pull_request_number == pull_request_number + ) + if work_order_status is not None: + statement = statement.where(ExecutionWorkOrder.status == work_order_status) + if run_status is not None: + statement = statement.where(latest_run.status == run_status) + if reuse_decision is not None: + statement = statement.where(latest_run.reuse_decision == reuse_decision) + if routing_policy is not None: + statement = statement.where( + ExecutionWorkOrder.routing_policy == routing_policy + ) + if publication_state is not None: + statement = statement.where( + GitHubValidationRequest.publication_state == publication_state + ) + if created_after is not None: + statement = statement.where( + GitHubValidationRequest.created_at >= _utc_naive(created_after) + ) + if created_before is not None: + statement = statement.where( + GitHubValidationRequest.created_at <= _utc_naive(created_before) + ) + return statement + + +def _bounded_browsers(value: object) -> list[str]: + """Return a small safe subset of valid persisted browser identifiers.""" + + if not isinstance(value, list): + return [] + browsers: list[str] = [] + for item in value: + if len(browsers) >= MAX_OPERATOR_BROWSER_COUNT: + break + if not isinstance(item, str): + continue + browser = item.strip() + if 1 <= len(browser) <= MAX_OPERATOR_BROWSER_LENGTH: + browsers.append(browser) + return browsers + + +def _worker_activity_state( # noqa: PLR0913 + *, + status: WorkerStatus, + active_run_count: int, + max_concurrency: int, + last_heartbeat_at: dt.datetime, + last_checkout_poll_at: dt.datetime | None, + now: dt.datetime, + heartbeat_freshness_seconds: int, + active_poll_freshness_seconds: int, +) -> Literal["active", "stale", "capacity_constrained", "unavailable"]: + if status in {WorkerStatus.DRAINING, WorkerStatus.OFFLINE}: + return "unavailable" + if status not in {WorkerStatus.ONLINE, WorkerStatus.BUSY}: + return "unavailable" + if ( + not _is_plain_int(active_run_count) + or not _is_plain_int(max_concurrency) + or not _is_plain_int(heartbeat_freshness_seconds) + or not _is_plain_int(active_poll_freshness_seconds) + or active_run_count < 0 + or max_concurrency < 1 + or active_run_count > max_concurrency + or heartbeat_freshness_seconds < 0 + or active_poll_freshness_seconds < 0 + or not isinstance(last_heartbeat_at, dt.datetime) + or not isinstance(now, dt.datetime) + ): + return "unavailable" + if not _fresh( + last_heartbeat_at, now=now, seconds=heartbeat_freshness_seconds + ) or not _fresh( + last_checkout_poll_at, now=now, seconds=active_poll_freshness_seconds + ): + return "stale" + if status == WorkerStatus.BUSY or active_run_count >= max_concurrency: + return "capacity_constrained" + return "active" + + +def _is_plain_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _network_policy_capability( + policy: NetworkPolicy, +) -> Literal["disabled", "worker_restricted"]: + if policy is NetworkPolicy.DISABLED: + return "disabled" + return "worker_restricted" + + +def _history_item( + request: GitHubValidationRequest, + order: ExecutionWorkOrder, + run: ExecutionRun | None, +) -> ExecutionHistoryItemOut: + fingerprint = None + duration = None + if run is not None: + if isinstance(run.evidence_metadata, dict): + candidate = run.evidence_metadata.get("fingerprint") + if isinstance(candidate, str) and len(candidate) == SHA256_HEX_LENGTH: + fingerprint = candidate + if run.started_at is not None and run.finished_at is not None: + duration = max(0.0, (run.finished_at - run.started_at).total_seconds()) + return ExecutionHistoryItemOut( + request_id=request.id, + repository_full_name=request.repository_full_name, + pull_request_number=request.pull_request_number, + tested_head_sha=request.head_sha, + manifest_name=request.manifest_name, + manifest_version=request.manifest_version, + work_order_id=order.id, + work_order_status=_enum_value(order.status), + routing_policy=_enum_value(order.routing_policy), + preferred_executor=order.preferred_executor, + maximum_cost_units=order.maximum_cost_units, + required_quota_units=order.required_quota_units, + run_id=run.id if run is not None else None, + run_status=_enum_value(run.status) if run is not None else None, + selected_worker_id=run.worker_id if run is not None else None, + estimated_cost_units=( + run.route_estimated_cost_units if run is not None else None + ), + reuse_decision=_enum_value(run.reuse_decision) if run is not None else None, + reused_from_run_id=run.reused_from_run_id if run is not None else None, + run_duration_seconds=duration, + evidence_fingerprint=fingerprint, + publication_state=request.publication_state, + publication_decision=request.publication_decision, + created_at=request.created_at, + updated_at=request.updated_at, + ) + + +__all__ = [ + "MAX_OPERATOR_LIMIT", + "MAX_OPERATOR_OFFSET", + "MAX_OPERATOR_WINDOW_DAYS", + "ExecutionHistoryPageOut", + "ExecutionOperatorOverviewOut", + "ExecutionOperatorProjection", + "ExecutionWorkerPageOut", +] diff --git a/server/github_adapter/schemas.py b/server/github_adapter/schemas.py index b789785..1fabc9c 100644 --- a/server/github_adapter/schemas.py +++ b/server/github_adapter/schemas.py @@ -7,6 +7,9 @@ from pydantic import BaseModel, ConfigDict, Field +from server.execution.enums import ReusePolicy, RoutingPolicy +from server.execution.routing import MAX_ROUTING_INTEGER + from .service import GitHubRequestStatus @@ -33,6 +36,15 @@ class GitHubValidationCreateIn(GitHubAdapterInput): ) pull_request_number: int = Field(ge=1, le=2_147_483_647) manifest: GitHubManifestReferenceIn + reuse_policy: ReusePolicy = ReusePolicy.NEVER + routing_policy: RoutingPolicy = RoutingPolicy.FIRST_AVAILABLE + maximum_cost_units: int | None = Field( + default=None, strict=True, ge=0, le=MAX_ROUTING_INTEGER + ) + required_quota_units: int = Field( + default=0, strict=True, ge=0, le=MAX_ROUTING_INTEGER + ) + preferred_executor: str | None = Field(default=None, min_length=1, max_length=128) class GitHubValidationRequestOut(BaseModel): @@ -60,6 +72,11 @@ class GitHubValidationRequestOut(BaseModel): operator_id: str = Field(min_length=1, max_length=128) work_order_id: int = Field(ge=1) work_order_status: str = Field(min_length=1, max_length=32) + reuse_policy: ReusePolicy + routing_policy: RoutingPolicy + maximum_cost_units: int | None = Field(default=None, ge=0) + required_quota_units: int = Field(ge=0) + preferred_executor: str | None = Field(default=None, max_length=128) terminal_run_id: int | None = Field(default=None, ge=1) terminal_run_status: str | None = Field(default=None, max_length=32) evidence_fingerprint: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$") diff --git a/server/github_adapter/service.py b/server/github_adapter/service.py index 7b2c574..641a94a 100644 --- a/server/github_adapter/service.py +++ b/server/github_adapter/service.py @@ -12,7 +12,12 @@ from sqlalchemy.exc import IntegrityError from server.execution.entities import WorkOrderDraft -from server.execution.enums import ApprovalPolicy, is_terminal_run +from server.execution.enums import ( + ApprovalPolicy, + ReusePolicy, + RoutingPolicy, + is_terminal_run, +) from server.execution.evidence import ExecutionEvidence from server.execution.exceptions import ExecutionNotFoundError, MalformedEvidenceError from server.execution.registry import ( @@ -74,6 +79,11 @@ class GitHubRequestStatus: operator_id: str work_order_id: int work_order_status: str + reuse_policy: ReusePolicy + routing_policy: RoutingPolicy + maximum_cost_units: int | None + required_quota_units: int + preferred_executor: str | None terminal_run_id: int | None terminal_run_status: str | None evidence_fingerprint: str | None @@ -115,13 +125,18 @@ def __init__( self._settings = settings self._clock = clock - async def request_validation( + async def request_validation( # noqa: PLR0913 self, *, repository_full_name: str, pull_request_number: int, manifest_name: str, manifest_version: str, + reuse_policy: ReusePolicy = ReusePolicy.NEVER, + routing_policy: RoutingPolicy = RoutingPolicy.FIRST_AVAILABLE, + maximum_cost_units: int | None = None, + required_quota_units: int = 0, + preferred_executor: str | None = None, ) -> GitHubRequestStatus: """Create or return one pending work order for the exact resolved head.""" @@ -146,8 +161,27 @@ async def request_validation( actor=actor, resolved=resolved, manifest=manifest, + reuse_policy=reuse_policy, + routing_policy=routing_policy, + maximum_cost_units=maximum_cost_units, + required_quota_units=required_quota_units, + preferred_executor=preferred_executor, ) existing = await self._repository.get_by_idempotency_key(idempotency_key) + if existing is None and _uses_legacy_default_policy( + reuse_policy=reuse_policy, + routing_policy=routing_policy, + maximum_cost_units=maximum_cost_units, + required_quota_units=required_quota_units, + preferred_executor=preferred_executor, + ): + legacy_key = _legacy_idempotency_key( + settings=self._settings, + actor=actor, + resolved=resolved, + manifest=manifest, + ) + existing = await self._repository.get_by_idempotency_key(legacy_key) if existing is not None: return await self._status(existing) @@ -179,8 +213,12 @@ async def request_validation( resource_metadata={}, network_policy=manifest.network_policy, repository_write_allowed=False, - preferred_executor=None, + preferred_executor=preferred_executor, cost_ceiling=0.0, + reuse_policy=reuse_policy, + routing_policy=routing_policy, + maximum_cost_units=maximum_cost_units, + required_quota_units=required_quota_units, ) ) request = await self._repository.create( @@ -223,6 +261,22 @@ async def request_validation( recovered = await self._repository.get_by_idempotency_key( idempotency_key, refresh=True ) + if recovered is None and _uses_legacy_default_policy( + reuse_policy=reuse_policy, + routing_policy=routing_policy, + maximum_cost_units=maximum_cost_units, + required_quota_units=required_quota_units, + preferred_executor=preferred_executor, + ): + recovered = await self._repository.get_by_idempotency_key( + _legacy_idempotency_key( + settings=self._settings, + actor=actor, + resolved=resolved, + manifest=manifest, + ), + refresh=True, + ) if recovered is None: raise request = recovered @@ -677,6 +731,11 @@ async def _status( operator_id=request.operator_id, work_order_id=request.work_order_id, work_order_status=work_order.status.value, + reuse_policy=work_order.reuse_policy, + routing_policy=work_order.routing_policy, + maximum_cost_units=work_order.maximum_cost_units, + required_quota_units=work_order.required_quota_units, + preferred_executor=work_order.preferred_executor, terminal_run_id=run.id if run is not None else None, terminal_run_status=(run.status.value if run is not None else None), evidence_fingerprint=fingerprint, @@ -696,12 +755,17 @@ async def _status( ) -def _idempotency_key( +def _idempotency_key( # noqa: PLR0913 *, settings: GitHubSettings, actor: GitHubActorIdentity, resolved: ResolvedPullRequest, manifest: TrustedManifest, + reuse_policy: ReusePolicy, + routing_policy: RoutingPolicy, + maximum_cost_units: int | None, + required_quota_units: int, + preferred_executor: str | None, ) -> str: if resolved.head_sha is None: # pragma: no cover - request requires it raise GitHubTransportError("github_head_unavailable") @@ -718,11 +782,65 @@ def _idempotency_key( "repository_node_id": resolved.repository_node_id, "schema_version": GITHUB_ADAPTER_SCHEMA_VERSION, "tested_head_sha": resolved.head_sha, + "execution_policy": { + "maximum_cost_units": maximum_cost_units, + "preferred_executor": preferred_executor, + "required_quota_units": required_quota_units, + "reuse_policy": reuse_policy.value, + "routing_policy": routing_policy.value, + "schema_version": 1, + }, } encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")) return hashlib.sha256(encoded.encode("utf-8")).hexdigest() +def _legacy_idempotency_key( + *, + settings: GitHubSettings, + actor: GitHubActorIdentity, + resolved: ResolvedPullRequest, + manifest: TrustedManifest, +) -> str: + """Return the exact pre-#136 key for immutable all-default fallback only.""" + + if resolved.head_sha is None: # pragma: no cover - request requires it + raise GitHubTransportError("github_head_unavailable") + payload = { + "github_api_url": settings.api_url, + "github_actor_id": actor.actor_id, + "github_actor_node_id": actor.node_id, + "manifest_digest": manifest.digest, + "manifest_name": manifest.name, + "manifest_version": manifest.version, + "pull_request_id": resolved.pull_request_id, + "pull_request_node_id": resolved.pull_request_node_id, + "repository_id": resolved.repository_id, + "repository_node_id": resolved.repository_node_id, + "schema_version": GITHUB_ADAPTER_SCHEMA_VERSION, + "tested_head_sha": resolved.head_sha, + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def _uses_legacy_default_policy( + *, + reuse_policy: ReusePolicy, + routing_policy: RoutingPolicy, + maximum_cost_units: int | None, + required_quota_units: int, + preferred_executor: str | None, +) -> bool: + return ( + reuse_policy == ReusePolicy.NEVER + and routing_policy == RoutingPolicy.FIRST_AVAILABLE + and maximum_cost_units is None + and required_quota_units == 0 + and preferred_executor is None + ) + + def _same_stable_identity( request: GitHubValidationRequest, resolved: ResolvedPullRequest ) -> bool: diff --git a/server/tests/test_execution_operator_projection.py b/server/tests/test_execution_operator_projection.py new file mode 100644 index 0000000..28c6653 --- /dev/null +++ b/server/tests/test_execution_operator_projection.py @@ -0,0 +1,309 @@ +# ruff: noqa: PLR0913, PLR0915 +"""Focused activity and redaction coverage for operator worker projections.""" + +from __future__ import annotations + +import datetime as dt +from http import HTTPStatus + +import pytest +from httpx import ASGITransport, AsyncClient +from pydantic import ValidationError +from sqlalchemy import select + +from server.app import app +from server.db import AsyncSessionLocal +from server.execution.entities import WorkerRegistration, WorkOrderDraft +from server.execution.enums import ( + ApprovalPolicy, + NetworkPolicy, + WorkerStatus, +) +from server.execution.operator_projection import ( + MAX_OPERATOR_BROWSER_COUNT, + ExecutionOperatorProjection, + ExecutionWorkerSummaryOut, + _worker_activity_state, +) +from server.execution.repository import ExecutionRepository +from server.execution.service import ExecutionService +from server.models import ExecutionWorker +from server.time_utils import utcnow_naive + + +def _worker( + worker_id: str, + *, + status: WorkerStatus = WorkerStatus.ONLINE, + browsers: tuple[str, ...] = (), + capabilities: dict[str, object] | None = None, +) -> WorkerRegistration: + return WorkerRegistration( + worker_id=worker_id, + display_name=f"Synthetic {worker_id}", + operating_system="linux", + architecture="x86_64", + python_version="3.11.14", + node_version=None, + docker_available=False, + browsers=browsers, + gpu_available=False, + unity_available=False, + desktop_available=False, + capabilities=capabilities or {}, + max_concurrency=1, + network_policy_capability=NetworkPolicy.WORKER_RESTRICTED, + repository_write_capability=False, + status=status, + ) + + +def _draft() -> WorkOrderDraft: + return WorkOrderDraft( + schema_version=1, + repository_full_name="Nobodyworld/dev-agent-switchboard", + commit_sha="d" * 40, + manifest_name="validate-switchboard", + manifest_version="1", + manifest_parameters={}, + required_capabilities={}, + permitted_paths=("server",), + forbidden_scope_notes="read-only operator projection proof", + expected_artifact_kinds=("command-log",), + approval_policy=ApprovalPolicy.EXPLICIT, + timeout_seconds=3600, + resource_metadata={}, + network_policy=NetworkPolicy.WORKER_RESTRICTED, + repository_write_allowed=False, + preferred_executor=None, + cost_ceiling=None, + ) + + +@pytest.mark.parametrize( + ( + "status", + "active_run_count", + "max_concurrency", + "heartbeat_age", + "poll_age", + "expected", + ), + [ + (WorkerStatus.ONLINE, 0, 1, 0, 0, "active"), + (WorkerStatus.BUSY, 1, 1, 0, 0, "capacity_constrained"), + (WorkerStatus.ONLINE, 1, 1, 0, 0, "capacity_constrained"), + (WorkerStatus.BUSY, 1, 1, 301, 61, "stale"), + (WorkerStatus.DRAINING, 0, 1, 0, 0, "unavailable"), + (WorkerStatus.OFFLINE, 0, 1, 0, 0, "unavailable"), + (WorkerStatus.ONLINE, 2, 1, 0, 0, "unavailable"), + ], +) +def test_worker_activity_state_precedence_is_deterministic( + status: WorkerStatus, + active_run_count: int, + max_concurrency: int, + heartbeat_age: int, + poll_age: int, + expected: str, +) -> None: + now = dt.datetime(2026, 8, 9, 12, 0, 0, tzinfo=dt.UTC).replace(tzinfo=None) + + result = _worker_activity_state( + status=status, + active_run_count=active_run_count, + max_concurrency=max_concurrency, + last_heartbeat_at=now - dt.timedelta(seconds=heartbeat_age), + last_checkout_poll_at=now - dt.timedelta(seconds=poll_age), + now=now, + heartbeat_freshness_seconds=300, + active_poll_freshness_seconds=60, + ) + + assert result == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "insertion_order", + [ + ( + "worker-stale", + "worker-offline", + "worker-active", + "worker-busy", + "worker-draining", + "worker-inconsistent-full", + ), + ( + "worker-inconsistent-full", + "worker-draining", + "worker-busy", + "worker-active", + "worker-offline", + "worker-stale", + ), + ], +) +async def test_busy_capacity_lifecycle_projects_bounded_stable_worker_state( + insertion_order: tuple[str, ...], +) -> None: + now = utcnow_naive() + browser_names = tuple(f"browser-{index:02d}" for index in range(10)) + registrations = { + "worker-active": _worker("worker-active"), + "worker-busy": _worker( + "worker-busy", + browsers=browser_names, + capabilities={ + "unlisted_flag": True, + "nested_metadata": {"count": 7}, + }, + ), + "worker-draining": _worker("worker-draining", status=WorkerStatus.DRAINING), + "worker-inconsistent-full": _worker("worker-inconsistent-full"), + "worker-offline": _worker("worker-offline", status=WorkerStatus.OFFLINE), + "worker-stale": _worker("worker-stale"), + } + expected_states = { + "worker-active": "active", + "worker-busy": "capacity_constrained", + "worker-draining": "unavailable", + "worker-inconsistent-full": "capacity_constrained", + "worker-offline": "unavailable", + "worker-stale": "stale", + } + + async with AsyncSessionLocal() as session: + repository = ExecutionRepository(session) + service = ExecutionService( + repository=repository, + clock=lambda: now, + lease_seconds=lambda: 60, + routing_freshness_seconds=lambda: (300, 60), + ) + for worker_id in insertion_order: + await service.register_worker(registrations[worker_id]) + + for worker_id in ( + "worker-active", + "worker-busy", + "worker-inconsistent-full", + "worker-stale", + ): + checkout = await service.checkout(worker_id) + assert not checkout.assigned + + stale_worker = await repository.get_worker("worker-stale") + inconsistent_worker = await repository.get_worker("worker-inconsistent-full") + assert stale_worker is not None + assert inconsistent_worker is not None + stale_worker.last_heartbeat_at = now - dt.timedelta(seconds=301) + stale_worker.last_checkout_poll_at = now - dt.timedelta(seconds=61) + inconsistent_worker.active_run_count = inconsistent_worker.max_concurrency + inconsistent_worker.status = WorkerStatus.ONLINE + await session.flush() + + order = await service.create_work_order(_draft()) + await service.approve_work_order(order.id) + assignment = await service.checkout("worker-busy") + assert assignment.assigned + session.expire_all() + + persisted_busy = ( + await session.execute( + select(ExecutionWorker) + .where(ExecutionWorker.worker_id == "worker-busy") + .execution_options(populate_existing=True) + ) + ).scalar_one() + assert persisted_busy.status == WorkerStatus.BUSY + assert persisted_busy.active_run_count == persisted_busy.max_concurrency == 1 + + projection = ExecutionOperatorProjection(session) + workers = await projection.list_workers( + limit=100, + offset=0, + heartbeat_freshness_seconds=300, + active_poll_freshness_seconds=60, + ) + overview = await projection.overview( + window_days=1, + heartbeat_freshness_seconds=300, + active_poll_freshness_seconds=60, + now=now, + ) + await session.commit() + + projected_states = {item.worker_id: item.activity_state for item in workers.items} + assert projected_states == expected_states + assert [item.worker_id for item in workers.items] == sorted(expected_states) + assert overview.workers.model_dump() == { + "total": 6, + "active": 1, + "stale": 1, + "capacity_constrained": 2, + "unavailable": 2, + } + assert ( + sum( + ( + overview.workers.active, + overview.workers.stale, + overview.workers.capacity_constrained, + overview.workers.unavailable, + ) + ) + == overview.workers.total + ) + + transport = ASGITransport(app=app, client=("operator-worker-summary", 12_346)) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/execution/workers?limit=100&offset=0") + overview_response = await client.get( + "/api/execution/operator/overview?window_days=1" + ) + assert response.status_code == HTTPStatus.OK + assert overview_response.status_code == HTTPStatus.OK + assert overview_response.json()["workers"] == { + "total": 6, + "active": 1, + "stale": 1, + "capacity_constrained": 2, + "unavailable": 2, + } + payload = response.json() + assert [item["worker_id"] for item in payload["items"]] == sorted(expected_states) + payload_by_id = {item["worker_id"]: item for item in payload["items"]} + busy_payload = payload_by_id["worker-busy"] + assert busy_payload["activity_state"] == "capacity_constrained" + assert busy_payload["python_version"] == "3.11.14" + assert busy_payload["node_version"] is None + assert busy_payload["docker_available"] is False + assert busy_payload["browsers"] == list(browser_names[:MAX_OPERATOR_BROWSER_COUNT]) + assert busy_payload["gpu_available"] is False + assert busy_payload["unity_available"] is False + assert busy_payload["desktop_available"] is False + assert busy_payload["network_policy_capability"] == "worker_restricted" + assert busy_payload["repository_write_capability"] is False + assert "capabilities" not in busy_payload + assert { + "local_root", + "environment", + "command", + "argv", + "hostname", + "username", + "credential", + }.isdisjoint(busy_payload) + + with pytest.raises(ValidationError): + ExecutionWorkerSummaryOut.model_validate( + { + **busy_payload, + "browsers": [ + *busy_payload["browsers"], + "one-browser-too-many", + ], + } + ) diff --git a/server/tests/test_github_adapter_service.py b/server/tests/test_github_adapter_service.py index 4dd6a39..fadad62 100644 --- a/server/tests/test_github_adapter_service.py +++ b/server/tests/test_github_adapter_service.py @@ -6,7 +6,7 @@ import asyncio import datetime as dt import json -from collections.abc import Callable +from collections.abc import AsyncGenerator, Callable from dataclasses import asdict, replace from http import HTTPStatus from typing import cast @@ -20,27 +20,42 @@ from server.api.dependencies import ( SessionDependency, get_github_adapter_service, + get_session, ) from server.app import app as server_app from server.application import build_execution_service from server.db import AsyncSessionLocal -from server.execution.entities import ExecutionCompletion, WorkerRegistration +from server.execution.entities import ( + ExecutionCompletion, + RoutingProfileDraft, + WorkerRegistration, +) from server.execution.enums import ( ExecutionRunStatus, NetworkPolicy, + ReuseDecision, + ReusePolicy, + RoutingPolicy, WorkerStatus, WorkOrderStatus, ) from server.execution.evidence import ( AuditSummary, + DependencyLockHash, EnvironmentIdentity, + EvidenceReuseIdentity, + EvidenceReuseProvenance, ExecutionEvidenceDraft, ParsedCoverage, ParsedResult, ParsedTestCounts, StepEvidence, + compute_result_contract_hash, + compute_reuse_identity_hash, finalize_evidence, ) +from server.execution.operator_projection import ExecutionOperatorProjection +from server.execution.registry import get_trusted_manifest from server.github_adapter.errors import ( GitHubAmbiguousWriteError, GitHubRateLimitedError, @@ -58,6 +73,7 @@ PUBLICATION_CLAIM_TTL, GitHubAdapterDependencies, GitHubAdapterService, + _legacy_idempotency_key, ) from server.github_adapter.transport import ( GitHubActorIdentity, @@ -297,6 +313,78 @@ def _step( ) +def _reuse_identity(order: ExecutionWorkOrder) -> EvidenceReuseIdentity: + manifest = get_trusted_manifest(order.manifest_name, order.manifest_version) + assert manifest is not None + return EvidenceReuseIdentity( + repository_full_name=order.repository_full_name, + tested_sha=order.commit_sha, + manifest_name=order.manifest_name, + manifest_version=order.manifest_version, + manifest_digest=order.manifest_digest, + worker_environment_fingerprint="d" * 64, + dependency_lock_hashes=[ + DependencyLockHash(relative_path=path, sha256="e" * 64) + for path in sorted(manifest.dependency_lock_paths) + ], + execution_policy_hash=order.execution_policy_hash, + result_contract_hash=compute_result_contract_hash( + fixed_step_metadata=manifest.fixed_step_metadata, + artifact_declarations=manifest.artifact_declarations, + dependency_lock_paths=manifest.dependency_lock_paths, + ), + ) + + +def _reuse_evidence( # noqa: PLR0913 + *, + order: ExecutionWorkOrder, + run_id: int, + worker_id: str, + identity: EvidenceReuseIdentity, + decision: str, + source_run_id: int | None = None, + source_fingerprint: str | None = None, +): + now = dt.datetime(2026, 8, 8, 12, tzinfo=dt.UTC) + return finalize_evidence( + ExecutionEvidenceDraft( + work_order_id=order.id, + run_id=run_id, + repository_full_name=order.repository_full_name, + tested_sha=order.commit_sha, + manifest_name=order.manifest_name, + manifest_version=order.manifest_version, + manifest_digest=order.manifest_digest, + worker_id=worker_id, + environment=EnvironmentIdentity( + operating_system="windows", + architecture="amd64", + python_version="3.11.14", + fingerprint=identity.worker_environment_fingerprint, + ), + dependency_lock_hashes=identity.dependency_lock_hashes, + started_at=now, + finished_at=now + dt.timedelta(seconds=7), + duration_seconds=7, + terminal_status="succeeded", + steps=[], + artifacts=[], + dependency_lock_status="succeeded", + artifact_finalization_status="succeeded", + source_cleanup_status="succeeded", + local_record_status="succeeded", + reuse_provenance=EvidenceReuseProvenance( + decision=decision, # type: ignore[arg-type] + reason=("exact_evidence_verified" if decision == "reused" else "fresh"), + reuse_identity_hash=compute_reuse_identity_hash(identity), + source_run_id=source_run_id, + source_evidence_fingerprint=source_fingerprint, + ), + ) + ) + + async def _complete_with_compact_evidence( service: GitHubAdapterService, request_id: int, @@ -494,6 +582,343 @@ async def test_duplicate_request_reuses_work_order_and_new_head_is_distinct() -> assert int(work_order_count or 0) == 2 +@pytest.mark.asyncio +async def test_every_adapter_policy_dimension_changes_request_identity() -> None: + transport = FakeGitHubTransport() + variants = ( + {}, + {"reuse_policy": ReusePolicy.ALLOW_EXACT}, + {"routing_policy": RoutingPolicy.CHEAPEST_CAPABLE}, + {"maximum_cost_units": 17}, + {"required_quota_units": 3}, + {"preferred_executor": "worker-preferred"}, + ) + async with AsyncSessionLocal() as session: + service = _service(session, transport) + await service._execution.register_worker(_worker("worker-preferred")) + statuses = [] + for policy in variants: + statuses.append( + await service.request_validation( + repository_full_name=REPOSITORY, + pull_request_number=125, + manifest_name="validate-switchboard", + manifest_version="1", + **policy, + ) + ) + records = ( + ( + await session.execute( + select(GitHubValidationRequest).order_by(GitHubValidationRequest.id) + ) + ) + .scalars() + .all() + ) + + assert len({status.request_id for status in statuses}) == len(variants) + assert len({status.work_order_id for status in statuses}) == len(variants) + assert len({record.idempotency_key for record in records}) == len(variants) + assert statuses[0].reuse_policy == ReusePolicy.NEVER + assert statuses[0].routing_policy == RoutingPolicy.FIRST_AVAILABLE + assert statuses[1].reuse_policy == ReusePolicy.ALLOW_EXACT + assert statuses[2].routing_policy == RoutingPolicy.CHEAPEST_CAPABLE + assert statuses[3].maximum_cost_units == 17 + assert statuses[4].required_quota_units == 3 + assert statuses[5].preferred_executor == "worker-preferred" + + +@pytest.mark.asyncio +async def test_default_policy_finds_exact_legacy_key_without_mutating_it() -> None: + transport = FakeGitHubTransport() + async with AsyncSessionLocal() as session: + service = _service(session, transport) + created = await service.request_validation( + repository_full_name=REPOSITORY, + pull_request_number=125, + manifest_name="validate-switchboard", + manifest_version="1", + ) + record = await session.get(GitHubValidationRequest, created.request_id) + manifest = get_trusted_manifest("validate-switchboard", "1") + assert record is not None + assert manifest is not None + legacy_key = _legacy_idempotency_key( + settings=_settings(), + actor=CREDENTIAL_ACTOR, + resolved=transport.resolved, + manifest=manifest, + ) + record.idempotency_key = legacy_key + await session.flush() + + recovered = await service.request_validation( + repository_full_name=REPOSITORY, + pull_request_number=125, + manifest_name="validate-switchboard", + manifest_version="1", + ) + unchanged = await session.get(GitHubValidationRequest, created.request_id) + non_default = await service.request_validation( + repository_full_name=REPOSITORY, + pull_request_number=125, + manifest_name="validate-switchboard", + manifest_version="1", + reuse_policy=ReusePolicy.ALLOW_EXACT, + ) + + assert recovered.request_id == created.request_id + assert unchanged is not None + assert unchanged.idempotency_key == legacy_key + assert non_default.request_id != created.request_id + + +@pytest.mark.asyncio +async def test_direct_completion_projects_fresh_then_reused_github_lifecycle() -> ( # noqa: PLR0915 + None +): + """Exercise projection formulas without representing the local-worker trust path.""" + + transport = FakeGitHubTransport() + cheap_worker = "worker-cheap" + expensive_worker = "worker-expensive" + async with AsyncSessionLocal() as session: + adapter = _service(session, transport) + execution = adapter._execution + await execution.register_worker(_worker(cheap_worker)) + await execution.register_worker(_worker(expensive_worker)) + await execution.create_routing_profile( + RoutingProfileDraft( + schema_version=1, + worker_id=cheap_worker, + enabled=True, + estimated_cost_units_per_run=3, + quota_capacity_units=20, + quota_remaining_units=20, + quota_reset_at=None, + routing_priority=0, + ) + ) + await execution.create_routing_profile( + RoutingProfileDraft( + schema_version=1, + worker_id=expensive_worker, + enabled=True, + estimated_cost_units_per_run=9, + quota_capacity_units=20, + quota_remaining_units=20, + quota_reset_at=None, + routing_priority=0, + ) + ) + + fresh_request = await adapter.request_validation( + repository_full_name=REPOSITORY, + pull_request_number=125, + manifest_name="validate-switchboard", + manifest_version="1", + reuse_policy=ReusePolicy.NEVER, + routing_policy=RoutingPolicy.CHEAPEST_CAPABLE, + required_quota_units=1, + ) + assert not (await execution.checkout(expensive_worker)).assigned + assert not (await execution.checkout(cheap_worker)).assigned + await execution.approve_work_order(fresh_request.work_order_id) + assert not (await execution.checkout(expensive_worker)).assigned + fresh_assignment = await execution.checkout(cheap_worker) + assert fresh_assignment.run_id is not None + await execution.heartbeat_run(fresh_assignment.run_id, worker_id=cheap_worker) + fresh_order = await execution.get_work_order(fresh_request.work_order_id) + fresh_identity = _reuse_identity(fresh_order) + fresh_evidence = _reuse_evidence( + order=fresh_order, + run_id=fresh_assignment.run_id, + worker_id=cheap_worker, + identity=fresh_identity, + decision="fresh", + ) + fresh_run = await execution.complete_run( + fresh_assignment.run_id, + worker_id=cheap_worker, + completion=ExecutionCompletion( + status=ExecutionRunStatus.SUCCEEDED, + evidence_metadata=fresh_evidence, + reuse_decision=ReuseDecision.FRESH, + reuse_reason="fresh_execution_completed", + reuse_identity=fresh_identity, + reuse_identity_hash=compute_reuse_identity_hash(fresh_identity), + evidence_retention_expires_at=( + fresh_evidence.started_at + dt.timedelta(days=14) + ), + ), + ) + fresh_run.started_at = dt.datetime(2026, 8, 8, 12, 0, 0, tzinfo=dt.UTC) + fresh_run.finished_at = dt.datetime(2026, 8, 8, 12, 0, 7, tzinfo=dt.UTC) + await session.flush() + published_current = await adapter.publish(fresh_request.request_id) + assert published_current.publication_state == "published_current" + republished_current = await adapter.publish(fresh_request.request_id) + assert republished_current.managed_comment_id == ( + published_current.managed_comment_id + ) + + reused_request = await adapter.request_validation( + repository_full_name=REPOSITORY, + pull_request_number=125, + manifest_name="validate-switchboard", + manifest_version="1", + reuse_policy=ReusePolicy.ALLOW_EXACT, + routing_policy=RoutingPolicy.CHEAPEST_CAPABLE, + required_quota_units=1, + ) + reused_order = await execution.get_work_order(reused_request.work_order_id) + assert reused_request.request_id != fresh_request.request_id + assert reused_order.execution_policy_hash == fresh_order.execution_policy_hash + assert not (await execution.checkout(expensive_worker)).assigned + assert not (await execution.checkout(cheap_worker)).assigned + await execution.approve_work_order(reused_request.work_order_id) + assert not (await execution.checkout(expensive_worker)).assigned + reused_assignment = await execution.checkout(cheap_worker) + assert reused_assignment.run_id is not None + reused_identity = _reuse_identity(reused_order) + assert reused_identity == fresh_identity + lookup = await execution.resolve_reuse_candidate( + reused_assignment.run_id, + worker_id=cheap_worker, + reuse_identity=reused_identity, + reuse_identity_hash=compute_reuse_identity_hash(reused_identity), + ) + assert lookup.candidate is not None + assert lookup.candidate.source_run_id == fresh_run.id + reused_evidence = _reuse_evidence( + order=reused_order, + run_id=reused_assignment.run_id, + worker_id=cheap_worker, + identity=reused_identity, + decision="reused", + source_run_id=fresh_run.id, + source_fingerprint=fresh_evidence.fingerprint, + ) + reused_run = await execution.complete_run( + reused_assignment.run_id, + worker_id=cheap_worker, + completion=ExecutionCompletion( + status=ExecutionRunStatus.SUCCEEDED, + evidence_metadata=reused_evidence, + reuse_decision=ReuseDecision.REUSED, + reuse_reason="exact_evidence_verified", + reuse_identity=reused_identity, + reuse_identity_hash=compute_reuse_identity_hash(reused_identity), + evidence_retention_expires_at=( + reused_evidence.started_at + dt.timedelta(days=14) + ), + ), + ) + assert reused_run.worker_id == cheap_worker + assert reused_run.reused_from_run_id == fresh_run.id + assert reused_run.evidence_metadata["steps"] == [] + + transport.resolved = _resolved(head_sha=MOVED_SHA) + published_stale = await adapter.publish(reused_request.request_id) + assert published_stale.publication_state == "published_stale" + + projection = ExecutionOperatorProjection(session) + overview = await projection.overview( + window_days=30, + heartbeat_freshness_seconds=300, + active_poll_freshness_seconds=60, + ) + history = await projection.list_history(limit=25, offset=0) + first_page = await projection.list_history(limit=1, offset=0) + second_page = await projection.list_history(limit=1, offset=1) + filter_now = dt.datetime.now(dt.UTC) + reused_history = await projection.list_history( + limit=25, + offset=0, + repository_full_name=REPOSITORY, + pull_request_number=125, + work_order_status=WorkOrderStatus.SUCCEEDED.value, + run_status=ExecutionRunStatus.SUCCEEDED.value, + reuse_decision=ReuseDecision.REUSED.value, + routing_policy=RoutingPolicy.CHEAPEST_CAPABLE.value, + publication_state="published_stale", + created_after=filter_now - dt.timedelta(days=1), + created_before=filter_now + dt.timedelta(days=1), + ) + workers = await projection.list_workers( + limit=25, + offset=0, + heartbeat_freshness_seconds=300, + active_poll_freshness_seconds=60, + ) + fresh_run.started_at = None + reused_run.route_estimated_cost_units = None + await session.flush() + missing_estimates = await projection.overview( + window_days=30, + heartbeat_freshness_seconds=300, + active_poll_freshness_seconds=60, + ) + + assert overview.requests.total == 2 + assert overview.runs.fresh_successful == 1 + assert overview.runs.reused_successful == 1 + assert overview.avoided_work.deterministic_executions_avoided == 1 + assert overview.avoided_work.reference_seconds_avoided == 7 + assert overview.avoided_work.comparison_units_avoided == 3 + assert overview.avoided_work.reuse_rate == 0.5 + assert overview.publications.current == 1 + assert overview.publications.stale == 1 + assert history.total == 2 + assert history.items[0].request_id == reused_request.request_id + assert history.items[0].reuse_decision == "reused" + assert history.items[1].reuse_decision == "fresh" + assert [item.request_id for item in first_page.items] == [reused_request.request_id] + assert [item.request_id for item in second_page.items] == [fresh_request.request_id] + assert reused_history.total == 1 + assert reused_history.items[0].request_id == reused_request.request_id + assert {item.worker_id for item in workers.items} == { + cheap_worker, + expensive_worker, + } + assert "capabilities" not in workers.items[0].model_dump() + assert "repository_full_name" not in workers.items[0].model_dump() + assert missing_estimates.avoided_work.deterministic_executions_avoided == 1 + assert missing_estimates.avoided_work.reference_seconds_avoided == 0 + assert missing_estimates.avoided_work.comparison_units_avoided == 0 + + +@pytest.mark.asyncio +async def test_operator_projection_routes_enforce_hard_query_bounds() -> None: + transport = ASGITransport( + app=server_app, client=("operator-projection-test", 12_345) + ) + async with AsyncClient(transport=transport, base_url="http://test") as client: + overview = await client.get("/api/execution/operator/overview") + history = await client.get("/api/execution/operator/history") + workers = await client.get("/api/execution/workers") + requests = await client.get("/api/execution/github/requests") + too_wide = await client.get( + "/api/execution/operator/overview", params={"window_days": 366} + ) + too_many = await client.get( + "/api/execution/operator/history", params={"limit": 101} + ) + too_far = await client.get( + "/api/execution/github/requests", params={"offset": 10_001} + ) + + assert overview.status_code == HTTPStatus.OK + assert overview.json()["window"]["days"] == 30 + assert history.status_code == HTTPStatus.OK + assert workers.status_code == HTTPStatus.OK + assert requests.status_code == HTTPStatus.OK + assert too_wide.status_code == HTTPStatus.UNPROCESSABLE_ENTITY + assert too_many.status_code == HTTPStatus.UNPROCESSABLE_ENTITY + assert too_far.status_code == HTTPStatus.UNPROCESSABLE_ENTITY + + @pytest.mark.asyncio async def test_token_rotation_to_another_actor_creates_distinct_request() -> None: transport = FakeGitHubTransport() @@ -1388,6 +1813,68 @@ def override(session: SessionDependency) -> GitHubAdapterService: assert forbidden_keys.isdisjoint(payload) +@pytest.mark.asyncio +async def test_unknown_preferred_executor_is_bounded_and_rolls_back( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SWITCHBOARD_ADMIN_TOKEN", ADMIN_TOKEN) + reload_admin_token() + transport = FakeGitHubTransport() + app = server_app + request_session = AsyncSessionLocal() + + async def isolated_session() -> AsyncGenerator[AsyncSession, None]: + yield request_session + + def override(session: SessionDependency) -> GitHubAdapterService: + return _service(session, transport) + + app.dependency_overrides[get_session] = isolated_session + app.dependency_overrides[get_github_adapter_service] = override + try: + async with AsyncClient( + transport=ASGITransport(app=app, raise_app_exceptions=False), + base_url="http://test", + ) as client: + response = await client.post( + "/api/execution/github/pull-requests/validate", + json={ + "repository_full_name": REPOSITORY, + "pull_request_number": 125, + "manifest": {"name": "validate-switchboard", "version": "1"}, + "preferred_executor": "worker-not-registered", + }, + headers={"Authorization": f"Bearer {ADMIN_TOKEN}"}, + ) + async with AsyncSessionLocal() as verification: + request_count = await verification.scalar( + select(func.count()).select_from(GitHubValidationRequest) + ) + work_order_count = await verification.scalar( + select(func.count()).select_from(ExecutionWorkOrder) + ) + finally: + app.dependency_overrides.clear() + await request_session.close() + monkeypatch.delenv("SWITCHBOARD_ADMIN_TOKEN", raising=False) + reload_admin_token() + + assert response.status_code == HTTPStatus.NOT_FOUND + assert response.json() == {"detail": "preferred_executor_not_found"} + assert int(request_count or 0) == 0 + assert int(work_order_count or 0) == 0 + assert not request_session.in_transaction() + for prohibited in ( + ADMIN_TOKEN, + GITHUB_TOKEN, + "ExecutionNotFoundError", + "Traceback", + "response_body", + r"C:\\", + ): + assert prohibited not in response.text + + @pytest.mark.asyncio async def test_missing_server_token_returns_bounded_api_error( monkeypatch: pytest.MonkeyPatch, @@ -1431,7 +1918,10 @@ async def test_api_rejects_caller_owned_injection_before_service_call( reload_admin_token() app = server_app - app.dependency_overrides[get_github_adapter_service] = lambda: object() + def invalid_service() -> object: + return object() + + app.dependency_overrides[get_github_adapter_service] = invalid_service payload = { "repository_full_name": REPOSITORY, "pull_request_number": 125, diff --git a/web/index.html b/web/index.html index 422efcb..3eb4489 100644 --- a/web/index.html +++ b/web/index.html @@ -11,9 +11,14 @@
-
-

Switchboard

-

Real-time task switchboard & live file host for agents.

+
+
+
+

Switchboard

+

Real-time task switchboard & local validation broker for agents.

+
+ Public developer preview — not production ready +
+
+
+
+

Local-first execution

+

Validation Broker

+

Resolve an exact GitHub head, route it to a trusted local worker, and publish compact evidence explicitly.

+
+
+ + + +
+
+

Loading validation broker…

+ +
+ +
+
+
+
+

New request

+

GitHub PR validation

+
+
+
+ + + +
+
+ + +
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ + + +

Comparison units are operator-defined routing values, not money or measured paid-agent credits.

+ +
+
+ +
+
+
+

Selected request

+

Lifecycle & evidence

+
+ +
+
Select a request from history or submit a new validation.
+
+
+ +
+
+
+
+

Operator-owned configuration

+

Worker routing profiles

+
+
+
+
+ +
+
+
+

Revision-safe edit

+

Create or replace profile

+
+
+
+ + +
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+ + +
+ + +
+

Choose a worker to author operator-owned cost and quota state.

+
+
+
+ +
+
+
+

Bounded newest-first records

+

Execution history

+
+
+ + + + + + + + + +
+
+
+
+

No history loaded.

+
+ + +
+
+
+
+
@@ -214,5 +421,6 @@

Tasks

+ diff --git a/web/static/app.js b/web/static/app.js index 28a36f8..319337e 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -1409,3 +1409,12 @@ document.addEventListener('click', (event) => { }); document.addEventListener('DOMContentLoaded', initialize); + +export { + apiFetch, + apiFetchJson, + copyToClipboard, + escapeHtml, + loadAdminToken, + showToast, +}; diff --git a/web/static/styles.css b/web/static/styles.css index 31f2afc..cbea7d6 100644 --- a/web/static/styles.css +++ b/web/static/styles.css @@ -1,3 +1,9 @@ +html, +body { + max-width: 100%; + overflow-x: hidden; +} + .toast-container { position: fixed; top: 1rem; @@ -199,6 +205,474 @@ justify-content: space-between; } +.preview-badge { + align-self: flex-start; + border: 1px solid #d97706; + border-radius: 999px; + color: #92400e; + background: #fffbeb; + padding: 0.35rem 0.75rem; + font-size: 0.7rem; + font-weight: 800; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.broker-shell { + --broker-bg: #07111f; + --broker-panel: #0d1a2b; + --broker-border: #26364d; + --broker-muted: #9fb0c6; + --broker-text: #eff6ff; + --broker-accent: #22d3ee; + --broker-success: #34d399; + --broker-warning: #fbbf24; + color: var(--broker-text); + background: var(--broker-bg); + border: 1px solid #17263a; + border-radius: 1rem; + padding: 1rem; + overflow: hidden; + box-shadow: 0 24px 60px -42px rgba(15, 23, 42, 0.9); +} + +.broker-shell * { + box-sizing: border-box; +} + +.broker-heading-row, +.broker-panel-heading, +.broker-pagination, +.broker-action-row, +.broker-toolbar { + display: flex; + align-items: center; + gap: 0.75rem; +} + +.broker-heading-row, +.broker-panel-heading, +.broker-pagination { + justify-content: space-between; +} + +.broker-heading-row h2 { + margin: 0; + font-size: clamp(1.5rem, 3vw, 2rem); + font-weight: 750; +} + +.broker-heading-row p, +.broker-panel p, +.broker-pagination p { + color: var(--broker-muted); +} + +.broker-eyebrow { + margin: 0 0 0.25rem; + color: var(--broker-accent) !important; + font-size: 0.68rem; + font-weight: 800; + letter-spacing: 0.12em; + text-transform: uppercase; +} + +.broker-toolbar label, +.broker-form label { + color: #dbeafe; + font-size: 0.75rem; + font-weight: 700; +} + +.broker-toolbar select, +.broker-form input, +.broker-form select, +.broker-history-filters input, +.broker-history-filters select { + min-width: 0; + border: 1px solid #34465f; + border-radius: 0.45rem; + color: var(--broker-text); + background: #07111f; + padding: 0.55rem 0.65rem; +} + +.broker-toolbar select:focus-visible, +.broker-form input:focus-visible, +.broker-form select:focus-visible, +.broker-history-filters input:focus-visible, +.broker-history-filters select:focus-visible, +.broker-button:focus-visible { + outline: 2px solid var(--broker-accent); + outline-offset: 2px; +} + +.broker-inline-status { + min-height: 1.25rem; + margin: 0.8rem 0; + color: var(--broker-muted); + font-size: 0.8rem; +} + +.broker-inline-status[data-tone="error"], +.broker-help[data-tone="error"] { + color: #fca5a5; +} + +.broker-inline-status[data-tone="success"], +.broker-help[data-tone="success"] { + color: #6ee7b7; +} + +.broker-metrics { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0.65rem; + margin-bottom: 0.85rem; +} + +.broker-metric { + min-width: 0; + border: 1px solid var(--broker-border); + border-radius: 0.65rem; + background: var(--broker-panel); + padding: 0.8rem; +} + +.broker-metric dt { + min-height: 2rem; + color: var(--broker-muted); + font-size: 0.7rem; + font-weight: 700; + line-height: 1rem; +} + +.broker-metric dd { + margin: 0.25rem 0 0; + color: #fff; + font-size: 1.5rem; + font-weight: 750; + font-variant-numeric: tabular-nums; +} + +.broker-workspace-grid { + display: grid; + gap: 0.85rem; + margin-bottom: 0.85rem; +} + +.broker-panel { + min-width: 0; + border: 1px solid var(--broker-border); + border-radius: 0.75rem; + background: var(--broker-panel); + padding: 0.9rem; +} + +.broker-panel h3 { + margin: 0; + color: #fff; + font-size: 1rem; + font-weight: 750; +} + +.broker-form { + display: grid; + gap: 0.6rem; + margin-top: 0.8rem; +} + +.broker-form label { + display: block; + margin-bottom: 0.25rem; +} + +.broker-form label span { + color: var(--broker-muted); + font-weight: 500; +} + +.broker-form > input, +.broker-form > select, +.broker-form-pair input, +.broker-form-pair select { + width: 100%; +} + +.broker-form-pair { + display: grid; + gap: 0.6rem; + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.broker-help { + margin: 0; + color: var(--broker-muted); + font-size: 0.72rem; + line-height: 1.1rem; +} + +.broker-checkbox { + display: inline-flex !important; + align-items: center; + gap: 0.5rem; +} + +.broker-button { + border: 1px solid transparent; + border-radius: 0.45rem; + padding: 0.55rem 0.75rem; + font-size: 0.75rem; + font-weight: 750; + cursor: pointer; +} + +.broker-action-row { + flex-wrap: wrap; +} + +.broker-button:disabled { + cursor: not-allowed; + opacity: 0.45; +} + +.broker-button--primary { + color: #06202a; + background: var(--broker-accent); +} + +.broker-button--secondary { + border-color: #3b516e; + color: #dbeafe; + background: #16253a; +} + +.broker-button--quiet { + border-color: #34465f; + color: #cbd5e1; + background: transparent; +} + +.broker-button--danger { + border-color: #7f1d1d; + color: #fecaca; + background: #3f1218; +} + +.broker-empty { + margin-top: 0.8rem; + border: 1px dashed #34465f; + border-radius: 0.6rem; + color: var(--broker-muted); + padding: 1rem; + font-size: 0.8rem; +} + +.broker-detail-list { + display: grid; + grid-template-columns: minmax(7rem, 0.7fr) minmax(0, 1.3fr); + gap: 0.45rem 0.75rem; + margin: 0.8rem 0; + font-size: 0.78rem; +} + +.broker-detail-list dt { + color: var(--broker-muted); +} + +.broker-detail-list dd { + min-width: 0; + margin: 0; + overflow-wrap: anywhere; +} + +.broker-code { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 0.72rem; +} + +.broker-badge { + display: inline-flex; + align-items: center; + border: 1px solid #40526c; + border-radius: 999px; + color: #dbeafe; + background: #17263a; + padding: 0.2rem 0.5rem; + font-size: 0.65rem; + font-weight: 750; + white-space: nowrap; +} + +.broker-badge[data-tone="success"] { + border-color: #166534; + color: #86efac; + background: #0c2f25; +} + +.broker-badge[data-tone="warning"] { + border-color: #92400e; + color: #fde68a; + background: #352309; +} + +.broker-badge[data-tone="fresh"] { + border-color: #0369a1; + color: #7dd3fc; + background: #082f49; +} + +.broker-worker-list { + display: grid; + gap: 0.55rem; + margin-top: 0.8rem; +} + +.broker-worker-card { + border: 1px solid #34465f; + border-radius: 0.55rem; + padding: 0.7rem; +} + +.broker-worker-card header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.5rem; +} + +.broker-worker-card dl { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0.35rem; + margin: 0.6rem 0; + color: var(--broker-muted); + font-size: 0.72rem; +} + +.broker-worker-card dd { + margin: 0; + color: var(--broker-text); + font-variant-numeric: tabular-nums; +} + +.broker-history-panel { + overflow: hidden; +} + +.broker-panel-heading--history { + align-items: flex-end; +} + +.broker-history-filters { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 0.45rem; +} + +.broker-history-filters input, +.broker-history-filters select { + max-width: 12rem; + font-size: 0.72rem; +} + +.broker-table-wrap { + width: 100%; + min-width: 0; + max-width: 100%; + margin-top: 0.8rem; + overflow-x: auto; + contain: inline-size layout paint; +} + +.broker-table { + width: 100%; + min-width: 62rem; + border-collapse: collapse; + font-size: 0.72rem; +} + +.broker-table th, +.broker-table td { + border-bottom: 1px solid #26364d; + padding: 0.55rem; + text-align: left; + vertical-align: top; +} + +.broker-table th { + color: var(--broker-muted); + font-size: 0.65rem; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.broker-table tbody tr:hover { + background: #111f32; +} + +.broker-table .copy-button { + color: #bae6fd; + background: #143147; +} + +.broker-pagination { + margin-top: 0.7rem; + font-size: 0.72rem; +} + +@media (min-width: 768px) { + .broker-metrics { + grid-template-columns: repeat(4, minmax(0, 1fr)); + } + + .broker-workspace-grid { + grid-template-columns: minmax(18rem, 0.85fr) minmax(22rem, 1.15fr); + } + + .broker-workspace-grid--profiles { + grid-template-columns: minmax(22rem, 1.2fr) minmax(18rem, 0.8fr); + } +} + +@media (min-width: 1200px) { + .broker-metrics { + grid-template-columns: repeat(8, minmax(0, 1fr)); + } +} + +@media (max-width: 767px) { + .broker-heading-row, + .broker-panel-heading--history, + .broker-pagination { + align-items: stretch; + flex-direction: column; + } + + .broker-toolbar, + .broker-history-filters { + align-items: stretch; + flex-direction: column; + } + + .broker-history-filters input, + .broker-history-filters select { + max-width: none; + width: 100%; + } + + .broker-form-pair { + grid-template-columns: 1fr; + } + + .toast-container { + right: 0.75rem; + left: 0.75rem; + max-width: none; + } +} + @media (prefers-reduced-motion: reduce) { .toast { transition: none; @@ -208,3 +682,4 @@ transition: none; } } +.toast-container { diff --git a/web/static/validation_broker.js b/web/static/validation_broker.js new file mode 100644 index 0000000..74bad4d --- /dev/null +++ b/web/static/validation_broker.js @@ -0,0 +1,799 @@ +import { + apiFetchJson, + escapeHtml, + loadAdminToken, + showToast, +} from './app.js'; + +const POLL_INTERVAL_MS = 4000; +const HISTORY_LIMIT = 25; +const ACTIVE_WORK_ORDER_STATUSES = new Set([ + 'pending_approval', + 'approved', + 'queued', + 'assigned', + 'running', +]); +const TERMINAL_WORK_ORDER_STATUSES = new Set([ + 'succeeded', + 'failed', + 'timed_out', + 'cancelled', + 'rejected', + 'expired', +]); + +const brokerState = { + overview: null, + manifests: [], + workers: [], + history: [], + historyTotal: 0, + historyOffset: 0, + selectedRequest: null, + selectedProjection: null, + selectedRoute: null, + selectedRun: null, + pollTimer: null, +}; + +function authorizedHeaders({ json = true } = {}) { + const headers = {}; + if (json) headers['Content-Type'] = 'application/json'; + const token = loadAdminToken(); + if (token) headers.Authorization = `Bearer ${token}`; + return headers; +} + +function setBrokerStatus(message, tone = '') { + const element = document.getElementById('brokerStatus'); + if (!element) return; + element.textContent = message; + if (tone) element.dataset.tone = tone; + else delete element.dataset.tone; +} + +function setProfileStatus(message, tone = '') { + const element = document.getElementById('profileStatus'); + if (!element) return; + element.textContent = message; + if (tone) element.dataset.tone = tone; + else delete element.dataset.tone; +} + +function formatInteger(value) { + if (value === null || value === undefined || value === '') return '—'; + return Number.isFinite(Number(value)) ? Number(value).toLocaleString() : '—'; +} + +function formatSeconds(value) { + if (!Number.isFinite(Number(value))) return '—'; + const seconds = Number(value); + if (seconds < 60) return `${seconds.toFixed(seconds % 1 ? 1 : 0)} s`; + return `${(seconds / 60).toFixed(1)} min`; +} + +function formatPercent(value) { + return Number.isFinite(Number(value)) ? `${(Number(value) * 100).toFixed(1)}%` : '—'; +} + +function formatDate(value) { + if (!value) return '—'; + const date = new Date(value); + return Number.isNaN(date.valueOf()) ? '—' : date.toLocaleString(); +} + +function formatQuotaReset(value) { + if (!value) return 'Not scheduled'; + const formatted = formatDate(value); + return formatted === '—' ? 'Not available' : formatted; +} + +function timezoneAwareIso(value) { + if (!value) return null; + const normalized = /(?:Z|[+-]\d{2}:\d{2})$/i.test(value) ? value : `${value}Z`; + const date = new Date(normalized); + return Number.isNaN(date.valueOf()) ? null : date.toISOString(); +} + +function formatLabel(value) { + return value ? String(value).replaceAll('_', ' ') : '—'; +} + +function shortSha(value) { + return value ? String(value).slice(0, 8) : '—'; +} + +function badge(value, tone = '') { + const attribute = tone ? ` data-tone="${tone}"` : ''; + return `${escapeHtml(formatLabel(value))}`; +} + +function stateTone(value) { + if (['succeeded', 'reused', 'published_current', 'active', 'online'].includes(value)) { + return 'success'; + } + if (['fresh'].includes(value)) return 'fresh'; + if ( + ['failed', 'timed_out', 'cancelled', 'expired', 'published_stale', 'stale'].includes(value) + ) { + return 'warning'; + } + return ''; +} + +function renderMetrics() { + const container = document.getElementById('brokerMetrics'); + if (!container) return; + const overview = brokerState.overview; + if (!overview) { + container.innerHTML = '

Metrics are unavailable.

'; + return; + } + const cards = [ + ['Deterministic executions avoided', overview.avoided_work.deterministic_executions_avoided], + ['Reference execution time avoided', formatSeconds(overview.avoided_work.reference_seconds_avoided)], + ['Comparison units avoided', overview.avoided_work.comparison_units_avoided], + ['Fresh successful runs', overview.runs.fresh_successful], + ['Reused successful runs', overview.runs.reused_successful], + ['Reuse rate', formatPercent(overview.avoided_work.reuse_rate)], + ['Current publications', overview.publications.current], + ['Stale publications', overview.publications.stale], + ]; + container.innerHTML = cards + .map( + ([label, value]) => ` +
+
${escapeHtml(label)}
+
${typeof value === 'number' ? formatInteger(value) : escapeHtml(value)}
+
+ ` + ) + .join(''); +} + +function renderManifestChoices() { + const select = document.getElementById('validationManifest'); + if (!select) return; + const previous = select.value; + select.innerHTML = brokerState.manifests.length + ? brokerState.manifests + .map( + (manifest) => + `` + ) + .join('') + : ''; + if (previous && Array.from(select.options).some((option) => option.value === previous)) { + select.value = previous; + } +} + +function renderWorkerChoices() { + const preferred = document.getElementById('validationPreferredExecutor'); + const profileWorker = document.getElementById('profileWorker'); + if (!preferred || !profileWorker) return; + const preferredValue = preferred.value; + const profileValue = profileWorker.value; + const options = brokerState.workers + .map( + (worker) => + `` + ) + .join(''); + preferred.innerHTML = `${options}`; + profileWorker.innerHTML = brokerState.workers.length + ? `${options}` + : ''; + if (preferredValue) preferred.value = preferredValue; + if (profileValue) profileWorker.value = profileValue; +} + +function renderWorkers() { + const container = document.getElementById('brokerWorkers'); + if (!container) return; + if (!brokerState.workers.length) { + container.innerHTML = '

No execution workers are registered.

'; + return; + } + container.innerHTML = brokerState.workers + .map((worker) => { + const profile = worker.profile; + const browsers = Array.isArray(worker.browsers) && worker.browsers.length + ? worker.browsers.join(', ') + : 'Not reported'; + const profileDetails = profile + ? ` +
+
Profile
${badge(profile.enabled ? 'enabled' : 'disabled', profile.enabled ? 'success' : 'warning')}
+
Comparison units
${formatInteger(profile.estimated_cost_units_per_run)}
+
Revision
${formatInteger(profile.revision)}
+
Quota
${formatInteger(profile.quota_remaining_units)} / ${formatInteger(profile.quota_capacity_units)}
+
Quota reset
${escapeHtml(formatQuotaReset(profile.quota_reset_at))}
+
Priority
${formatInteger(profile.routing_priority)}
+
+ ` + : '

No operator routing profile.

'; + return ` +
+
+
+ ${escapeHtml(worker.display_name)} +

${escapeHtml(worker.worker_id)}

+
+
${badge(worker.status, stateTone(worker.status))} ${badge(worker.activity_state, stateTone(worker.activity_state))}
+
+
+
Platform
${escapeHtml(worker.operating_system)} / ${escapeHtml(worker.architecture)}
+
Capacity
${formatInteger(worker.active_run_count)} active / ${formatInteger(worker.max_concurrency)} maximum
+
Last heartbeat
${escapeHtml(formatDate(worker.last_heartbeat_at))}
+
Last checkout poll
${escapeHtml(formatDate(worker.last_checkout_poll_at))}
+
+
+
Python
${escapeHtml(worker.python_version || 'Not reported')}
+
Node
${escapeHtml(worker.node_version || 'Not reported')}
+
Docker
${worker.docker_available ? 'Available' : 'Unavailable'}
+
Browsers
${escapeHtml(browsers)}
+
GPU
${worker.gpu_available ? 'Available' : 'Unavailable'}
+
Unity
${worker.unity_available ? 'Available' : 'Unavailable'}
+
Desktop automation
${worker.desktop_available ? 'Available' : 'Unavailable'}
+
Network
${escapeHtml(formatLabel(worker.network_policy_capability))}
+
Repository writes
${worker.repository_write_capability ? 'Enabled' : 'Disabled'}
+
+ ${profileDetails} + +
+ `; + }) + .join(''); +} + +function populateProfileForm(workerId) { + const worker = brokerState.workers.find((item) => item.worker_id === workerId); + const form = document.getElementById('routingProfileForm'); + if (!form || !worker) return; + const profile = worker.profile; + form.elements.worker_id.value = worker.worker_id; + form.elements.estimated_cost_units_per_run.value = + profile?.estimated_cost_units_per_run ?? ''; + form.elements.quota_capacity_units.value = profile?.quota_capacity_units ?? ''; + form.elements.quota_remaining_units.value = profile?.quota_remaining_units ?? ''; + form.elements.routing_priority.value = profile?.routing_priority ?? 0; + form.elements.enabled.checked = profile?.enabled ?? true; + form.elements.expected_revision.value = profile?.revision ?? ''; + document.getElementById('resetProfileQuota').disabled = !profile; + setProfileStatus( + profile + ? `Editing revision ${profile.revision}. A newer server revision will return a conflict.` + : 'Creating the first operator-owned profile for this worker.' + ); +} + +function currentProjectionForRequest(requestId) { + return brokerState.history.find((item) => item.request_id === requestId) || null; +} + +async function optionalDetail(url) { + try { + return await apiFetchJson(url, { + headers: authorizedHeaders({ json: false }), + }); + } catch (error) { + if ([404, 409].includes(error?.response?.status)) return null; + throw error; + } +} + +async function refreshSelectedDetails() { + const request = brokerState.selectedRequest; + brokerState.selectedRoute = null; + brokerState.selectedRun = null; + if (!request) return; + const runId = brokerState.selectedProjection?.run_id || request.terminal_run_id; + if (runId) { + brokerState.selectedRun = await optionalDetail(`/api/execution/runs/${runId}`); + brokerState.selectedRoute = brokerState.selectedRun?.route_provenance || null; + return; + } + if (request.work_order_status === 'queued') { + brokerState.selectedRoute = await optionalDetail( + `/api/execution/work-orders/${request.work_order_id}/route-assessment` + ); + } +} + +function measuredDuration(run, projection) { + if (run?.started_at && run?.finished_at) { + const started = new Date(run.started_at); + const finished = new Date(run.finished_at); + if (!Number.isNaN(started.valueOf()) && !Number.isNaN(finished.valueOf())) { + return Math.max(0, (finished.valueOf() - started.valueOf()) / 1000); + } + } + return projection?.run_duration_seconds ?? null; +} + +function renderRequestDetail() { + const container = document.getElementById('brokerRequestDetail'); + const refresh = document.getElementById('refreshRequest'); + if (!container || !refresh) return; + const request = brokerState.selectedRequest; + if (!request) { + refresh.disabled = true; + container.className = 'broker-empty'; + container.textContent = 'Select a request from history or submit a new validation.'; + stopRequestPolling(); + return; + } + refresh.disabled = false; + container.className = ''; + const projection = brokerState.selectedProjection || {}; + const run = brokerState.selectedRun; + const route = brokerState.selectedRoute || run?.route_provenance || null; + const evidence = run?.evidence_metadata || null; + const status = request.work_order_status; + const canApprove = status === 'pending_approval'; + const canQueue = status === 'approved'; + const canCancel = !TERMINAL_WORK_ORDER_STATUSES.has(status); + const canExpire = ['approved', 'queued'].includes(status); + const canPublish = Boolean(request.evidence_fingerprint); + const selectedWorker = route?.selected_worker_id || run?.worker_id || projection.selected_worker_id; + const reuseDecision = run?.reuse_decision || projection.reuse_decision || 'pending'; + const sourceRunId = run?.reused_from_run_id || evidence?.reuse_provenance?.source_run_id; + const sourceFingerprint = + run?.source_evidence_fingerprint || evidence?.reuse_provenance?.source_evidence_fingerprint; + const evidenceFingerprint = evidence?.fingerprint || request.evidence_fingerprint; + const duration = measuredDuration(run, projection); + const executedSteps = Array.isArray(evidence?.steps) ? evidence.steps.length : null; + container.innerHTML = ` +
+
Request
#${request.request_id}
+
Repository
${escapeHtml(request.repository_full_name)}
+
Pull request
#${formatInteger(request.pull_request_number)}
+
Created
${escapeHtml(formatDate(request.created_at))}
+
Last head resolution
${escapeHtml(formatDate(request.last_resolved_at))}
+
Exact head
${escapeHtml(request.tested_head_sha)}
+
Base SHA
${escapeHtml(request.base_sha)}
+
Manifest
${escapeHtml(`${request.manifest_name}@${request.manifest_version}`)} ${escapeHtml(request.manifest_digest)}
+
Work order
#${request.work_order_id} ${badge(status, stateTone(status))}
+
Reuse policy
${badge(request.reuse_policy)}
+
Routing policy
${badge(request.routing_policy)}
+
Selected route
${escapeHtml(selectedWorker || 'Not available')}
+
Route reason
${escapeHtml(formatLabel(route?.reason))}
+
Comparison units
${formatInteger(route?.estimated_cost_units ?? projection.estimated_cost_units)}
+
Eligible candidates
${formatInteger(route?.eligible_candidate_count)}
+
Explicit pin
${route ? (route.explicit_pin_applied ? 'Applied' : 'Not applied') : 'Not available'}
+
Profile revision
${formatInteger(route?.selected_routing_profile_revision)}
+
Quota
${formatInteger(route?.required_quota_units ?? request.required_quota_units)} required / ${formatInteger(route?.reserved_quota_units)} reserved · ${escapeHtml(formatLabel(route?.quota_reservation_state))}
+
Reuse decision
${badge(reuseDecision, stateTone(reuseDecision))}
+
Source run
${sourceRunId ? `#${formatInteger(sourceRunId)}` : 'Not available'}
+
Source evidence
${escapeHtml(sourceFingerprint || 'Not available')}
+
Run
${run ? `#${run.id} ${badge(run.status, stateTone(run.status))}` : 'Not available'}
+
Assigned
${escapeHtml(formatDate(run?.assigned_at))}
+
Started
${escapeHtml(formatDate(run?.started_at))}
+
Finished
${escapeHtml(formatDate(run?.finished_at))}
+
Measured duration
${duration === null ? 'Not available' : escapeHtml(formatSeconds(duration))}
+
Executed steps
${executedSteps === null ? 'Not available' : formatInteger(executedSteps)}
+
Cleanup
${escapeHtml(formatLabel(run?.cleanup_status))}
+
Terminal reason
${escapeHtml(formatLabel(run?.terminal_reason))}
+
Evidence
${escapeHtml(evidenceFingerprint || 'Not available')}
+
Publication
${badge(request.publication_state, stateTone(request.publication_state))} · ${escapeHtml(formatLabel(request.publication_decision))}
+
Updated
${escapeHtml(formatDate(request.updated_at))}
+
+
+ + + + + + +
+

Actions are enabled only when the persisted lifecycle permits them. Publication always rechecks the GitHub head.

+ `; + syncRequestPolling(); +} + +function renderHistory() { + const container = document.getElementById('brokerHistory'); + const summary = document.getElementById('brokerHistorySummary'); + const previous = document.getElementById('historyPrevious'); + const next = document.getElementById('historyNext'); + if (!container || !summary || !previous || !next) return; + if (!brokerState.history.length) { + container.innerHTML = '

No validation requests match these filters.

'; + } else { + const rows = brokerState.history + .map( + (item) => ` + + + ${escapeHtml(item.repository_full_name)} #${item.pull_request_number} + ${escapeHtml(shortSha(item.tested_head_sha))} + ${badge(item.work_order_status, stateTone(item.work_order_status))} + ${item.reuse_decision ? badge(item.reuse_decision, stateTone(item.reuse_decision)) : '—'} + ${escapeHtml(item.selected_worker_id || '—')} + ${formatInteger(item.estimated_cost_units)} + ${badge(item.publication_state, stateTone(item.publication_state))} + ${escapeHtml(formatDate(item.created_at))} + + ` + ) + .join(''); + container.innerHTML = ` + + + ${rows} +
RequestSourceExact SHALifecycleDecisionWorkerUnitsPublicationCreated
+ `; + } + const first = brokerState.historyTotal ? brokerState.historyOffset + 1 : 0; + const last = Math.min( + brokerState.historyOffset + brokerState.history.length, + brokerState.historyTotal + ); + summary.textContent = `Showing ${first}–${last} of ${brokerState.historyTotal} requests`; + previous.disabled = brokerState.historyOffset === 0; + next.disabled = brokerState.historyOffset + HISTORY_LIMIT >= brokerState.historyTotal; +} + +async function refreshOverview() { + const days = document.getElementById('brokerWindow')?.value || '30'; + brokerState.overview = await apiFetchJson( + `/api/execution/operator/overview?window_days=${encodeURIComponent(days)}`, + { headers: authorizedHeaders({ json: false }) } + ); + renderMetrics(); +} + +async function refreshManifests() { + brokerState.manifests = await apiFetchJson('/api/execution/manifests', { + headers: authorizedHeaders({ json: false }), + }); + renderManifestChoices(); +} + +async function refreshWorkers() { + const page = await apiFetchJson('/api/execution/workers?limit=100&offset=0', { + headers: authorizedHeaders({ json: false }), + }); + brokerState.workers = Array.isArray(page.items) ? page.items : []; + renderWorkerChoices(); + renderWorkers(); + const selectedWorker = document.getElementById('profileWorker')?.value; + if (selectedWorker) populateProfileForm(selectedWorker); +} + +function historyQuery() { + const form = document.getElementById('brokerHistoryFilters'); + const params = new URLSearchParams({ + limit: String(HISTORY_LIMIT), + offset: String(brokerState.historyOffset), + }); + if (form) { + for (const [key, rawValue] of new FormData(form).entries()) { + const value = String(rawValue).trim(); + if (value) params.set(key, value); + } + } + return params; +} + +async function refreshHistory() { + const page = await apiFetchJson(`/api/execution/operator/history?${historyQuery()}`, { + headers: authorizedHeaders({ json: false }), + }); + brokerState.history = Array.isArray(page.items) ? page.items : []; + brokerState.historyTotal = Number(page.total) || 0; + if (brokerState.selectedRequest) { + brokerState.selectedProjection = currentProjectionForRequest( + brokerState.selectedRequest.request_id + ); + await refreshSelectedDetails(); + renderRequestDetail(); + } + renderHistory(); +} + +async function refreshSelectedRequest({ silent = false } = {}) { + if (!brokerState.selectedRequest) return; + const requestId = brokerState.selectedRequest.request_id; + try { + brokerState.selectedRequest = await apiFetchJson( + `/api/execution/github/requests/${requestId}`, + { headers: authorizedHeaders({ json: false }) } + ); + brokerState.selectedProjection = currentProjectionForRequest(requestId); + await refreshSelectedDetails(); + renderRequestDetail(); + } catch (error) { + console.error('Failed to refresh selected validation request', error); + if (!silent) setBrokerStatus('Unable to refresh the selected request.', 'error'); + } +} + +async function selectRequest(requestId) { + brokerState.selectedProjection = currentProjectionForRequest(requestId); + brokerState.selectedRequest = await apiFetchJson( + `/api/execution/github/requests/${requestId}`, + { headers: authorizedHeaders({ json: false }) } + ); + await refreshSelectedDetails(); + renderRequestDetail(); + document.getElementById('brokerRequestDetail')?.scrollIntoView({ + behavior: 'smooth', + block: 'nearest', + }); +} + +function stopRequestPolling() { + if (brokerState.pollTimer) { + clearInterval(brokerState.pollTimer); + brokerState.pollTimer = null; + } +} + +function syncRequestPolling() { + stopRequestPolling(); + const status = brokerState.selectedRequest?.work_order_status; + if (!ACTIVE_WORK_ORDER_STATUSES.has(status)) return; + brokerState.pollTimer = setInterval(async () => { + await refreshSelectedRequest({ silent: true }); + await refreshHistory(); + await refreshOverview(); + }, POLL_INTERVAL_MS); +} + +async function refreshBrokerWorkspace({ announce = false } = {}) { + setBrokerStatus('Refreshing bounded operator projections…'); + const results = await Promise.allSettled([ + refreshOverview(), + refreshManifests(), + refreshWorkers(), + refreshHistory(), + ]); + const failures = results.filter((result) => result.status === 'rejected').length; + if (failures) { + setBrokerStatus( + `${failures} broker surface${failures === 1 ? '' : 's'} could not be loaded. Enter the admin token above if access is protected.`, + 'error' + ); + } else { + const windowDays = brokerState.overview?.window?.days || 30; + setBrokerStatus( + `Operator projections refreshed. Metrics cover the last ${windowDays} days.`, + 'success' + ); + if (announce) showToast('Validation broker refreshed.', 'success'); + } +} + +async function handleValidationRequest(event) { + event.preventDefault(); + const data = new FormData(event.currentTarget); + const [manifestName, manifestVersion] = String(data.get('manifest') || '').split('@'); + const maximumCost = String(data.get('maximum_cost_units') || '').trim(); + const preferredExecutor = String(data.get('preferred_executor') || '').trim(); + const payload = { + repository_full_name: String(data.get('repository_full_name') || '').trim(), + pull_request_number: Number(data.get('pull_request_number')), + manifest: { name: manifestName, version: manifestVersion }, + reuse_policy: data.get('reuse_policy'), + routing_policy: data.get('routing_policy'), + maximum_cost_units: maximumCost ? Number(maximumCost) : null, + required_quota_units: Number(data.get('required_quota_units') || 0), + preferred_executor: preferredExecutor || null, + }; + try { + brokerState.selectedRequest = await apiFetchJson( + '/api/execution/github/pull-requests/validate', + { + method: 'POST', + headers: authorizedHeaders(), + body: JSON.stringify(payload), + } + ); + setBrokerStatus( + `Validation request #${brokerState.selectedRequest.request_id} resolved exact head ${shortSha(brokerState.selectedRequest.tested_head_sha)}.`, + 'success' + ); + showToast('GitHub validation request resolved.', 'success'); + await Promise.all([refreshHistory(), refreshOverview()]); + brokerState.selectedProjection = currentProjectionForRequest( + brokerState.selectedRequest.request_id + ); + renderRequestDetail(); + } catch (error) { + console.error('Failed to request GitHub validation', error); + setBrokerStatus( + error?.details + ? `Validation request failed: ${error.details}` + : 'Validation request failed. Check GitHub adapter configuration and retry.', + 'error' + ); + } +} + +async function handleProfileSubmit(event) { + event.preventDefault(); + const form = event.currentTarget; + const data = new FormData(form); + const workerId = String(data.get('worker_id') || ''); + const worker = brokerState.workers.find((item) => item.worker_id === workerId); + if (!worker) return; + const base = { + enabled: form.elements.enabled.checked, + estimated_cost_units_per_run: Number(data.get('estimated_cost_units_per_run')), + quota_capacity_units: Number(data.get('quota_capacity_units')), + quota_remaining_units: Number(data.get('quota_remaining_units')), + quota_reset_at: timezoneAwareIso(worker.profile?.quota_reset_at), + routing_priority: Number(data.get('routing_priority')), + }; + const existing = worker.profile; + const url = existing + ? `/api/execution/routing-profiles/${encodeURIComponent(workerId)}` + : '/api/execution/routing-profiles'; + const payload = existing + ? { ...base, expected_revision: existing.revision } + : { ...base, schema_version: 1, worker_id: workerId }; + try { + const profile = await apiFetchJson(url, { + method: existing ? 'PUT' : 'POST', + headers: authorizedHeaders(), + body: JSON.stringify(payload), + }); + setProfileStatus(`Saved profile revision ${profile.revision}.`, 'success'); + showToast(`Routing profile for ${workerId} saved.`, 'success'); + await refreshWorkers(); + populateProfileForm(workerId); + } catch (error) { + if (error?.response?.status === 409) { + await refreshWorkers(); + populateProfileForm(workerId); + setProfileStatus('Profile revision conflicted. Latest server state was reloaded.', 'error'); + } else { + console.error('Failed to save routing profile', error); + setProfileStatus('Profile could not be saved.', 'error'); + } + } +} + +async function resetProfileQuota() { + const form = document.getElementById('routingProfileForm'); + const workerId = form?.elements.worker_id.value; + const worker = brokerState.workers.find((item) => item.worker_id === workerId); + if (!form || !worker?.profile) return; + const remaining = Number(form.elements.quota_remaining_units.value); + if (!window.confirm(`Reset ${workerId} quota to ${remaining} units?`)) return; + try { + const profile = await apiFetchJson( + `/api/execution/routing-profiles/${encodeURIComponent(workerId)}/quota-reset`, + { + method: 'POST', + headers: authorizedHeaders(), + body: JSON.stringify({ + expected_revision: worker.profile.revision, + quota_remaining_units: remaining, + quota_reset_at: new Date().toISOString(), + }), + } + ); + setProfileStatus(`Quota reset recorded at revision ${profile.revision}.`, 'success'); + await refreshWorkers(); + populateProfileForm(workerId); + } catch (error) { + if (error?.response?.status === 409) { + await refreshWorkers(); + populateProfileForm(workerId); + setProfileStatus('Quota reset conflicted. Latest profile was reloaded.', 'error'); + } else { + console.error('Failed to reset routing quota', error); + } + } +} + +async function mutateSelectedRequest(action) { + const request = brokerState.selectedRequest; + if (!request) return; + let url; + let body; + let confirmMessage; + if (action === 'approve' || action === 'approve-queue') { + url = `/api/execution/work-orders/${request.work_order_id}/approve`; + body = { queue: action === 'approve-queue' }; + } else if (action === 'queue') { + url = `/api/execution/work-orders/${request.work_order_id}/queue`; + } else if (action === 'cancel' || action === 'expire') { + confirmMessage = `${formatLabel(action)} work order #${request.work_order_id}?`; + url = `/api/execution/work-orders/${request.work_order_id}/${action}`; + body = { reason: `operator_${action}` }; + } else if (action === 'publish') { + confirmMessage = `Publish compact evidence for request #${request.request_id}? The GitHub head will be rechecked.`; + url = `/api/execution/github/requests/${request.request_id}/publish`; + } + if (!url || (confirmMessage && !window.confirm(confirmMessage))) return; + try { + await apiFetchJson(url, { + method: 'POST', + headers: authorizedHeaders({ json: body !== undefined }), + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + showToast(`${formatLabel(action)} completed.`, 'success'); + await Promise.all([ + refreshSelectedRequest({ silent: true }), + refreshHistory(), + refreshOverview(), + refreshWorkers(), + ]); + } catch (error) { + console.error(`Failed request action ${action}`, error); + if (error?.response?.status === 409) { + setBrokerStatus('Lifecycle action conflicted. Latest request state was reloaded.', 'error'); + await refreshSelectedRequest({ silent: true }); + } + } +} + +function initBrokerEvents() { + document.getElementById('refreshBroker')?.addEventListener('click', () => { + refreshBrokerWorkspace({ announce: true }); + }); + document.getElementById('brokerWindow')?.addEventListener('change', refreshOverview); + document.getElementById('validationRequestForm')?.addEventListener( + 'submit', + handleValidationRequest + ); + document.getElementById('routingProfileForm')?.addEventListener( + 'submit', + handleProfileSubmit + ); + document.getElementById('profileWorker')?.addEventListener('change', (event) => { + if (event.currentTarget.value) populateProfileForm(event.currentTarget.value); + }); + document.getElementById('resetProfileQuota')?.addEventListener('click', resetProfileQuota); + document.getElementById('refreshRequest')?.addEventListener('click', () => { + refreshSelectedRequest(); + }); + document.getElementById('brokerHistoryFilters')?.addEventListener('submit', (event) => { + event.preventDefault(); + brokerState.historyOffset = 0; + refreshHistory(); + }); + document.getElementById('historyPrevious')?.addEventListener('click', () => { + brokerState.historyOffset = Math.max(0, brokerState.historyOffset - HISTORY_LIMIT); + refreshHistory(); + }); + document.getElementById('historyNext')?.addEventListener('click', () => { + brokerState.historyOffset += HISTORY_LIMIT; + refreshHistory(); + }); + document.getElementById('validation-broker')?.addEventListener('click', (event) => { + const selectButton = event.target.closest('[data-select-request]'); + if (selectButton) { + selectRequest(Number(selectButton.dataset.selectRequest)); + return; + } + const editButton = event.target.closest('[data-profile-edit]'); + if (editButton) { + populateProfileForm(editButton.dataset.profileEdit); + document.getElementById('profileWorker')?.focus(); + return; + } + const actionButton = event.target.closest('[data-request-action]'); + if (actionButton && !actionButton.disabled) { + mutateSelectedRequest(actionButton.dataset.requestAction); + } + }); + window.addEventListener('beforeunload', stopRequestPolling, { once: true }); +} + +function initializeBroker() { + if (!document.getElementById('validation-broker')) return; + initBrokerEvents(); + renderMetrics(); + renderWorkers(); + renderHistory(); + renderRequestDetail(); + refreshBrokerWorkspace(); +} + +document.addEventListener('DOMContentLoaded', initializeBroker); diff --git a/web/tests/test_ui.py b/web/tests/test_ui.py index b4c2a31..88f0d06 100644 --- a/web/tests/test_ui.py +++ b/web/tests/test_ui.py @@ -24,6 +24,7 @@ HTTP_OK = 200 SERVER_WAIT_INTERVAL = 0.2 +EXPECTED_PUBLICATION_DIALOGS = 2 ROOT = Path(__file__).resolve().parents[2] if str(ROOT) not in sys.path: @@ -90,7 +91,7 @@ def app_server(tmp_path_factory: pytest.TempPathFactory) -> Generator[str, None, sys.executable, "-m", "uvicorn", - "server.app:app", + "web.tests.ui_test_app:app", "--host", "127.0.0.1", "--port", @@ -233,8 +234,7 @@ def test_two_agent_dependency_flow_updates_dashboard( page.goto(f"{app_server}/", wait_until="domcontentloaded") page.wait_for_selector("#tasks") - task_ids = page.evaluate( - """async () => { + task_script = """async () => { const jsonHeaders = { 'Content-Type': 'application/json', }; @@ -275,7 +275,7 @@ def test_two_agent_dependency_flow_updates_dashboard( return { taskAId: taskA.id, taskBId: taskB.id }; }""" - ) + task_ids = page.evaluate(task_script) page.wait_for_selector('tr:has-text("Task A")') page.wait_for_selector('tr:has-text("Task B")') @@ -361,3 +361,438 @@ def test_two_agent_dependency_flow_updates_dashboard( expect(page.locator("#planVersion")).not_to_have_text(initial_version) finally: page.close() + + +def _post_json(page, url: str, payload: dict | None = None) -> dict: + return page.evaluate( + """async ({ url, payload }) => { + const options = { method: 'POST' }; + if (payload !== null) { + options.headers = { 'Content-Type': 'application/json' }; + options.body = JSON.stringify(payload); + } + const response = await fetch(url, options); + const body = await response.json(); + if (!response.ok) { + const detail = JSON.stringify(body); + throw new Error(`${url}: ${response.status} ${detail}`); + } + return body; + }""", + {"url": url, "payload": payload}, + ) + + +def _put_json(page, url: str, payload: dict) -> dict: + return page.evaluate( + """async ({ url, payload }) => { + const response = await fetch(url, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + const body = await response.json(); + if (!response.ok) { + const detail = JSON.stringify(body); + throw new Error(`${url}: ${response.status} ${detail}`); + } + return body; + }""", + {"url": url, "payload": payload}, + ) + + +def test_validation_broker_operator_workflow_is_accessible_and_responsive( # noqa: PLR0915 + app_server: str, + browser: object, +) -> None: + page = browser.new_page(viewport={"width": 1440, "height": 1000}) + console_errors: list[str] = [] + page.on( + "console", + lambda message: ( + console_errors.append(message.text) + if message.type == "error" and "status of 409" not in message.text + else None + ), + ) + try: + page.goto(f"{app_server}/", wait_until="domcontentloaded") + expect(page.locator("h1")).to_have_text("Switchboard") + expect(page.locator("#validation-broker-heading")).to_have_text( + "Validation Broker" + ) + expect(page.locator("#validation-broker")).to_be_visible() + page.wait_for_function( + "() => document.querySelector('#brokerStatus')?.textContent" + ".includes('refreshed')" + ) + page.evaluate( + "() => localStorage.setItem('switchboardAdminToken', 'ui-admin-sentinel')" + ) + page.click("#refreshBroker") + page.wait_for_function( + "() => document.querySelector('#brokerStatus')?.textContent" + ".includes('refreshed')" + ) + assert "ui-admin-sentinel" not in page.locator("body").inner_text() + assert "ui-admin-sentinel" not in page.locator("body").inner_html() + + worker_base = { + "display_name": "Synthetic local worker", + "operating_system": "linux", + "architecture": "x86_64", + "python_version": "3.11.14", + "node_version": None, + "docker_available": False, + "browsers": [], + "gpu_available": False, + "unity_available": False, + "desktop_available": False, + "capabilities": {}, + "max_concurrency": 1, + "network_policy_capability": "worker_restricted", + "repository_write_capability": False, + "status": "online", + } + _post_json( + page, + "/api/execution/workers", + { + **worker_base, + "worker_id": "ui-worker-cheap", + "display_name": "Local small", + "browsers": ["chromium"], + "capabilities": { + "internal_marker": "not-for-operator-ui", + }, + }, + ) + _post_json( + page, + "/api/execution/workers", + { + **worker_base, + "worker_id": "ui-worker-expensive", + "display_name": "Local large", + }, + ) + _post_json( + page, + "/api/execution/workers", + { + **worker_base, + "worker_id": "ui-worker-new", + "display_name": "Local reserve", + }, + ) + for worker_id, cost in (("ui-worker-cheap", 3), ("ui-worker-expensive", 9)): + _post_json( + page, + "/api/execution/routing-profiles", + { + "schema_version": 1, + "worker_id": worker_id, + "enabled": True, + "estimated_cost_units_per_run": cost, + "quota_capacity_units": 20, + "quota_remaining_units": 20, + "quota_reset_at": ( + "2026-08-10T12:00:00Z" + if worker_id == "ui-worker-cheap" + else None + ), + "routing_priority": 0, + }, + ) + _post_json(page, "/api/execution/checkout", {"worker_id": worker_id}) + _post_json(page, "/api/execution/checkout", {"worker_id": "ui-worker-new"}) + + page.click("#refreshBroker") + page.wait_for_selector('[data-worker-id="ui-worker-cheap"]') + expect(page.locator("#brokerWorkers")).to_contain_text("Local small") + expect(page.locator("#brokerWorkers")).to_contain_text("3") + expect(page.locator('[data-worker-id="ui-worker-cheap"]')).to_contain_text( + "online" + ) + expect(page.locator('[data-worker-id="ui-worker-cheap"]')).to_contain_text( + "active" + ) + expect(page.locator('[data-worker-id="ui-worker-cheap"]')).to_contain_text( + "linux / x86_64" + ) + expect(page.locator('[data-worker-id="ui-worker-cheap"]')).to_contain_text( + "0 active / 1 maximum" + ) + expect(page.locator('[data-worker-id="ui-worker-cheap"]')).to_contain_text( + "Last heartbeat" + ) + expect(page.locator('[data-worker-id="ui-worker-cheap"]')).to_contain_text( + "Last checkout poll" + ) + expect(page.locator('[data-worker-id="ui-worker-cheap"]')).to_contain_text( + "enabled" + ) + cheap_worker_card = page.locator('[data-worker-id="ui-worker-cheap"]') + expect(cheap_worker_card).to_contain_text("Python") + expect(cheap_worker_card).to_contain_text("3.11.14") + expect(cheap_worker_card).to_contain_text("Node") + expect(cheap_worker_card).to_contain_text("Not reported") + expect(cheap_worker_card).to_contain_text("Docker") + expect(cheap_worker_card).to_contain_text("Unavailable") + expect(cheap_worker_card).to_contain_text("Browsers") + expect(cheap_worker_card).to_contain_text("chromium") + expect(cheap_worker_card).to_contain_text("GPU") + expect(cheap_worker_card).to_contain_text("Unity") + expect(cheap_worker_card).to_contain_text("Desktop automation") + expect(cheap_worker_card).to_contain_text("worker restricted") + expect(cheap_worker_card).to_contain_text("Repository writes") + expect(cheap_worker_card).to_contain_text("Disabled") + expect(cheap_worker_card).to_contain_text("Quota reset") + expect(cheap_worker_card).to_contain_text("2026") + expensive_worker_card = page.locator('[data-worker-id="ui-worker-expensive"]') + expect(expensive_worker_card).to_contain_text("Quota reset") + expect(expensive_worker_card).to_contain_text("Not scheduled") + assert "internal_marker" not in page.locator("#brokerWorkers").inner_text() + assert "not-for-operator-ui" not in page.locator("#brokerWorkers").inner_text() + + page.select_option("#profileWorker", "ui-worker-new") + page.fill("#profileCost", "12") + page.fill("#profilePriority", "2") + page.fill("#profileCapacity", "20") + page.fill("#profileRemaining", "15") + page.click('#routingProfileForm button[type="submit"]') + expect(page.locator("#profileStatus")).to_contain_text("revision 1") + page.fill("#profileRemaining", "14") + page.once("dialog", lambda dialog: dialog.accept()) + page.click("#resetProfileQuota") + expect(page.locator("#profileStatus")).to_contain_text("Editing revision 2") + expect(page.locator("#profileRemaining")).to_have_value("14") + + page.click('[data-profile-edit="ui-worker-cheap"]') + expect(page.locator("#profileRevision")).to_have_value("1") + page.fill("#profileCost", "4") + page.click('#routingProfileForm button[type="submit"]') + expect(page.locator("#profileStatus")).to_contain_text("revision 2") + + _put_json( + page, + "/api/execution/routing-profiles/ui-worker-cheap", + { + "expected_revision": 2, + "enabled": True, + "estimated_cost_units_per_run": 3, + "quota_capacity_units": 20, + "quota_remaining_units": 20, + "quota_reset_at": None, + "routing_priority": 0, + }, + ) + page.fill("#profileCost", "5") + page.click('#routingProfileForm button[type="submit"]') + expect(page.locator("#profileStatus")).to_contain_text("conflicted") + expect(page.locator("#profileRevision")).to_have_value("3") + + page.fill("#validationRepository", "missing-owner") + page.fill("#validationPullRequest", "0") + page.fill("#validationCostCeiling", "-1") + page.fill("#validationQuotaUnits", "-1") + assert not page.locator("#validationRequestForm").evaluate( + "form => form.checkValidity()" + ) + assert page.locator("#validationReusePolicy option").evaluate_all( + "options => options.map(option => option.value)" + ) == ["never", "allow_exact", "require_exact"] + assert page.locator("#validationRoutingPolicy option").evaluate_all( + "options => options.map(option => option.value)" + ) == ["first_available", "cheapest_capable"] + + page.fill("#validationRepository", "Nobodyworld/dev-agent-switchboard") + page.fill("#validationPullRequest", "137") + page.fill("#validationCostCeiling", "") + page.select_option("#validationReusePolicy", "never") + page.select_option("#validationRoutingPolicy", "cheapest_capable") + page.fill("#validationQuotaUnits", "1") + page.click('#validationRequestForm button[type="submit"]') + page.wait_for_function( + "() => document.querySelector('#brokerRequestDetail')?.textContent" + ".includes('7d3a91c')" + ) + request_detail = page.locator("#brokerRequestDetail") + expect(request_detail).to_contain_text("pending approval") + expect( + request_detail.locator("dt", has_text="Repository").locator( + "xpath=following-sibling::dd[1]" + ) + ).to_have_text("Nobodyworld/dev-agent-switchboard") + expect( + request_detail.locator("dt", has_text="Pull request").locator( + "xpath=following-sibling::dd[1]" + ) + ).to_have_text("#137") + expect( + request_detail.locator("dt", has_text="Reuse policy").locator( + "xpath=following-sibling::dd[1]" + ) + ).to_contain_text("never") + expect( + request_detail.locator("dt", has_text="Routing policy").locator( + "xpath=following-sibling::dd[1]" + ) + ).to_contain_text("cheapest capable") + page.click('[data-request-action="approve-queue"]') + page.wait_for_function( + "() => document.querySelector('#brokerRequestDetail')?.textContent" + ".includes('queued')" + ) + expect(page.locator("#brokerRequestDetail")).to_contain_text("ui-worker-cheap") + expect(page.locator("#brokerRequestDetail")).to_contain_text("routing selected") + expect(page.locator("#brokerRequestDetail")).to_contain_text( + "Eligible candidates" + ) + expect(page.locator("#brokerRequestDetail")).to_contain_text("Not applied") + first_request_id = page.evaluate( + "() => Number(document.querySelector('[data-history-request]')" + "?.dataset.historyRequest)" + ) + fresh = _post_json(page, f"/__test__/complete/{first_request_id}") + assert fresh["reuse_decision"] == "fresh" + page.click("#refreshRequest") + page.wait_for_function( + "() => document.querySelector('#brokerRequestDetail')?.textContent" + ".includes('succeeded')" + ) + fresh_detail = page.locator("#brokerRequestDetail") + expect(fresh_detail).to_contain_text("1 required / 1 reserved") + expect(fresh_detail).to_contain_text("consumed") + expect(fresh_detail).to_contain_text("Measured duration") + expect(fresh_detail).to_contain_text("7 s") + expect(fresh_detail).to_contain_text("Cleanup") + expect(fresh_detail).to_contain_text("succeeded") + expect( + fresh_detail.locator("dt", has_text="Reuse decision").locator( + "xpath=following-sibling::dd[1]" + ) + ).to_contain_text("fresh") + + dialogs: list[str] = [] + + def accept_dialog(dialog) -> None: + dialogs.append(dialog.message) + dialog.accept() + + page.on("dialog", accept_dialog) + page.click('[data-request-action="publish"]') + page.wait_for_function( + "() => document.querySelector('#brokerRequestDetail')?.textContent" + ".includes('published current')" + ) + expect(page.locator("#brokerRequestDetail")).to_contain_text("current") + + page.select_option("#validationReusePolicy", "allow_exact") + page.click('#validationRequestForm button[type="submit"]') + page.wait_for_function( + "() => document.querySelectorAll('[data-history-request]').length === 2" + ) + page.click('[data-request-action="approve-queue"]') + second_request_id = page.evaluate( + "() => Number(document.querySelector('[data-history-request]')" + "?.dataset.historyRequest)" + ) + assert second_request_id != first_request_id + reused = _post_json(page, f"/__test__/complete/{second_request_id}") + assert reused["reuse_decision"] == "reused" + _post_json(page, f"/__test__/github/head/{'c' * 40}") + page.click("#refreshRequest") + page.wait_for_function( + "() => !document.querySelector('[data-request-action=publish]')?.disabled" + ) + reused_detail = page.locator("#brokerRequestDetail") + expect(reused_detail).to_contain_text("reused") + expect(reused_detail).to_contain_text(f"#{fresh['run_id']}") + expect(reused_detail).to_contain_text(fresh["evidence_fingerprint"]) + expect(reused_detail).to_contain_text("Executed steps") + expect(reused_detail).to_contain_text("0") + expect( + reused_detail.locator("dt", has_text="Reuse policy").locator( + "xpath=following-sibling::dd[1]" + ) + ).to_contain_text("allow exact") + expect( + reused_detail.locator("dt", has_text="Reuse decision").locator( + "xpath=following-sibling::dd[1]" + ) + ).to_contain_text("reused") + page.click('[data-request-action="publish"]') + page.wait_for_function( + "() => document.querySelector('#brokerRequestDetail')?.textContent" + ".includes('published stale')" + ) + assert len(dialogs) == EXPECTED_PUBLICATION_DIALOGS + + page.click("#refreshBroker") + page.wait_for_function( + "() => document.querySelector('#brokerHistory')?.textContent" + ".includes('reused')" + ) + expect(page.locator("#brokerHistory")).to_contain_text("fresh") + expect(page.locator("#brokerHistory")).to_contain_text("reused") + expect(page.locator("#brokerHistory")).to_contain_text("published current") + expect(page.locator("#brokerHistory")).to_contain_text("published stale") + expect(page.locator("#brokerMetrics")).to_contain_text( + "Deterministic executions avoided" + ) + expect(page.locator("#brokerMetrics")).to_contain_text( + "Reference execution time avoided" + ) + expect( + page.locator("#brokerMetrics .broker-metric").first.locator("dd") + ).to_have_text("1") + + page.select_option("#historyReuseDecision", "reused") + page.click('#brokerHistoryFilters button[type="submit"]') + page.wait_for_function( + "() => document.querySelectorAll('#brokerHistory tbody tr').length === 1" + ) + expect(page.locator("#brokerHistory")).to_contain_text("reused") + + page.locator("#validationRepository").focus() + page.keyboard.press("Tab") + assert ( + page.evaluate("() => document.activeElement?.id") == "validationPullRequest" + ) + expect(page.locator('[data-request-action="approve-queue"]')).to_be_disabled() + assert "pending" in ( + page.locator('[data-request-action="approve-queue"]').get_attribute("title") + or "" + ) + + page.set_viewport_size({"width": 390, "height": 844}) + expect(page.locator("#validation-broker")).to_be_visible() + overflow_script = """() => ({ + contained: document.documentElement.scrollWidth <= window.innerWidth, + offenders: [...document.querySelectorAll('body *')] + .filter( + (node) => node.getBoundingClientRect().right + > window.innerWidth + 1 + ) + .slice(0, 10) + .map((node) => ({ + tag: node.tagName, + id: node.id, + className: String(node.className), + right: node.getBoundingClientRect().right, + })), + })""" + overflow = page.evaluate(overflow_script) + assert overflow["contained"], overflow["offenders"] + assert "ui-admin-sentinel" not in page.url + assert all("ui-admin-sentinel" not in message for message in console_errors) + page.evaluate( + "() => document.querySelectorAll('.toast.is-visible')" + ".forEach((toast) => toast.click())" + ) + expect(page.locator(".toast.is-visible")).to_have_count(0) + assert console_errors == [] + finally: + page.close() diff --git a/web/tests/ui_test_app.py b/web/tests/ui_test_app.py new file mode 100644 index 0000000..888e289 --- /dev/null +++ b/web/tests/ui_test_app.py @@ -0,0 +1,317 @@ +"""Real Switchboard app with synthetic completion for UI-only browser acceptance. + +Worker-local execution and cryptographic reuse verification are intentionally proved by +the real ``ExecutionClient``/``LocalWorker`` acceptance in the Python client suite. +""" + +from __future__ import annotations + +import datetime as dt + +from fastapi import HTTPException + +from server.api.dependencies import SessionDependency, get_github_adapter_service +from server.app import app +from server.application import build_execution_service +from server.execution.entities import ExecutionCompletion +from server.execution.enums import ExecutionRunStatus, ReuseDecision +from server.execution.evidence import ( + DependencyLockHash, + EnvironmentIdentity, + EvidenceReuseIdentity, + EvidenceReuseProvenance, + ExecutionEvidenceDraft, + compute_result_contract_hash, + compute_reuse_identity_hash, + finalize_evidence, +) +from server.execution.registry import get_trusted_manifest +from server.github_adapter.repository import GitHubAdapterRepository +from server.github_adapter.service import ( + GitHubAdapterDependencies, + GitHubAdapterService, +) +from server.github_adapter.transport import ( + GitHubActorIdentity, + GitHubComment, + GitHubCommentListing, + GitHubTransport, + ResolvedPullRequest, +) +from server.settings import GitHubSettings +from server.time_utils import utcnow_naive + +REPOSITORY = "Nobodyworld/dev-agent-switchboard" +INITIAL_HEAD = "7d3a91c" + "0" * 33 +BASE_SHA = "223df77" + "0" * 33 +CHEAP_WORKER = "ui-worker-cheap" +EXPENSIVE_WORKER = "ui-worker-expensive" +SHA1_HEX_LENGTH = 40 + + +class OfflineGitHubTransport(GitHubTransport): + """Minimal stateful transport that never performs network I/O.""" + + def __init__(self) -> None: + self.head_sha = INITIAL_HEAD + self.comments: list[GitHubComment] = [] + self.actor = GitHubActorIdentity(actor_id=700, node_id="U_ui_operator") + + async def resolve_authenticated_actor(self) -> GitHubActorIdentity: + return self.actor + + async def resolve_pull_request( + self, + repository_full_name: str, + pull_request_number: int, + *, + require_head: bool = True, + ) -> ResolvedPullRequest: + _ = require_head + return ResolvedPullRequest( + repository_full_name=repository_full_name, + repository_id=100, + repository_node_id="R_ui_repo", + pull_request_number=pull_request_number, + pull_request_id=200, + pull_request_node_id="PR_ui_request", + state="open", + draft=True, + merged=False, + base_ref="main", + base_sha=BASE_SHA, + head_ref="feat/operator-validation-command-center", + head_sha=self.head_sha, + head_repository_full_name=repository_full_name, + head_repository_id=100, + ) + + async def list_comments( + self, repository_full_name: str, pull_request_number: int + ) -> GitHubCommentListing: + return GitHubCommentListing( + comments=tuple( + comment + for comment in self.comments + if comment.repository_full_name == repository_full_name + and comment.pull_request_number == pull_request_number + ), + complete=True, + ) + + async def get_comment( + self, repository_full_name: str, comment_id: int + ) -> GitHubComment: + for comment in self.comments: + if ( + comment.repository_full_name == repository_full_name + and comment.comment_id == comment_id + ): + return comment + raise AssertionError("offline UI transport comment not found") + + async def create_comment( + self, + repository_full_name: str, + pull_request_number: int, + body: str, + ) -> GitHubComment: + comment = GitHubComment( + comment_id=500 + len(self.comments), + body=body, + author=self.actor, + repository_full_name=repository_full_name, + pull_request_number=pull_request_number, + ) + self.comments.append(comment) + return comment + + async def update_comment( + self, + repository_full_name: str, + comment_id: int, + body: str, + ) -> GitHubComment: + current = await self.get_comment(repository_full_name, comment_id) + updated = GitHubComment( + comment_id=current.comment_id, + body=body, + author=current.author, + repository_full_name=current.repository_full_name, + pull_request_number=current.pull_request_number, + ) + self.comments[self.comments.index(current)] = updated + return updated + + +offline_transport = OfflineGitHubTransport() +offline_settings = GitHubSettings( + api_url="https://api.github.com", + operator_id="synthetic-ui-operator", + token="offline-ui-placeholder", # noqa: S106 +) + + +def get_offline_github_adapter(session: SessionDependency) -> GitHubAdapterService: + """Build the real adapter around a stateful no-network transport.""" + + return GitHubAdapterService( + dependencies=GitHubAdapterDependencies( + repository=GitHubAdapterRepository(session), + execution=build_execution_service(session), + transport=offline_transport, + ), + settings=offline_settings, + clock=utcnow_naive, + ) + + +app.dependency_overrides[get_github_adapter_service] = get_offline_github_adapter + + +def _identity(order) -> EvidenceReuseIdentity: + manifest = get_trusted_manifest(order.manifest_name, order.manifest_version) + assert manifest is not None + return EvidenceReuseIdentity( + repository_full_name=order.repository_full_name, + tested_sha=order.commit_sha, + manifest_name=order.manifest_name, + manifest_version=order.manifest_version, + manifest_digest=order.manifest_digest, + worker_environment_fingerprint="d" * 64, + dependency_lock_hashes=[ + DependencyLockHash(relative_path=path, sha256="e" * 64) + for path in sorted(manifest.dependency_lock_paths) + ], + execution_policy_hash=order.execution_policy_hash, + result_contract_hash=compute_result_contract_hash( + fixed_step_metadata=manifest.fixed_step_metadata, + artifact_declarations=manifest.artifact_declarations, + dependency_lock_paths=manifest.dependency_lock_paths, + ), + ) + + +def _evidence( # noqa: PLR0913 + *, + order, + run_id: int, + identity: EvidenceReuseIdentity, + decision: str, + source_run_id: int | None = None, + source_fingerprint: str | None = None, +): + now = dt.datetime(2026, 8, 8, 12, tzinfo=dt.UTC) + return finalize_evidence( + ExecutionEvidenceDraft( + work_order_id=order.id, + run_id=run_id, + repository_full_name=order.repository_full_name, + tested_sha=order.commit_sha, + manifest_name=order.manifest_name, + manifest_version=order.manifest_version, + manifest_digest=order.manifest_digest, + worker_id=CHEAP_WORKER, + environment=EnvironmentIdentity( + operating_system="linux", + architecture="x86_64", + python_version="3.11.14", + fingerprint=identity.worker_environment_fingerprint, + ), + dependency_lock_hashes=identity.dependency_lock_hashes, + started_at=now, + finished_at=now + dt.timedelta(seconds=7), + duration_seconds=7, + terminal_status="succeeded", + steps=[], + artifacts=[], + dependency_lock_status="succeeded", + artifact_finalization_status="succeeded", + source_cleanup_status="succeeded", + local_record_status="succeeded", + reuse_provenance=EvidenceReuseProvenance( + decision=decision, # type: ignore[arg-type] + reason=("exact_evidence_verified" if decision == "reused" else "fresh"), + reuse_identity_hash=compute_reuse_identity_hash(identity), + source_run_id=source_run_id, + source_evidence_fingerprint=source_fingerprint, + ), + ) + ) + + +@app.post("/__test__/github/head/{sha}") +async def set_offline_head(sha: str) -> dict[str, str]: + if len(sha) != SHA1_HEX_LENGTH or any( + character not in "0123456789abcdef" for character in sha + ): + raise HTTPException(status_code=422, detail="invalid_test_sha") + offline_transport.head_sha = sha + return {"head_sha": sha} + + +@app.post("/__test__/complete/{request_id}") +async def complete_offline_validation( + request_id: int, session: SessionDependency +) -> dict[str, object]: + request = await GitHubAdapterRepository(session).get(request_id) + if request is None: + raise HTTPException(status_code=404, detail="github_request_not_found") + execution = build_execution_service(session) + await execution.checkout(EXPENSIVE_WORKER) + assignment = await execution.checkout(CHEAP_WORKER) + if assignment.run_id is None: + raise HTTPException(status_code=409, detail=assignment.reason or "not_assigned") + await execution.heartbeat_run(assignment.run_id, worker_id=CHEAP_WORKER) + order = await execution.get_work_order(request.work_order_id) + identity = _identity(order) + identity_hash = compute_reuse_identity_hash(identity) + decision = ReuseDecision.FRESH + source_run_id = None + source_fingerprint = None + if order.reuse_policy.value != "never": + lookup = await execution.resolve_reuse_candidate( + assignment.run_id, + worker_id=CHEAP_WORKER, + reuse_identity=identity, + reuse_identity_hash=identity_hash, + ) + if lookup.candidate is None: + raise HTTPException(status_code=409, detail=lookup.reason) + decision = ReuseDecision.REUSED + source_run_id = lookup.candidate.source_run_id + source_fingerprint = lookup.candidate.expected_source_evidence_fingerprint + evidence = _evidence( + order=order, + run_id=assignment.run_id, + identity=identity, + decision=decision.value, + source_run_id=source_run_id, + source_fingerprint=source_fingerprint, + ) + run = await execution.complete_run( + assignment.run_id, + worker_id=CHEAP_WORKER, + completion=ExecutionCompletion( + status=ExecutionRunStatus.SUCCEEDED, + evidence_metadata=evidence, + reuse_decision=decision, + reuse_reason=( + "exact_evidence_verified" + if decision == ReuseDecision.REUSED + else "fresh_execution_completed" + ), + reuse_identity=identity, + reuse_identity_hash=identity_hash, + evidence_retention_expires_at=(evidence.started_at + dt.timedelta(days=14)), + ), + ) + if decision == ReuseDecision.FRESH: + run.started_at = dt.datetime(2026, 8, 8, 12, 0, 0, tzinfo=dt.UTC) + run.finished_at = dt.datetime(2026, 8, 8, 12, 0, 7, tzinfo=dt.UTC) + await session.commit() + return { + "run_id": run.id, + "reuse_decision": decision.value, + "evidence_fingerprint": evidence.fingerprint, + }