diff --git a/.github/actions/setup-workspace/action.yml b/.github/actions/setup-workspace/action.yml index 27f27493..ba940f08 100644 --- a/.github/actions/setup-workspace/action.yml +++ b/.github/actions/setup-workspace/action.yml @@ -1,7 +1,7 @@ name: Setup workspace description: >- Select Node from .nvmrc and install the workspace via the exact-key dependency cache contract - (specs/arch/test-runner/github-ci.md#dependency-install). Restores + (specs/arch/test-runner/github-ci.md#dependency-and-browser-caches). Restores node_modules under a key covering the declared inputs; on a hit npm ci is skipped entirely; on a miss npm ci runs and the path is saved. No restore-keys fallback, ever. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a44a537c..c771cc68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - # Tree-hash attestation decision (specs/arch/test-runner/attestation.md#skip-semantics-in-ci): + # Tree-hash attestation decision (specs/arch/test-runner/attestation.md#skipping-in-ci): # cheap, no install, always runs. Expected negative outcomes (no ref, # unreachable origin, disabled policy, malformed ref) complete successfully # with skip=false; continue-on-error keeps even an unexpected crash @@ -111,7 +111,7 @@ jobs: # !cancelled(): a crashed attest-check must never suppress validation — # the implicit success() gate would skip the heavy tier and let the join # green a run with zero heavy validation - # (specs/arch/test-runner/attestation.md#skip-semantics-in-ci). + # (specs/arch/test-runner/attestation.md#skipping-in-ci). if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} name: ${{ matrix.suite.name }} runs-on: ubuntu-latest @@ -131,7 +131,7 @@ jobs: # the concrete required check contexts (unit, e2e:desktop) are emitted, # and under a skip every real step below is guarded and the job # fast-succeeds as an explicit no-op - # (specs/arch/test-runner/attestation.md#skip-semantics-in-ci). + # (specs/arch/test-runner/attestation.md#skipping-in-ci). - name: Attested skip no-op if: ${{ needs.attest-check.outputs.skip == 'true' && needs.attest-check.outputs.authorized == 'true' }} run: 'echo "attested skip: ${{ matrix.suite.name }} already proven for this tree"' @@ -176,7 +176,7 @@ jobs: # Never assume a tool: jobs that reach the runner's cleanup paths need # ss and a readable /proc - # (specs/arch/test-runner/github-ci.md#tool-prerequisites-on-ci-workers). + # (specs/arch/test-runner/test-runner.md#processes-started-by-tests). - if: ${{ (needs.attest-check.outputs.skip != 'true' || needs.attest-check.outputs.authorized != 'true') && matrix.suite.cleanup }} name: Assert or install cleanup tools run: | @@ -253,7 +253,7 @@ jobs: # One deduplicated round of the surfaces' preCommand builds # (test.prebuilt.mjs recipes), fanned out to the e2e shard jobs as a run # artifact so no shard rebuilds them - # (specs/arch/test-runner/github-ci.md#build-once-fan-out). + # (specs/arch/test-runner/github-ci.md#build-once). build: needs: [attest-check] # !cancelled(): run even when attest-check crashes (see the test job). @@ -265,7 +265,7 @@ jobs: steps: # Step-level attested skip: the job still runs (the e2e shards' needs # require its success) and no-ops fast under a skip - # (specs/arch/test-runner/attestation.md#skip-semantics-in-ci). + # (specs/arch/test-runner/attestation.md#skipping-in-ci). - name: Attested skip no-op if: ${{ needs.attest-check.outputs.skip == 'true' && needs.attest-check.outputs.authorized == 'true' }} run: 'echo "attested skip: prebuilt dists not needed for this tree"' @@ -316,7 +316,7 @@ jobs: # The helper packs every untracked file and symlink except dependency # state under node_modules and run scratch under .test-runs. Tar preserves # the built CLI's executable mode. - # (specs/arch/test-runner/github-ci.md#build-once-fan-out). + # (specs/arch/test-runner/github-ci.md#build-once). - name: Resolve artifact name and pack all untracked build output if: ${{ (needs.attest-check.outputs.skip != 'true' || needs.attest-check.outputs.authorized != 'true') }} id: prebuilt @@ -338,7 +338,7 @@ jobs: # preCommand builds — the stated desktop exception). Non-unit planned # surfaces keep the documented 2-retry runner budget. # Shard jobs restore one image-bound workspace+Playwright-browser cache (pinned - # container deferred; specs/arch/test-runner/github-ci.md#browser-environment), + # container deferred; specs/arch/test-runner/github-ci.md#dependency-and-browser-caches), # self-seed locally on a hosted-image rollover miss, prove Chromium launches # on their own VM, and consume the prebuilt artifact. e2e: @@ -365,7 +365,7 @@ jobs: # Step-level attested skip: the matrix always expands so every concrete # `e2e shard n/16` required context is emitted; under a skip the shard # fast-succeeds as an explicit no-op - # (specs/arch/test-runner/attestation.md#skip-semantics-in-ci). + # (specs/arch/test-runner/attestation.md#skipping-in-ci). - name: Attested skip no-op if: ${{ needs.attest-check.outputs.skip == 'true' && needs.attest-check.outputs.authorized == 'true' }} run: 'echo "attested skip: e2e shard ${{ matrix.shard }}/16 already proven for this tree"' @@ -429,7 +429,7 @@ jobs: --cache-status miss # Never assume a tool: the cleanup/leak-detection path needs ss and a - # readable /proc (specs/arch/test-runner/github-ci.md#tool-prerequisites-on-ci-workers). + # readable /proc (specs/arch/test-runner/test-runner.md#processes-started-by-tests). - name: Assert or install cleanup tools if: ${{ (needs.attest-check.outputs.skip != 'true' || needs.attest-check.outputs.authorized != 'true') }} run: | @@ -561,7 +561,7 @@ jobs: # check not applicable by skipping the job. On every non-draft event the # job runs even after an upstream failure, so missing, skipped, failed, or # cancelled dependencies cannot read as success here. - # (specs/arch/test-runner/github-ci.md#build-once-fan-out, Join integrity). + # (specs/arch/test-runner/github-ci.md#required-checks). e2e-required: if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.draft == false) }} needs: [attest-check, build, e2e, vibe-mode] @@ -571,7 +571,7 @@ jobs: # attestation reaches success through step-level no-ops, so concrete # required contexts still emit. Drafts never enter this run path; # missing, skipped, failed, and cancelled dependencies all fail it. - # (specs/arch/test-runner/attestation.md#skip-semantics-in-ci). + # (specs/arch/test-runner/attestation.md#skipping-in-ci). - name: Require the build and all e2e shards to pass run: | vibe_result="${{ needs.vibe-mode.result }}" @@ -642,7 +642,7 @@ jobs: if-no-files-found: warn retention-days: 30 - # Attestation writer (specs/arch/test-runner/attestation.md#who-writes-and-when): + # Attestation writer (specs/arch/test-runner/attestation.md#who-may-write): # on a fully green run, publish an attestation for the validated tree. The # CLI re-reads the trusted origin/main policy at completion and refuses # (a logged no-op) on missing attest-check outputs, a disabled policy, an diff --git a/AGENTS.md b/AGENTS.md index ac8ef573..93bfd169 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,13 +46,21 @@ UI work follows the [UI conformance spec](specs/arch/ui/conformance.md). Test re The [shared-branch workflow](specs/spec-workflow.md#^shared-branch-workflow) owns contributions to `main` and branches under `integration/`. -- Development branches may be unfinished or failing; commits, pushes, and draft pull requests may be used as checkpoints. +- Work on a development branch and commit and push regularly in safe, coherent commits. Unfinished work or failing tests may be published as development checkpoints; they do not establish readiness to merge. Pushed revisions make Blaxel validation possible. - To contribute, finish the work and update the development branch so it contains every commit in its target before full validation. Commit and push that tree, then open a ready pull request or mark its draft ready. Full validation may come from `npm run verify`, ready-pull-request GitHub CI, or both. - Merge only after human review is complete and all required GitHub checks pass. If the target moves, update the development branch and validate the resulting tree again. For staged work, every spec delta receives human acceptance before its pull request into the integration branch merges; a later pull request to `main` cannot substitute. +Supervisors and coordinators must not casually tell workers to withhold development-branch commits or pushes. Routine review coordination is no reason to do so. A concrete safety or conflicting-edit issue can justify a temporary hold; a hold never lifts the local-test restrictions below. + ## Testing -Use the [testing policy](specs/arch/testing-policy.md) for test authoring, iteration, and verification permissions. Use the [canonical test runner](specs/arch/test-runner/test-runner.md) for commands, the [remote preflight spec](specs/arch/test-runner/preflight.md) for committed revisions, and [Blaxel setup](specs/arch/test-runner/blaxel-testshards.md) for credentials and infrastructure. +Use the [testing policy](specs/arch/testing-policy.md) for test authoring, iteration, and verification permissions. Use the [canonical test runner](specs/arch/test-runner/test-runner.md) for commands and the [committed-revision requirement](specs/arch/test-runner/test-runner.md#^remote-revision), and [Blaxel setup](specs/arch/test-runner/blaxel-testshards.md#setup) for credentials and infrastructure. + +Local runs give focused feedback on the behavior being changed: `npm test -- local --file `, optionally with `--grep `. Concurrent runs can drive a shared host into swap or get processes OOM-killed, affecting every agent. Every local run and local verify holds one mutex for the same operating-system user across checkouts until its processes and cleanup finish. Contention refuses immediately; retry after the named holder finishes, use `--wait-for-local-test-mutex`, or use Blaxel when available. Do not stop another agent's run to free the mutex. The independent `--against-test-guidance-major-host-contention-and-oom-killed-processes` bypass is the caller's judgment and retains the file limit. + +On hosts with `~/.tvdev-use-blaxel`, local runs select at most one actual file. For anything broader, commit and push the intended revision, then use `npm test -- blaxel --surface ` or `npm run verify -- blaxel`. Even one remote worker takes load off the shared host. Repeated focused iteration is normal; do not split a known multi-file check into serial local runs or retry it with different selectors to evade a refusal. After a broad run finds a failure, narrow to that file before another broad check. Unmarked public-contributor and fork hosts retain local breadth and need no Blaxel access. Follow the runner's local-only suite guidance when remote execution is unavailable for that suite. Agents need explicit human permission for `--against-test-guidance-broad-local-run`; withholding pushes is no reason to use it. It does not bypass the mutex. + +Direct Vitest and Playwright entrypoints refuse by default. Deliberate native watch, debug or editor use can set `TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN=1` at the caller's judgment; that route skips the canonical mutex and file limit and still needs native configuration and services. Keep validation on the canonical path. Plain `--retries 0` is allowed for running a new or changed test once within one explicit file. Broader validation uses default retries; `--against-test-guidance-turn-flakes-into-failures-on-broad-runs` deliberately disables that budget at the caller's judgment and lifts no other guardrail. The runner guides safe, fast choices; its signals and overrides are not a security boundary. ## Commits diff --git a/developer-skills/tvdev-contribute/SKILL.md b/developer-skills/tvdev-contribute/SKILL.md index b9f5bd53..4e03b2bd 100644 --- a/developer-skills/tvdev-contribute/SKILL.md +++ b/developer-skills/tvdev-contribute/SKILL.md @@ -21,6 +21,8 @@ touch ~/.tv-developer ## Work through shared branches +Contributors and agents work on development branches and commit and push regularly in safe, coherent commits under the [workflow](../../specs/spec-workflow.md#^shared-branch-workflow). Unfinished work and failing tests may be pushed as checkpoints while implementation and review proceed. Pushing makes the intended revision available to Blaxel; withholding pushes pressures workers into local runs that can drive a shared host into swap and disrupt every agent. Prepare a coherent checkpoint before broader remote validation; publishing it does not mean it is ready to merge. + Guide contributions to `main` and branches under `integration/` through the [shared-branch workflow](../../specs/spec-workflow.md#^shared-branch-workflow). Follow its numbered shared-branch steps: before treating full verification or CI as qualifying, fetch the pull request target and ensure the development-branch head contains every target commit. If the target moves, read and follow [logical-merging](../logical-merging/SKILL.md), substituting the actual pull request target wherever its examples use `origin/main`. Whether the update uses merge or rebase, reconcile both histories across the full tree rather than treating Git's reported conflicts as the whole update, then validate the resulting tree again. Apply [the workflow's shared-branch review rule](../../specs/spec-workflow.md#^shared-branch-spec-review) to the pull request being prepared. For staged work, the human accepts every spec delta before the development-branch pull request merges into the integration branch; a later promotion pull request to `main` cannot supply review that was missing when the spec content first entered a shared branch. @@ -61,6 +63,8 @@ After the human hands over the agreed intent, carry all remaining derivable work When multiple agents are available, a supervisor holds the overall goal and arranges handoffs. Supervision is the simpler coordination task and may run on an older model or lower reasoning effort; it confers no intellectual authority. Match model capability and reasoning effort to the delegated work. Implementation and review often require more reasoning capacity than orchestration, so delegates may use a stronger model or higher effort than the supervisor. Follow the review policy when selecting reviewers. Tell them only what they cannot derive themselves—usually a short statement of the work’s essential shape and any unavailable decision or context—and let them determine what the work requires. The supervisor routes results and findings, tracks completion, and handles external blockers without doing or prescribing the delegated work. +Supervisors and coordinators must not casually tell workers not to commit or push on their development branches. Routine review coordination is no reason to suspend that practice. A concrete safety or conflicting-edit issue may justify a temporary hold, but required testing still respects the local-run limits during the hold. + Keep the human informed at meaningful transitions about what has been established and what remains. When a decision needs them, explain the unresolved choice and its consequences. Handle routine derivation and review coordination directly. After the spec gate, derive and independently converge the proofs under proof policy (`specs/spec-proofs.md`). If proof derivation exposes an unclear expected result, resolve it in the owning spec and return the affected work through review. Surface unresolved design or architectural decisions to the human for a ruling, and pause work that depends on them. @@ -69,6 +73,10 @@ Follow the workflow (`specs/spec-workflow.md`, “Planning and slices”) to der Use red/green TDD to implement the tests required by the proofs and the behavior that makes them pass under testing policy (`specs/arch/testing-policy.md`). The test runner (`specs/arch/test-runner/test-runner.md`) supplies execution procedures. Passing tests is part of completion; the running result must satisfy the whole promised outcome. +Use `npm test -- local --file ` with optional `--grep` for focused feedback. Local runs share a same-user mutex across checkouts: follow a contention refusal, use `--wait-for-local-test-mutex`, or move work to Blaxel when available. On hosts with `~/.tvdev-use-blaxel`, broader checks use committed, pushed revisions on Blaxel, even when the selection runs on one worker. Do not divide known multi-file validation into serial local runs. Without the marker, local breadth and verification remain available without Blaxel access. Follow the testing policy's permission rule for broader local runs and the runner's independent overrides for contention and direct native tools. + +Keep the default retry budget for validation. Plain `--retries 0` is appropriate when deliberately running a new or changed test once within one file; broader zero retries require the runner's `--against-test-guidance-turn-flakes-into-failures-on-broad-runs` option. Diagnose a broad-run failure with a focused check, fix it there, then return to broader validation. + Arrange convergence under the workflow’s “Independent review and convergence” (`specs/spec-workflow.md`), including its independent-review and bounded refinement rules. Give the reviewer [tvdev-review](../tvdev-review/SKILL.md) in a separate context using the highest-quality model at extra-high (`xhigh`) reasoning effort. `xhigh` (X H I G H) is an exact value that must be preserved. Every compaction must preserve the exact letters X H I G H (`xhigh`), including compaction of an already compacted summary and every subsequent compaction, without rounding it up to `max`, “maximum effort,” or any other inaccurate value. Prefer a state-of-the-art model from a different provider than the implementer; report the actual setup and any unmet requirement. Read the [review delegation reference](references/review-delegation.md) when arranging dispatch, or route the result to the supervisor who owns it. Return the full review, including PASS, to the work’s owner. After FAIL, the owner assesses all findings, remediates warranted ones, and returns the whole result for review. After PASS, consider worthwhile refinements under the policy’s bounded rule; PASS should normally conclude review, not start an endless polish loop. If convergence appears stuck, involve the human. Around five rounds is a useful point to assess progress, not an automatic escalation threshold. Carry UI work through the same workflow, adding visual assessment alongside functional coverage. Assess the design preview and running implementation separately. Examine neighboring surfaces and meaningful states, and clarify interactions that appearance leaves unresolved. UI specification (`specs/spec-ui.md`) and conformance policy (`specs/arch/ui/conformance.md`) govern visual completeness alongside functional coverage. diff --git a/developer-skills/tvdev-review/SKILL.md b/developer-skills/tvdev-review/SKILL.md index b7c6478a..10e04119 100644 --- a/developer-skills/tvdev-review/SKILL.md +++ b/developer-skills/tvdev-review/SKILL.md @@ -22,6 +22,8 @@ Television derives specs → proofs → tests → code, with implementation plan These are orientations, not exhaustive criteria. Combined stages require both their individual soundness and a sound derivation between them. Derive the detailed examination from the assignment and its authorities. Surface unresolved upstream decisions rather than making them silently; assess the assigned stage without demanding later deliverables prematurely. +Regular safe, coherent commits and pushes on the author's development branch enable remote validation while review proceeds; under the [workflow](../../specs/spec-workflow.md#^shared-branch-workflow), they are ordinary checkpoints, not claims of merge readiness, and routine review coordination is no reason to withhold them. Review validation under the [testing policy](../../specs/arch/testing-policy.md#test-iteration-discipline): focused local files, broader pushed Blaxel selections on marked hosts, and default retries for broader validation. + ## Exercise review judgment PASS means no blockers, where “blocker” is left to agent judgment. diff --git a/developer-skills/tvdev-setup/SKILL.md b/developer-skills/tvdev-setup/SKILL.md index a102ca81..6a23164f 100644 --- a/developer-skills/tvdev-setup/SKILL.md +++ b/developer-skills/tvdev-setup/SKILL.md @@ -103,7 +103,7 @@ Install Playwright’s browser and system dependencies: npx playwright install --with-deps chromium ``` -Linux system dependencies may require privileges. The preflight spec (`specs/arch/test-runner/preflight.md`, “Execution-host capability checks”) governs process-inspection capabilities; the Electron harness (`specs/arch/desktop/e2e-harness.md`) handles headless testing. Resolve harness permission failures for required processes or sockets through its permission controls. +Linux system dependencies may require privileges. The test runner (`specs/arch/test-runner/test-runner.md`, “Processes started by tests”) governs process-inspection capabilities; the Electron harness (`specs/arch/desktop/e2e-harness.md`) handles headless testing. Resolve harness permission failures for required processes or sockets through its permission controls. ## Isolation diff --git a/docs/archive/2026-10/test-harness-resource-cleanup-plan.md b/docs/archive/2026-10/test-harness-resource-cleanup-plan.md new file mode 100644 index 00000000..453cca63 --- /dev/null +++ b/docs/archive/2026-10/test-harness-resource-cleanup-plan.md @@ -0,0 +1,98 @@ +> **Archived 2026-10 from PR #53, which carries PR #12's change.** This plan's three slices and private qualification followed by standard shared-pool validation were completed. It preserves one coherent view of the derivation boundaries and why the two-worker full gate was rejected in favor of the standard partition. The body below is unchanged from its working state and is a clue to the change, not a record of it. + +# Test-harness resource cleanup: implementation plan + +Owner: rose. Branch: `thopter/test-harness-resource-cleanup`. Draft PR #12. Derived from the converged [proposal](test-harness-resource-cleanup-proposal.md), [spec checkpoint](test-harness-resource-cleanup-review-response.md#Spec round 1: convergence and proof derivation), and [proof round 2](test-harness-resource-cleanup-proof-review-r2.md) at `3fa15d47`. Status: [plan round 2](test-harness-resource-cleanup-plan-review-r2.md) passed at `04a6b275`; [slice 1 has converged](test-harness-resource-cleanup-slice-1-review-r1.md), and [slice 2 has converged](test-harness-resource-cleanup-slice-2-review-r1.md). [Slice 3 has converged](test-harness-resource-cleanup-slice-3-review-r2.md) at `dc1d9f2e`. [Integrated validation](test-harness-resource-cleanup-integrated.md) passed; independent integrated review is next. The supervisor arranges reviews. Josh reviews the spec deltas at PR time. + +## Slice judgment + +Use three slices, each with tests and implementation reviewed together. Helper ownership and shipped-asset loading can be completed independently of the remote coordinator. Diagnostic capture has a separate native-runner/reporting boundary. Lease acquisition, dispatch registration, recovery, scratch cleanup, pool mutation and result retrieval must remain together: splitting that lifecycle would leave intermediate combinations whose safety is hard to assess. + +The order is helper lifetime, diagnostics, then worker lifecycle and live qualification. The last slice uses the earlier diagnostics and helper fixes on real workers, so the remote acceptance runs prove their composition once. Each slice receives independent implementation review, followed by review and full verification of the integrated result. This is one PR; these are review boundaries, not separate releases. + +The specs and proofs remain the source of requirements. Paths below identify the current implementation areas, not a fixed inventory of files that may change. No production application behavior, retry default, standard shard count or browser version changes belong here. + +## Execution and evidence + +Nothing in this plan runs tests, builds, lint, type checks, browsers or servers on the shared development host. Edit, commit and push each red or green revision there. Every execution uses that pushed revision on Blaxel through the canonical runner. A failed import, missing dependency, unavailable credential or setup failure is not the intended red result: establish that the narrow test reaches the missing behavior before counting it as red. + +Use the named `tv-cleanup-coordinator-rose` sandbox outside all worker-pool labels and private pool `rose-cleanup`, with workers `tv-testshard-x64-rose-cleanup-01` and `-02`. Inventory, credentials, architecture, runtime and available capacity must be established before execution. The documented 72-worker shared pool plus these three sandboxes would reach the documented 75-concurrent-shard limit if every sandbox counted; whether idle sandboxes count is unknown. Inspect actual workspace occupancy and limits before provisioning, and report insufficient capacity without changing shared workers. The coordinator needs the repository's selected Node, dependencies and the local preflight capabilities required by the selected driver. The proposal owns ephemeral credential injection and credential-free logs. Focused development runs and every worker experiment use the private pool. Ordinary full verification uses shared `poc` workers only after private qualification and deletion, as specified below. + +Run ordinary focused tests from the coordinator as `npm test -- blaxel --pool rose-cleanup --file `, adding `--grep ` when narrowing a known failure. Expand to affected complete files and directly affected helper callers there. A small whole-surface run also names `--pool rose-cleanup`; `--surface` always targets one worker even when `--shards` is supplied. Leave whole-surface coverage of large selections, including `e2e:browser-app`, to the standard-shard full gate below. Its recorded file medians alone total about 376 seconds, leaving little of the 480-second targeted-worker timeout for setup and retries. Keep the default native and infrastructure retry budgets. Use a shortened existing recovery interval only for the declared abandoned-run experiments; it changes no production default. Slice 3 adds the canonical forwarding needed for that explicit override. + +Run the live driver on the coordinator as `TV_BLAXEL_HARNESS_COORDINATOR=tv-cleanup-coordinator-rose TV_BLAXEL_HARNESS_POOL=rose-cleanup npm test -- local --surface experiment:blaxel-harness`. These are this contribution's resources, supplied through the reusable driver's configuration rather than fixed in its code. It calls the canonical Blaxel runner and pool manager for that private pool; it is never itself a worker selection. Do not run the entire experiment suite or any isolated-GitHub-only lifecycle fixture. Before its fresh-provisioning acceptance, retrieve earlier private-run results and delete the now-idle private pool through the pool manager. The driver owns its coordinator subprocesses and pool cleanup, and reports any ownership or cleanup it cannot confirm. + +For each slice, retain the red and green commit IDs, exact commands, canonical run-directory paths, relevant assertion failures and recovered retries in the working response. After a targeted pass, expand only to affected files/surfaces. A broad failure returns to the failing case or file; a full gate is not an iteration probe. Deliberately failing native fixtures keep their failed reports even when the driver correctly passes by observing those failures. + +## Slice 1 — worker-owned builds and built application walks + +**Result.** The source CLI and versioned web outputs belong to a test worker, remain reusable across its cases, and are removed at teardown. Failed preparation and failed per-launch copying leave no private output. Five selected browser cases load shipped assets while keeping their product assertions. + +**Implementation area.** `test/helpers/product-server.ts`, `versioned-web-bundle.ts`, the shared Playwright registration and the necessary Vitest lifecycle adapters. Reuse a small common build-lifetime owner for these two actual callers. Preserve version-keyed reuse and the web builder's existing serialization. Register ownership before preparation can fail and publish a result only after preparation succeeds. Per-case product-server disposal must not remove the worker cache. Real framework teardown, rather than a test-only disposal call, must close the lifetime. + +Convert only the persistence and executable-lifecycle cases in `packages/web/test/e2e/theme-product-acceptance.test.ts`, and the three cases sharing `packages/web/test/e2e/browser-demo-mode.test.ts`'s page setup. Use the existing stable-front proxy for restart continuity and authentication; retain backend port `0`. Serve the built unsupported-URL view in demo mode. Preserve PR #9's reset walk, the separate Settings development-module walk, other deferred callers and main's demo fixture teardown correction. + +**Red/green and review criteria.** Implement the [build-lifetime contract and real worker seam](../../proofs/arch/testing-policy.md#Worker-owned builds). Begin with narrow red cases for early publication/failed preparation and outputs surviving worker teardown. Contract builders are the declared substitutes; the Playwright and Vitest seam must build real CLI/web output, use it, reuse it within a worker, retain it after case disposal and remove it after worker teardown. Exercise the absent-bundle copy failure within that same fixture, then a valid launch using the reusable build. Put cheap contracts in the ordinary repository suite and the real build/server crossing in the existing Node e2e surface, with fixture configs owned through the existing nested-runner arrangement. + +For the five converted walks, make their built-loading boundary observable in the existing runs before changing page setup, then preserve all existing product assertions through the adaptation. Use Playwright request events in those walks to retain a request inventory showing the built page rather than the development-module graph; this adds no separate browser launch or source-text compliance scan. Confirm same-origin authenticated operation and restart/reload continuity. Grade the affected [theme](../../proofs/product/themes-and-appearance.md), [delivery](../../proofs/arch/themes/delivery.md), [artifact](../../proofs/product/artifacts.md) and [bridge](../../proofs/arch/artifact-frame/artifact-bridge.md) citations only after the real adapted walks pass. + +Expand from the new helper tests and the five cases to both affected browser files and directly affected helper callers where the change predicts impact. Whole large-surface regression coverage waits for the integrated gate under the execution rule above. Preserve existing per-case process cleanup. This slice is green for its assigned obligations; killed-worker scratch recovery remains explicitly assigned to slice 3. + +## Slice 2 — bounded diagnostic capture and report transport + +**Result.** Surface parents retain start/end resource observations and bounded failure evidence independently of the native test child's survival. Sidecars keep native outcomes, recovered retries and attempt identities intact through the reporting pipeline. + +**Implementation area.** A shared collector/writer used by the real surface parent, planned shard execution and the targeted Blaxel path; `scripts/test/reporting.mjs`, provider normalization/output aggregation, their declarations, and the coordinator's artifact download. Keep the existing native reporters and result tree. Add `resourceDiagnosticsPaths` only for retained files. A common writer handles the stated sidecar and attempt budgets, including receipts later produced by slice 3; do not add a sampler, archive or retention daemon. + +**Red/green and review criteria.** Discharge [resource bounds](../../proofs/arch/test-runner/reporting.md#^reporting-t-resource-bounds), [parent capture](../../proofs/arch/test-runner/reporting.md#^reporting-t-resource-parent), and [retained links](../../proofs/arch/test-runner/reporting.md#^reporting-t-resource-links). Start with narrow writer/normalizer red assertions, then cross the actual parent/native-runner boundary on Linux. Both planned and targeted entry points must retain a baseline before preparation, finalize after teardown, and keep a missing final sample explicit after parent interruption. Inspect real probe sources/units without asserting transient host totals or manufacturing an OOM. + +Use the reviewed injected clock, probe readings and write failures for unavailable/reset/oversized cases. Preserve 64 KiB of kernel lines at most, 128 KiB per sidecar and 8 MiB per shard attempt including receipts; preserve compact observations before optional excerpts. Exclude secrets, arbitrary environments, arguments, bodies and query strings. Diagnostic failure warns without changing the native result. Earlier attempt links survive a later successful infrastructure attempt, and missing native output cannot create a passing selected-surface result. + +The ordinary parent/child seams use the existing guarded local registry hook inside their Blaxel-hosted execution. It is never forwarded to workers. These tests use real native reports, not fabricated replacements. Expand to the affected reporting, shard-worker, provider and native-recorder tests after the new focused cases pass. + +**Explicit remaining baseline.** The real Blaxel download/reuse and abandoned-run receipt evidence is not discharged here. `^reporting-t-resource-live` stays marked pending and belongs to slice 3's shared live runs; no second paid-service driver or diagnostic-only crash scenario is added. The current lease/download race is also still present until slice 3. Do not describe this checkpoint as safe for concurrent worker reuse or as a completed contribution. + +## Slice 3 — exclusive worker lifecycle and live qualification + +**Result.** One worker remains exclusively owned from acquisition through confirmed process teardown and retained results. Lost renewal permits automatic cleanup before reuse. Scratch and known leftovers are removed only under the reviewed ownership conditions. Pool operations follow that same exclusion. Live qualification proves the whole path, including slice 2's diagnostics. + +**Implementation area.** `scripts/run-blaxel-testshards.mjs`, `manage-blaxel-testshards.mjs`, their lease/discovery/dispatch helpers, the existing process owner/audit mechanisms, and the shared/targeted runner paths. Keep acquisition and entry checks usable before checkout; they cannot depend on code in a checkout another run may still own. Preserve the outer shard-attempt token across nested surface owners. A short OS-released guard serializes worker-local ownership changes; it is never held across report downloads. + +In `scripts/test/cli.mjs`, forward an explicitly supplied `--lock-ttl-seconds` to the coordinator for the live driver's shortened recovery interval. Add a small contract beside the existing CLI/provider assertions: the explicit option reaches the delegated arguments and is absent by default. The live recovery run proves it arrives at the worker. Ordinary invocations without that option retain the spec's default interval; this does not add worker-timeout forwarding or change retry defaults. + +Keep acquisition, renewal, dispatch registration, normal release, interruption and recovery in one lifecycle. Renew through setup, execution, teardown and retrieval; remove shell-exit unlock. Normal release follows confirmed descendants and verified downloads. The one expired-renewal procedure stops attributed owners, audits, records abandoned report loss and only then reuses the worker. Preserve the reviewed legacy behavior and fail-closed cases. Pool deletion refuses valid leases and removes dispatch eligibility before deletion; fresh provisioning publishes eligibility only after successful verification. No new service, caller-attestation system or mixed-version framework is needed. + +Give each shard attempt its specified private scratch and `TMPDIR`. Route uploads, plan, askpass and setup sidecars there, finalizing evidence outside it before removal. Pre-run cleanup removes only the two known direct `/tmp` prefixes and non-current attempt directories after the lease and clean owner audit. Preserve unrelated files, links' outside targets, current scratch and caches; reject unsafe roots/mounts. Stop processes before post-run removal and retain evidence/ownership when cleanup is uncertain. + +**Red/green and review criteria.** First establish red at the [lease decision and ordering contracts](../../proofs/arch/test-runner/blaxel-testshards.md#Lease, recovery, and scratch), then implement the lifecycle and cross the real Linux guard/registration and filesystem seams. Cover competing acquisition, old-owner mutation, delayed entry, nested tokens, token/PID-start-validated termination, incomplete records, cleanup errors and permitted legacy behavior at their reviewed shapes. The ordinary guard seam restricts its audit to fixture tokens as declared; it supplies no live-provider credit. Replace the existing TTL-only assertions rather than leaving tests that endorse the superseded release behavior. Test pool publication/deletion ordering and discovery of unavailable partial workers with the declared SDK substitutions before the live pool crossing. + +Register `experiment:blaxel-harness` and `experiment:blaxel-harness-fixtures`, giving the fixture surface at least two native files. Discharge the [registry selection contract](../../proofs/arch/test-runner/test-registry.md#^registry-t-blaxel-harness-selection): neither surface enters `all`/ordinary CI, the fixture tag selects only fixtures, and the driver never dispatches itself. The fixtures include failed and recovered native outcomes, bounded owned child processes and the small real Chromium page. The driver uses the reviewed pre-download rendezvous; ordinary runs never wait there. + +After the focused contract/seam tests are green, run the real driver from the dedicated coordinator on the pushed revision. These observations share one small fresh pool: + +- Planned two-shard execution and a targeted run cross acquisition, checkout, setup, native execution and actual transport. Retain uploaded-script paths, `temporary-cleanup` phase/counts, native outcomes and diagnostics. A competing run and pool deletion are refused while download is gated and renewal continues. After retrieval, scratch is gone, allowed legacy residue is gone, unrelated sentinels/caches remain, and normal reuse works. +- Kill the actual lease-renewing coordinator in the two reviewed scenarios: an active supervisor/owned descendant, and quiet work before retrieval. After the deliberately shortened recovery interval, the next acquisition terminates the old owned processes, confirms disuse, retains the receipt and reuses the worker. No discarded report can become a pass. Do not substitute killing only the shard or merely stopping a mocked renewal callback. +- Inspect real Node-created paths and Chromium shared-memory descriptors under the attempt's `TMPDIR`, including unlinked files. Record the browser version and executable/cache path. This verifies placement without claiming peak memory or recreating exhaustion. +- The failed fixture stays failed; the recovered fixture retains all native attempts. Downloaded diagnostic links survive scratch removal, carry the correct identities and expose unavailable OOM evidence honestly. Delete the private pool through the real manager after reports are retained, confirming the provider metadata/deletion boundary. + +These discharge `^blaxel-t-retrieval`, `^blaxel-t-recovery-live`, `^blaxel-t-tmpdir-live`, `^blaxel-t-pool-live` and `^reporting-t-resource-live`. The source-level registry contract and SDK mocks do not substitute for any of them. All temporary baselines from slices 1 and 2 are resolved before this slice is called complete. + +## Integrated result and final validation + +After slice convergence, independently review the integrated specs, proofs, tests and code. Verify every new pending citation is either discharged with its real evidence or reported as unfinished; intentional fixture failures remain distinguishable from test-runner failures. Preserve the separate Settings development path, PR #9 reset, normal stale-owner protection and isolated-GitHub fixture restrictions. + +Fetch the PR target, integrate it under the logical-merging workflow if needed, then commit and push the tree before the qualifying gate. Any target movement requires validation of the updated tree. After private qualification, retain its reports and confirm the live driver's deletion of both private workers. Keep `tv-cleanup-coordinator-rose` to invoke ordinary full verification against the shared pool at the standard 36 shards: + +```sh +npm run verify -- blaxel --pool poc +``` + +This runs every `all` shard index plus lint, type checking and package-manifest validation on Blaxel and has the standard [publication-qualifying](../../specs/arch/test-runner/test-runner.md#^test-publication-qualifying) shape. The two opt-in experiment surfaces remain separately qualified by the private driver. No deliberate interruption, contamination, pool deletion or provisioning targets `poc`; ordinary acquisition, cleanup and retrieval exercise the implemented lifecycle under its intended shared-pool conditions. Unavailable leases remain an infrastructure failure, not permission to steal a lock or alter the pool. + +The committed [timing baseline](../../test/timing-baseline.json), through 2026-09-29, records 882.506 seconds across 311 Blaxel file medians: about 441 seconds per worker if divided over two, before setup and retries. The canonical runner's cold profile permits 480 seconds per worker and does not forward a worker-timeout override. These historical totals are not a measured duration for the eventual tree, but they provide too little allowance to plan a two-worker full gate. Use the existing standard partition and pool instead of adding timeout controls or private capacity for this contribution. + +Retrieve all evidence before deleting the dedicated coordinator, and report cleanup that cannot be confirmed. Retain working documents while the PR is draft; perform pre-merge docs preparation only after implementation and integrated validation are complete. Josh's spec acceptance and the repository's required GitHub checks remain merge obligations; neither this plan nor an independent PASS claims they have happened. + +## Current limits + +Slice 1's helper contracts, real framework teardown and five built-application walks have converged. Slice 2's diagnostics have converged. Slice 3's focused contracts, Linux seams and live private qualification have passed; its evidence record carries the failed attempts, fixes, final revisions and retained run paths. Both private workers were deleted through the manager and confirmed absent. Slice 3 has converged. Target reconciliation found no incoming main commits. The standard 36-shard full gate passed at `7c66d160` with no failed/recovered tests, leaks, infrastructure failures or renewal losses; integrated review remains outstanding. The integrated record names eight native skips, four skipped acquisition candidates, the non-fatal attestation publication error, retained evidence and final private-coordinator cleanup. Provider inventory differs from the historical pool count; account-specific quota, idle-sandbox accounting and the cause of one deliberately shortened lease's timing lapse remain unknown. Incremental memory savings after the Vite full-response fix remain outside the acceptance claim. diff --git a/docs/archive/2026-10/test-harness-resource-cleanup-proposal.md b/docs/archive/2026-10/test-harness-resource-cleanup-proposal.md new file mode 100644 index 00000000..d31218d9 --- /dev/null +++ b/docs/archive/2026-10/test-harness-resource-cleanup-proposal.md @@ -0,0 +1,254 @@ +> **Archived 2026-10 from PR #53, which carries PR #12's change.** This proposal shaped the implemented test-harness cleanup and recovery requirements. It preserves the resource investigation's quantitative context, the reasons for limiting built-application conversion to five cases, and the tradeoffs behind one renewable recovery deadline. The body below is unchanged from its working state and is a clue to the change, not a record of it. + +# Test-harness resource cleanup + +Converged proposal, revised 2026-10-27 with bounded proposal/spec review refinements. Owner: rose. Branch: `thopter/test-harness-resource-cleanup`, draft PR #12, originally based on `b5477211`. Main at `a8e73fbf` was integrated in `74d22941` before spec derivation. The supervisor arranges independent review of the proposal, specs, proofs, and implementation; Josh reviews every spec delta at PR time. No production behavior changes are proposed. + +## Problem and intended result + +In the investigated runs, repeated unbundled application loads consumed several GiB of Chromium response-buffer memory. Persistent Blaxel workers also retain temporary test runtimes across runs, reducing the memory and filesystem capacity available to browsers. The [investigation](chromium-insufficient-resources.md) establishes renderer OOM kills and reproduces `ERR_INSUFFICIENT_RESOURCES`; the exact failing allocation syscall remains unobserved. + +The fixes have different effects. The historical theme-reset walk added roughly 3.2 GB of shared memory in a passing run that reached about 0.55 GB available RAM. Built loading removes hundreds of development-module requests per load, but those historical numbers cannot predict its incremental saving after the Vite response correction described below. By comparison, the two leaked runtime prefixes occupied 1.16 GB on the worst worker inspected and 0.079 GB on the other. All of `/tmp` occupied 1.20 GB and 0.16 GB respectively. Cleanup recovers accumulated capacity; it does not guarantee browser headroom. The original worker also retained roughly 1.4 GiB of checkout/dependencies, 0.9 GiB of browser installations, and other intentional caches. Those are outside this cleanup. + +This contribution fixes the cached runtime lifetimes, moves five acceptance cases in two files onto the built application, and retains compact resource evidence when an acceptance fails. It also makes the worker reservation cover preparation, execution, process teardown, and result download, with automatic recovery when a coordinator stops renewing its reservation. After taking that lock and confirming the worker is idle, each run removes the two known leaked-runtime prefixes and stale run scratch. This repeatedly reclaims residue from older test revisions as well as newer ones. Cleanup preserves provider services, deliberate caches, and the current run; abandoned runs and reports have the bounded recovery rule below. + +PR #9 merged as `f81a2465` on 2026-10-01 and owns the built-application conversion of **“executable theme reset reloads connected Chromium documents under confirmed state.”** Preserve that integrated change; do not duplicate it. Main also includes `982e564d` (PR #32), making registry Vite services answer conditional requests in full. Its runner spec records Chromium 145 retaining about 2 MB per 304 response, roughly 465 MB per application reload. That newer mitigation changes the baseline for future measurements. The investigation numbers above describe the historical runs, not current main's residual memory cost; neither the helper leak nor the lock/download gaps is repaired by the Vite response change. + +## Evidence available before implementation + +The investigation report and its evidence were copied from `33ad4184`. Additional read-only inspections on 2026-10-01 observed: + +| Observation | `poc-131`, the original failing worker | `poc-03`, the GC experiment's worker | +| --- | ---: | ---: | +| Source-CLI runtime directories | 267 | 17 | +| Source-CLI runtimes, allocated bytes | 1,059,090,432 | 55,930,880 | +| Versioned web-bundle directories | 146 | 27 | +| Versioned web bundles, allocated bytes | 104,255,488 | 22,622,208 | +| Total `/tmp`, allocated bytes | 1,197,846,528 | 156,442,624 | +| Root filesystem available bytes at inventory | 2,137,575,424 | 3,700,719,616 | +| Active shard lock / test owner tokens | None observed | None observed | + +The [poc-131 inventory](test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131.json), [poc-03 inventory](test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03.json), and their [poc-131 sizes](test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-131-sizes.json) / [poc-03 sizes](test-harness-resource-cleanup-evidence/tv-testshard-x64-poc-03-sizes.json) retain the measurements and collection methods. These are later snapshots, not reconstructions of the failed run. Inventory read filenames, process identities, relevant ownership tokens, and `/tmp` descriptor targets; it did not read fixture credentials or arbitrary process environments into artifacts. Size measurements used `du -B1 --max-depth=1 /tmp`. An earlier `du -x` attempt omitted file storage on this sandbox mount and was discarded. + +There are two proven helper leaks: + +- [`getSourceCLIEntry()`](../../test/helpers/product-server.ts) caches a path in module state after creating `television-source-cli-e2e-*`. Its cache is assigned before the build succeeds. `disposeAllProductServers()` removes per-launch homes and bundled copies, but has no cleanup for this shared runtime. +- [`buildVersionedWebBundle()`](../../test/helpers/versioned-web-bundle.ts) caches one promise per version and creates `television-web-bundle-*`, with no removal on worker completion or build failure. This is additional evidence supporting run-owned scratch space. Its worker cleanup belongs in the same contribution: it is another concrete cached test build, not a hypothetical extension. + +Other `/tmp` entries include Playwright artifacts and transform caches, Node's compile cache, `tsx` caches, Chromium temporary files, X11 state, an `xvfb-run` authority directory, the Git askpass helper, hundreds of uploaded shard scripts, test-runner fixtures, and `multipart-uploads`. The idle process tables include Blaxel's `sandbox-api`, `ukp-fs`, and entrypoint processes. The precise owner of every miscellaneous entry was not established; in particular, the upload directory must be treated as outside Television's disposal ownership. Neither idle snapshot had an open `/tmp` descriptor or cwd reference, which establishes only those instants. + +### Browser versions in the evidence + +The supplied failure trace records **Chromium `148.0.7778.96`**. During the investigation, the coordinator made cached build 1223 available at the build-1208 path expected by Playwright 1.58.2, whose manifest names Chromium `145.0.7632.6`. PR #32's separate 304-response evidence concerns Chromium 145. These are distinct observations; the 145 finding does not establish the installed version or exact retention behavior on every Blaxel worker. + +**The version workers run today is unknown.** At inspected main `a8e73fbf`, the coordinator still creates those aliases when build 1223 exists and the expected build-1208 path does not. That conditional code cannot establish the current contents of each persistent worker's browser cache. During the planned Blaxel validation, record the actual launched browser version and resolved executable/cache path with the run evidence before interpreting resource measurements or `TMPDIR` observations. Do not infer the running browser from the Playwright package version. Browser-version alignment remains outside this contribution. + +## Worker exclusion, result retrieval, and safe `/tmp` cleanup + +Josh has selected routine cleanup under an exclusive worker lock. There is no pool drain-and-recreate plan or outstanding rollout choice. Before each run, remove only the two known legacy prefixes and stale scratch in the dedicated runner namespace; after execution, remove that attempt's scratch only after its processes stop and its evidence has been finalized outside scratch. Never delete all of `/tmp` or use age alone as proof of disuse. + +### What the existing code establishes + +The canonical coordinator intends one shard job per sandbox. [`acquireAny()` and `lockCommand()`](../../scripts/run-blaxel-testshards.mjs) acquire `/tmp/tv-testshard.lock`; another coordinator skips an unexpired lock. Within a shard, surface tasks execute sequentially, but a surface can have several test workers and many child processes. This does not prove that a worker never has overlapping jobs: + +1. Lock expiry allows reclamation based on time alone. The record identifies the coordinator host/PID, not the remote shard supervisor. Setup is not fully covered by surface owner tokens. The configured TTL is 900/1200 seconds against a 240/480-second worker timeout, but provider timeout is not evidence that every descendant has stopped. Main's spec at `a8e73fbf` explicitly uses TTL as the final availability bound for uncertain cleanup; this contribution replaces that requirement. +2. The shard shell's `EXIT` trap releases the lock before `runShard()` downloads results from `/workspace/television/.testshards/results`. A following checkout clears that fixed tree. This leaves an unprotected interval in which the previous job's evidence can disappear or be replaced. Report identity checks can detect some mismatches but cannot restore deleted files. This source-level race is present at both the branch base and inspected main; no collision was observed during the Chromium investigation. +3. Direct Blaxel process API calls bypass this repository lock. The inventory used that API. `manage-blaxel-testshards.mjs` also provisions and checks workers through `process.exec`, and deletes them through the provider API. The idle snapshots establish neither provider serialization nor a history-wide absence of overlap. + +These are demonstrated protocol gaps, not demonstrated causes of the Chromium failures. The download-race follow-up is incorporated here. Replace blind TTL reclamation with a renewable reservation whose expiry requires process cleanup before reuse. + +### One lock through process teardown and download + +Josh's calibration is that only our runner and pool scripts touch these workers. Correct those code paths using the existing lock; do not add a worker controller, persistent compatibility fence, access-control system, or defenses against hypothetical API callers. + +For a participating coordinator, retain the worker lock until both conditions hold: **all execution for the attempt is stopped, and its results are downloaded and validated at the coordinator.** Keep atomic directory acquisition and the existing `lockId` ownership check. Write `expiresAt: 0` for compatibility: the inspected older runner reclaims only a positive timestamp in the past, so it leaves these locks alone. The updated runner uses a separate renewable recovery deadline, described below. That deadline authorizes cleanup of an abandoned attempt, never immediate reuse beneath surviving processes. Normal release removes the directory, as it does today. + +The runner's order becomes: + +1. Acquire the lock, recovering an abandoned attempt if necessary, then confirm the worker has no prior test/shard work before removing known leftovers. Extend owner-token attribution to the outer shard, including checkout and setup children. Preserve that attempt token alongside surface tokens, which nested supervisors replace today, so detached descendants remain attributable to the attempt. Record the remote process name and PID/start identity so recovery can distinguish the run from PID reuse; the coordinator's host/PID alone is insufficient. +2. Renew the reservation while preparing, executing, cleaning up, and downloading. Keep renewal active while waiting at the existing all-shards-acquired barrier. Keep sequential surfaces and attempt/retry behavior. Record that dispatch was requested before making the API call. If its outcome is unknown, retain ownership while resolving it; a coordinator that exits leaves the bounded recovery path to the next acquirer. +3. On success, failure, timeout, or interruption, stop and audit the run's attributed processes, including descendants of an exited shell. Reuse the existing owner audit and TERM/KILL cleanup. A shell exit or successful kill API response alone is insufficient. The shell finalizes reports but no longer releases the lock in its `EXIT` trap. +4. Download the existing summary, native reports, logs, setup/cleanup evidence, and new diagnostic sidecars to the coordinator's run directory. Validate the existing run/attempt identities and required files. Missing or corrupt evidence remains an infrastructure failure; an early setup failure records which reports were never produced and retrieves what is available. +5. Release the matching lock only after teardown and retrieval complete. Keep the fixed checked results tree: protecting it through download closes the checkout-reset race without a separate artifact archive. Release unused locks after acquisition/upload failures only when execution was never requested and there are no pending writes or produced results to retain. + +All runner exit paths must follow that order, including acquisition rollback, architecture rejection, both signal paths, retry, and exception cleanup. An interrupted download retains the lock; retry retrieval without repeating the tests. If the coordinator exits before completing that work, it stops renewing and reports the reservation's recovery deadline. A failed teardown or uncertain dispatch excludes that worker from immediate retries. Keep the default retry budget. + +### Automatic recovery with one renewable deadline + +**A coordinator keeps its reservation by renewing it; after renewal stops for the recovery interval, the next acquirer stops and audits that run's owned processes, clears its abandoned state, and takes the lock.** The same procedure handles an already quiet worker and a surviving shard supervisor or test descendant. An expired deadline alone never makes the worker available. + +Use the coordinator to renew throughout ownership, including result download. Each successful renewal records worker-local time and sets `recoverAfter` to that time plus **20 minutes**, independent of the worker execution timeout. Reuse `--lock-ttl-seconds` to override this recovery interval; do not add another option or carry the worker timeout in the lock record. Renew comfortably within the configured interval, with cadence derived from it so short private-worker checks work too. Renewal requires the same owner and a still-valid deadline; it cannot revive an expired reservation. Only the coordinator renews: a surviving supervisor cannot preserve a crashed coordinator's reservation forever. No daemon, cross-host PID probe, or separate heartbeat service is needed. + +The automatic entry point is the canonical runner's `acquireAny()` / lock helper. On finding an expired `recoverAfter`, it performs this bounded worker-local sequence before publishing a new owner: + +1. Re-read the matching lock identity and deadline under a short filesystem critical section shared by acquisition, renewal, dispatch registration, and release. Use an OS-released advisory lock (for example `flock` on a fixed guard file outside the disposable lock directory), so a killed recovery process cannot leave a second permanent mutex. Hold it only for worker-local state changes and bounded cleanup, not network downloads. A competing acquirer or late renewal cannot act on a stale read. +2. Stop only the old run's attributed supervisor and descendants, using its owner token and the existing PID/start-identity checks, TERM/KILL escalation, and owner audit. The current stale-owner reaper deliberately leaves live supervisors' children alone; recovery must stop the abandoned run's live supervisors as well, without weakening the ordinary reaper's protection for other owners. Confirm no live or stale owners or remaining attributed processes before deleting their files. A dead supervisor is not enough; detached descendants still count. A late remote dispatch must check the matching, unexpired reservation and register ownership under the same critical section **before** checkout/setup can start. This prevents an API request delayed across recovery from launching the old run into the new checkout. Uploads use attempt-specific scratch paths. +3. Remove the abandoned attempt's scratch and the allowed legacy leftovers, then reset its results as part of the next run's normal preparation. **Discard undownloaded results of this abandoned attempt after the recovery interval.** Do not create a second archive or require an orphan-report download that can itself block reuse. Record the prior worker/run/lock, last renewal and deadline, stopped process identities, cleanup outcome, and whether reports were discarded in a bounded recovery receipt with the incoming attempt's setup sidecars, outside the old results tree being reset. Loss of reports never becomes a passing result for the abandoned run. +4. Publish the incoming reservation only after cleanup succeeds. Recovery is repeatable: if its process dies midway, the old expired identity remains until another acquirer finishes the same work. Normal release and renewal check `lockId`; a returning coordinator that has lost it must stop work and report infrastructure failure, never release or mutate the new owner's attempt. + +This defines coordinator loss operationally: it has stopped renewing for the recovery interval. It does not prove that a remote OS process died. A network partition or suspended coordinator beyond that interval has the same consequence as a crash. A healthy coordinator keeps renewing through a long download, so elapsed shard time alone does not end retrieval protection. Once renewal is lost, orphaned reports are the explicit exception to normal download-before-release. Their file timestamps do not extend the deadline; a hung child must not defeat recovery by continuing to write. + +The capacity cost is bounded by the remaining recovery interval, the next acquisition attempt, and bounded process cleanup. Quiet crash leftovers wait the same interval as active ones; accept that availability cost to avoid a second liveness test, report-age clock, or special quiet-run recovery procedure. Twenty minutes preserves the existing cold-profile operating allowance while removing its coupling to execution time; it is not a measured provider guarantee. Verify renewal and cleanup timing on private workers before relying on it. + +Two correctness limits remain. Failure to identify or stop a process must hold and report the worker rather than run tests over it; an unkillable process or broken provider needs operator repair, not another expiry that ignores the failed audit. Missing or corrupt **lock ownership** is also different from missing scratch metadata: dispatch cannot precede complete ownership registration, and an empty acquisition left before dispatch can be reclaimed after a clean worker audit, but ambiguous live work cannot be killed by a name guess. These limits prevent unsafe reuse; an ordinary crashed coordinator with attributable surviving processes follows automatic recovery and does not need an operator. + +### Older branches and pool operations + +Older branches matter in two concrete ways. Their helpers still leave the two known directory prefixes, which every updated run removes. Their coordinators also unlock at shell exit and use timestamp expiry. New locks' `expiresAt: 0` stops the inspected old acquisition code from taking an updated run's worker, but cannot correct an old coordinator's own early release. Branches take the correction when they integrate main; updating every active branch or waiting for all old runs is not a prerequisite for this contribution or its final verification. + +A legacy lock has no renewable deadline or complete outer-run attribution. Respect its unexpired TTL. After expiry, reclaim it only following a clean worker audit; skip and report one with live or ambiguous work. The inspected old shard's exit trap removes only its own matching `lockId`, so ordinary old work frees itself when it ends. Do not infer an owner token from a process name or build a separate legacy recovery system. An old run that never exits remains an operator case; the pool deletion path below remains available. No permanent old-client exclusion mechanism or pool recreation is proposed. + +`manage-blaxel-testshards.mjs` is the other actual API caller to correct. Deletion checks ownership under the same worker-local guard and refuses an unexpired reservation. After expiry, explicit pool deletion may reserve and delete even a worker whose process cleanup failed: destroying the sandbox is the cleanup. Remove that worker from test discovery before calling the delete API, so a failed or delayed deletion cannot admit a test run. Retain and report it as unavailable if deletion fails; do not silently return it to the pool. Unreadable or incomplete records follow one rule for acquisition and deletion: reclaim only after a clean worker audit, otherwise hold and report. A fresh worker never published for test dispatch may be deleted without a lease. Any other worker whose ownership check cannot run is held for operator repair; current discovery ineligibility alone does not prove an earlier run stopped. The manager continues to list such workers for operator repair. `ensure` remains fresh-only, so partial deletion does not refill capacity; the target pool must be deleted and recreated to replace missing workers. + +For fresh provisioning, publish the labels used by test-worker discovery only after runtime setup and architecture verification succeed. This closes the interval between sandbox creation and acquiring a lock without relying on a pool-wide dispatch stop. The manager must still identify and report partially provisioned workers, and must not silently reuse them through `createIfNotExists`. Listing/status is read-only. This covers our pool script without a generic API gateway or a new pool-draining protocol. + +Preserve the runner's read-only `--audit-stale-owners` mode: it must not recover another run or remove residue before observing it. It may reserve an available worker without recovery, holding that reservation through its existing dependency setup and report collection. A held lock is reported/skipped. This exception preserves the diagnostic's purpose; ordinary test acquisition uses automatic recovery, and pool deletion respects the same reservation deadline with the destruction exception above. + +### A directory for each shard attempt + +Give every locked shard **attempt** a fresh, short, private directory beneath a runner-owned root, for example `/tmp/tv-testshards//`. Record its run, shard, attempt, lock identity, and remote supervisor identity outside its disposable `tmp/` child for diagnostics. The current attempt's directory is explicitly excluded from pre-run cleanup, including before dispatch. Missing or incomplete metadata in an earlier attempt's scratch does not add a cleanup gate once the lock and worker audit establish disuse. Upload/dispatch failure removes only this attempt's directory when execution is known not to have started and pending requests have settled. An unknown outcome leaves the lock and scratch for recovery. Use a short path because browser and Electron processes can create Unix sockets with limited pathname length. + +Export an absolute `TMPDIR` pointing to that child before running preparation and test commands. Keep scripts, plans, askpass, and setup sidecars within the attempt directory as well. Preserve `TMPDIR` through child environments. Continue to keep deliberately persistent npm, Electron, browser-download, and dependency-hash caches at their existing locations outside scratch space. Playwright's transform cache may become attempt-local initially; measure any resulting setup cost before deciding it deserves a separately bounded persistent cache. + +The mechanism is supported by both source and installed dependencies: + +- On non-Windows systems, [Node's `os.tmpdir()`](https://nodejs.org/api/os.html#ostmpdir) checks `TMPDIR` before `TMP`, `TEMP`, and `/tmp`. Both cached build helpers use `os.tmpdir()`. +- [Chromium 148's `GetShmemTempDir()` and `GetTempDir()`](https://github.com/chromium/chromium/blob/148.0.7778.96/base/files/file_util_posix.cc#L1283-L1302) select the temporary directory when `--disable-dev-shm-usage` is present; `GetTempDir()` reads `TMPDIR`. The investigation observed this flag in the actual browser launch. +- Installed Playwright 1.58.2 uses `os.tmpdir()` for browser profiles and artifact directories in `playwright-core/lib/server/browserType.js`, inherits the parent environment by default, and includes that Chromium flag. Its transform cache also defaults to `os.tmpdir()`. + +This establishes the intended wiring; a live Blaxel test must still observe Node-created paths and actual Chromium shared-memory descriptor paths under the attempt directory. We have not run that check at the proposal stage. Setting `TMPDIR` cannot relocate hard-coded `/tmp` paths or X11's global socket directory; the runner's explicit temporary paths must be moved deliberately, and external infrastructure remains untouched. Chromium also unlinks shared-memory files while holding them open, so directory deletion cannot substitute for stopping the owning processes. + +### Exact deletion scope and order + +Before preparation for **every run**, while holding the worker reservation and after confirming no prior execution or retrieval remains, remove: + +- Real directories directly under `/tmp` whose basenames match `television-source-cli-e2e-*` or `television-web-bundle-*`. The helper source and inventories establish these as leaked test builds; Josh has authorized their recurring removal. Missing per-directory metadata on these legacy copies is expected. The whole-worker exclusion and idle check supply the disuse evidence that the prefix alone cannot supply. +- Every attempt directory directly beneath the dedicated `/tmp/tv-testshards/` root except the current one, after the worker owner audit confirms no prior live or stale owners remain. These directories belong to our runner. As with the legacy prefixes, the held lock and clean worker audit establish disuse; missing, half-written, or inconsistent per-directory metadata does not prevent removal. This does not reclaim another run's held lock or decide how dead-coordinator locks are recovered. + +Revalidate directory identity immediately before deletion; reject symlink roots, unexpected mount boundaries, or paths escaping the exact allowed parent. Recursive removal must not follow nested links to another tree. Preserve the lock and guard, provider uploads, X11 state, unrelated temporary files, checkouts, and deliberate caches. Reports stay outside scratch; only abandoned-attempt recovery can discard them before download. Do not broaden the legacy patterns to other Television-looking names. Report per-prefix counts, allocated bytes removed, filesystem free space before/after, and skips or errors; those two space measurements need not be equal when files remain open or the filesystem accounts storage differently. + +After execution, finalize evidence outside scratch, stop all attributed processes, then remove this attempt's scratch while the reservation remains held through download. Setup failure, acceptance failure, timeout, and interruption use the same ownership rule. If teardown is uncertain, preserve scratch and keep the worker reserved. If deletion fails after confirmed teardown, preserve the original test outcome and add cleanup failure evidence; the worker cannot start another test until pre-run cleanup succeeds. Evidence must still be downloaded. A partial deletion must never be reported as reclaimed capacity for the whole directory. + +An inconclusive worker audit or a live consumer means skip deletion **and exclude that worker from test dispatch**, with a recoverable infrastructure failure. Missing scratch metadata alone does not make that audit inconclusive. There is no fallback to deleting unrelated files to obtain headroom. Known residue left by old test revisions is removed on subsequent updated runs without recreating workers or losing their caches. This proposal revision performs no worker cleanup or configuration changes. + +## Worker lifetime for cached runtimes + +Keep one lazily built source-CLI runtime per test worker and reuse it across launches and test cases. Publish the cached entry only after the build and asset installation succeed. A partial build removes its own directory and leaves the cache retryable. Apply the equivalent ownership rule to the cached versioned web bundles, retaining their per-version reuse. + +Test cleanup continues to stop each case's product servers promptly. Worker teardown then stops any remaining servers and removes the shared cached runtime and versioned bundles. Cleanup is idempotent and waits for pending builds before removing their outputs. The worker teardown must cover both the shared Playwright registration helper and the Vitest callers; a per-test deletion would discard reuse and add unnecessary builds. A failed server teardown cannot justify removing files beneath a surviving server. The outer shard's process ownership and scratch cleanup provide the backstop when a worker is killed and cannot execute its own teardown. + +The relevant entry points are `test/helpers/playwright.ts`, the two cached build helpers, and the Node/Vitest callers of those helpers. The design should use the runners' existing teardown facilities rather than a general-purpose cleanup registry. Failure during a per-launch runtime copy must also dispose that copy and its newly created home: the current bundle preparation occurs before the launch disposer is registered. + +The helper fix matters even with shard scratch: it releases files between test workers within a shard and protects callers on hosts without the Blaxel shard script, including ordinary CI and developer environments. The reviewer independently counted 1,388 source-CLI and 727 versioned-bundle directories on this development host; that is reviewer evidence, not a new inspection by this contributor. Cleaning those existing host files is the host owner's decision and outside this contribution. The helper fix prevents new leakage on updated branches; it does not retrospectively remove unowned files. + +## Built application for product acceptances + +### Assessment after PR #32 + +**Convert five cases in two files, rather than the earlier 17-file set.** Josh delegated the scope judgment to this contribution. PR #32 (`982e564d`, merged 2026-10-09) removes conditional request headers at the registry Vite service boundary, so those services return full responses. Its recorded Chromium 145 evidence identifies the 2 MB buffer retained per 304 until context closure. That change directly addresses the repeated-reload mechanism for tests using those services, including tests that remain on Vite. Routine cleanup also recovers the leaked runtime capacity on each updated run. Neither mechanism now justifies a broad application-loader migration. + +The two remaining source-loaded theme lifecycle cases are worth converting because they already claim built-application behavior in parts of their proofs and exercise repeated loads and server restarts. Browser demo mode is a small adjacent conversion: its three cases share one product-page helper, with repeated application reloads in two cases. Both files already have a real `ProductServer` and built assets available. This improves their shipped-asset boundary and removes hundreds of module requests per load without a new browser harness. The incremental memory and time benefit after #32 is unmeasured. Main's `f9180241` (PR #30) fixes the demo-mode flake by closing an unused HTTP connection that held fixture teardown open; that flake does not support the resource rationale for this conversion. + +Diminishing returns set in after these two files. The other product walks could gain packaged-asset coverage, but reviewing their loader, authentication, restart, and proof changes adds regression work without evidence of a remaining resource failure that those conversions would cure. Keep #32 for every remaining registry Vite service. Preserve the assertions and #9's merged theme-reset conversion. Validate the selected paths functionally and confirm their request inventory; any before/after resource comparison must use a baseline containing #32. This scope is based on source and proof inspection, not a post-#32 memory experiment. + +### Conversion boundaries + +Use the already-built web application served by `ProductServer`. Existing registry pre-commands build the web and artifact documents before both browser-app surfaces, and `installProductRuntimeAssets()` already copies the application index and assets. Preserve the tests' assertions, real browser behavior, CSS-motion declarations, authentication, and server lifecycle. + +The complete conversion set under `packages/web/test/e2e/` is: + +| Tests | Reason and boundary | +| --- | --- | +| `theme-product-acceptance.test.ts`: “theme settings persist rendered app and artifact outcomes through a built-server restart” and “executable themes run only in the Chromium application lifecycle and isolate failures” | Keep the two real documents, executable isolation, appearance, package watching, and server-restart assertions. Current main's [product proof](../../proofs/product/themes-and-appearance.md) calls the executable and appearance walks built-application acceptances, while its selection description still names Vite loading. The [delivery proof](../../proofs/arch/themes/delivery.md) likewise calls the executable frame walk built. Make the setup and proof descriptions agree on shipped assets. PR #9's theme-reset case and the already-built cases are untouched. | +| All three cases in `browser-demo-mode.test.ts` | Change the shared `openApp` helper. The marker-on/off walk reloads twice; return-to-original reloads once. The bridge-inert case shares the same helper and real marker/server boundary, so keeping its loader identical needs no separate harness or additional conversion machinery. Preserve external navigation, unsupported-view behavior, and the inert bridge assertions. | + +A server URL is sufficient where a backend stays up. Where restart is part of the walk, reuse the existing [`startStableFrontProxy()`](../../test/helpers/stable-front-proxy.ts) to serve the built document and retarget HTTP/WebSocket traffic after restart. Do not weaken a reconnect test by navigating to a replacement origin or clearing browser state. Keep development loading available explicitly for module and fixture seams; avoid changing every `appURL()` caller implicitly, including PR #9's test and the Electron dispatcher seams. + +The product runtime also needs the built `views/url-unsupported` assets for the existing external-navigation walk, which currently obtains that view from Vite. Add the missing test-runtime asset copy, not a production routing change. This is a prerequisite for preserving that assertion honestly. + +For new or substantially changed **browser acceptances using `ProductServer`**, prefer its built application unless the assertion needs development modules. This testing directive does not require retroactive conversion of other files or apply to in-process `Server` walks using main's `appURLForServer()`, component fixtures, or Electron module/dispatcher seams. Those boundaries remain as authored. Existing built-app acceptances remain built-app acceptances. + +### Remaining development-page exposure + +At `b5477211`, a read-only inventory found 38 e2e test files with a literal `/packages/web/src/index.html` reference and two more using `guerilla-artifact-sharing.helpers.ts`: 40 test files, or 41 source-loading files when that helper is counted. That historical count is not today's loader count or a measurement of concurrent pages or memory. Repository tests that inspect HTML as text and component fixtures importing individual modules are excluded. The narrowed scope deliberately leaves the following exposures: + +| Remaining tests | Observed reload/page behavior and why deferred | +| --- | --- | +| `channel-product.test.ts`, `channel-crud.test.ts`, `channel-sidebar-remote.test.ts`, `channel-switcher-context-menu.test.ts`, `tab-pages.test.ts`, `keyboard-navigation-product.test.ts`, `artifact-view-wheel.test.ts`, product walks in `top-bar.test.ts` and `auth.01.test.ts` | Deferred from the earlier conversion set. Channel, tab, and keyboard walks include observer pages; the tab walk also reloads. They remain legitimate conversion candidates, but their present product assertions do not require this contribution to change their delivery path. The channel drag flake has no established resource cause. | +| `markdown-content-sync.spec.ts`, `path-artifact-saga.test.ts`, `path-artifact-real-stack-coverage.test.ts`, `navigable-history-real-stack.01/.02.test.ts`, `onboarding-browser.test.ts` | Deferred from the earlier set. These cover real editing, persistence, navigation, or boot/restart; onboarding in particular requires stable-origin continuity. After #32 and recurring cleanup, a larger migration is not justified by the resource evidence. Keep their proof boundaries and assertions intact. | +| Main's web `artifact-isolation.test.ts`, `artifact-move.test.ts`, and desktop `artifact-isolation.test.ts` | Additional `appURL()` users identified by round 2 review on main. They are explicitly outside the two-file conversion. Source inspection finds a reload in web isolation and a second page in web move; no headroom claim is made. | +| Web `telemetry-session.test.ts` | One case reloads the application; another opens two tabs in one persistent context, closes it, then opens a new context. This is a concrete remaining multi-page exposure. It uses an in-process server with an injected telemetry sink; converting that boundary needs its own proof review. | +| Web `artifact-navigation.01.test.ts`, `artifact-navigation-history.test.ts` | Their `beforeEach` loads the development app, clears storage, then reloads it: two full graph loads per case. They use directly controlled `Store`/`Server` instances. These are sensible next conversion candidates, not proven-safe exceptions. | +| Web `auth.02.test.ts` | The silent reconnect case explicitly reloads the app. Other cases inject authentication failures into an in-process server. Reconcile this base-revision inventory against PR #9's merged changes before implementation. | +| Web `reload-after-reconnect.01/.02.test.ts`, `guerilla-artifact-sharing.test.ts`, `guerilla-artifact-sharing-resilience.test.ts` | These open one application page and reload/recover artifact documents or reconnect the event socket. The reconnect continuity case explicitly expects no top-level navigation. They control server internals/producer-consumer seams; an artifact reload is not evidence of a full application graph reload. | +| Web `settings.test.ts`, `platform-marker.test.ts`, `artifact-navigation.02.test.ts`, `appearance-delivery.test.ts`, `theme.test.ts`, `artifact-full-screen.test.ts` | The inspected source-app cases use one application page. Other cases in these files use fixtures or built assets. Retain their in-process server, platform, appearance, and navigation boundaries for this contribution; no claim that development modules are inherently necessary. | +| Desktop `all-webview-dispatcher.01/.02.test.ts`, `webview-event-parity.test.ts` | The executable-theme parity walk explicitly waits for a top-level document reset and later closes/relaunches Electron. The navigation-history walk quits and relaunches. These remain exposed; application resets, fresh Electron processes, and guest-webview reloads must be distinguished in any follow-up conversion. | +| Desktop `artifact-view-webview-wheel.test.ts`, `guerilla-artifact-sharing.test.ts`, `path-webview-content-reload.test.ts`, `artifact-navigation.test.ts`, `native-navigation-key.test.ts`, `artifact-navigation-external.test.ts` | Launch the development application in Electron; several reload or navigate guest documents. Native input, preload, and webview seams are outside this browser-product conversion. No measurement establishes their peak memory. | + +This contribution does not establish a post-#32 memory saving, eliminate unbundled loading, or certify every remaining test's headroom. Preserve the remaining walks in full validation and use the new diagnostics if they fail. The testing directive is limited to new or substantially changed `ProductServer` browser acceptances. + +### Reconciliation with current main + +Main at `a8e73fbf` is integrated in `74d22941`. Preserve PR #9's theme-reset conversion, the stable-front proxy's current API, SDK runtime assets, and single-origin application loading. The inventory above is explicitly tied to `b5477211`, not a claim that all listed lines remain on today's main. Do not convert an already converted case again. The five selected cases remain source-loaded at this checkpoint; the derived proofs now describe their built-application boundary and explicitly mark the adaptation pending. + +Main also now owns full responses from registry Vite services and selection of a contributor's origin repository for remote checkout. Preserve both. Measure built-app memory savings against this new baseline rather than attributing PR #32's benefit to PR #12. + +## Failure diagnostics and retention + +The minimum that would have identified this incident is **available RAM, shared memory, free bytes on the effective temporary filesystem, and the Linux OOM-kill counter at surface start and end, plus kernel OOM lines when failure occurs**. Commit to that small set. Read it from the parent surface process so a browser or test-worker crash does not destroy the initial observation. Include the surface preparation/teardown interval in both planned and targeted paths; retain the start record immediately, and mark a missing final observation as unknown after interruption. + +Keep one resource sidecar per surface execution, linked to the native attempt evidence already recording file/title/project/retry and failures. Retain it when a test attempt fails even if a later retry passes. When available, the shared browser failure hook can add a snapshot at the failed attempt boundary; this is an enhancement, not the minimum contract. Start/end samples provide context and an OOM delta, not a peak or exact allocation-time measurement. A system OOM-counter increase alone cannot attribute a kill to a browser; readable kernel lines provide process identities. If native results are missing, retain the sidecar without inventing an acceptance result. + +Read kernel output through a bounded capture and retain at most **64 KiB of relevant OOM lines** for the execution interval, with explicit truncation and unavailable-probe reasons. The two snapshots and bounded excerpt produce at most **128 KiB per surface sidecar and 8 MiB of resource diagnostics per shard attempt**. Enforce these limits when writing; a cap is necessary for bounded retention, but a continuous sampler/ring-buffer framework is not. Retain each surface's small start/end record before using the remaining budget for excerpts. Omit credentials, request query strings, bodies, and arbitrary process environments. Probe failure adds an observability warning and preserves the original test outcome. + +Periodic sampling, per-process descriptor inventories, cgroup probes, detailed browser/request event records, and browser-revision comparison remain optional investigation tools. They are not new merge obligations or spec requirements in this contribution. Do not add default videos, heap snapshots, or NetLogs. Existing native retry evidence and trace policy remain unchanged. + +Write sidecars under the existing checked `.testshards/results` tree, outside disposable scratch, and download them with the other reports. Keep at most one attempt's bounded resource diagnostics per worker. A failed download leaves that attempt locked while the coordinator retries retrieval and renews; it does not start a growing queue of retained runs. Normally the next results reset follows download and release. If the coordinator stops renewing past the recovery deadline, automatic recovery may discard those abandoned reports and records that loss. Its recovery receipt shares the incoming attempt's 8 MiB diagnostic budget, with explicit truncation; it needs no separate size limit. Coordinator-side `.test-runs/` retention is unchanged. No cross-run archive or retention daemon is needed. + +## Authority and derivation + +After proposal convergence, derive the smallest changes in the existing owners: + +- [`testing-policy.md`](../../specs/arch/testing-policy.md): built application as the default for new or substantially changed `ProductServer` browser acceptances unless development-module behavior is under test, excluding in-process `Server`/`appURLForServer()` walks; worker-owned reusable test runtimes and cleanup expectations. +- [`blaxel-testshards.md`](../../specs/arch/test-runner/blaxel-testshards.md): exclusive reservation through setup, process teardown and result retrieval; renewal, automatic recovery after lost renewal, owned-process cleanup before reuse, and the explicit abandoned-report exception; actual older-runner behavior; pool-script locking; attempt temporary directories, recurring exact-prefix cleanup and bounded worker retention. Replace its blind TTL escape and shell-exit unlock expectations rather than leaving conflicting requirements. +- [`reporting.md`](../../specs/arch/test-runner/reporting.md): minimum resource observations and artifact references, bounded content, unavailable evidence, retention alongside failed/recovered native attempts, and recovery receipts that disclose discarded orphan reports. + +Use existing lifecycle and reporting contracts in `test-runner.md` and `sharded-execution.md`, adding the cross-references or ownership corrections required by the outer Blaxel lock. The proof review also exposed missing authority for the automated live driver and its deliberate-failure fixtures: the `test-registry.md` and `test-runner.md` delta places them in separately selected experiment surfaces, with the driver on a dedicated credentialed Blaxel coordinator and the fixtures on its private pool. This keeps repeatable live coverage without adding a kind or suite. Proof review round 2 approved that placement delta; its dependent proof binding is complete. Existing isolated-GitHub-only lifecycle fixtures keep their placement. No `preflight.md` delta is planned. Optional investigation instrumentation remains optional. + +Update the mirrored proofs after independent spec convergence. Test setup changes may also require proof-only corrections to descriptions of the affected browser walks; they do not change their product promises. Do not prescribe a new framework or adopt unrelated harness internals into spec authority. + +## Validation and contribution boundaries + +All execution is on Blaxel at committed, pushed revisions. The shared development host performs file inspection, edits, Git operations, and lightweight remote coordination only. No tests, builds, browser launches, lint, or type checks have run for this proposal. Apply the test-iteration discipline: red/green at the narrowest relevant file, then affected surfaces, then full verification with the default retry budget. + +Derive proofs for the principal paths without a combinatorial fault matrix: + +- One run excludes a competing coordinator through preparation, teardown, and a deliberately delayed result download while renewing. Passing the former TTL never admits a second run beneath that valid reservation. Exercise both targeted and planned dispatch through the real provider. +- Stopping coordinator renewal permits automatic acquisition after the recovery interval, both with a quiet worker and with an owned supervisor/descendant still alive. Reuse follows confirmed termination; the receipt records any discarded reports. Contract coverage uses a controlled clock for renewal/deadline calculation, the recovery-interval override, and error paths. A private-worker seam crosses real dispatch, recovery, and delayed retrieval, including a late dispatch rejected after ownership changes. It must not reduce to mocked API responses or simulate a coordinator crash by killing only the shard. +- Concurrent recovery attempts cannot both acquire; delayed renewal/release cannot alter a successor's reservation. Keep this at the lock helper's real filesystem boundary. The inspected older runner does not reclaim a new held lock with `expiresAt: 0`. Legacy helper residue is removed on the next updated run; live legacy work is skipped and failed process cleanup prevents reuse. Pool deletion respects the same deadline but can destroy an abandoned worker that failed cleanup. Fresh workers are unavailable to tests until provisioning succeeds. +- Exact-prefix and stale-scratch cleanup preserves unrelated files, live consumers, and the current attempt directory, while removing earlier scratch even when its metadata is incomplete. Real runtime builds demonstrate once-per-worker reuse, final disposal, and partial-build cleanup. A Blaxel observation verifies Node temporary paths and Chromium shared-memory descriptors under the attempt's `TMPDIR`, recording the actual browser version used. +- A failed or recovered acceptance retains bounded start/end/OOM evidence through retrieval, including when its worker exits. Contract coverage handles unavailable probes, unchanged original test outcomes, truncation, and limits. + +The live recovery checks use small, disposable, owned processes on private workers. No deliberate OOM, process storm, or invocation of the repository's isolated-GitHub-only destructive fixtures is authorized. If a required assertion cannot be proven safely within Blaxel-only execution, report that specific blocker at proof review; do not silently substitute local or GitHub execution. + +Use a dedicated two-worker pool named **`rose-cleanup`**, with workers `tv-testshard-x64-rose-cleanup-01` and `-02`, for cleanup, orphan, and interruption experiments. Explicitly pass `--pool rose-cleanup` through the canonical runner; reduced planned runs also need the documented shard-count override. Exercise interruption with disposable owned subprocesses there, without intentionally exhausting memory or running repository fixtures restricted to isolated GitHub hosts. Never run those experiments on shared `poc` workers. Two workers permit a distinct replacement after a contaminated attempt; they do not guarantee capacity for every retry after repeated contamination, which remains an infrastructure failure with the default retry budget intact. + +Re-run the converted acceptances without weakening their user-visible assertions. A recorded built-page request inventory should show that they no longer load the unbundled application graph. Validate restart continuity and authenticated navigation directly. Repeated use of one isolated worker should leave no generated runtime directories after cleanup and no growing sequence of resource reports. + +For the final gate, retain the separate sandbox named **`tv-cleanup-coordinator-rose`**, excluded from testshard pool labels. After private qualification and deletion of the two experimental workers, it will fetch the committed, pushed branch and run `npm run verify -- blaxel --pool poc` at the standard 36 shards, so the invoking machine's lint/type-check/package-manifest phases also execute on Blaxel. The two experimental workers and coordinator are a plan, not currently provisioned capacity. Provision and verify their identities before starting validation; if unavailable, report blocked rather than run anything on this host or silently substitute GitHub execution. The user has already authorized outside-pool Blaxel hosts. + +Supply the coordinator's credentials through the Blaxel SDK's `process.exec` environment: `BL_WORKSPACE` and an authorized `BL_API_KEY` for its outbound Blaxel calls, plus `BLAXEL_TV_GH_TOKEN` for repository access. The launching process reads the existing authorized secrets into memory and passes them directly in that environment object; do not interpolate their values into shell commands, uploaded scripts, output, or reports. The inspected SDK accepts the Blaxel pair without a CLI login/config file, and `readBlaxelGithubToken()` accepts `BLAXEL_TV_GH_TOKEN` when no `.blaxel-gh-token` file is present. Use that environment path in the fresh coordinator, with `GIT_ASKPASS` reading the environment for its initial fetch; the existing runner passes only the GitHub token onward to workers as `GH_TOKEN`. Do not copy local credential files or persist tokens in the coordinator's filesystem, sandbox configuration, or Git configuration. Confirm that the required credential is available and valid for the run before dispatch; its availability is not established by this proposal. End the credential-bearing process after downloading its reports and dispose the dedicated coordinator. This describes ephemeral process injection, not a claim about provider-internal retention. + +The experiment plan adds at most three sandboxes. The spec's pool-size and tier examples are not a current capacity inventory: inspect actual workspace usage and provider limits before provisioning/dispatch. Do not claim slots are reserved or alter shared workers to obtain them. The [implementation plan](test-harness-resource-cleanup-plan.md#Integrated result and final validation) keeps focused development and all worker experiments on the named private hosts, then uses the standard shared pool for ordinary full verification. The committed file medians leave inadequate setup/retry allowance for a two-worker full run within the canonical runner's 480-second worker timeout. Retrieve private reports before the pool-deletion acceptance and all remaining reports before deleting the coordinator. Record the exact tested revision, failures, recovered flakes, and any measurement limits. Ready-PR CI remains the repository's merge requirement; the contribution's execution evidence stays on Blaxel as instructed. + +Keep browser-version alignment, the default shard count, browser request limits, retry defaults, and production memory behavior outside this contribution. Forced GC is an investigation tool, not a proposed mitigation. Preserve main's demo-mode teardown fix; the drag flake has no established resource-exhaustion cause. + +## Review status + +Proposal round 1 at `1652787e`: FAIL on lease scope and mixed-version consequences. The response at `99b7ed93` narrowed the proposal while those human scope decisions were pending. Josh's 2026-10-27 instruction brings lock correction and recurring legacy cleanup into this PR and removes the pool-recreation choice. The [review response](test-harness-resource-cleanup-review-response.md) records the current dispositions without changing the independent review's historical verdict. + +Proposal round 2 at `d3570bf4`: FAIL because dead-coordinator lock recovery is undefined. The [round 2 review](test-harness-resource-cleanup-review-r2.md) is preserved unchanged. Josh selected automatic recovery, required that an orphaned active run cannot hold a worker forever, and delegated the recovery mechanism and built-app conversion scope. His supervisor clarified that the suggested predicate and grace-period wording were illustrations, not requirements. The revision at `57a701d0` supplied one renewable recovery deadline and narrowed conversion to two files. + +Proposal round 3 at `57a701d0`: **PASS**, relayed by the supervisor. The [review](test-harness-resource-cleanup-review-r3.md) is preserved unchanged. Its bounded refinements are applied here: fixed recovery interval, deletion of abandoned workers despite failed process cleanup, ordinary skipping of live legacy work, and trimmed receipt/provisioning language. The [response](test-harness-resource-cleanup-review-response.md#Round 3: convergence and spec derivation) records the judgments. The proposal and spec deltas have converged: spec round 1 at `cd8aa453` also passed. Its bounded refinements and the derived proof checkpoint are recorded in the response; independent proof review is next. + +Proof round 1 at `9fd51f49`: **FAIL** on missing spec-owned placement for live qualification and deliberate-failure fixtures. The [review](test-harness-resource-cleanup-proof-review-r1.md) and [response](test-harness-resource-cleanup-review-response.md#Proof round 1: placement finding and refinements) retain that history. Proof round 2 at `3fa15d47`: **PASS**, including the placement delta. Its [report](test-harness-resource-cleanup-proof-review-r2.md) permits the bounded proof binding now applied. Plan round 1 at `70f27ccb`: **FAIL** on the two-worker full-gate budget. The [review](test-harness-resource-cleanup-plan-review-r1.md) and [response](test-harness-resource-cleanup-review-response.md#Plan round 1: full-gate placement and refinements) record the correction to ordinary shared-pool verification and the bounded refinements. The [implementation plan](test-harness-resource-cleanup-plan.md) is ready for independent re-review; tests and implementation have not begun. No tests, builds, remote experiments, or worker operations were performed for this checkpoint. Unknowns are live provider behavior during failed dispatch/recovery, today's worker browser versions, actual `TMPDIR` placement and post-mitigation memory savings, current capacity/credential availability, the implemented tree's full-gate duration, and observed recovery timing. Recovery waits at most 20 minutes after the last renewal by default, then acquisition and bounded cleanup; failed cleanup prevents test reuse and explicit pool deletion remains available. Josh's spec acceptance remains due at PR time; there is no pending shared-pool cleanup rollout decision. diff --git a/docs/archive/2026-10/test-runner-local-guardrails/test-runner-local-guardrails-plan.md b/docs/archive/2026-10/test-runner-local-guardrails/test-runner-local-guardrails-plan.md new file mode 100644 index 00000000..8e6340fd --- /dev/null +++ b/docs/archive/2026-10/test-runner-local-guardrails/test-runner-local-guardrails-plan.md @@ -0,0 +1,83 @@ +> **Archived 2026-10 from PR #53, which carries PR #49's change.** This plan organized the implementation into file/retry admission, mutex ownership and native-entrypoint guidance, with isolated crash validation and a final full gate. It preserves the complete view of the work’s dependencies, fixture-safety constraints and validation responsibilities across those parts. The body below is unchanged from its working state and is a clue to the change, not a record of it. + +# Local test-runner guardrails — implementation plan + +This plan derives from the converged [proposal](test-runner-local-guardrails-proposal.md), [runner spec](../../specs/arch/test-runner/test-runner.md), [testing policy](../../specs/arch/testing-policy.md), and [runner proof](../../proofs/arch/test-runner/test-runner.md), including the proof review at `3bdfa431`. It groups implementation and validation into three reviewable slices. The supervisor coordinates independent review; Josh reviews the specs on the pull request. + +## Why three slices + +Selection and retries share the actual one-file boundary. Mutex ownership crosses CLI execution, verify orchestration, and process cleanup. Native admission crosses separately wired configs and launchers, and must agree with the instruction layer. Each group has enough distinct behavior and evidence to warrant its own review under [Planning and slices](../../specs/spec-workflow.md#planning-and-slices). The integrated review then checks their interactions. + +The order establishes command admission before attaching process ownership, then enables native refusals once canonical execution is complete. Host health remains the primary outcome throughout; targeted Blaxel fan-out is outside this contribution. + +## Implementation boundaries + +The existing code offers the necessary seams, with a few consequential gaps: + +- `scripts/test/cli.mjs` resolves surface ownership before running local or remote work. `file-inventory.mjs` can read a selected Git tree, but its working-tree inventory currently omits untracked files. Reuse these boundaries for one shared file-resolution result consumed by local admission, retry admission, and exact native selection. +- Local and targeted commands currently forward native file filters, which can widen. Planned workers already have exact-file adapters. Reuse that machinery where it fits and cross each distinct local and targeted handoff with the proof's real native tests. +- Verify currently runs phases with `spawnSync` and direct local paths call `process.exit`. Mutex ownership must surround those paths through cleanup. Keep phase order, exit propagation, reporting, and attestation behavior intact while making ownership release explicit. The direct verify test child needs a deliberate ownership handoff; a generally inherited exemption is insufficient. +- Planned Blaxel forwarding currently omits a zero retry budget. Pass the resolved numeric budget explicitly through the existing adapters. Attestation fact construction must also see the standalone broad zero-retry override. +- Registered configs include `staging/vitest.config.ts`, outside the usual discovery roots. Use the registry to cover entrypoints. Native context is `TV_TEST_RUNNER_CONTEXT=1`; the non-plan shard path needs it without acquiring new lifecycle responsibilities. + +Keep the implementation within these modules and their callers. The mutex uses existing process identity and stale-owner cleanup, a stable same-user location and compatible record, and atomic acquisition. It needs no expiry, background service, waiter queue, resource monitor, or authentication scheme. Exact record fields and internal helper boundaries can be settled while implementing the proof-derived tests. + +If implementation requires a changed promise, including registry wording for additional fixture surfaces, return it through spec review and proof derivation before dependent work; Josh's human spec review remains due on the pull request. + +## Slice 1 — file admission and retry policy + +Implement marked-host one-file admission, exact native file selection on local and targeted remote paths, actionable redirection, the broad-local flag rename, and selection-dependent zero retries. Resolve local files from the current working tree, including untracked files; resolve remote files and ownership from the selected committed tree. Preserve unmarked local breadth, placement restrictions, and the independent full-`all` shortcut guardrail. + +Keep parsing, verify phase forwarding, native retry arguments, remote numeric budgets, and attestation eligibility in this slice. Update CLI help for the behavior delivered here. Reuse one command-admission path for direct tests and verify where their semantics agree, without broadening unrelated argument parsing. + +**Red/green and evidence.** First select the repository's `.nvmrc` runtime and supported npm, then install this worktree's missing dependencies with `npm ci`. Start with failing CLI assertions in `test/repo/test-runner-guardrails.test.ts` and focused file-resolution/native-selection tests. Prove real Vitest and Playwright collection and retry attempts with small authored fixtures; extend the worker and attestation tests at their existing seams. The obligations are [file selection and guidance](../../proofs/arch/test-runner/test-runner.md#file-selection-and-guidance-at-the-command-boundary), [retry selection and execution](../../proofs/arch/test-runner/test-runner.md#retry-selection-and-execution), and [retry eligibility facts](../../proofs/arch/test-runner/attestation.md#^attest-retry-options), plus [marked verify refusal](../../proofs/arch/test-runner/test-runner.md#^t-marked-verify-refusal), [explicit local verify](../../proofs/arch/test-runner/test-runner.md#^t-marked-local-verify), and [host-marker behavior](../../proofs/arch/test-runner/test-runner.md#^t-blaxel-host-marker). Shared help and override-independence assertions cover the implemented options now and are completed by slices 2 and 3; their proof markers stay until slice 3 completes each whole assertion. + +After focused red/green work, commit and push, then run the affected repository-test surface on Blaxel. Record targeted and planned remote zero-retry evidence using small non-unit native fixtures, including the adversarial sibling on the targeted path. These runs establish file/retry behavior here; native-context evidence is completed after slice 3. + +**Expected baseline.** All tests introduced or changed in this slice pass; update existing expectations for the renamed flag rather than carrying failures forward. The mutex/wait and native refusal obligations remain unimplemented, explicitly assigned to slices 2 and 3. Full instruction-layer updates remain assigned to slice 3. No known failing-test baseline is planned. + +## Slice 2 — mutex ownership, waiting, and recovery + +Implement the stable same-user mutex, the gated private-location hook, immediate refusal, opt-in wait, and independent contention bypass. Perform usage, selection, and placement checks before waiting. Dry-run reports the same resolved lock path supplied to real acquisition without taking the production lock. + +Integrate ownership into direct local execution and the whole local verify. Keep the direct test-phase child protected if its verify parent exits; unrelated and arbitrarily nested commands still contend. Release follows test-process exit and cleanup. Reuse identity checks and stale-owner recovery, preserving cleanup-then-failure when stale test processes are found. Adapt every existing nested CLI test that executes work, including reporting, toolchain, and lifecycle cases, to a private lock path in this same slice. + +**Red/green and evidence.** Derive the ordinary mutex tests from [Local mutex](../../proofs/arch/test-runner/test-runner.md#local-mutex). Organize direct-run and verify ownership tests into independently runnable files, using small temporary checkouts and cooperative native fixtures. Include routine dead-owner recovery, real acquisition races, cancellation statuses, bypass independence, service teardown, and the preflight-bounded remote exemption. Update CLI help and complete the mutex portion of the shared assertions. Run the edited file locally; after pushing, run the affected repository-test surface on Blaxel to check isolation under an outer worker. + +Add the interruption, surviving-child, orphan, and stale-cleanup cases only to the existing isolated lifecycle surface. At the start of slice 2, prepare the exact branch-only `Process Lifecycle Fault Injection` workflow with its cleanup, timeout, and artifact configuration, and send that concrete proposal to the supervisor for any still-needed operator authorization while implementation proceeds. Validate the cases through that authorized workflow before declaring this slice complete; do not run them on this host or Blaxel. The contribution must not retain that workflow. Record which operating-system backend actually ran. If only this authorization or run remains pending after the ordinary mutex checks pass, independent slice 3 work may proceed while slice 2 stays open; both slices must converge before integrated review. + +**Expected baseline.** Slice 1 remains green, all ordinary mutex tests pass, and the isolated evidence covers the destructive cases. No failing-test baseline is carried into slice 3. Native entrypoint refusal and the final guidance remain outstanding; no mutex guarantee is deferred to final integration. + +## Slice 3 — native entrypoints and reachable guidance + +Add the shared native check to every registered Vitest and Playwright config, before service-dependent setup can mask its diagnostic. Wire context through local, planned, targeted supervised, and non-plan diagnostic launchers. Preserve declared native fixtures and the deliberate exact-value direct-native override. Complete the help and independent-option assertions. + +Update `AGENTS.md` and affected invocation guidance from the converged specs. The entry instructions must state focused local iteration, the same-user mutex and wait, shared-host risk, independent overrides, broader Blaxel validation after committing and pushing, and the narrow/broad retry distinction. Carry regular safe development-branch commits and pushes into the repository sources of `tvdev-contribute` and `tvdev-review`, including the coordinator constraint. Edit those sources as contribution content; do not adopt the principal-agent skill. Preserve the testing policy's current broad-local permission rule and the current `--force` and `--ignore-uncommitted` names unless the supervisor relays a further Josh decision. + +**Red/green and evidence.** Implement [Native entrypoints](../../proofs/arch/test-runner/test-runner.md#native-entrypoints) in a dedicated, independently runnable test file. Measure the real config-loading sweep; cover marker-specific message variants once per native tool. Launcher tests remove inherited context before exercising the production launcher. Use real small native fixtures for override and context admission. Review written guidance under the [testing-policy proof](../../proofs/arch/testing-policy.md), including desktop/macOS and local-only suite commands. Regenerate indexes and run the relevant documentation checks through the prescribed test route. + +After committing and pushing, validate the affected repository tests remotely and complete the live targeted/planned handoff evidence with guarded configs. The integrated full Blaxel verify supplies planned-worker context evidence; it is not an extra special-purpose run. + +**Expected baseline.** Every implemented test passes and every new committed-test obligation has its evidence citation. All functionality and guidance are present. Integrated full verification and final review remain the completion gate below, with no accepted failing-test baseline. + +## Test execution and fixture safety + +For each behavior, write its proof-derived test, observe the intended failure, implement, and rerun that focused check. An import or setup failure alone does not establish the red behavior. Locally run only the one test file just edited, optionally narrowed with grep; do not serially loop over files for a known multi-file check. Use the default retry budget for validation, with plain `--retries 0` only for a deliberate focused check. Push safe, coherent development checkpoints before broader remote validation. + +A refusal test must remain bounded if the refusal is missing, during red/green work or a later regression: the native sweep uses a files-only listing or verified no-match file filter that loads the real config without executing tests, and other refusal cases use dry-run or small authored fixtures so none can start broad work. + +Nested canonical fixture launchers supply separate private lock paths per independent scenario and share a path only for deliberate contenders. Production-location checks remain read-only. Readiness and explicit release coordinate the small competing processes; no workload stresses the host. Tests own and clean up their fixtures even after failed assertions. Verify uses an authored small registry and substituted downstream phase programs, never the real repository's full gate inside a test. These limits apply under local, Blaxel, and GitHub outer runners. + +Reuse committed native fixture inputs for local seams and live remote checks. Where canonical live dispatch needs registry entries, use small opt-in non-unit fixture surfaces excluded from `all`; they test native attempts and collection and never dispatch Blaxel themselves. Keep intentional fail-first behavior distinguishable from product failures in the validation record. Destructive lifecycle fixtures retain their isolated-only placement and cannot be repurposed for these remote checks. + +At each slice handoff, report the pushed revision, relevant red/green evidence, remote checks, recovered flakes, and remaining obligations. Replace proof test markers with honest citations as coverage lands. Keep commands, tested revisions, outcomes, and retained live report locations in the contribution's validation record; standing proofs describe which live evidence is required. + +## Integrated result and pull request + +After all three slice reviews converge, review the complete interaction of selection, retries, native context, mutex ownership, reporting, and guidance. Resolve all outstanding proof obligations and any failures discovered by wider validation. Report Linux and macOS evidence separately; Linux process tests alone do not establish the macOS backend. + +Before final full validation, update this development branch to contain the current target history under the repository's logical-merging guidance, reconcile changed policies, and push the resulting tree. Run `npm run verify -- blaxel` on that revision with the default retries and await the complete result. Retain the targeted/planned live evidence and isolated lifecycle reports alongside that full-gate result. Changes after validation require checks appropriate to the changed tree; no earlier run is represented as validation of a later revision. + +Converge the integrated review through the supervisor. Prepare the draft pull request for Josh's spec review, with its description written under [developer-skills/pr-writing](../../developer-skills/pr-writing/SKILL.md). Keep working documents while the PR is a draft; perform [pre-merge docs prep](../../specs/spec-docs.md#^pre-pr-docs-prep) before treating the completed contribution as ready to merge. This task ends with the contribution ready for Josh's review, without merging it. + +The draft-PR handoff must list production GitHub CI as outstanding until the PR is marked ready: its unit job exercises the production local mutex around the nested tests, and its desktop job exercises canonical local execution through the Electron wrapper. Blaxel validation does not establish those CI handoffs. Required GitHub checks must pass before merge under the shared-branch workflow. diff --git a/docs/archive/2026-10/test-runner-local-guardrails/test-runner-local-guardrails-proposal.md b/docs/archive/2026-10/test-runner-local-guardrails/test-runner-local-guardrails-proposal.md new file mode 100644 index 00000000..bbbe53f4 --- /dev/null +++ b/docs/archive/2026-10/test-runner-local-guardrails/test-runner-local-guardrails-proposal.md @@ -0,0 +1,123 @@ +> **Archived 2026-10 from PR #53, which carries PR #49's change.** This proposal set the host-health priority, independent overrides and instruction-layer scope carried into the runner specs and implementation. It preserves the single coherent statement of intent and the rationale for the boundaries between local iteration, remote validation and deliberate departures from guidance. The body below is unchanged from its working state and is a clue to the change, not a record of it. + +# Local test-runner guardrails + +Proposal converged under independent review, with Josh's subsequent decisions on deliberate overrides recorded below. Josh will review the authoritative spec deltas on the pull request. This document records intent; the [runner spec](../../specs/arch/test-runner/test-runner.md) will own the command behavior. + +## Purpose and decisions + +Several agents can run tests on one development host. Independent local runs compete for CPU and memory and can cause the operating system to kill processes. Josh's first priority is keeping test runs from overwhelming that shared host. Every Blaxel test run moves its test workload off the host, including a surface run on a single remote worker. + +Faster iteration is the second priority. Blaxel's planned execution distributes files across workers; it cannot split the work inside one file. Adding fan-out to targeted surface runs is a secondary speed improvement. Its absence is never a reason to run broad work locally. + +The canonical runner is the supported front door for testing with good wall-clock time and host health. Clear defaults, actionable refusals, and written guidance steer callers toward that path, including when an agent reaches for a familiar command or tries to work around a refusal. Josh's interface principle is to make callers reflect before leaving the guidance, then let them do what they need through deliberately named overrides. These guardrails are not a security boundary: callers can bypass repository entrypoints or exhaust host resources with other commands. Where repository entrypoints can reasonably recognize a departure from the guidance, they should stop it and explain how to proceed. Size those checks to the real invocation paths; deliberate circumvention does not justify tamper resistance, access control, or host-wide resource policing. + +Josh's decisions are: + +1. Every canonical local test run, including local verify, takes one mutex for the same operating-system user across the host, outside every checkout, and holds it until its test processes exit. By default, a second run is refused immediately with exit `2`, identifying the holder and explaining the available next steps; `--wait-for-local-test-mutex` opts into waiting instead. A separate, conspicuously named flag allows the caller to bypass this protection at the caller's judgment. +2. On a host where the invoking user's `~/.tvdev-use-blaxel` exists, ordinary local iteration selects one test file, optionally narrowed with `--grep`. Broader selections are refused with exit `2` and redirection toward Blaxel: an equivalent command where supported, an explanation where the selection is unsupported, and instructions to commit and push first. `--against-test-guidance-broad-local-run` overrides this restriction, including for local verify, and replaces `--allow-extreme-inefficiency`. +3. Without that marker, canonical local selection breadth and the local verify default are preserved. Public contributors and forks need no Blaxel credentials or access. The mutex, native-entrypoint checks, and deliberate retry-disabling option apply on all hosts. +4. Registered native-tool entrypoints refuse direct test invocations by default on both marked and unmarked hosts, directing callers to the canonical runner. This also changes direct native execution for public contributors; it does not require Blaxel. `TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN=1` deliberately permits lower-level tool use at the caller's judgment. +5. Plain `--retries 0` is within the guidance for one explicitly selected file, optionally narrowed with `--grep`. Broader validation keeps the default budget; disabling retries there requires `--against-test-guidance-turn-flakes-into-failures-on-broad-runs`. + +## Local concurrency + +The lock coordinates canonical local runs by the same operating-system user across worktrees and separate clones on the same host. The production lock location is outside checkouts and is not selected by ordinary home-directory or temporary-directory overrides. Acquisition is atomic; contention refuses immediately by default. An active holder is identified by enough information to find its run: process identity, command, and checkout. The location and ownership record must remain compatible between participating runner revisions. Checkouts that have not picked up the mutex implementation do not participate; this change cannot coordinate their unmodified runners. + +A local verify acquires the lock before executing its phases and retains it through completion and cleanup. Its test-phase child participates in that same ownership; it must neither contend with its parent nor grant an exemption to unrelated or arbitrarily nested runs. Verify passes the applicable override options to that child. Direct local execution takes the same lock before starting preflight or test work. Informational commands such as `list`, `help`, and `verify --plan` do not acquire it. The existing self-test dry run checks selection guardrails but acquires no lock because it starts no test work. Blaxel dispatch does not acquire the submitting host's local-run lock. + +The runner's own reporting, toolchain, and lifecycle tests launch real nested canonical runs. They and the mutex tests will supply private lock locations through a hook accepted only with `TV_TEST_RUNNER_SELFTEST=1`; supplying that hook without the gate is a usage error. This is the explicit test-only exception to the production location rule. Each independent test scenario uses its own location, while processes intentionally testing contention share one. The real acquisition and release logic still runs, including under local verify, GitHub CI, and Blaxel workers. This isolates fixture runs from the outer run's lock and from parallel tests without making ancestry a general ownership exemption. The existing self-test warning identifies the substitution, and proofs declare its coverage limits. This hook does not relax destructive-fixture placement or process-cleanup rules. + +Release follows test-process exit and the runner's existing lifecycle cleanup, including failure and handled interruption. If verify's parent exits while its test-phase supervisor remains alive, that child still keeps the run protected; parent death alone cannot make the lock available. An abruptly killed owner cannot run cleanup; recovery must establish that its test processes have exited or complete the existing stale-owner cleanup before admitting another run. Finding stale test processes retains the existing outcome: cleanup followed by a failed invocation, with a later clean invocation able to proceed. Elapsed time alone is not evidence that a holder is safe to replace. Reuse the existing process identity and cleanup facilities. + +`--wait-for-local-test-mutex` waits until acquisition or caller interruption, with no time limit or ordering guarantee. Usage, selection, and placement checks precede waiting. The waiting message names the holder; SIGINT and SIGTERM cancel only the waiter, exiting `130` and `143` respectively. Waiting and the contention bypass are mutually exclusive. + +The proposed bypass spelling is `--against-test-guidance-major-host-contention-and-oom-killed-processes`. A caller passing it still takes and holds the mutex when it is free; when occupied, the flag permits concurrent local execution without displacing the holder. Its name states why concurrent local execution is avoided by default. The refusal tells callers to try after the named run finishes, pass the wait option, use Blaxel when available, or deliberately use this bypass with its stated consequence. It cautions against stopping another agent's run just to free the lock. Unmarked hosts receive a usable local next step without needing Blaxel. + +## One local file on marked hosts + +An allowed selection supplies `--file` and resolves to exactly one test file in the current working tree. An additional `--grep` restricts cases within that file. Other selectors may disambiguate or restrict the file's owning surface; selecting a surface, suite, package, runner, or tag alone does not satisfy the file restriction, even if that selection happens to contain one file. + +Counting owning surfaces is insufficient: one surface can contain many files. The runner must establish the file boundary before running tests, including for native substring or regular-expression filters that could match several paths. It must account for uncommitted and newly created test files; remote planning's committed-file inventory cannot alone prove a local selection is narrow. A directory or filter that would run multiple files is refused before those tests start. Invalid or empty selections retain their ordinary selection errors. + +The marker's lookup remains the invoking user's home-directory file, with contents ignored. The file limit applies to local test execution, not standalone preflight or list commands. Local verify is inherently broader than one file and retains its existing refusal and explicit local override. These decisions occur before expensive execution so a refused command is quick. + +When Blaxel accepts the same selection, a refusal prints a shell-usable command with the provider changed and the selection and grep preserved, plus the requirement to commit the intended changes and push the revision to `origin`. Otherwise it explains which selection or placement Blaxel cannot run and directs the caller to supported remote selections. For example, the current target path cannot take multiple surfaces together with grep. The diagnostic need not translate every unsupported combination into equivalent commands or broaden the remote runner's capabilities. Full verification points to `npm run verify -- blaxel`. The runner does not automatically commit, push, or dispatch a refused local request. + +Existing placement restrictions remain authoritative. In particular, the live PostHog suite needs a local secret and daemon acceptance mutates the designated local host. For a selection containing either surface, however selected, the diagnostic explains the local requirement and gives a one-file local invocation where supported, or the explicit broad-local override. Agent tests also need a local route because the pool does not provide their live-agent setup. Their invocation guidance will show the one-file forms usable on marked hosts. The mutex still applies. + +The desktop harness's operations guidance will explain that its full local surface command follows the same guardrails: Blaxel supplies Linux coverage; running the complete surface on macOS requires the documented broad-local override on a marked host. Blaxel setup will state that creating the marker enables both automatic remote verification and the one-file local limit. + +## Guide the choices agents actually make + +The instruction layer and refusals will address common attempts to get a test run started, with active checks where the repository can reasonably recognize them: + +- **Direct `npx vitest` or `npx playwright test`:** use lightweight checks at the registered runner-config entrypoints to refuse recognizable execution outside the canonical runner before tests start. A fixed message gives the canonical local file form and, on marked hosts, the commit-and-push prerequisite and broader Blaxel form; it need not translate native arguments. Native tool invocations made by the canonical runner, shard workers (including the non-plan diagnostic path), and declared test fixtures must continue to work. Reuse runner context and the existing self-test conventions; context is an invocation signal, not a credential to authenticate. Unmarked hosts are directed to the canonical local path without requiring Blaxel. +- **Widening `--file`:** establish the actual file selection before execution. If a directory or filter selects several files on a marked host, explain that a file filter is not necessarily one file and point to one concrete local file or the broader Blaxel run. Counting surfaces or trusting the flag's spelling would miss this case. +- **Looping over files:** guidance will say to commit and push a known multi-file validation selection and run it on Blaxel, rather than split it into serial local commands to evade the restriction. Repeated single-file runs while investigating or editing a particular behavior remain normal iteration. The mutex catches overlapping local commands; a serial loop's intent is addressed in guidance and broad-run refusal messages, without adding cross-invocation history to guess that intent. +- **Retrying broad work:** a refused broad local request continues to be refused on retry, including with `--force`. Its message explains the better path instead of inviting another selector spelling. After a broad Blaxel run finds a failure, narrow to the failing file for diagnosis and return to broader remote validation after the focused check passes. + +The runner spec owns the checks and messages. The testing policy owns the judgment about file loops and narrowing after broader failures; AGENTS.md provides a brief entry point to both. The contribution covers the registered test entrypoints people and agents ordinarily reach, with no claim to intercept every possible native-tool command. + +The native refusal also names `TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN=1` for deliberate direct use, such as native watch, headed/debug mode, or an editor's test explorer. Only the exact value `1` opts in. It is the caller's judgment, with no human permission requirement. The message explains that direct execution skips the canonical runner's mutex and marked-host file limit; it does not claim to supply services or other native setup. This environment option works with either native tool without adding a native-argument translation layer. + +## Independent overrides + +Each override uses the shared phrase `against-test-guidance` so the departure is recognizable. The native-tool form is an environment variable because the repository does not own either tool's argument parser. + +| Option | What it permits | +| ------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| `--against-test-guidance-broad-local-run` | A broader local selection on a marked host, including local verify. The host mutex still applies. | +| `--against-test-guidance-major-host-contention-and-oom-killed-processes` | Local execution despite another local run. The marked-host file restriction still applies. | +| `TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN=1` | Direct native-tool execution outside the canonical runner's protections, with native setup still the caller's responsibility. | +| `--against-test-guidance-turn-flakes-into-failures-on-broad-runs` | A zero runner-level retry budget for a selection broader than one explicit file. | + +`--allow-extreme-inefficiency` is renamed, without a working alias: using it refuses with exit `2` and names the replacement. This keeps the shared phrase on every route through these protections while making the transition actionable. Unrecognized options beginning with `--against-test-guidance` are usage errors, so a mistyped consequence suffix cannot silently leave retries enabled during an explicitly requested broad zero-retry run. The existing `--force` bypasses only the full-`all` provider-shortcut guardrail; it lifts neither local protection. The broad-local override does not lift that full-`all` guardrail either, so a full-`all` local shortcut on a marked host requires both flags. + +A caller needing both local exceptions must request both. Using the contention bypass is the caller's judgment, including for agents; it requires no human permission. The existing testing-policy requirement for explicit human permission before an agent uses the broad-local bypass remains in force. During this contribution, the narrower task instruction applies: locally execute only the one test file just edited; send broader validation to Blaxel on a pushed revision. + +## Validation retries + +Running a new or changed test once is normal focused iteration. Plain `--retries 0` is allowed for an explicit `--file` selecting exactly one actual file, optionally narrowed by grep, on either provider and on marked or unmarked hosts. Resolve against the local working tree or the selected remote commit; a directory, multi-file filter, surface alone, or grep without the file boundary does not qualify. + +Broader validation keeps the default budget. Plain zero retries on a broader selection refuses with exit `2` and names `--against-test-guidance-turn-flakes-into-failures-on-broad-runs`. The override sets zero on its own or alongside `--retries 0`; pairing it with a positive count is a usage error. It is the caller's judgment, with no human permission requirement, and lifts none of the other guardrails. Verify is broader than one file and checks this requirement before running phases, then forwards the selected retry policy. + +AGENTS.md will state the distinction briefly, with contributor guidance owned by the testing policy and exact mechanics by the runner spec and help. Unit defaults remain zero, positive numeric overrides retain their semantics, and per-test flaky annotations and infrastructure retries stay separate. Recovered flakes remain visible; newly written flaky tests still require repair. The attestation proof must cover the broad zero-retry override as a nonstandard budget even when `--retries` is absent. + +## Guidance agents reach from AGENTS.md + +The contribution includes written guidance as well as enforcement. The repository's [AGENTS.md](../../AGENTS.md) is the entry point every agent receives, so its Testing section will state the practical rules directly and link to their authoritative owners. The intended summary is: + +> Use the canonical runner for fast, narrow local feedback on the behavior you are changing; direct Vitest and Playwright commands are refused by default. Concurrent runs can drive a shared host into swap or cause OOM kills. Every local test run and local verify takes a mutex for the same operating-system user across checkouts; retry after its holder finishes, pass `--wait-for-local-test-mutex`, or use Blaxel when available. The runner documents the independent bypasses: the contention bypass is the caller's judgment, while agent use of the broad-local bypass follows the testing policy's permission rule. +> +> On hosts with `~/.tvdev-use-blaxel`, select one local file with optional grep. Repeated focused runs are normal iteration; for a known multi-file check, commit, push, and use Blaxel. Follow the runner's guidance for local-only suites. Unmarked hosts need no Blaxel access. Plain `--retries 0` is normal for running a new or changed test once within one file. Broader validation keeps the default retries; disabling them requires the runner’s deliberate override. + +The [testing policy's iteration guidance](../../specs/arch/testing-policy.md#Test iteration discipline) will explain how to widen validation without widening local work on a marked host: start with the file under change, optionally use grep, then commit and push for broader Blaxel checks. It will state the shared-host safety priority and link to the runner's checks and independent bypasses. Its provider guidance will preserve ordinary local development for unmarked public-contributor and fork hosts. The [remote preflight spec](../../specs/arch/test-runner/preflight.md#Contributor branches and remote revisions) remains the owner of the committed, pushed revision requirement. + +The runner spec owns exact command behavior, file eligibility, lock lifetime, refusal messages, and bypass semantics. CLI help will expose the same practical choices at invocation time. This uses the existing instruction path from AGENTS.md to policy and command authority. The existing contribution and review skills will reinforce the development-branch practice below. + +## Regular development-branch commits and pushes + +Josh's standard way of working is for contributors and agents to work on their own development branch and commit and push regularly, in safe, coherent commits. A commit should capture a meaningful unit of the assigned work and be safe to publish; it need not make unfinished work ready to merge. Development branches may carry incomplete work or failing tests while the contribution follows its review and validation stages. Shared-branch merge requirements remain in force. + +Pushing makes the current work available to Blaxel. Withholding pushes prevents remote workers from testing that revision and pressures agents into broad local runs, which can drive a shared host into swap and make it unusable for every agent. The normal response is to prepare and push a coherent development-branch checkpoint, then run the broader checks on Blaxel. Withheld or blocked pushes never themselves justify broad local execution. + +A supervisor or coordinator must not casually instruct a worker not to commit or push on its development branch. Routine review coordination is not a reason to suspend this practice. A concrete safety or conflicting-edit issue can justify a temporary hold; required validation must still respect the local-run restrictions during that hold. + +The [workflow](../../specs/spec-workflow.md#Pull requests and human revisions) will own this regular development-branch practice and its coordination guidance. The contribution will carry it into the instructions each role reads: + +- **AGENTS.md:** the shared-branch section will directly tell contributors and agents to work on a development branch, commit and push safe, coherent changes regularly, and preserve workers' ability to do so during coordination. Its testing section will connect pushed checkpoints to Blaxel and shared-host protection. +- **[tvdev-contribute](../../developer-skills/tvdev-contribute/SKILL.md):** contribution and delegation guidance will make regular development-branch commits and pushes part of ordinary progress, including before broader Blaxel validation. It will explain the shared-host consequence of withholding pushes and the restriction on casual coordinator instructions to do so. +- **[tvdev-review](../../developer-skills/tvdev-review/SKILL.md):** one sentence will distinguish routine pushed development checkpoints, which enable remote validation while review proceeds, from readiness to merge. This clarifies how the existing stage-specific review guidance applies without adding a reviewer procedure or giving reviewers a commit/push role. + +These instructions describe normal practice without a fixed commit interval or a separate approval step for each push. They distinguish publishing a development checkpoint from merging into a shared branch, so review gates do not prevent the remote validation needed to satisfy them. Repository guidance reaches a coordinator working outside the checkout when it reads AGENTS.md or uses the contribution skill; this change does not update external supervisor skills. Copied skill installations need the existing reinstall procedure to receive source edits. + +## Scope and derivation + +The contribution includes the AGENTS.md guidance above, authoritative runner, testing-policy, and workflow changes, and the corresponding contribution and review skill updates. Proof derivation will establish that local protections enforce the stated selection and lifetime guarantees and that the runner's own tests remain isolated. Test shape and fixtures follow the testing policy's cheapest honest coverage and the existing placement restrictions on destructive lifecycle tests. + +Implementation should reuse the existing CLI, registry, marker, process lifecycle, and registered runner-config entrypoints. This contribution does not require a queue, configurable concurrency levels, a background lock service, or changes to public-contributor provider selection. Help and affected invocation guidance will show the one-file local path and the committed-revision remote path. + +The current Blaxel CLI dispatches `--surface`, `--file`, and `--grep` through a single-worker target path. Suite, package, runner, and tag selections use planned file distribution. This proposal preserves those remote execution paths: redirecting a broad local selection to even one remote worker meets the primary goal of taking that test load off the shared host. Fan-out for targeted surface runs remains a separate speed improvement and is neither a prerequisite for these guardrails nor grounds for a broad local exception. + +After proposal review, spec deltas and proofs receive their own independent review before implementation. The mutex, file-selection gate, native-entrypoint checks, and guidance may warrant separate implementation slices; derive a plan after proof convergence to settle their order and validation. Completion requires broad validation on the pushed revision, integrated implementation review, documentation cleanup, and a pull request describing the final change for Josh's spec review under the repository workflow. diff --git a/docs/archive/2026-10/test-runner-spec-rewrite/change-detail.md b/docs/archive/2026-10/test-runner-spec-rewrite/change-detail.md new file mode 100644 index 00000000..407942a2 --- /dev/null +++ b/docs/archive/2026-10/test-runner-spec-rewrite/change-detail.md @@ -0,0 +1,132 @@ +> **Archived 2026-10 from PR #53.** This is the detailed description of PR #53, written when the pull request was ready for review and moved out of its description, which keeps only an orientation; the merged change is the one it describes. It holds how the guardrails, worker leases, cleanup and diagnostics work, the judgment calls in the spec rewrite, how the proofs and tests were brought into line with the rewritten specs, and the validation evidence, including the live Blaxel probes and the evidence carried over from #49 and #12, which the description only summarizes. The body below is unchanged since it was written and is a clue to the change, not a record of it. + +# PR #53 in detail + +PR #53 carries #49's local guardrails and #12's Blaxel worker leases, cleanup and diagnostics, integrated with each other, together with a rewrite of the specs under `specs/arch/test-runner/` and the repairs that bring the proofs, references and tests into line with it. The ledger (`ledger.md`) and removal account (`removal-account.md`) in this folder cover the spec rewrite statement by statement; this document covers the whole change. + +## Local runs on shared hosts + +Each guardrail refuses with exit `2`, says what to run instead, and has a deliberately named override containing `against-test-guidance`. They steer ordinary use and are not a security boundary. + +- **One local run per operating-system user.** `npm test -- local` and local verify take a mutex shared by every checkout of that user on the host. They take it before preflight and hold it until the test processes have exited and cleanup has finished; verify's test phase shares verify's hold. A second run is refused at once with the holder's process, command and checkout. `--wait-for-local-test-mutex` waits instead, without a time limit, and `SIGINT` or `SIGTERM` cancels the wait with exit `130` or `143`. The mutex is a single row in a SQLite database at `~/.cache/television/local-test-mutex.sqlite3`, using Node's built-in SQLite (`scripts/test/local-test-mutex.mjs`). The home directory comes from the operating-system account, so tests that override `HOME` still share it. A holder is stale only when none of its recorded processes is alive by PID and start time; elapsed time never releases it. +- **One explicit file on marked hosts.** On a host with `~/.tvdev-use-blaxel`, a local run must resolve `--file` to exactly one actual test file, optionally narrowed with `--grep`. Untracked files count. A broader selection is refused with the equivalent `npm test -- blaxel` command and a reminder to commit and push first. Agent, PostHog and daemon surfaces and the Blaxel harness driver cannot run on Blaxel, so their refusal gives the one-file local command. Hosts without the marker, such as public contributors' machines, keep full local breadth and need no Blaxel access. +- **Exact files to the native tool.** The runner resolves `--file` itself and hands Vitest or Playwright exactly those files (`scripts/test/target-native.mjs`), because Vitest treats a file argument as a substring and Playwright as a regular expression. The fixtures in `test/runner-fixtures/guidance/` include a sibling named `one.test.ts.test.ts` that a substring match would collect. +- **Retries follow the same boundary.** Defaults stay at two runner retries for non-unit surfaces and none for unit surfaces. Plain `--retries 0` is accepted for one explicit file on either provider. A broader zero-retry run, including verify, needs `--against-test-guidance-turn-flakes-into-failures-on-broad-runs`. Per-test `flaky:` annotations and Blaxel's infrastructure retries keep their own budgets. +- **Direct native entrypoints refuse.** All 38 registered Vitest and Playwright configs import `scripts/test/native-entrypoint.mjs`, which refuses a direct `vitest` or `playwright` invocation and names the canonical command. Every path by which the runner starts a native tool (local runs, planned and targeted shards, diagnostics and Electron launch) marks its environment so that its own runs load. + +| Override | What it permits | +| --- | --- | +| `--against-test-guidance-broad-local-run` | Broader local runs, including local verify, on a marked host. The testing policy requires explicit human permission for agents. The mutex still applies. | +| `--against-test-guidance-major-host-contention-and-oom-killed-processes` | Running alongside the mutex holder. It cannot be combined with waiting, and the file limit still applies. | +| `TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN=1` | Loading a registered config directly, for watch mode, a debugger or an editor. Such a run is outside the mutex and the file limit, and native setup is the caller's job. | +| `--against-test-guidance-turn-flakes-into-failures-on-broad-runs` | A zero runner retry budget for a broad selection. | + +`--allow-extreme-inefficiency` is renamed `--against-test-guidance-broad-local-run`; the old spelling refuses and names the replacement. `--force` still permits only the full-suite shortcut. + +## Blaxel worker leases + +A shard attempt leases one worker from before checkout until its processes are confirmed gone and its reports have been downloaded and checked (`scripts/test/blaxel-worker-lease.mjs`, `scripts/test/blaxel-lease-lifecycle.mjs`). + +- The lease is `/tmp/tv-testshard.lock/owner.json` on the worker. Every change to it happens under a short `flock` on `/tmp/tv-testshard.guard` and rechecks the current owner first, so a delayed renewal or release cannot overwrite a newer owner. The guard is never held during downloads. +- Only the coordinator renews. It records each dispatch before requesting it, and the remote entry point registers its PID and start time before touching the checkout, and only while the lease still belongs to that coordinator. +- Every process the attempt starts carries `TV_TEST_SHARD_OWNER`, the lease's `lockId`, beside each surface's `TV_TEST_SURFACE_OWNER` token. Release requires an audit that finds no process carrying the attempt's tokens; survivors are stopped after their PID and start time are validated. An unreadable audit, an unknown dispatch outcome or a surviving owner leaves the worker leased and out of the run's retries. +- A lease not renewed for 20 minutes (`--lock-ttl-seconds` changes this) is abandoned. The next acquisition stops only the abandoned attempt's processes, confirms that no test owner remains, removes the abandoned attempt's scratch, discards its undownloaded reports, and writes a recovery receipt into the new attempt's evidence. If the owners cannot be identified or stopped, the worker is held for an operator. +- Lease records set `expiresAt: 0`, which runners predating the lease never treat as passed, so branches still on those runners leave current leases alone. A current runner respects a legacy lock until its deadline and then acquires only after a clean audit. +- Pool provisioning makes a worker eligible for test runs only after it is provisioned and verified. Deletion refuses leased workers, reserves an idle worker after a clean audit, and removes its eligibility before calling Blaxel (`scripts/test/blaxel-pool-lifecycle.mjs`). + +## Temporary files and generated runtimes + +Worker `/tmp` is memory-backed and holds Chromium's shared memory, so temporary files on a worker cost memory. + +- Each shard attempt gets a private directory under `/tmp/tv-testshards/`, and `TMPDIR` points into it. Holding the lease, and after a clean audit, a run first removes only directories directly under `/tmp` named `television-source-cli-e2e-*` or `television-web-bundle-*`, and earlier attempts' scratch. After the attempt it stops the attempt's processes and removes its scratch. Persistent caches and unrelated files are kept. +- Test helpers give reusable builds the lifetime of their test worker (`test/helpers/worker-builds.ts`). A source-CLI or versioned web build is shared within its Vitest or Playwright worker, published only after preparation succeeds, and removed at worker teardown once servers have stopped and pending builds have finished. Failed preparation and per-launch copies remove themselves. The worker cleanup above reclaims what killed test workers leave behind, and what test revisions without these helpers leave behind. +- Five browser acceptances (two theme lifecycle walks and three demo-mode cases) load the built application through `ProductServer` and keep their authentication, restart, navigation and rendering assertions. `test/helpers/application-requests.ts` records their request paths to show they loaded built assets. The testing policy makes the built application the default for new or substantially changed `ProductServer` browser acceptances. + +## Resource diagnostics + +Each Blaxel surface execution records available memory, shared memory in use, free temporary space and the kernel's out-of-memory kill counter at its start and end, and adds the kernel's out-of-memory messages when a native attempt failed or the evidence is incomplete. The records are linked from the surface result, survive retries and download, and are bounded at 64 KiB of kernel lines, 128 KiB per record and 8 MiB per shard attempt (`scripts/test/resource-diagnostics.mjs`). They never change a test outcome, and they do not measure peak memory. + +## Refusals and run directories + +A refusal leaves no run directory, as `specs/arch/test-runner/reporting.md` states. In `scripts/test/cli.mjs`, `runBlaxel` makes its selection checks, one surface for a targeted run and a supported suite otherwise, before remote preflight and before it creates the run directory, and an invalid `--run-dir-output` value is rejected before either provider creates one. The arguments passed to the Blaxel coordinator are unchanged. A caller that passed `--run-dir-output` therefore finds either a run directory with its summary or no directory at all. This ordering is the only behavior change the spec rewrite made to the code. + +## Live Blaxel qualification + +`test/runner-fixtures/blaxel-harness/` is an automated qualification of the lease, recovery, deletion, delayed retrieval, diagnostics and temporary-file placement, run against disposable private workers. It has two experiment surfaces outside `all` and CI: `experiment:blaxel-harness`, the driver, and `experiment:blaxel-harness-fixtures`, which the driver dispatches. The driver runs only on a dedicated, credentialed coordinator sandbox outside every worker pool, with `TV_BLAXEL_HARNESS_COORDINATOR` and `TV_BLAXEL_HARNESS_POOL` set. The runner refuses it on Blaxel, inside a leased worker, or without both variables. The driver surface's registry root is its one test file, so on a marked host its command is `npm test -- local --file test/runner-fixtures/blaxel-harness/driver/harness.test.ts`. The fixture README gives the full usage. + +## Contributor guidance + +`AGENTS.md`, the testing policy, the spec workflow and the contributor and reviewer skill sources describe the practice the guardrails support. Contributors iterate on one file locally and commit and push regularly so that Blaxel can test current work. They run known multi-file checks on Blaxel rather than as a series of local one-file runs, keep default retries for validation, and narrow a broad failure to its file before the next broad run. The workflow adds that supervisors and coordinators must not casually tell workers to withhold development-branch commits or pushes. The testing policy says which overrides need human permission, and `specs/terms.md` defines *marked host*. The desktop, artifact, PostHog, daemon and Blaxel setup instructions use commands the guardrails accept. + +## Test-runner specs + +The nine specs under `specs/arch/test-runner/`, 28,168 words, become six specs of 10,627 words. They were written from an empty page under `specs/spec-policy.md`. A statement is included only where an agent deriving behavior from the code cannot be trusted to get it acceptably right, or where something outside the runner depends on it. Testing guidance is included only where removing it would change a correct proof derivation. + +| Spec | Owns | +| --- | --- | +| `test-runner.md` | The command, selectors and exit codes; providers; the test registry; the shared-host guardrails and overrides; retries; verify; the processes and network addresses that tests create. | +| `blaxel-testshards.md` | The pool, repository and credentials, the lease protocol, temporary files, how Blaxel runs selections, and setup. | +| `sharded-execution.md` | Shard plans, the duration baseline, and how shard outcomes combine. | +| `reporting.md` | The run directory, honest status, the compact summary, timing records, keeping secrets out, and resource diagnostics. | +| `github-ci.md` | When CI runs, the required checks, and the build, cache and artifact arrangements. | +| `attestation.md` | Tree attestations: the claim, who may write, the policy file, skipping in CI, pruning and operations. | + +`preflight.md`, `flaky-tests.md` and `test-registry.md` are deleted. The rules for remote revisions are at `test-runner.md#^remote-revision`, retries and flaky annotations under `test-runner.md#retries`, and the registry under `test-runner.md#the-test-registry`. Flake discipline is left to the testing policy, Electron preflight and runtime preparation to `specs/arch/desktop/e2e-harness.md`, and the PostHog key to `specs/arch/telemetry/sink.md`. + +The specs pin the contracts that other runner revisions, people and external services depend on: + +- the local mutex's location and record, the surface owner token, and the Blaxel worker lease with its deletion reservation, which runners of different revisions read on one host or worker; +- the `summary.json` fields that people and scripts read, and the `events.ndjson` envelope and fields that `npm test -- baseline update` validates in earlier runs' streams, with the rule for changing them; +- the job names the repository ruleset requires (`lint`, `type-check`, `package-manifests`, `unit`, `e2e:desktop` and `e2e-required`), the attestation ref and policy file, and the GitHub Actions behaviors the attested skip must work around, such as `continue-on-error` making a failed job look successful to its dependents and job-level conditions hiding a matrix job's check names; +- measured facts that derivation would not recover, such as the Chromium shared-memory cost of `304` responses and the single Playwright worker for `e2e:browser-app-real-stack`. + +Internal types, algorithms, option tables and tuned settings are left to the code. The specs also state an existing gap: Blaxel workers have no Firefox, so a Blaxel run covers Chromium only, and an attestation from a Blaxel verify does not cover the Firefox projects, which CI then skips for that tree (TV-1011). + +These are the judgment calls in the rewrite a reviewer is most likely to question: + +- **Shapes pinned for the first time.** The mutex record, the owner token, the worker lease, and the summary and event fields were called contracts in the earlier specs without a shape. Each is pinned only as far as another revision or reader relies on it. +- **Left to the code.** The normalized result and provider report types, the baseline file's format, the shard-balancing arithmetic and the coordinator's option table. +- **What crosses runner revisions is named, and its shapes are left to the code by a stated exception.** In a Blaxel run with `--commit`, the coordinator is the checkout's runner and the workers run the tested commit's. `blaxel-testshards.md#runs` says that the registry, baseline, plan, worker command and shard summary cross revisions, and that a mismatch must fail as infrastructure and never be read as results. Spec policy asks for pinned types for data another revision reads, so the paragraph states an exception: both ends validate what they receive, and the worker checks the plan against a digest of its whole content, so no subset of fields would be the whole contract. +- **The attestation policy read comes before the run's tests, not before every phase.** CI's lint, type-check and manifest jobs start alongside the decision job that first reads the policy, while every test job waits for it. Requiring the read before every phase, as the earlier attestation spec did, would mean reordering CI. +- **Measurements left out.** The 36-shard timing figures, the timeout profiles, the `e2e:browser-app` two-worker setting and the cost-model revision triggers are not in the specs. The settings remain in the code. +- **Left to other owners.** Electron preflight and runtime preparation, the PostHog key, flake discipline and the `tv serve` test recipe. Each is right only if the named owner states it. +- **Small rules left out.** The quarantine-file allowance, `--fail-fast`, the `--allow-extreme-inefficiency` refusal, `--audit-stale-owners` and the record of retiring Blacksmith runners. The two options and the refusal remain in the code. + +The removal account goes through the 430 passages of the nine earlier specs, as they stood with #49 and #12 integrated, and says what became of each: 2 kept, 257 reworded, 60 moved to another spec, 35 now derivable from a named statement, and 76 dropped. 77 of the reworded or moved passages also lose a detail, which the entry names. + +## Proofs, references and tests + +The proofs follow the six-spec layout. The proofs for the registry, preflight and flaky tests become sections of `proofs/arch/test-runner/test-runner.md`, keeping every assertion, hook and anchor, so tests that cite them change only the file name. The proofs were brought into line with the rewritten specs, not derived again from them: + +- Links to removed headings and anchors point at where that content now lives. +- Statements the specs now contradict are corrected: attestation eligibility is by tree, not by `--commit` naming another commit; CI's build-recipe check covers surfaces in the `all` suite; the shard-count recommendation allows exactly two minutes and falls back to the pool size. +- Assertions on detail the specs leave to the code, such as the shard-assignment algorithm, the weighting constants and pinned worker settings, are kept and marked *superfluous*. +- Contracts the specs pin for the first time are cited to the tests that already cover them: the owner-token format, the worker's digest check on the shard plan, refusal of a worker reserved for deletion, `--run-dir-output`, and one rule of the timing-event envelope. + +Two promises in the rewritten specs had no test, so tests were added. In a disposable checkout with the production CLI, `test/repo/test-runner-guardrails.test.ts` refuses a local full-suite shortcut, a two-surface targeted Blaxel selection, an unsupported Blaxel suite and an empty `--run-dir-output`, and shows that the checkout has no `.test-runs/` afterwards. `test/repo/test-runner-reporting.test.ts` checks the `summary.json` fields the spec pins. Two existing tests compared the code with sentences of the earlier specs; they now read the sentences in `github-ci.md` and `blaxel-testshards.md` that state the same rules. + +References elsewhere follow the new files. `AGENTS.md`, the spec workflow, the testing policy, the desktop, licensing, Node-version and telemetry specs, the migration map, `developer-skills/tvdev-setup/SKILL.md` and code comments link to the sections that now hold their content. Two of these spec edits change wording rather than only a link: + +- `specs/product/desktop-app.md` explained its required Mac verify run by citing preflight's testing guidance. The sentence now says what the run exercises: the runner's macOS process backend. +- `specs/terms.md` points each test-runner term at its new owner, defines *recovered flake* and *failed shard* by pointing at the rules that define them, and drops *execution group* and *normalized surface result*, which no spec now defines. + +## Validation + +`npm run verify -- blaxel` passed at `1945b6c5` (tree `b1d2a3d0a29355c94d99b0aea2908482eaacbc66`), the head before this document was added: lint, type-check and package manifests, then 22 surfaces on 36 shards with 3,812 tests passed, 8 skipped, and no failed tests, recovered flakes, process leaks or infrastructure failures, then the vibe-mode check. It published `refs/testpass/1/b1d2a3d0a29355c94d99b0aea2908482eaacbc66`. [GitHub CI run 40779402162](https://github.com/telepath-computer/television/actions/runs/40779402162) passed at that commit, with the heavy jobs skipping on that attestation. Blaxel runs Chromium only, so the Firefox projects ran in neither (TV-1011). + +The full verify divides suites into planned shards, so it never takes the targeted Blaxel path where the guardrail and diagnostics code meet: `target-native.mjs` writes the targeted Vitest runner config and attempt file, and `supervised-command.mjs` reads that attempt file for resource diagnostics. Four fixture probes crossed the real Blaxel API and transport on `8e6cfcb2`, whose code differs from `1945b6c5` only in the run-directory ordering above and in comments. Under the default budget, a Vitest and a Playwright file selection each recovered a deliberate first-attempt failure, and the native report held only the selected file. `--retries 0` made the failure final. A planned two-shard run with the broad zero-retry override kept the failure final while the separately annotated case still retried. Each kept complete resource diagnostics with resolving links. + +Each probe ran with `--commit 8e6cfcb2` on the `poc` pool with the default infrastructure retry budget of 2. Run directories were under `.test-runs/` in the checkout that ran them. + +| Probe | Selection | Observed | Run directory | +| --- | --- | --- | --- | +| A | `--file test/runner-fixtures/guidance/vitest/one.test.ts --grep 'deliberate first-attempt failure\|sibling'` | exit 0; runner budget 2; native report holds only `one.test.ts`; attempts failed, then passed; generated runner config, attempt file and native report downloaded and readable; diagnostic record complete with the failed attempt and recovery marked, kernel evidence available, both links resolving | `2026-10-28T10-23-13-752Z-p1710376-r9118b10c699014bb` | +| B | as A, with `--retries 0` | exit 1; budget 0; one failed attempt; the same artifacts kept; diagnostic marks the failed attempt, no recovery | `2026-10-28T10-23-11-962Z-p1710373-r9ceb449ad7bda74d` | +| C | `--file test/runner-fixtures/guidance/playwright/one.test.ts --grep 'deliberate first-attempt failure\|sibling'` | exit 0; budget 2; native report holds only `one.test.ts`; diagnostic complete with the recovery marked | `2026-10-28T10-24-08-127Z-p1711445-rb2d4c679f21425bd` | +| D | `--tag runner-guidance --runner playwright --against-test-guidance-turn-flakes-into-failures-on-broad-runs --shards 2 --allow-shard-count-override 1` | exit 1; test retries 0, infrastructure retries 2; shard 1 ran `one.test.ts`, its deliberate failure final after one attempt and the annotated case recovered; shard 2 ran the sibling file; both shards' test lists, native reports and diagnostics kept | `2026-10-28T10-24-42-747Z-p1711676-r9d77334ea5dd8cef` | + +The lease, recovery, provisioning, deletion and scratch-cleanup code is unchanged from #12, and the mutex and lifecycle code from #49, so their live evidence applies here: + +- The Blaxel qualification passed on disposable private workers: competing acquisition and deletion, delayed retrieval, active and quiet coordinator crashes, recovery receipts, diagnostic transport, and the files Node and Chromium actually create under the attempt's `TMPDIR`. The worker-lease and lease-contract tests passed all 20 cases on private Blaxel workers at `2156f266`, including a root-only case with a same-user nondumpable child. [GitHub CI run 40594589067](https://github.com/telepath-computer/television/actions/runs/40594589067) passed at `9787188d`. +- A full Blaxel verify on #12's branch removed 1,510 generated runtime directories, 5,003,157,504 allocated bytes, and kept 231 diagnostic records with complete start and end readings and no change in the out-of-memory kill counter. Peak memory and how often renewals are lost over time remain unmeasured. +- [Isolated Linux lifecycle run 40502488567](https://github.com/telepath-computer/television/actions/runs/40502488567) passed at `eeef2dab`: the direct mutex tests, verify's handoff, four mutex crash and recovery cases and thirteen existing lifecycle cases. No macOS run is claimed. [GitHub CI run 40514829404](https://github.com/telepath-computer/television/actions/runs/40514829404) passed at `ea587122`. diff --git a/docs/archive/2026-10/test-runner-spec-rewrite/ledger.md b/docs/archive/2026-10/test-runner-spec-rewrite/ledger.md new file mode 100644 index 00000000..d1a54019 --- /dev/null +++ b/docs/archive/2026-10/test-runner-spec-rewrite/ledger.md @@ -0,0 +1,443 @@ +> **Archived 2026-10 from PR #53.** This was the ledger of the test-runner spec rewrite; the specs merged as revised by the review responses at its end, and the proofs as described in its repair-phase section. It records why each statement in the rewritten specs is there and how each review finding was answered, including the alternatives rejected, which is found nowhere else. The body below is unchanged from its working state and is a clue to the change, not a record of it. + +# Test-runner spec rewrite: ledger + +This ledger records why each statement in the rewritten `specs/arch/test-runner/` specs is there. The specs were rewritten from an empty page under `specs/spec-policy.md`, following the spec-rewrite method's steps 1 and 3. The old specs are those at `bcc65d97`, the merge of #49 and #12. Later commits on this branch change no spec text. + +The test for each statement: derivation from the code and the rest of the specs could not be trusted to get it acceptably right, or something outside this version of the product depends on it staying as it is. "Outside" here is mostly one of four things: + +- other checkouts and branches running other revisions of the runner on the same host or the same Blaxel pool; +- people and agents who type the commands and read the output; +- GitHub settings and services that people maintain; +- Blaxel. + +## Layout + +The area is rewritten as a set, so that each fact has one owner. + +| New file | Holds | Absorbs | +|---|---|---| +| `test-runner.md` | the command, guardrails, retries, verify, the registry, test processes and addresses | old `test-runner.md`, `test-registry.md`, `preflight.md`, `flaky-tests.md` | +| `blaxel-testshards.md` | the pool, repository and credentials, the lease protocol, recovery, temporary files, setup | old `blaxel-testshards.md` | +| `sharded-execution.md` | shard plans, the duration baseline, shard outcomes | old `sharded-execution.md` | +| `reporting.md` | the run directory, honest status, the compact summary, timing records, resource diagnostics | old `reporting.md` | +| `github-ci.md` | triggers, required checks, the join, build transfer, caches | old `github-ci.md` | +| `attestation.md` | the claim, writers, policy, skip wiring, pruning | old `attestation.md` | + +Once the rest was cut, the registry, preflight and flaky-test content was each a few paragraphs, all about what the command does before and around a run. Keeping them as separate files would have made readers hunt across four files for one command's behavior. Their proofs under `proofs/arch/test-runner/` are now orphans, which the spec-link test reports until the proofs are re-derived. + +## test-runner.md + +**R1. One command; package scripts and direct native runs are not supported.** This is the design intent the whole area serves. Without it, derivation adds per-package scripts and invocations, which is the drift the runner exists to stop. + +**R2. Ownership and pointers to the companion specs and the testing policy.** The policy asks for ownership where it would not be obvious. The testing policy owns permissions, and readers expect them here. + +**R3. Guidance, not enforcement; the shared-host and retry overrides carry `against-test-guidance`; not a security boundary.** This design intent decides how far the checks go. Without it, an implementer would plausibly harden checks into authentication or host-wide policing. The shared phrase is a naming convention people and agents rely on, scoped to those overrides because `--force` and others do not carry it (review round 1, finding 10). + +**R4. Providers: local runs the working tree; Blaxel runs a committed revision; CI is not a provider; only verify chooses automatically.** These define what a result means, which callers depend on. + +**R5. Blaxel needs a clean tree and a commit on an origin branch; commit and push first; `--ignore-uncommitted`, `--commit`.** People must act on this, and the workflow spec links to it. The flag meanings are a command contract. Anchored `^remote-revision`. + +**R6. Commands table.** These commands are the contract with people, agents, CI and the docs that name them. Each entry gives only its purpose and its owner. + +**R7. Selectors, including how `--file` matches, that its matches must belong to one selected surface, that `--grep` alone does not narrow, and which commands default to `all`.** These are command-line contract. That `--file` is a substring filter unless exact is non-obvious, and people rely on it. Without the default sentence a reader would assume that `list` and `preflight` also start from `all` (per-file review round 1). + +**R8. The runner resolves files itself, because Vitest and Playwright filters are not exact.** This is knowledge that would otherwise be lost. A derivation that passes the user's string through would silently run sibling files, breaking both the one-file limit and plan assignment. + +**R9. Exit codes `0`, `1`, `2`, `130`, `143`, and `verify --plan` exiting `0` without verifying.** Scripts, CI and agents branch on these codes. `2` separates a refusal from a test failure. + +**R10. A Blaxel command succeeds only with a passed normalized report.** This decision is about honesty. Trusting the coordinator's exit alone is a plausible derivation that would turn missing evidence into a pass. + +**R11. The registry is the single declaration of surfaces; classification is never inferred from names.** This architectural decision makes everything else selectable. A derivation could plausibly infer kinds from paths. + +**R12. Every test file under `packages/` and `test/` has exactly one owner, and an undeclared `vitest.config.ts`, `vitest.e2e.config.ts` or `playwright.config.ts` is an error.** This invariant guarantees that no test silently drops out of every run. The names matter because fixture configs with other names are deliberately unregistered (per-file review round 1). + +**R13. `all` is unit plus end-to-end and means "the full suite" for verify, CI and attestation; agent, PostHog, daemon and experiment surfaces are outside it.** This defines the gate other specs depend on, and it records why each excluded class is excluded. + +**R13a. Firefox projects are omitted with a warning outside GitHub Actions when Firefox is missing; GitHub CI requires it; Blaxel workers have none, so Blaxel runs cover Chromium only.** This is an accepted limitation that changes what a passing local or Blaxel result proves, which a reader would not derive from the registry. It was observed in Blaxel shard logs and native reports (review round 1, finding 4). + +**R14. Daemon acceptance mutates the real global `tv` and user service, runs only on a designated host, and refuses without `TV_DAEMON_TEST_HOST=1`.** This is a contract with the people who run it, and a safety decision. The variable is set by hand. + +**R15. `isolated-github-only` placement, the uncommitted operator workflow's name and variables, and that workflow's obligations.** The workflow is outside the repository and maintained by an operator, who depends on these exact names. The obligations are what make destructive fixtures safe there. + +**R16. Shared-host intent: host health first; small, one-at-a-time local runs; broader work on Blaxel.** This design intent explains the guardrail family and sets its priority, so a human should own it. + +**R17. The marker `~/.tvdev-use-blaxel`, found through `HOME`; unmarked hosts keep full local use.** People create the file and setup skills tell them to. Fork and contributor hosts depend on working without Blaxel. Anchored `^blaxel-host-marker`; `terms.md` cites it. + +**R18. The one-file local limit on marked hosts, and that it counts uncommitted and untracked files.** This is the core guardrail decision. "Untracked counts" is a correction a derivation from committed inventories would miss. Anchored `^local-one-file`; the testing policy and the desktop harness cite it. + +**R19. What the refusal offers: an equivalent Blaxel command, a local route for selections Blaxel cannot run, never acting for the caller.** This contract with agents is what makes the refusal usable. The local-only route was a real integration defect, found in review of this branch. + +**R20. The broad-local override, and that `--force` does not lift the limit.** The testing policy and docs name this flag. Without the `--force` clause, a reader could plausibly treat `--force` as the master override. + +**R21. The mutex: scope, when it is taken and held, and who does not take it.** These decisions shape every local run. Taking it on Blaxel dispatch, or releasing it before cleanup, are plausible wrong derivations. Anchored `^local-test-mutex`; the testing policy cites it. + +**R22. Contention behavior: immediate refusal, the wait option, the contention bypass, advice not to kill another run, and how the options combine.** The flag names and behavior are command contract, cited by the testing policy and AGENTS.md. The combinations are where a derivation would most plausibly differ. + +**R23. Staleness is by process identity, never elapsed time; the next run reaps leftovers and fails.** A time-to-live is the obvious derivation and would break long runs. The failure outcome matches the stale-owner rule. + +**R24. The mutex's location and record are a cross-revision contract; the path rule ignoring `$HOME`; the pinned SQLite table and record; atomic acquisition, identity-checked changes, no transaction held during tests.** Checkouts at different revisions on one host read and write this record. The policy says to pin stored data another version reads. That the path ignores `$HOME` is non-obvious and load-bearing. The update method is left open (review round 1, finding 9). + +**R25. Native-entrypoint refusal and `TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN=1`; runner launches mark their environment.** People set the variable by hand, and the testing policy cites the route. The last clause keeps implementers from breaking the runner's own launches. + +**R26. The full-suite shortcut is refused in favor of verify; `--force`; verify passes `--force`.** This decision keeps broad validation behind lint and type-check, and the forms that trip it are command contract. + +**R27. The runner retry budget: 2 for non-unit surfaces, 0 for unit; `--retries`; recovered flakes pass and are reported.** These decisions define what passing means. Attestation and CI depend on the standard budget. + +**R28. The zero-retry rule and `--against-test-guidance-turn-flakes-into-failures-on-broad-runs`.** This decision and flag are command contract, cited by the testing policy and contributor docs. + +**R29. The per-test annotation: `FLAKY_TEST_RETRIES` (5 under the runner and CI unless the variable is set), the `flaky:` prefix that reports read, one case only, and only after investigation.** Test authors would not derive that the title prefix is load-bearing for reports. The "one case after investigation" rule counters a known agent tendency to annotate broadly to get a green run. + +**R30. Infrastructure retries rerun only infrastructure failures.** This decision keeps test failures from being hidden. Detail lives in `sharded-execution.md`. + +**R31. No retry clears a process leak.** A derivation that treats leaks as flakes would retry them away. + +**R32. Ordinary runs impose no automatic failure cap.** This answers an objection an implementer can be expected to raise (a `maxFailures` cap), and it keeps evidence complete. It is scoped to ordinary runs, so it does not forbid the shard worker's deliberate opt-in early stop (review round 1, finding 8). + +**R33. Verify's phases for each provider (Blaxel tests the selected committed revision, `HEAD` by default), its success line, publication, and `--plan`, which runs no phases but still selects a provider.** The contribution workflow, people and agents rely on the gate's composition and output. That `--plan` can contact Blaxel on a marked host is what a caller would not expect (per-file review round 1). Publication rules live in `attestation.md`. + +**R34. Verify's provider choice: unmarked means local; marked means Blaxel if its checks pass, and refusal otherwise, never a silent fallback.** This decision's behavior people depend on, and silent fallback is the plausible wrong derivation. + +**R35. The vibe-mode check: what it detects and why, that it runs last, and that CI applies it.** No other spec owns the check, and the vibe-waiver template depends on this behavior. Running last so that other results stay real is a non-obvious decision. Anchored `^verify-vibe-mode`; code and tests cite it. + +**R36. Test process scopes and their owner as the final reaper.** This defines the term the rules below use. Anchored `^test-surface-owner`. + +**R37. Token inheritance, and that helpers must preserve the token.** Test authors would not derive the helper obligation. Breaking it makes processes undiscoverable. + +**R38. Teardown in its implemented order, leak detection, start-time revalidation, and that leaks fail the surface.** This strict decision has visible consequences. Warning instead of failing is a plausible derivation. Anchored `^test-process-leak`. + +**R39. Discovery only by token through `/proc`/`ss` or `ps`/`lsof`, never by port or name; local runs and planned workers check the backend, and other paths such as targeted Blaxel runs do not.** This prohibition is stated on purpose: `pkill`-style cleanup is the obvious derivation and would kill concurrent runs on shared hosts. The preflight clause prevents silently running without cleanup where it applies. Its scope is stated so that it does not promise a check targeted runs lack (review round 1, finding 7). + +**R40. Stale owners are reaped and fail the run; live ones are left alone.** This decision is about safety beside concurrent runs and about not running in a contaminated state. + +**R41. The token format is a cross-revision contract; the pinned `SurfaceOwner`.** Hosts decode tokens written by other revisions to decide whether to kill processes. + +**R42. Port `0` everywhere, full URLs from the binder, no derived ports or numeric literals, and why.** A fixed or reserved port is the common derivation and races concurrent runs. Anchored `^test-dynamic-ports`; the CLI specs and tests cite it. + +**R43. The daemon acceptance exception.** Persisted serve rejects port `0`. Without the exception, the rule above would contradict a required suite. + +**R44. Vite services bound by the runner because Vite treats port `0` as unset; pre-transforms off for teardown.** This is knowledge that would otherwise be lost, both times a trap a reimplementation would fall into again. + +**R45. Vite services never answer `304`, and the Chromium shared-memory regression behind it.** This regression case says why it exists. Anchored `^test-service-full-responses`; code cites it. + +**R46. The Cursor agent environment workaround.** This environment fact a test author would not know. The desktop harness spec cites the rule. + +**R47. Testing guidance: destructive lifecycle tests go in isolated surfaces.** A proof writer could plausibly put crash and orphan cases into ordinary unit tests, which run on shared hosts. + +## blaxel-testshards.md + +**B1. Blaxel is the recommended broad validation; a hot persistent pool; a full verify in about three minutes; even one worker relieves the host.** This design intent is the reason for the provider and for its cost. + +**B2. Workers are shared by branches at other revisions and by several repositories, so the protocol must stay compatible.** This frames the cross-version contracts below, which a reader would not otherwise recognize as contracts. + +**B3. Ownership pointers to sharding, reporting and node versions.** These are where readers would expect the content. + +**B4. Pool labels, `DEPLOYED`, the x64 check, and failing when too few workers exist.** Every revision discovers workers by these labels, which are Blaxel-side state. + +**B5. The 75-worker tier limit, a pool of 72, standard runs of 36 shards, and the override.** The external service's limit is knowledge derivation cannot get. The numbers are the decision that keeps two runs within it. + +**B6. Fresh-only provisioning, and why: Blaxel's create-if-absent does not update; eligibility only after verification.** This is knowledge about the external service plus an operator procedure. In-place reconciliation is the plausible wrong derivation. + +**B7. Workers install nvm and the Node version from `.nvmrc`.** This is how the node-versions decision reaches workers. That spec links here for it. + +**B8. Repository selection from `origin`, normalized without credentials; non-GitHub URLs fail.** People rely on public and private checkouts working without options. Stripping credentials is a security decision. The coordinator script's `--repo-url` is left to the code, because `npm test -- blaxel` does not pass it (per-file review round 1). + +**B9. One pool, several repositories; one checkout switched per run; the shared trust boundary.** This is an accepted limitation with security weight, stated so that it reads as a decision. + +**B10. Token sources, the access the token needs, and that it is never written to files or Git configuration.** People configure the sources. The never-written rule is a security decision. + +**B11. The lease lasts until download, and why.** This decision fixes a real race: the next checkout overwriting results. + +**B12. The lease path, the `flock` guard, recheck before change, and no guard during downloads.** Other revisions use the same paths and guard. These are the cross-version protocol. + +**B13. Only the coordinator renews; the 20-minute interval; `--lock-ttl-seconds`; no reviving an expired lease.** This decision defines abandonment. The flag is command contract. + +**B14. `TV_TEST_SHARD_OWNER` equals `lockId`; dispatch registration before checkout; delayed dispatch cannot start work.** Recovery by other revisions finds processes by this variable. The ordering prevents obsolete work after recovery. + +**B15. Release needs confirmed process exit and retrieved reports; a shell exit is not proof; holding on doubt.** This decision rests on a non-obvious fact: a Blaxel process stop leaves descendants running. + +**B16. Interrupt handling and exit `130`.** This is command contract, and the guarantee that interruption keeps retrieved evidence. + +**B17. The pinned `WorkerLeaseRecord`: the `WorkerLease` fields that decide ownership, and the `DeletionReservation` that every acquisition refuses.** Other revisions read these fields to decide ownership. A revision that did not know the reservation could treat it as an incomplete record, reclaim it after a clean audit and start work on a worker being deleted (per-file review round 1). It is pinned only as far as they rely on it. + +**B18. The abandoned-run recovery sequence, the receipt, and holding when unsure.** This decision is the automatic recovery. Its ordering (stop, confirm, clean, discard, take) is what makes it safe beside live processes. + +**B19. Older runners and `expiresAt: 0`, marked transitional.** This cross-version compatibility with branches still on an older lock exists only while such branches exist, and the policy says to mark transitional code. + +**B20. Deletion follows the lease rules, reserves the worker with a `DeletionReservation` and removes dispatch eligibility first; never-published workers need no lease.** This decision about safety alongside concurrent revisions of the runner would not be derived. + +**B21. Memory-backed `/tmp`; private attempt directories; `TMPDIR`; short paths; persistent caches untouched.** These decisions rest on knowledge (memory cost, socket path limits). Anchored `^blaxel-scratch`; the testing policy cites the backstop. + +**B22. Removing only the named leftovers and earlier attempt directories, rejecting unexpected mount boundaries, with no link following and everything else kept.** This decision is a deliberate allowlist. The prefixes are what helpers on older branches leave behind, so they are a cross-version fact. + +**B23. Teardown removes scratch while holding the lease; deleting files does not free Chromium's memory; uncertain teardown keeps scratch.** This rests on knowledge (unlinked shared memory) and is a fail-safe decision. + +**B24. Workers keep only the current attempt's results; no archive.** This decision bounds worker storage. + +**B25. Targeted and planned selection forms on Blaxel.** This is command contract: what Blaxel accepts. + +**B26. Blaxel runs use the same registry declarations as local runs.** This design intent is what lets a remote result stand for a local one. + +**B27. Stale owners on a worker: no tests, excluded from the run, retry elsewhere.** This decision about safety and retry behavior would not be derived from the generic stale rule. + +**B31. What a Blaxel run's coordinator and workers exchange crosses revisions when they differ, as with `--commit`: the registry and baseline read from the tested commit, the plan and worker command sent, the shard summary read back. A change keeps working with revisions in use, or the mismatch fails as infrastructure and is never read as results.** The code presents this exchange as internal: the coordinator plans and validates with the checkout's runner, and workers run the tested commit's shard worker. The shapes are left to the code, as an exception to the spec policy's rule on pinning types, stated in the spec itself (per-file review round 2). Both ends already validate what they receive, so a mismatch fails closed. The worker checks the plan against a digest of its whole content, so no subset of the plan's fields is a complete contract, and the exchange includes the worker command line and environment as well as the data shapes; pinning some of them would suggest the rest may change freely (per-file review round 1). + +**B28. Setup: the CLI login, the 1Password item, the marker, the provisioning commands, and the `default` pool trap.** Operational procedure, which the policy places in arch specs. The setup skill links here. + +**B29. Testing guidance: which live run each kind of change needs; dedicated pools for experiments; observing temporary files live.** These are facts about the environment a proof writer could not derive: what a targeted run does and does not exercise, and the risk to shared workers. + +**B30. The live harness: placement, variable names, what the runner refuses, and the README.** This is an operator contract and a safety placement. A proof writer would not know the driver must not run on shared hosts or leased workers. Anchored `^blaxel-harness-placement`. + +## sharded-execution.md + +**X1. The purpose: complete, reproducible division, so that a passing result accounts for every selected file.** This design intent is the one guarantee everything else in the file serves. It is not "ran exactly once": an infrastructure retry can run a file again (per-file review round 1). + +**X2. Every selected file is assigned to exactly one shard, and files are never split.** This decision underlies the guarantee. + +**X3. The inventory comes from the tracked files of the tested commit and registry ownership, never from imports or the coordinator's tree.** This decision guarantees that independent jobs agree. Using the working tree is the plausible wrong derivation. + +**X4. Each shard runs exactly its files and checks what was collected; a shard with no files for a surface skips that surface.** The collection check is the runtime guarantee. Skipping empty surfaces is a decision with real cost impact. + +**X5. Plans are deterministic and identified by digest; why (CI jobs plan independently); how Blaxel distributes them, and that the plan crosses revisions (B31).** This is the decision that makes the CI join meaningful. + +**X6. Balancing by duration; unknown files weighted as expensive; predictions are advisory.** "Unknown means expensive" is a decision a derivation might get wrong. The advisory note is an accepted limitation. The algorithm and constants are left to the code. + +**X7. The plan is fixed; no queue; retries reuse the assignment.** This decision is about reproducibility, and a dynamic queue is a plausible alternative. + +**X8. The baseline holds Blaxel medians of planning duration from complete, finally passing observations, and all plans use them, CI included.** What is measured is a decision: an ordinary elapsed median would let failures and flake overhead into the weights (review round 1, finding 3). Using Blaxel time for CI would otherwise look like an error to a reader of the CI plan. The baseline file's format is left to the code. It does cross revisions in a `--commit` run, under B31's rule (per-file review round 1). + +**X9. The baseline is manual-only, from complete Blaxel full-suite runs, and fails without such runs, with the reason.** This decision guards against erasing measurements, which is non-obvious. + +**X10. The shard-count recommendation, bounded by the pool's size, and that people change the counts.** This decision is against automatic scaling, with the target that makes the recommendation meaningful. The bound keeps the sentence from promising a count that meets the target when none does (per-file review round 1). + +**X11. The three shard outcomes, retry rules, supersession, and how contradictions are treated.** This is the core honesty vocabulary, and the line between retrying and not retrying is a decision. + +**X12. Final results come from each shard's latest attempt; failures from superseded attempts do not reappear; the run stays incomplete while a shard is still infrastructure-failed; CI has no retry coordinator.** This is the combination decision, which a reader could plausibly get wrong. + +**X13. Testing guidance: only live planned and CI runs show plan transfer and agreement.** This environment fact is not derivable: a targeted run carries no plan. + +## reporting.md + +**P1. One result shape everywhere, and the uses: the summary, debugging, planning.** This is design intent. + +**P2. The run directory's location and files; the summary path printed; atomic writes; a summary on failure once the directory exists; when local and Blaxel runs create it; refusals leave none.** People, agents and CI read these locations. The early-failure rule is a decision for diagnosability (review round 1, finding 5). A Blaxel run creates its directory once its remote preflight and selection checks have passed. Two selection checks in `runBlaxel` ran after the directory existed; they were moved ahead of remote preflight and the directory rather than promising less (per-file review rounds 1 and 2). + +**P3. `--run-dir-output`, written when the directory exists, read by callers of `npm test`, and why `.test-runs/` is not searched.** This is a contract with orchestration, and the trap of enumerating directories is non-obvious. + +**P4. The three status fields, in the summary's `run` block, and their relations.** This contract with every reader of results is the honesty mechanism the other specs refer to. + +**P5. Reports state only what was observed.** This design intent is the rule a derivation under completion bias would bend. + +**P6. Leaks fail the surface and mark it incomplete.** This consequence is non-obvious: a leak makes the run incomplete, not merely failed. + +**P7. Recovered flakes, and the Vitest attempt recorder as the only evidence.** This is knowledge that would otherwise be lost: Vitest's JSON lacks attempts, and reading failure messages is the trap. + +**P8. The compact summary for agents: what is listed, what is not, and what each entry keeps.** This design intent is about agents' context budget. The leak fields are what diagnosis needs. Passing tests are neither listed nor counted, and a recovered Playwright flake can have no error excerpt (per-file review round 1). + +**P8a. The pinned `RunSummary` fields.** People, agents and `scripts/test/failing-set.mjs` read summaries, including from earlier runs, so the status block and the identifying fields of listed tests and leaks are command output to pin. The rest of the summary is left to the writer (review round 1, finding 1). + +**P9. Timing records: their contents, determinism, the phases covered, and that they feed the baseline.** The baseline update depends on them. Phase coverage is the decision that makes setup cost visible. + +**P9a. Event streams are read across revisions; the rule for changing them; the envelope every event carries (version, `runId`, contiguous `sequence`, a recognized status except on leak events); the Blaxel timing-provider identity; the pinned fields baseline update uses, including the meaning of `planningDurationMs`.** `baseline update` validates streams that earlier revisions left in `.test-runs/` and aborts on one it cannot read, so everything its validator checks is persisted data another version reads, not only the fields its calculation uses (review rounds 1 and 2, finding 1). + +**P10. The event stream and initial summary are required; optional diagnostics (resource probes, workflow metrics, later phase appends such as verify's) are best-effort whenever they are collected and never change a result.** This decision prevents metrics from turning green red or the reverse, without making the required report optional (review round 1, finding 6; round 2, finding 2). + +**P11. No secrets in run directories, because CI artifacts are readable by anyone who can read the repository.** This security requirement comes from an external fact: the repository is public. + +**P12. Resource diagnostics: why, what is observed, the saved start, kernel lines on failure, unavailable is not zero, and the limits on what it proves.** This decision defines what evidence exists for memory investigations, with accepted limitations. Exact byte limits are left to the code. Anchored `^resource-diagnostics`. + +**P13. Records are linked, survive retries and download, are bounded, never change outcomes, and include recovery receipts.** These decisions mean evidence is neither lost nor allowed to harm the worker. + +## github-ci.md + +**G1. Triggers and ownership.** The workflow spec depends on when CI runs. + +**G2. Drafts run nothing; superseded pull-request runs are cancelled; pushes are not.** The contribution workflow and people depend on these decisions. + +**G3. What always runs, and which jobs may skip.** This defines CI's composition alongside attestation. + +**G4. The required check names as a contract with the ruleset maintained in GitHub.** These names come from GitHub's settings, not from the code. The old spec named required contexts that the live ruleset does not require. + +**G5. The join's conditions, including that shard agreement applies when shards ran and that an authorized attested skip requires no shard reports, and that a skip is never read as success.** Branch protection depends on this decision about integrity. The conditions are what make a green join mean all tests ran once (review round 1, finding 2). + +**G6. What the unit and desktop jobs run; shards use `e2e-ci` and the worker directly, so they pass the retry budget; 16 shards and why; the total changes together.** These are decisions with reasons. The retry clause prevents CI from silently dropping retries. + +**G7. Build once, recipes covering each build step of the surfaces in `all`, ship every untracked file, no output list, a tar archive for modes, no fallback build.** This architectural decision is one the team should discuss if it changed. The mode-stripping fact is knowledge. Anchored `^gha-prebuilt-partition`; `licensing.md` cites it. + +**G8. The build's rules and the transfer's limits: tracked files unchanged; no deletions, empty directories or `node_modules` writes; desktop independent.** These are the obligations the design places on builds, which a builder could not derive. + +**G9. Exact-keyed caches with no near matches, and a hit equal to `npm ci`.** This is knowledge: restore-keys fallbacks are the common pattern and produce stale trees. + +**G10. The combined shard archive keyed on the concrete image; the build as sole writer after launch proof; fallback on image skew; per-shard launches; raising the namespace.** This is knowledge about GitHub's gradual image rollout and why one machine's launch proves nothing for another, plus an obligation (raising the namespace) nothing else would reveal. + +**G11. Every test job uploads its run directory, and finds it from the identity its runner reports: `--run-dir-output` for jobs that run `npm test`, the shard worker's summary for shard jobs.** The upload is a decision for diagnosability, and the identity rule is non-obvious. + +**G12. One Playwright worker for real-stack browser tests, with the observed failure.** This regression case says why it exists. + +**G13. Testing guidance: CI mechanics need a live isolated graph copy, with scratch namespaces deleted afterwards; production names are checked after merge.** These environment facts are not derivable: no test can reach the services, the cache quota is shared, and isolated copies cannot report production names. + +## attestation.md + +**A1. The purpose, and that attestation is an optimization, never a gate, resolving doubt to running tests.** This design intent decides every failure mode. + +**A2. The ref name, the claim, recovered flakes counting, and that a Blaxel attestation does not cover Firefox projects.** The ref is shared across runs and revisions on GitHub. The claim is what readers trust, so it must say what a Blaxel-sourced attestation leaves out. + +**A3. Tree hashing pins tracked content; what it does not pin; the 7-day bound; future dates rejected; the version.** This decision comes with its accepted limitation. Future-dated rejection is a non-obvious edge. + +**A4. Bare refs checked by readers; atomic create-only writes; existing valid and malformed refs.** This cross-run contract on GitHub is where races are, which the push form avoids. + +**A5. Lab refs are never honored.** The testing guidance depends on this boundary. + +**A6. The two writers and their identities; forks cannot write.** This decision defines trust, and the identity checks are what keep other contexts out. + +**A7. The eligibility conditions.** Each condition is a decision that could plausibly be derived differently: dirty trees, mismatched trees, skipped surfaces, or a version changing mid-run. The writer compares trees, not commits, and the first policy read precedes the run's tests rather than every phase (per-file review round 1). + +**A8. Local runs never write; a miss is not an error; a write failure does not change the result.** These decisions keep attestation optional. + +**A9. The trust model.** This answers the objection an implementer can be expected to raise about forged attestations. + +**A10. The pinned policy file, read from `origin/main` only, with anything unexpected meaning disabled.** People write this file and every branch reads it. Reading from the tested checkout is the plausible wrong derivation that would let a branch enable itself. + +**A11. The decision job's checks and log.** This is the contract for auditable skips. + +**A12. GitHub Actions wiring: expected errors succeed; `continue-on-error`; `authorized`; `!cancelled()`; step-level skips on matrix jobs.** Each item is GitHub behavior that the obvious wiring gets wrong. All of it was measured, and none of it is derivable from the code's intent. + +**A13. Only heavy jobs skip; the join passes skips; main pushes skip and stay green for publication.** Publication depends on this composition decision. + +**A14. Revocation by deleting refs or with the flag, and in-flight skips.** This is the operator's contract, including its limitation. + +**A15. Prune modes and safety rules.** This is command contract, and the safety rules are decisions. + +**A16. The version-change procedure.** This operational procedure has an ordering that matters. Reusing a version is the trap. + +**A17. Testing guidance: the isolated live CI graph against lab refs; what no test can reach; the operator's first checks and rollback.** These environment facts are not derivable. + +## Inbound citations affected + +The method defers repairing citations until after review. These references outside the rewritten files point at files or headings that no longer exist; block refs that survive are omitted. + +- `AGENTS.md`, `specs/spec-workflow.md`, `specs/arch/testing-policy.md`, `developer-skills/tvdev-setup/SKILL.md`, `specs/arch/desktop/index.md`, `specs/arch/desktop/runtime.md`, `specs/arch/node-versions.md`, `specs/arch/telemetry/sink.md` and `specs/product/desktop-app.md` link to `preflight.md`. The remote-revision content is now `test-runner.md#^remote-revision`; process-backend checks are in `test-runner.md#processes-started-by-tests`. +- `specs/arch/testing-policy.md` links to `flaky-tests.md`; the content is in `test-runner.md#retries`. +- `specs/arch/desktop/e2e-harness.md` links to `test-registry.md` and `test-registry.md#Surface services`; the content is in `test-runner.md#the-test-registry`. +- `specs/arch/explainer-desktop-app.md` cites `preflight.md#^preflight-electron-runtime`. That statement is dropped as owned by the desktop harness spec. +- `specs/arch/testing-policy.md` links to `test-runner.md#Verify orchestration` (now `#verify`), `blaxel-testshards.md#Local setup` (now `#setup`) and `blaxel-testshards.md#Temporary files` (now `#temporary-files`). +- `specs/terms.md` entries name `test-registry.md`, `preflight.md` and `flaky-tests.md` as owners, and define `attempt` and `shard` against the old specs. +- Proofs under `proofs/arch/test-runner/` and test-code comments cite many anchors and headings of the old specs. They are re-derived later. +- `test/repo/test-runner-guardrails.test.ts` comments cite `github-ci.md#dependency-install` and `#test-assertions` headings that no longer exist. + +## Responses to second opinions + +### Round 1 + +The review is `second-opinion-r1.md` in this folder, on `f71b1f7d`. + +1. **Persisted reporting contract.** Agreed in part, and changed. `reporting.md` now places the statuses in the summary's `run` block. It pins the summary fields that people, agents and `failing-set.mjs` read. It states that event streams are read across revisions by `baseline update`, gives the rule for changing them, and pins the fields the update uses, with the meaning of `planningDurationMs`. I did not pin `test/timing-baseline.json`: it is committed in the same tree as the planner and updater that read and write it, so a format change lands together with a regenerated file. Run directories, by contrast, stay in `.test-runs/` from runs at earlier revisions. +2. **The join and the attested skip.** Agreed, and changed. Shard agreement now applies when shards ran tests. A new sentence says that an authorized attested skip needs the build and shard no-ops and the vibe-mode check, and no shard reports. This matches `ci.yml` and `attestation.md`. +3. **What the baseline measures.** Agreed, and changed. `sharded-execution.md` now says the baseline is the median planning duration of complete observations that finally passed, recovered flakes included, and that failed and incomplete observations supply no samples. `planningDurationMs` is defined with the pinned event fields in `reporting.md`. The median arithmetic and the unknown-file constants stay in the code. +4. **Firefox outside GitHub CI.** Agreed, and changed. The registry section of `test-runner.md` now states the omission outside GitHub Actions and the failure inside it. Checking this found a wider consequence: Blaxel workers have no Firefox. In the full verify of `8e6cfcb2`, every browser-app shard ran only the `chromium` project, and the shard logs carry the skip warning. The spec now says that Blaxel runs cover Chromium only, and `attestation.md` says that a Blaxel attestation does not cover Firefox and lets CI skip it for that tree. Both statements describe current behavior; whether that is acceptable is a question for the owner. +5. **Run-directory promises.** Agreed, and changed. The summary-on-failure promise now holds once a directory exists. Local runs create theirs before local preflight; Blaxel runs create theirs after the dispatch checks pass. Refusals leave no directory, and `--run-dir-output` is written when the directory exists. +6. **Best-effort scope.** Agreed, and changed. The event stream and summary are now part of finalizing a run, and failing to write them fails the command. Only later additions are best-effort: verify's phases, which are added to the summary after the test command, workflow metrics and resource probes. Verify is no longer listed among the event stream's phases. +7. **Process-backend preflight scope.** Agreed, and changed. Canonical local runs and planned shard workers check the backend, and other paths such as targeted Blaxel runs skip it, with worker setup supplying the tools on Blaxel. I did not name the shard worker's diagnostic mode without a plan, which the specs mention nowhere else; "such as" keeps the list open. +8. **Failure cap versus the opt-in early stop.** Agreed, and changed. The sentence now says that ordinary runs impose no automatic cap on failures. I did not document the shard worker's `--fail-fast` option: nothing in the repository passes it, and the policy leaves out mechanisms nothing uses. +9. **The mutex update algorithm.** Agreed, and changed. The compare-and-swap sentence is gone. What remains is atomic acquisition, identity-checked changes, the shared representation, and no transaction held while tests run. +10. **Scoped defaults.** Agreed, and changed all three. The `against-test-guidance` naming is limited to the shared-host and retry overrides. `FLAKY_TEST_RETRIES` is 5 under the runner and CI unless the variable is set. Blaxel verify tests the selected committed revision, `HEAD` by default. + +### Round 2 + +The review is `second-opinion-r2.md` in this folder, on `28039415`. + +1. **The event contract omits fields the reader requires.** Agreed, and changed. `validateTimingEvents` (`scripts/test/timing-events.mjs`) rejects a stream whose events lack `schemaVersion: 1`, a shared `runId` or a `sequence` equal to the line position, or whose events other than leak events lack a recognized status. It also checks the timing provider, and `baseline update` reads only streams with the Blaxel identity. `reporting.md` now states that envelope and the Blaxel identity in a sentence each. It pins them in the type: a shared envelope, the status union, the shard status, and the `test-attempt` and `process-leak` kinds as far as the envelope and status reach them. Attempt payloads, hash formulas and the normalized result model stay in code. +2. **Best-effort described by timing.** Agreed, and changed. The paragraph now says that the event stream and initial summary are required, and that optional diagnostics are best-effort whenever they are collected. That covers resource probes, including the start observation saved before a surface prepares, workflow metrics and later phase appends such as verify's. + +The reviewer agreed that `test/timing-baseline.json` can stay unpinned and confirmed that the Firefox statements are accurate (TV-1011). + + +## Responses to per-file reviews + +### Round 1 + +The reviews are in `file-reviews/`, one per spec, all on `ee599ee8`. Each finding was checked against the code and the neighbouring specs before changing anything. + +#### test-runner.md + +1. **`--file` must resolve to one surface.** Agreed, and changed. `resolveFileSelection` (`scripts/test/test-guidance.mjs`) refuses matches that span surfaces even when each file has one owner. The `--file` bullet now says that matched files must belong to one selected surface, and that another selector such as `--surface` narrows a match that spans surfaces. R7; removal account test-runner 19. +2. **The default selection.** Agreed, and changed. Only `npm test -- local` and `npm test -- blaxel` supply `all`; `list` and `preflight` pass their options straight to `selectSurfaces`, which starts from every registered surface. The default sentence now says so, and the preflight row of the commands table takes selectors, with `--suite all` as its example. R7. +3. **Which configs are registry errors.** Agreed, and changed. Discovery recognizes only `vitest.config.ts`, `vitest.e2e.config.ts` and `playwright.config.ts`, and fixture configs with other names are deliberately unregistered. The sentence now names those three. Discovery is not broadened. R12; removal account test-registry 22. +4. **The grace period before the leak scan.** Agreed, and changed. After a normal exit there are no running commands left to stop, and `checkForLeaks` scans at once; the wait the code has is between `SIGTERM` and `SIGKILL`. The bullet now says that the owner stops the services and any commands still running, then scans. A wait before the scan is not intended: a process that exits before any signal is sent is not a leak (`isReportableProcessLeakCleanup`), which already covers a helper exiting just after its runner. R38; removal account test-runner 84 and 87. +5. **`verify --plan` still selects a provider.** Agreed, and changed. `resolveVerifyProvider` runs the Blaxel preflight on a marked host before `--plan` is checked. The sentence now says that `--plan` runs no phases and that choosing the provider on a marked host still runs its Blaxel checks. R33; removal account test-runner 48. +6. **The reason for dropping package-level zero retries.** Agreed. Grouped `unit:workspaces` runs pass no retry count and rely on the package configs' `retry: 0`. Removal account flaky-tests 4 now gives the runner-budget bullet's requirement, no runner retries for unit surfaces, as the reason, and leaves how it is met to the code. No spec change. + +#### blaxel-testshards.md + +1. **The deletion reservation is part of the shared lease contract.** Agreed, and changed. Deletion writes `{ lockId, deleting: true }` (`scripts/test/blaxel-worker-lease.mjs`), and acquisition refuses it before any audit. The pinned type is now `WorkerLeaseRecord`, either a `WorkerLease` or a `DeletionReservation`, and Deleting workers says that every acquisition refuses a reserved worker whatever its audit finds. B17, B20; removal account blaxel-testshards 13 and 24. +2. **`--repo-url` is not an option of the canonical command.** Agreed on the fact, and changed differently. `runBlaxel` (`scripts/test/cli.mjs`) does not pass `--repo-url` to the coordinator, and nothing in the repository uses it. The spec now says that workers use the checkout's `origin` and leaves the coordinator-script option to the code, rather than naming an option that `npm test -- blaxel` cannot reach. B8; removal account blaxel-testshards 4 and 39. +3. **Mount boundaries.** Agreed, and changed. A recursive removal can cross a mount point without following a link, and the cleaner checks mount points and devices separately. The sentence now reads "Removal rejects unexpected mount boundaries and does not follow links out of these locations." B22; removal account blaxel-testshards 28. + +#### sharded-execution.md + +1. **Plans, the baseline and shard summaries cross revisions.** Agreed in part. The boundary is real, and the ledger and removal account were wrong to treat these as read only within one revision. With `--commit`, the checkout's runner reads the tested commit's registry and baseline, plans, and validates the summary that the tested commit's shard worker returns. Changed: blaxel-testshards.md#runs now says that what the coordinator and workers exchange crosses revisions when they differ. A change should keep working with revisions still in use, and where it cannot, the mismatch must fail as infrastructure and never be read as test results. sharded-execution.md#plans points there. Not changed: the shapes stay in the code. The exchange is wider than the three types named (the worker's command line and environment cross too), both ends already validate what they receive, so a mismatch fails closed, and pinning three types would suggest the rest may change freely. B31, X5, X8; removal account sharded-execution 4, 18 and 56. +2. **The recommendation can exceed two minutes.** Agreed, and changed. `recommendShardCount` searches up to the pool's capacity and returns the capacity when no count meets the target. The sentence now says both. X10; removal account sharded-execution 45. +3. **Unique assignment is not exactly-once execution.** Agreed, and changed. An infrastructure retry can run a file again. The introduction now says that a passing result accounts for every selected file. X1. +4. **Retention of generated inputs.** Agreed. Removal account sharded-execution 39 is now Dropped, left to the implementation, and 40 says that only the mismatch behavior is kept. This moves one passage from Derivable to Dropped in the counts. No spec change. + +#### reporting.md + +1. **Some Blaxel refusals leave a directory without a summary.** Agreed on the fact; I kept the promise. In `runBlaxel` the directory is created after remote preflight, and the one-surface check for targeted selections and the suite check then exit through `fail()` with no summary. The spec now says that a Blaxel run creates its directory once its remote preflight and selection checks have passed, and keeps the promises that a refusal before dispatch leaves no directory and that an existing directory always gets a summary. Qualifying the promise would make every reader of `--run-dir-output` handle a directory without a summary, only to describe two checks that run in the wrong order. Moving them before the directory is created is a small code change for the repair phase. Until then the tree has this gap, and the PR description says so. P2; removal account reporting 5. +2. **The summary has no passing-test count, and not every listed test has an error.** Agreed, and changed. The summary counts surfaces, failed tests and recovered flakes, and a recovered Playwright flake takes its error from the passing attempt. The text now says that passing tests are not listed and that each listed test has a short error excerpt when one is available. P8; removal account reporting 22. +3. **`selection.suite` can be `null`.** Agreed, and changed. `selectionSummary` writes `suite: null` for selections without `--suite` or `--all`, and the run event copies it. The pinned type is now `suite?: string | null`. P9a. + +#### github-ci.md + +1. **Recipe coverage is for the `all` suite.** Agreed, and changed. `checkManifestCoversRegistry` checks only surfaces in `all`, and `daemon-acceptance:cli` declares a build step that no recipe covers. The sentence now covers the build steps of surfaces in `all`. G7; removal account github-ci 25. +2. **Shard jobs do not use `--run-dir-output`.** Agreed, and changed. Shard jobs read `runDir` from the shard worker's summary. The run-artifacts paragraph now says that a job takes its run directory from the identity its runner reports: `--run-dir-output` for jobs that run `npm test`, the shard worker's summary for shard jobs. reporting.md#run-directory now names callers of `npm test` instead of "orchestration such as CI". G11, P3; removal account github-ci 19. + +#### attestation.md + +1. **CI's first policy read does not precede every phase.** Agreed on the mismatch; I changed the spec rather than CI. The static-check matrix has no dependency on `attest-check`, so the old "before the first gate phase" never held in CI, and the removal account was wrong to call it kept. The condition now says that the first read comes before any of the run's tests. That holds on both paths: CI's test, build and end-to-end jobs all need `attest-check`, and a Blaxel verify reads the policy before its first phase. The remaining exposure is a version change in the seconds between the static checks starting and the decision job's read, on a run that also spans a version change made without the operations procedure's steps of disabling writes and letting in-flight runs finish. Lint and type-check also depend least on the outside conditions a version records. Reordering CI mechanics would need an isolated live CI run under github-ci.md#testing-guidance. The stronger rule remains available as a dependency of the static-check job on `attest-check`. A7; removal account attestation 10 and 24. +2. **The writer checks trees, not commits.** Agreed, and changed. `beginVerifyAttestationFacts` and the eligibility check compare `HEAD^{tree}` at start and end and the tested commit's tree, and `--commit` is not itself disqualifying. The condition now says "still clean, with the same tree", and that a verify whose tests target a commit with a different tree cannot write. A7; removal account attestation 19 and 20. + +### Round 2 + +The reviews are the `*-r2.md` files in `file-reviews/`, on `db88693c`. `test-runner.md`, `blaxel-testshards.md`, `github-ci.md` and `attestation.md` passed with no findings. + +#### sharded-execution.md + +1. **The decision to leave cross-revision formats to the code is stated only in working documents.** Agreed, and changed by stating the exception in the spec. The spec policy's rule on pinning types covers data another revision reads, and the policy lets a more specific spec state an exception, but only in the spec. Pinning "the minimal fields the other revision consumes" has no small answer here. The worker checks the plan against a digest of its whole canonical content (`validateShardPlan`), so every field of the plan is consumed. The coordinator builds its whole report from the shard summary. The Runs paragraph of `blaxel-testshards.md`, which owns the coordinator–worker exchange, now states the exception: the shapes of these exchanges are left to the code, under the compatibility and infrastructure-failure rule. `sharded-execution.md#plans` names the plan, the baseline and the shard summary and points to it. This adds two sentences to `blaxel-testshards.md` after it passed; they change no rule there. B31. + +#### reporting.md + +1. **The run-directory code repair is outstanding.** Done. `runBlaxel` (`scripts/test/cli.mjs`) now computes its selection arguments, including the one-surface check for targeted selections and the supported-suite check, before remote preflight and before the run directory exists. The arguments passed to the coordinator are unchanged. The same review of the path found one more refusal after the directory existed: an invalid `--run-dir-output` value was rejected only when the identity was written. That check now runs first in `runWithSelection`, for local and Blaxel runs. `test/repo/test-runner-guardrails.test.ts` adds cases showing that each refusal comes before remote preflight and writes no run-directory identity. P2; removal account reporting 5. + +## Repair phase + +Josh approved the converged specs at `85b82b0d`. The repair phase brings everything that cites them into line, without re-deriving proofs: Josh asked for surgical proof edits that repoint citations, fold the proofs as the specs were folded, and correct what the new specs contradict, keeping coverage of anything the specs no longer promise and marking it *superfluous*. No proof, test or covered code is removed. Test changes are limited to the two tests that read old spec text, citation comments, and tests a new spec promise plainly lacks. + +### Citations outside the proofs (`961967df`) + +Every row of "Inbound citations affected" above is repaired, along with the removal account's table of old targets: + +- Spec links in `AGENTS.md`, `spec-workflow.md`, `testing-policy.md`, the desktop, licensing, Node-version and telemetry specs, the migration map and `developer-skills/tvdev-setup/SKILL.md` now point at `test-runner.md` sections and anchors. `specs/product/desktop-app.md` cited preflight's testing guidance as the reason for its Mac verify run; that guidance is gone, so the sentence now says what the run exercises: the runner's macOS process backend. +- `specs/terms.md`: owners updated; *shard*, *attempt*, *infrastructure failure* and *test failure* restated against `sharded-execution.md`; *flaky test* replaced by *recovered flake*, the term `test-runner.md#retries` defines. *Execution group* and *normalized surface result* are removed: no spec defines them now. +- Workflow, script and test comments that named old headings now name current ones. Three pre-existing broken wikilinks in test comments (`^gha-prebuilt-consumer`, `^gha-prebuilt-cli-outputs`) became checked path citations. +- The two tests that read old spec text (`test-runner-guardrails.test.ts` "keeps CI-planned e2e surfaces registry-owned…", `test-runner-provider.test.ts` "keeps canonical stale-owner retry authority…") now read the sentences in `github-ci.md#jobs` and `blaxel-testshards.md#runs` that state the same rules. + +### Proofs + +- **Fold.** `proofs/arch/test-runner/test-registry.md`, `preflight.md` and `flaky-tests.md` are folded into `test-runner.md`'s proof as "Test registry", "Preflight checks" and a paragraph under "Retry selection and execution". Their hooks join its Test hooks. Every assertion and anchor is kept, so test citations change only their file (`^registry-committed-normalization`, `^preflight-t-electron-planner-handoff`). +- **Citations.** Every proof link to a removed heading or anchor is repointed, following the removal account's table. +- **Contradictions corrected.** Attestation eligibility is by tree, not by `--commit` ≠ `HEAD`. CI recipe coverage is for surfaces in `all`. The shard-count recommendation is "at most" two minutes, with the pool-size fallback. Two references to spec testing directives that no longer exist (a CLI-boundary directive, native-config loading "as directed by the spec") are removed, and the proof now names where it honors the one directive the spec has: destructive lifecycle tests in isolated surfaces. +- **Superfluous, kept.** Marked where an assertion covers detail the spec leaves to the code: the `tv serve` test recipe, `--allow-extreme-inefficiency`, execution groups and their order, invalid Vite service forms, the `node` check's name, the coordinator script's `--repo-url`, nvm's pinned version, image and memory size, worker Electron repair, the normalized result type, the LPT algorithm and weighting constants, combined Vitest invocation, diagnostic byte bounds, and the `e2e:browser-app` worker pin. +- **New promises cited to existing tests.** The owner-token format (`process-lifecycle.test.ts`), plan validation by digest (`test-runner-shard-plan.test.ts`), refusal of a reserved worker (`blaxel-lease-contracts.test.ts`), `--run-dir-output` and refusals before preflight (`test-runner-reporting.test.ts`, `test-runner-guardrails.test.ts`), and one rule of the event envelope (`test-runner-timing-baseline.test.ts`, with the rest of the envelope stated as having no cases of its own). +- **New tests for promises that plainly lacked one.** A local guardrail refusal leaves no run directory (`test-runner-guardrails.test.ts`). The summary keeps the fields `RunSummary` pins (expectations added to two existing tests in `test-runner-reporting.test.ts`). Both pass locally, one file each. + +Primitive flagged `dec_8e3d38a8` (crash cases belong on the isolated lifecycle surface) on the new guardrails test. That test is a plain CLI refusal, not a crash case; recorded as followed. + +### Responses to repair review round 1 + +The review is `repair-review-r1.md` in this folder, on `609a1b6d`. + +1. **The refusal tests observed the identity file, not the run directory.** Agreed, and changed. Creating the directory and writing its identity are separate steps in `scripts/test/cli.mjs`, so the identity file's absence could not show that no directory was made. The four refusal cases in `test-runner-guardrails.test.ts` now run in the disposable checkout that `guidanceCheckout()` provides, with a second fixture surface committed there, because Blaxel selection reads the registry from the tested commit. Each asserts that the checkout has no `.test-runs/` directory afterwards. They keep the diagnostic, tripwire and identity-file assertions. The same fixture creates `.test-runs//` for an admitted run, so the observation can fail. The proof's `^reporting-t-refusal-no-run-dir` now describes that observation. +2. **The *broad run* entry added a routing rule.** Agreed, and changed to the suggested definition and pointer. +3. **The *test failure* entry equated a failed test with the `failed` classification.** Agreed. The entry is now *failed shard*, a pointer to the classification in `sharded-execution.md#shard-outcomes`. No spec uses *test failure* as a term. + +While building the fixture I noticed that `--surface` silently ignores an ID that names no surface: `npm test -- list --surface unit:root,unit:nonexistent` lists `unit:root` alone. This predates the branch and the repair scope does not cover it; it is reported to Josh rather than changed. diff --git a/docs/archive/2026-10/test-runner-spec-rewrite/removal-account.md b/docs/archive/2026-10/test-runner-spec-rewrite/removal-account.md new file mode 100644 index 00000000..91405be0 --- /dev/null +++ b/docs/archive/2026-10/test-runner-spec-rewrite/removal-account.md @@ -0,0 +1,585 @@ +> **Archived 2026-10 from PR #53.** This was the removal account of the test-runner spec rewrite, corrected after the per-file reviews, and the merged specs follow it. For each passage of the nine earlier test-runner specs it says whether the passage was kept, reworded, moved, judged derivable or dropped, and why, which the squash diff shows only as deletions. The body below is unchanged from its working state and is a clue to the change, not a record of it. + +# Removal account: test-runner specs + +This account compares the nine old specs in `specs/arch/test-runner/` at `d2837551` with the six new ones at `92574976`, and says what became of each passage of the old specs, in order. Entries marked "per-file review round 1" were corrected after the per-file reviews of `ee599ee8`, and describe the specs as revised in that round. It is step 5 of the spec-rewrite method. The ledger (`ledger.md`) justifies what the new specs say; this account justifies what they no longer say. + +A *passage* here is a paragraph, a list or table that makes one point, a code block, or a list item that makes a separate point of its own. Each passage has one primary outcome: + +- **Kept**: the same statement, in the same file, with its substance and nearly all of its wording unchanged. +- **Reworded**: the same meaning in new words, in the same file. +- **Moved**: now stated in another new file, which the entry names. +- **Derivable**: no longer stated, because it follows from a named statement in the new specs, the testing policy or the code. +- **Dropped**: no longer stated, because it does not belong under `specs/spec-policy.md`. The reason is given. + +Where a passage splits, the entry gives the primary outcome first and says what happened to the rest. New specs are named without their directory; anchors are given as `file#section`. "Cited by" lists references outside the rewritten files that pointed at a heading, anchor or file that no longer exists; the full inventory is at the end. The proofs under `proofs/arch/test-runner/` are re-derived after human review, so their citations are listed only in the inventory. + +## attestation.md + +1. **Opening paragraph (purpose; draft pull requests use a separate skip).** **Reworded**: attestation.md's introduction keeps the purpose, cheap checks always running, and doubt resolving to running tests. The draft-PR skip is stated in github-ci.md#when-it-runs. +2. **Ownership paragraph.** **Reworded**: the introduction's second paragraph names github-ci.md and test-runner.md#verify. +3. **"What an attestation claims", the claim in bold.** **Reworded**: attestation.md#the-claim, first paragraph. +4. **Tree hash definition.** **Reworded**: first bullet of #the-claim. +5. **Full validation gate (lint, type-check, manifests, complete `all` with desktop, standard retries; recovered flake is a pass).** **Reworded**: first paragraph of #the-claim. It now also says that a Blaxel attestation does not cover Firefox (review round 1). +6. **Self-pinning for everything the tree controls.** **Reworded**: first bullet of #the-claim. +7. **What the tree does not pin, and the two bounds.** **Reworded**: second bullet of #the-claim. +8. **Attestation version introduction.** **Reworded**: #the-claim (the version in the ref) and #policy (only current-version refs are honored). +9. **Version domain and encoding; malformed disables both.** **Reworded**: the comment on `testpassVersion` in #policy, and "anything unexpected means disabled". +10. **Version bound to the validation interval (read before the first phase and at completion).** **Reworded** in part: the last eligibility condition in #who-may-write, and the closing sentence of #operations. "Before the first gate phase" is narrowed to "before any of its tests". In CI the cheap checks start alongside `attest-check`, which makes the first read, so the old wording never held there; the operations procedure, which disables writing and lets in-flight runs finish before a version change, is what keeps a run validated under old conditions from writing (per-file review round 1). +11. **Safe version-change procedure.** **Reworded**: attestation.md#operations, five steps. The sentence that the disabled flag is re-checked by a run's writer is derivable from the eligibility condition that reads the policy at the end. "Policy history on main is the audit trail" is dropped as derivable from Git. +12. **Ref shape code block.** **Reworded**: the first sentence of #the-claim names the ref and its target. +13. **Bare ref, no metadata.** **Reworded**: third bullet of #the-claim. +14. **Create-only compare-and-swap push; benign outcomes; lease rejection handling.** **Reworded**: fourth bullet of #the-claim keeps the exact push form, existing-valid success and malformed-reported. The measured "Everything up-to-date" transport behavior is dropped as Git implementation detail. +15. **Consumer fetches by name and verifies the invariant; malformed refs ignored and deletable.** **Reworded**: third bullet of #the-claim, and `--delete-ref` in #revoking-and-pruning. +16. **Lab refs.** **Reworded**: last bullet of #the-claim. +17. **"Who writes, and when" introduction (every clause; a miss is not an error).** **Reworded**: #who-may-write, "A run that misses a condition simply does not write". Cited by `.github/workflows/ci.yml` and `scripts/test/cli.mjs` (`#who-writes-and-when`). +18. **Clause 1, eligible source.** **Reworded**: the two source bullets and "A local run never writes" in #who-may-write. +19. **Clause 2, one tree.** **Reworded** in part: first condition bullet. A verify whose tests target a commit with a different tree cannot write. The old exclusion of every `--commit` other than `HEAD` is narrowed to what the writer checks, the tested commit's tree (per-file review round 1). +20. **Clause 3, clean and unchanged, bracketed.** **Reworded**: first condition bullet, "still clean, with the same tree"; the old "same HEAD" was checked by tree (per-file review round 1). The remark that a recorded `workingTreeDirty: false` alone is insufficient is dropped as an implementation note. +21. **Clause 4, complete scope, including the `SKIP_ACP_TESTS` example.** **Reworded**: second condition bullet keeps `SKIP_ELECTRON_E2E`. The `SKIP_ACP_TESTS` example is derivable: that knob guards suites outside the gate. +22. **Clause 5, canonical retry policy.** **Reworded**: third condition bullet. +23. **Clause 6, not itself attested-skipped.** **Reworded**: fourth condition bullet. +24. **Clause 7, policy permits at start and end.** **Reworded**: fifth condition bullet, with the first read "before any of its tests" (see item 10). +25. **`blaxel-verify` writer.** **Reworded**: first source bullet, which now also holds the definition of a publication-qualifying run that used to live in test-runner.md. +26. **`github-ci` writer, command-boundary identity, `contents: write`, forks.** **Reworded**: second source bullet. The `contents: write` permission is dropped as workflow wiring the job needs anyway. +27. **The production writer refuses other sources; GitHub test jobs use `provider=local`.** **Reworded**: the identity requirement in the second source bullet, and "A local run never writes" in #who-may-write. +28. **Trust model.** **Reworded**: last paragraph of #who-may-write. +29. **`attest-check` computation and auditable log.** **Reworded**: first paragraph of attestation.md#skipping-in-ci. Cited by `.github/workflows/ci.yml` (`#skip-semantics-in-ci`). +30. **Expected negative outcomes are successes.** **Reworded**: first bullet of #skipping-in-ci. +31. **Unexpected failure is non-gating; outputs written before a failing step survive.** **Reworded**: second bullet of #skipping-in-ci. +32. **Skip authorization through `authorized`, with the measured run and exact expressions; accepted post-action residual.** **Reworded**: second bullet of #skipping-in-ci keeps the mechanism, the reason and the accepted residual. The run ID and literal condition strings are dropped as history and workflow text. +33. **Heavy jobs skip only on the literal `'true'`; the join's third condition requires it.** **Derivable**: from "any doubt resolves to running the tests" (attestation.md introduction) and from trusting a skip only together with `authorized=true` (#skipping-in-ci). +34. **Never-skip-on-doubt in the job graph: `!cancelled()`; shards require build success.** **Reworded**: third bullet of #skipping-in-ci. +35. **Step-level skips on matrix jobs, with the required contexts named.** **Reworded**: fourth bullet of #skipping-in-ci, corrected. The live ruleset requires the matrix checks `unit` and `e2e:desktop`, not each `e2e shard n/16`. +36. **What always runs; only heavy jobs skip; drafts skip everything.** **Reworded**: "Only the heavy jobs skip" in #skipping-in-ci, and github-ci.md#when-it-runs. +37. **The join's behavior on drafts, non-drafts and attested runs.** **Moved**: github-ci.md#required-checks, including the attested-skip exception (review round 1). +38. **Main pushes skip the same way; a novel tree runs everything.** **Reworded**: last paragraph of #skipping-in-ci. The novel-tree case is derivable from the tree-hash claim. +39. **"Failure modes" introduction.** **Reworded**: "any doubt resolves to running the tests" in the introduction. +40. **Failure-mode bullets (unreachable origin; bad policy; broken invariant or age; decision job failure; write failures non-fatal).** **Reworded**, spread over #skipping-in-ci (expected problems give `skip=false`; heavy jobs run when the decision job crashes), #policy (anything unexpected means disabled), #the-claim (readers ignore mismatched refs) and #who-may-write (a failed write leaves the result unchanged). +41. **Staleness: definition changes self-invalidate.** **Reworded**: first bullet of #the-claim. +42. **Environment drift: the 7-day window, the committer-time proxy, future timestamps rejected, the bound in the log.** **Reworded**: second bullet of #the-claim keeps the window and the rejection of future dates. The explanation of why committer time is conservative is dropped as derivable, and the bound in the log line as implementation detail. +43. **Revocation: delete the ref; the flag as kill switch; in-flight skips; refs inert.** **Reworded**: attestation.md#revoking-and-pruning, first paragraph. +44. **Activation policy: default-deny, the type, the trusted read from `origin/main`, unavailable conditions, lab refs not consulting the policy.** **Reworded**: attestation.md#policy. The statement that lab writes ignore the policy is dropped as detail of the experiment tooling. Cited by `test/repo/test-runner-attestation.test.ts` (`#activation-policy`). +45. **Prune command block.** **Reworded**: the option bullets in #revoking-and-pruning. Cited by `scripts/test/cli.mjs`, `test/repo/test-runner-attestation.test.ts` and `test/repo/test-runner-guardrails.test.ts` (`#prune-and-retention`). +46. **Prune safety semantics.** **Reworded**: #revoking-and-pruning keeps the dry-run default, refreshing from origin, zero deletions without a policy, strictly-older cutoffs, the version refusals and stopping at the first failure. Rejecting globs and non-`refs/testpass/` names is kept in the `--delete-ref` bullet. The note that custom namespaces need an explicit fetch is dropped as Git knowledge. +47. **Three prune modes.** **Reworded**: the three option bullets. +48. **Attestations are pure optimization state; growth is bounded.** **Reworded**: "Pruning is always safe…". The growth bound is dropped as derivable. +49. **Testing guidance: real Git for writer and prune tests; workflow-file inspection proves contents only.** **Dropped**: it restates the testing policy's seam rule and honesty rule (`testing-policy.md#^shape-seam`, `#Policy`), so it changes no proof derivation. +50. **Testing guidance: the isolated CI graph's four demonstrations.** **Reworded**: attestation.md#testing-guidance keeps the write, the skip with checks green and the decision-job failure cases. The absent-attestation case is derivable as an ordinary run. The join turning red after a build failure is derivable from github-ci.md#required-checks together with github-ci.md#testing-guidance. Cited by `test/repo/test-runner-attestation.test.ts` and `test/repo/test-runner-guardrails.test.ts` (`#test-assertions`, a heading that also did not exist in the old spec). +51. **Lab-mode verify publishes as part of the command; lab refs pruned.** **Kept**: attestation.md#testing-guidance. +52. **What no test can exercise; the operator's first checks; rollback.** **Reworded**: last paragraph of #testing-guidance. + +## blaxel-testshards.md + +1. **Opening paragraph (recommended path, speed, hot standby, the principal reason for a single runner).** **Reworded**: first paragraph of blaxel-testshards.md. The claim that Blaxel routing is the principal reason for a single runner is dropped, because test-runner.md's introduction states the runner's purpose. +2. **Ownership paragraph.** **Reworded**: the third introduction paragraph. Classification moved to sharded-execution.md#shard-outcomes. +3. **The only remote provider; the coordinator and pool scripts.** **Reworded**: last sentence of the first paragraph. +4. **Repository selection, normalization, failing before dispatch, public and private checkouts.** **Reworded**: blaxel-testshards.md#repository-and-credentials, first paragraph. `--repo-url` is dropped: it is an option of the coordinator script, which `npm test -- blaxel` does not pass, so the canonical command always uses `origin` (per-file review round 1). Cited by `scripts/test/blaxel-repository.mjs` (`#Repository`). +5. **One pool for every repository; one checkout switched per run; dependency reuse.** **Reworded**: second paragraph of #repository-and-credentials. +6. **Shared trust boundary.** **Reworded**: same paragraph. +7. **GitHub token sources, `GH_TOKEN`, `GIT_ASKPASS`, never written.** **Reworded**: third paragraph of #repository-and-credentials. The variable name `GH_TOKEN` is dropped as worker wiring. +8. **Pool labels, `DEPLOYED`, too few workers fails, 72 workers, 36 shards, the 75 limit, the override, architecture checks.** **Reworded**: blaxel-testshards.md#pool, first two paragraphs. The constant names are dropped. +9. **The 36-shard measurements (42.957 s, 21.479 s, 30 s) and two verify lanes.** **Dropped**: the measurements are history behind a setting; the two-lanes rationale is **reworded** in #pool. +10. **`ensure` naming, default pool, size, image, memory, nvm tag, `.nvmrc` runtime.** **Reworded** in part: #pool keeps nvm and the `.nvmrc` Node. Sandbox names, the image, the memory size and the nvm tag are dropped as provisioning parameters that the pool manager holds. +11. **Fresh-only provisioning and the `createIfNotExists` reason.** **Reworded**: third paragraph of #pool. +12. **Labels published only after verification; partial workers reported; a partly deleted pool is not refilled.** **Reworded**: third paragraph of #pool keeps eligibility after verification. The rest is derivable from "refuses a pool that already has workers" and "delete the pool and ensure it again". +13. **Locking: the lock file, record fields, the guard, rechecks, no guard during downloads (`^blaxel-reservation`).** **Reworded**: first bullet of blaxel-testshards.md#leases, and the `WorkerLeaseRecord` type, which pins the fields other revisions read, including the deletion reservation (per-file review round 1). The anchor is gone. +14. **At most 32 concurrent acquisitions; waiting for every lease; one infrastructure result per shard when acquisition fails.** **Dropped**: concurrency and per-shard failure bookkeeping are implementation. "Fails if fewer workers than shards" is kept in #pool, and infrastructure classification is in sharded-execution.md#shard-outcomes. +15. **Only the coordinator renews; `recoverAfter`; 20 minutes; `--lock-ttl-seconds`; no revival.** **Reworded**: second bullet of #leases. Its independence from the execution timeout is dropped as derivable. +16. **Outer shard attempt token; dispatch recorded first; the entry point registers before checkout; attempt-specific uploads; unknown dispatch outcomes keep the lease.** **Reworded**: third and fourth bullets of #leases. Attempt-specific upload paths are dropped as implementation. +17. **Release conditions; the shell never unlocks; process stop is not proof; audit and signals; audit in the log; nvm before nounset; unreadable audits keep the lease (`^blaxel-release`).** **Reworded**: fourth bullet of #leases. The shell ordering detail (nvm before nounset) and the audit's log placement are dropped as code-level detail. The anchor is gone. +18. **Download while holding the lock; identity checks; early setup failures; retrieval retry; release of an unused lease; no reset before the boundary.** **Reworded** in part: the opening paragraph of #leases keeps the lease lasting until reports are downloaded and checked. Infrastructure classification of missing evidence moved to sharded-execution.md#shard-outcomes. The unused-lease release conditions are dropped as implementation. +19. **Interruption: retained evidence, authoritative interrupted snapshot, exit `130`, a second signal exits.** **Reworded**: fifth bullet of #leases. The detailed list of retained evidence is derivable from reporting.md#honest-status, "Reports state only what was observed". +20. **Abandoned-run recovery introduction (`^blaxel-recovery`).** **Reworded**: blaxel-testshards.md#abandoned-runs, first paragraph. The anchor is gone. +21. **Recovery under the guard; stopping live abandoned supervisors; the receipt; the exception to download-before-release.** **Reworded**: the numbered steps and the receipt paragraph of #abandoned-runs. +22. **Interrupted recovery; a returning coordinator; holding on failure; unreadable records; operator action.** **Reworded**: last paragraph of #abandoned-runs. That interrupted recovery leaves the old identity in place is derivable from the guarded sequence. +23. **Older runners (transitional).** **Reworded**: blaxel-testshards.md#older-runners. +24. **Pool deletion rules.** **Reworded**: blaxel-testshards.md#deleting-workers, which now names the `DeletionReservation` that every acquisition refuses (per-file review round 1). +25. **`--audit-stale-owners`.** **Dropped**: a coordinator diagnostic option not reached through `npm test`; the code governs it. +26. **Temporary files: private attempt directory, `TMPDIR`, uploads in the attempt directory, sidecars finalized, short paths, caches (`^blaxel-scratch`).** **Reworded**: first paragraph of blaxel-testshards.md#temporary-files, anchor kept. Upload placement is dropped as implementation. +27. **The two removal targets.** **Reworded**: the removal list. +28. **Disuse established by lease and audit; link, mount and escape checks; what is preserved; what is reported (`^blaxel-known-leftovers`).** **Reworded**: the paragraph after the list. The mount-boundary and link checks are kept as "rejects unexpected mount boundaries and does not follow links out of these locations" (per-file review round 1). The anchor is gone. +29. **Teardown after the attempt; uncertain teardown; failed cleanup blocks dispatch; download despite cleanup failure; Chromium's unlinked shared memory.** **Reworded**: last paragraph of #temporary-files. Download despite a cleanup failure is derivable from release requiring retrieved reports (#leases). +30. **Only the current attempt's evidence on the worker; no archive; contents owned by reporting.** **Reworded**: last sentence of #temporary-files. +31. **Shard lifecycle: uploads before starting; a rejected upload becomes `transport/worker-dispatch`.** **Dropped**: implementation. That a transport failure is an infrastructure failure is in sharded-execution.md#shard-outcomes. Cited by proofs (`#Shard lifecycle`). +32. **The worker script's steps, Electron repair (`^blaxel-electron-repair`), runtime activation, system packages, owner lifecycle, targeted runs, `CI=1`.** **Reworded** in part: blaxel-testshards.md#runs keeps activating the checked-out `.nvmrc` Node, stale-owner handling, and running the same registry declarations, including wrapper commands. Step names, Electron repair mechanics (desktop runtime preparation is owned by `desktop/e2e-harness.md`) and environment exports are dropped as implementation. Nothing outside the area cited the anchor. +33. **A synthesized infrastructure summary for setup failures; path normalization.** **Derivable**: from sharded-execution.md#shard-outcomes, where a missing summary is `infra-failed`. +34. **Dependency-hash caching: inputs, marker path, `npm ci` flags.** **Reworded** in part: "dependencies are reinstalled only when the dependency declarations differ" in #repository-and-credentials. The marker path and flags are dropped as implementation. Cited by proofs (`#Dependency-hash caching`). +35. **Classification rules.** **Moved**: sharded-execution.md#shard-outcomes, generalized for both Blaxel and CI. +36. **Plan identity validation after download.** **Moved**: sharded-execution.md#plans (the worker checks the plan) and #shard-outcomes (a plan mismatch is `infra-failed`). +37. **Failure attribution (`worker-execution`, `timeout`, `transport`, `report-download`).** **Dropped**: internal failure-kind vocabulary that nothing outside the runner reads. +38. **`stale-owner` fails closed at the sandbox.** **Reworded**: second paragraph of blaxel-testshards.md#runs. +39. **Options table.** **Dropped** in large part: the coordinator script's options are code-governed. The user-facing ones are kept where their meaning matters: `--pool poc` (#pool), `--allow-shard-count-override 1` (#pool), `--lock-ttl-seconds` (#leases), `--commit` and `--ignore-uncommitted` (test-runner.md#^remote-revision), and the two infrastructure retries (test-runner.md#retries). `--repo-url` is a coordinator-script option only (item 4). Cited by proofs (`#Options`). +40. **Timeout profiles and the default-profile note.** **Dropped**: tuning values. +41. **Output directory layout and exit codes.** **Reworded** in part: exit codes are in test-runner.md's exit-code list and the interruption bullet of #leases, and provider output in `provider/` is in reporting.md#run-directory. The layout is dropped as internal. +42. **Local setup: install and log in.** **Kept**: blaxel-testshards.md#setup. +43. **Token item in 1Password, cross-repository access, env or file, the marker.** **Reworded**: #setup. Cross-repository access is in #repository-and-credentials. Cited by `specs/arch/testing-policy.md` (`#Local setup`, now `#setup`). +44. **Pool provisioning commands and the `default` pool trap.** **Reworded**: #setup. Cited by proofs (`#Pool provisioning`). +45. **Opt-in local suite exclusions (PostHog and daemon never on Blaxel).** **Moved**: test-runner.md#the-test-registry (both suites outside `all`, local-only placement) and blaxel-testshards.md#runs (other suites refused). +46. **Testing guidance: a live run before main; one targeted run suffices; real API and transport.** **Reworded**: first paragraph of #testing-guidance. +47. **A targeted run is insufficient for plan transfer.** **Reworded**: first bullet. +48. **Fresh workers for provisioning changes; a dedicated pool.** **Reworded**: second bullet. +49. **Recovery experiments on a dedicated pool; harness placement; observing temporary paths live.** **Reworded**: the following paragraphs. The harness placement moved here from test-runner.md. +50. **Two repositories for repository-switching changes.** **Reworded**: third bullet. + +## flaky-tests.md + +The file is removed. Its content now lives in test-runner.md#retries and reporting.md#honest-status. Cited by `specs/arch/testing-policy.md` (the file). + +1. **Opening paragraph (a two-layer mechanism).** **Moved**: the introduction and bullets of test-runner.md#retries. +2. **Ownership paragraph.** **Dropped**: the file no longer exists, and its owners are now named where the content lives. +3. **Definition of a *flaky test*.** **Moved** in part: test-runner.md#retries defines a *recovered flake*, and the annotation bullet confines annotations to investigated flakiness in the harness or environment. `specs/terms.md` still defines *flaky test* against this file. +4. **Retry budget (two for non-unit surfaces, none for unit; recoveries reported; package configs at zero).** **Moved**: the runner-budget bullet of test-runner.md#retries. Package-level zero is dropped: the bullet requires unit surfaces to get no runner retries, and how that is met is left to the code. Grouped unit runs pass no count and rely on the package configs' `retry: 0` (per-file review round 1). +5. **Infrastructure retries are separate; may repeat unit work; observations from incomplete attempts stay diagnostic; no flake count.** **Moved**: the infrastructure-retries bullet of test-runner.md#retries and sharded-execution.md#shard-outcomes. The counter detail is derivable from recovered flakes being native retries (reporting.md#honest-status). +6. **`FLAKY_TEST_RETRIES` type and resolution (parsing, `CI`, defaults).** **Moved** in part: the annotation bullet gives 5 under the runner and CI unless the variable is set. Parsing rules are dropped as code. +7. **The runner injects 5 unless set; direct runs get 0.** **Moved**: same bullet. The direct-run default is derivable from the code. +8. **One test case only.** **Moved**: same bullet ("never on a whole file or suite"). +9. **Playwright annotation pattern (code block).** **Derivable**: from `test/flaky-retries.ts` and existing annotated tests. The `flaky:` title prefix is kept in the annotation bullet. +10. **Vitest annotation pattern (code block).** **Derivable**: as above. +11. **A quarantine file allowed when its name says so.** **Dropped**: no quarantine file exists, and nothing uses the allowance. +12. **Policy: investigate first; fix real failures; annotate only investigated harness flakiness; the prefix sets `flakyAnnotated`.** **Moved**: the annotation bullet of test-runner.md#retries. +13. **Recovered-flake reporting (Vitest attempt recorder authoritative; Playwright `flaky`; listed with attempts).** **Moved**: reporting.md#honest-status, for Vitest. Playwright's native status is derivable from its report. +14. **Lifecycle failures are not recoverable flakes.** **Moved**: "No retry clears a process leak" in test-runner.md#retries, and sharded-execution.md#shard-outcomes (a leak is `failed`, not retried). +15. **Network resets are harness defects, not grounds for annotation.** **Dropped**: flake discipline, including the proxy reset rule, belongs to the testing policy (`testing-policy.md#timing-observable-status-cleanup-and-flakiness`). +16. **Testing: annotations are judged in review.** **Dropped**: it restates the testing policy's review-based coverage judgment. + +## github-ci.md + +1. **Opening paragraph (what CI validates, drafts skip, fan-out, caches, building once).** **Reworded**: github-ci.md's introduction and #when-it-runs. +2. **Ownership paragraph.** **Reworded**: the introduction. +3. **What CI runs is owned under the runner's retry contract; attestation owns the skip.** **Reworded**: the introduction. +4. **The developer-to-CI reuse path through verify's attestation; `--no-publish`.** **Moved**: attestation.md#who-may-write and #skipping-in-ci, and the `--no-publish` sentence in test-runner.md#verify. +5. **GitHub CI as the other attestation source; `TV_TEST_REF_PUBLICATION`.** **Moved**: attestation.md#who-may-write. +6. **Superseded pull-request runs (`^gha-pr-concurrency`).** **Reworded**: github-ci.md#when-it-runs. The anchor is gone. +7. **Every job selects Node from `.nvmrc` once; which action selects it; metric recorders on the runner default.** **Dropped**: workflow wiring. `.nvmrc` authority is owned by `node-versions.md`. +8. **The install contract: what is cached, the key, no `restore-keys`, a hit skips `npm ci`, a miss saves, version bumps miss.** **Reworded**: first paragraph of github-ci.md#dependency-and-browser-caches. The version-bump cost and a possible later refinement are dropped as an accepted trivial cost and backlog. Cited by `.github/actions/setup-workspace/action.yml` and `test/repo/test-runner-guardrails.test.ts` (`#dependency-install`). +9. **The npm store cache as miss-path accelerator.** **Dropped**: implementation. +10. **The readiness namespace must increment; `shard-readiness-v2`.** **Reworded**: last sentence of #dependency-and-browser-caches. +11. **The combined shard archive and its key on `ImageOS`/`ImageVersion`; the build as sole producer after launch proof.** **Reworded**: second paragraph of #dependency-and-browser-caches. +12. **Consumer fallback on a miss, and image rollout skew.** **Reworded**: same paragraph. +13. **Browser environment: `setup-playwright`, no Node selector, per-shard launch probes, the operator's removal condition.** **Reworded** in part: the per-shard launch and its reason are in #dependency-and-browser-caches. The composite action's internals are dropped as workflow detail, and the probes' removal condition as backlog. Cited by `.github/workflows/ci.yml` (`#browser-environment`). +14. **Deferred pinned Playwright container, with runs and findings.** **Dropped**: history and a deferred plan (spec-policy, "What the product is now"). +15. **Jobs without browsers use neither path; desktop needs Electron and builds for itself.** **Dropped**: workflow detail. Desktop's independence from the build is kept in github-ci.md#build-once. +16. **Desktop runtime preparation (`^gha-electron-runtime-preparation`).** **Dropped**: owned by `desktop/e2e-harness.md`, which the paragraph itself deferred to. Cited by `specs/arch/explainer-desktop-app.md` (`#desktop-runtime-preparation`). +17. **The matrix plans `e2e-ci`; a registry assertion; desktop covered separately.** **Reworded**: github-ci.md#jobs. The required registry assertion is dropped as the proof's to derive. +18. **Each shard computes the plan with the VM timing provider and Blaxel weights.** **Reworded**: #jobs, and sharded-execution.md#plans and #duration-baseline. The timing-provider identity is dropped as code. +19. **Phases recorded; run directory uploaded through `--run-dir-output`; the join's agreement checks.** **Reworded**: github-ci.md#run-artifacts and #required-checks. Run artifacts keep the old scope: jobs that run `npm test` use `--run-dir-output`, and shard jobs read the run directory from the shard worker's summary (per-file review round 1). Phase recording moved to reporting.md#timing-records. +20. **Overlapping builds as the reason to build once.** **Reworded**: the first sentence of #build-once states the arrangement. The CLI build's sub-builds are dropped as derivable. +21. **The artifact is the build's output, from three sources (`^gha-prebuilt-partition`).** **Reworded**: #build-once, anchor kept. +22. **The artifact is every untracked file except three paths, with no output list.** **Reworded**: #build-once. +23. **The promise is bounded; four things out of scope (`^gha-prebuilt-bounds`).** **Reworded**: the second limit bullet in #build-once (adds files only; no deletions, empty directories or `node_modules` writes). Repository metadata differing between checkouts is derivable from taking tracked files from the shard's own checkout. Nothing cited the anchor. +24. **The one rule on builds (`^gha-prebuilt-tracked-rule`), with the `LICENSE` example.** **Reworded**: the first limit bullet. The `LICENSE` example is dropped. The anchor is gone. +25. **The manifest declares recipes; the `PrebuiltManifest` type; the coverage check.** **Reworded** in part: #build-once keeps the recipes and the rule that each build step of a surface in the `all` suite is covered exactly once (per-file review round 1). The type is dropped: it is read only by the same tree's build job. +26. **One-day artifact retention; a tar archive keeping modes.** **Reworded** in part: the tar archive and its reason. Retention is dropped as a workflow setting. +27. **Consumers fail loudly, with no fallback (`^gha-prebuilt-consumer`).** **Reworded**: #build-once. The anchor is gone. Cited by a comment in `test/repo/test-runner-guardrails.test.ts`. +28. **`--skip-pre-commands` and `--plan` worker inputs.** **Reworded** in part: shards "skip their surfaces' build steps". The option names are dropped as internal to the workflow and worker. +29. **The build job produces the readiness cache before building.** **Dropped**: workflow ordering. +30. **Fan-out scope: desktop builds for itself.** **Reworded**: third limit bullet. +31. **Join integrity: draft guard, `always()`, success rules, vibe-mode, the attested path, the checkout, an induced-failure live run.** **Reworded**: github-ci.md#required-checks. The join's own checkout is dropped as derivable. The live induced failure is derivable from #testing-guidance together with the join's rules. +32. **Tool prerequisites on CI workers (`ss`, procfs; changes land together).** **Derivable**: planned shard workers check the process backend and fail without it (test-runner.md#processes-started-by-tests). The rule that changes land together is dropped as a maintenance obligation. Cited by `.github/workflows/ci.yml` (`#tool-prerequisites-on-ci-workers`). +33. **Sixteen shards as the measured knee, with run IDs; the total changes together.** **Reworded**: #jobs, without run IDs. Cited by `packages/web/playwright.config.ts`, `packages/web/playwright.real-stack.config.ts` and `test/repo/test-runner-guardrails.test.ts` (`#ci-worker-counts`). +34. **The `e2e:browser-app` two-worker pin, inconclusive measurements, `TV_PW_WORKERS`.** **Dropped**: a configuration value in `packages/web/playwright.config.ts` with no settled reason; the code governs it. +35. **`e2e:browser-app-real-stack` runs one worker, with the observed failures.** **Reworded**: github-ci.md#browser-worker-counts. Run IDs and the ticket are dropped as history. +36. **Testing: no test reaches the cache and artifact services; an isolated live run; scratch namespaces; deleting entries.** **Reworded**: github-ci.md#testing-guidance. Cited by proofs (`#Testing`). +37. **The four required pieces of live evidence.** **Derivable**: each follows from the promises in #dependency-and-browser-caches and #required-checks once a live run is required. +38. **Production check names can be confirmed only after merge.** **Reworded**: last sentence of #testing-guidance. +39. **Retirement record for the Blacksmith provider.** **Dropped**: history (spec-policy, "What the product is now"). Cited by `test/repo/test-runner-guardrails.test.ts` (`#retirement-record`). + +## preflight.md + +The file is removed. Cited by `AGENTS.md`, `developer-skills/tvdev-setup/SKILL.md` (by name, including "Execution-host capability checks"), `specs/arch/desktop/e2e-harness.md`, `specs/arch/desktop/index.md`, `specs/arch/desktop/runtime.md`, `specs/arch/node-versions.md` and `specs/arch/telemetry/sink.md`. Section and anchor citations are listed with their passages below. + +1. **Opening paragraph and ownership.** **Dropped**: the file is gone. Its owners are now test-runner.md (remote revision checks, process backend), `node-versions.md` (toolchain) and `telemetry/sink.md` (the PostHog key). +2. **Definition of a *preflight*: named checks, a result each, in order, stopping at the first failure.** **Derivable**: from the code. `test-runner.md` still lists `preflight` as a command, and verify's phases include it. `specs/terms.md` still defines *preflight* against this file. +3. **Required checks: `node` always, `process-lifecycle` for local and planned runs, the union of surface checks, unit-only not exempt, exceptions for targeted and non-plan paths.** **Moved** in part: the process-backend scope is in test-runner.md#processes-started-by-tests, and surface-declared checks in test-runner.md#the-test-registry ("checks that must pass before it runs"). The reason unit-only runs are not exempt is derivable from "canonical local runs … check". +4. **Coordinator checks (`blaxel-github-token`, `blaxel-auth`); workers run the host check; capability checks are local only.** **Moved** in part: the Blaxel checks (credentials, a clean tree, a pushed commit) in test-runner.md#verify and #providers. The rest is derivable from the backend scope and from the token rule in blaxel-testshards.md#repository-and-credentials. +5. **Plain GitHub CI gets the host check without a `ci` provider.** **Derivable**: planned workers check the backend (test-runner.md#processes-started-by-tests), and CI is not a provider (#providers). +6. **`node` delegates to `node-versions.md`; its name is stable.** **Dropped**: owned by `node-versions.md`. The check's name is internal. +7. **`process-lifecycle` backends on Linux and macOS; never degrade to names or ports.** **Moved**: test-runner.md#processes-started-by-tests. +8. **`playwright-chromium`.** **Derivable**: from the registry's surface checks and the code. +9. **`playwright-firefox`: skipped with a reason outside GitHub Actions, required inside it.** **Moved**: the Firefox paragraph in test-runner.md#the-test-registry (review round 1), extended to say that Blaxel runs cover Chromium only. +10. **`electron` check against the shared environment planner; never installs (`^preflight-electron-runtime`).** **Dropped**: Electron runtime preparation and its planner are owned by `desktop/e2e-harness.md`. Cited by `specs/arch/explainer-desktop-app.md`. +11. **`daemon-test-host`, with its direct-run backstop.** **Moved**: the daemon-acceptance bullet in test-runner.md#the-test-registry ("refuses to start unless `TV_DAEMON_TEST_HOST=1`"), which covers the backstop. +12. **`posthog-test-key`, its sources and prefix.** **Dropped**: owned by `telemetry/sink.md#real-posthog-integration-test-surface`, which states the key, its sources and the prefix; test-runner.md links there. +13. **`blaxel-github-token`, `blaxel-auth`.** **Derivable**: from the token rule in blaxel-testshards.md#repository-and-credentials and verify's Blaxel checks. +14. **`fuser` is not a dependency; provider images must include the backend; a missing tool fails.** **Moved** in part: test-runner.md#processes-started-by-tests (the backend tools; worker setup installs them on Blaxel). The `fuser` note is dropped. +15. **Blaxel remote preflight introduction.** **Moved**: test-runner.md#providers (`^remote-revision`). +16. **The four git-safety checks.** **Moved**: test-runner.md#providers (a clean tree, a commit on an `origin` branch, `--ignore-uncommitted`). The exact Git commands are dropped as code. +17. **The `git` summary only when all checks pass.** **Dropped**: an internal type. +18. **After dispatch, planned workers run the host check and stale-owner cleanup; a failure is a pre-test infrastructure failure.** **Moved**: test-runner.md#processes-started-by-tests (stale tokens fail the run as infrastructure) and sharded-execution.md#shard-outcomes. +19. **Contributors commit and push before Blaxel; development branches.** **Moved**: test-runner.md#^remote-revision. The branch workflow is owned by `spec-workflow.md`. Cited by `specs/spec-workflow.md` (`#contributor-branches-and-remote-revisions`) and `specs/arch/testing-policy.md` (`#Contributor branches and remote revisions`). +20. **Reachability, not naming; `--ignore-uncommitted` gives no evidence for local edits; macOS and Linux alike.** **Moved**: test-runner.md#^remote-revision ("on a branch on `origin`"; "still tests only the committed revision"). The platform sentence is dropped as derivable. +21. **Contract types (`PreflightResult`, `RemoteGitPreflight`, `RemotePreflightResult`).** **Dropped**: internal types written and read by the same revision. +22. **Testing guidance: contract coverage for missing capabilities; the Linux seam; macOS needs a macOS host.** **Derivable**: from `testing-policy.md#^platform-breadth` and the backend statement. +23. **Testing guidance: the Electron planner handoff.** **Dropped**: owned by `desktop/e2e-harness.md`. +24. **Testing guidance: a real Git seam for git safety.** **Derivable**: from the testing policy's seam rule. +25. **Testing guidance: live evidence for Blaxel capability changes.** **Moved**: blaxel-testshards.md#testing-guidance (a live run for changes to remote operation). Cited by `specs/product/desktop-app.md`, `proofs/product/desktop-app.md` and proofs (`#testing-guidance`). + +## reporting.md + +1. **Opening paragraph.** **Reworded**: reporting.md's introduction. +2. **Ownership paragraph.** **Reworded**: the introduction links duration planning to sharded-execution.md. The list of what reporting owns is derivable from the sections. +3. **Run-directory location and the exact run-ID format, with its reason.** **Reworded** in part: reporting.md#run-directory keeps uniqueness across processes and hosts. The exact format is dropped: nothing parses it. +4. **Directory tree.** **Reworded**: the list in #run-directory. +5. **The `outputs` block; atomic writes; early exits still write the top-level files.** **Reworded**: #run-directory, scoped after review round 1. The promise holds once a directory exists, and refusals leave none. A Blaxel run creates its directory once its remote preflight and selection checks have passed (per-file review round 1); two selection checks that ran after the directory existed were moved ahead of it (per-file review round 2). The `outputs` block is dropped as internal. +6. **Which process owns the directory (local CLI; the shard worker for CI shards; the Blaxel finalizer).** **Reworded**: #run-directory. +7. **CI matrix jobs keep distinct runs, and the join validates without rewriting them.** **Derivable**: from "Each GitHub CI test job writes its own" (#run-directory) and github-ci.md#required-checks. +8. **`--run-dir-output` and why `.test-runs/` is not enumerated.** **Reworded**: #run-directory. Its write is now tied to the directory's creation, not to "before preflight" (review round 1). +9. **Normalized surface result definition and types (`Counts`, `ProcessLeakSocket`, `NormalizedProcessLeak`, `NormalizedSurfaceResult`).** **Dropped** in large part: these are internal normalization types. The fields readers use are pinned instead in `RunSummary` (reporting.md#the-compact-summary). Cited by proofs (`#Normalized surface result`). +10. **Normalization rules: Vitest failures; recoveries only from the attempt recorder; never from aggregate messages; Playwright `flaky`; a missing report is incomplete evidence.** **Reworded**: reporting.md#honest-status. +11. **Duration sources for standalone, combined and provider parts.** **Dropped**: internal accounting. +12. **`TimingProvider` reference; `buildReports`.** **Dropped**: internal. +13. **`ReportSummary` and `ReportResults` types.** **Reworded** in part: the `RunSummary` pin keeps the fields readers use. The rest is dropped as writer-owned. +14. **`failedSurfaces` declared once; generated declarations must match; `surfacesRun`/`surfacesSkipped` counting; provider records.** **Dropped**: internal counting rules. "Missing work is never counted as run" is kept in #honest-status. +15. **`selection` is the requested scope; results only where evidence exists; nothing synthesized; idle shards.** **Reworded**: #honest-status, and the `results.json` line in #run-directory. The idle-shard detail is dropped. +16. **The three status fields.** **Reworded**: #honest-status, now placed in the summary's `run` block (review round 1). +17. **What an incomplete run may and may not report.** **Reworded**: #honest-status. +18. **`failedSurfaces`/`skippedSurfaces` semantics; partial aggregates.** **Dropped**: internal. +19. **An interrupted provider run is incomplete and keeps its evidence.** **Derivable**: interruption is an infrastructure failure (sharded-execution.md#shard-outcomes), so the run is incomplete (#honest-status). The coordinator keeps what it retrieved (blaxel-testshards.md#leases). +20. **A passed summary with a nonzero exit.** **Moved**: sharded-execution.md#shard-outcomes. +21. **Process-leak status fields; stale owners as synthetic lifecycle surfaces; counting unique leaks.** **Reworded** in part: #honest-status (a leak fails its surfaces and marks them incomplete). Failure-kind names and the synthetic-surface representation are dropped as internal. +22. **Compact discipline: no passed tests; the compact fields; 800-character errors; leaks in full; `results.json`.** **Reworded**: reporting.md#the-compact-summary and its pinned type. Passing tests are not listed, and the error excerpt is there when one is available (per-file review round 1). The 800-character limit is dropped as a tuning value. +23. **`events.ndjson` as the exception to compact discipline.** **Reworded**: #the-compact-summary. +24. **Resource diagnostics: what is observed, when, and as unavailable rather than zero (`^resource-diagnostics`).** **Reworded**: reporting.md#resource-diagnostics, anchor kept. +25. **The sidecar's identity, retention, links, attempt history; no invented outcomes.** **Reworded**: the second paragraph of #resource-diagnostics. +26. **Kernel OOM lines, byte limits, reserved space, stated truncation, warnings (`^resource-diagnostic-bounds`).** **Reworded** in part: bounded per surface and per attempt, never changing outcomes. The byte limits are dropped as tuning values. The anchor is gone. +27. **What the observations do not prove; no periodic sampler or heap snapshots required.** **Reworded**: first paragraph of #resource-diagnostics. The list of things not required is dropped, since nothing asks for them. +28. **Recovery receipts.** **Reworded**: last sentence of #resource-diagnostics. +29. **No credentials, query strings, bodies or environments; identities without arguments; outside scratch; out of the event stream; native retry and trace policy unchanged.** **Reworded** in part: reporting.md#nothing-secret. The remaining placement notes are derivable from blaxel-testshards.md#temporary-files (evidence moves out of scratch). +30. **Event stream encoding (UTF-8 NDJSON, RFC 8785) and completion.** **Reworded** in part: "The same input gives the same bytes" in reporting.md#timing-records. The canonicalization scheme is dropped as the means of that determinism. +31. **The event hierarchy; leak events; the synthetic shard; fail-fast and `not-assigned` surfaces.** **Reworded** in part: the event kinds are in the pinned `TimingEvent` type. The hierarchy rules are dropped as writer-owned. +32. **The full event types.** **Reworded** in part: the envelope and the fields `baseline update` reads are pinned, with the rule for changing the stream (review rounds 1 and 2). The rest is dropped as writer-owned. +33. **Phase metrics as standing output, mirrored in the summary and printed.** **Reworded**: "The summary repeats the phases" in #timing-records. +34. **Stable phase categories; costly steps; cache status; provider-specific names.** **Reworded** in part: the phase coverage sentence. The category vocabulary is dropped: no dashboard reads it. +35. **GitHub jobs measuring schedule delay; sidecar ingestion; the join's metrics artifact.** **Dropped**: workflow implementation. Their best-effort status is kept in #timing-records. +36. **Blaxel phases; surface phases; verify's metrics.** **Reworded**: the phase coverage sentence, and the verify example in the best-effort paragraph. +37. **Report finalization; `verify:tests` and `verify:total` appended to the summary only.** **Reworded**: the best-effort paragraph of #timing-records (review round 1). +38. **`phaseMetrics` is load-bearing; later appends best-effort; wrappers re-raise.** **Reworded**: the best-effort paragraph (review rounds 1 and 2). +39. **Duration integers; POSIX paths; stripping the remote root; the `testId` digest.** **Dropped**: internal. Repository-relative paths are kept in the pinned file event. +40. **Leak records: `leakId` digest, sorting, executable only, sockets, `survived`.** **Reworded** in part: #nothing-secret keeps the executable without arguments. The rest is dropped as internal. +41. **File `durationMs`, `planningDurationMs`, `complete: false`.** **Reworded**: the comments on the pinned file event, and sharded-execution.md#duration-baseline (review round 1). +42. **One event per native attempt; `recoveredFlake` flags; preserve unknowns.** **Derivable**: from #honest-status ("never becomes a failed test"; only observed outcomes). +43. **Playwright JSON suffices; Vitest's does not; the attempt recorder; never from `failureMessages` (`^vitest-attempt-recording`).** **Reworded**: #honest-status. The anchor is gone. Cited by proofs (sharded-execution). +44. **Deterministic ordering; contiguous `sequence`.** **Reworded** in part: the contiguous `sequence` is pinned, and determinism is stated. The ordering rules are dropped. +45. **What the event stream never contains; the environment allowlist.** **Reworded**: #nothing-secret. +46. **GitHub Actions artifacts (name, `if: always()`, error if missing, 30 days; no credentials).** **Moved**: github-ci.md#run-artifacts. The credentials rule is in reporting.md#nothing-secret. +47. **Testing: a seam for the Vitest attempt recorder; fixtures from recorded reports.** **Derivable**: from the testing policy's seam and composition rules, given the recorder statement in #honest-status. + +## sharded-execution.md + +1. **Opening paragraph.** **Reworded**: sharded-execution.md's introduction. +2. **Ownership paragraph.** **Reworded**: the introduction. +3. **One committed rollup at `test/timing-baseline.json`; a planning input measured only on Blaxel; every plan uses it.** **Reworded**: sharded-execution.md#duration-baseline. +4. **`TimingProvider` union and the baseline types.** **Dropped**: the baseline's format is left to code. A `--commit` run reads the tested commit's baseline with the checkout's planner, so the file crosses revisions; blaxel-testshards.md#runs states that boundary, its rule, and the exception that leaves the shape to code (review round 1, finding 1; per-file review rounds 1 and 2). The Blaxel identity is pinned in reporting.md#timing-records. +5. **The `providers` map holds only the Blaxel entry.** **Dropped**: format detail, as above. +6. **The canonical empty baseline; the planner requires the file; the 60-second no-history weight.** **Dropped**: format and a constant. Unknown files being weighted as expensive is kept in #plans. +7. **Timing-provider identities (local, Blaxel, GitHub, a reserved container ID).** **Dropped**: an internal identity scheme, except the Blaxel identity pinned in reporting.md. +8. **`TV_TEST_TIMING_PROVIDER` defaults; GitHub fails without one.** **Dropped**: wiring between the workflow and the reporter. +9. **What the median is taken over, and the sample and count fields.** **Reworded** in part: #duration-baseline (planning duration; complete, finally passing observations; recovered flakes included; review round 1). The count fields are dropped. +10. **Median arithmetic; units; ownership must match.** **Dropped**: arithmetic. Registry ownership of files is in test-runner.md#the-test-registry. +11. **Key order, serialization and byte-identical output.** **Dropped**: formatting that the code owns. +12. **The only writer is `baseline update`.** **Reworded**: #duration-baseline. Cited by `scripts/test/timing-baseline.mjs` (`#Manual update`). +13. **Input-run criteria.** **Reworded**: "it reads only complete Blaxel runs of the full suite". +14. **The `all` label is not evidence; validation; recompute; printed changes; `--dry-run`.** **Reworded**: #duration-baseline. Stream validation is in reporting.md#timing-records. +15. **Aborts on unreadable runs; fails with none; never commits; never automatic; a person reviews.** **Reworded**: #duration-baseline. +16. **Nothing is carried forward.** **Reworded**: same paragraph, with the reason. +17. **Shard plan definition; atomic files; every surface present.** **Reworded**: sharded-execution.md#plans. +18. **Plan types.** **Dropped**: the shape is left to code. The plan crosses revisions when the coordinator and workers differ, under the rule and the stated exception in blaxel-testshards.md#runs (per-file review rounds 1 and 2). +19. **A valid plan has at least one file; idle per-shard surfaces.** **Reworded** in part: a shard with no files for a surface skips it (#plans). The at-least-one-file rule is dropped as code. +20. **Ordering, canonical JSON, the `planId` digest, the baseline digest, no time or randomness; retained.** **Reworded** in part: "Plans are deterministic … identified by a digest of its content". The formulas are dropped. +21. **Inventory from `git ls-files`; a temporary index for another commit; untracked files cannot matter.** **Reworded**: first bullet of #plans. The Git mechanics are dropped. +22. **The candidate filename pattern; exactly one owner; configuration failures; no config imports.** **Reworded** in part: the first bullet of #plans. The one-owner rule is in test-runner.md#the-test-registry. The filename pattern is dropped as code. +23. **Workers load configs only to execute; the assigned-versus-collected guard.** **Reworded**: second bullet of #plans. Cited by proofs (`#Explicit file execution`). +24. **Baseline medians are used whatever the plan's provider; predictions are in Blaxel time.** **Reworded**: #duration-baseline and the balancing bullet of #plans. Cited by `scripts/test/shard-plan.mjs` (`#Provider weights and unknown files`). +25. **The generous unknown-file weight formula.** **Reworded** in part: "weighted as expensive". The formula is dropped. +26. **Nearest-rank p90; `weightSource`.** **Dropped**: arithmetic. +27. **LPT assignment steps.** **Dropped**: algorithm. "Balanced by expected duration" is kept. +28. **LPT prevents heavy surfaces piling up; predicted sums are advisory.** **Reworded**: the balancing bullet ("guide the balance but do not predict wall time"). +29. **A static plan; no queue; retries reuse it.** **Reworded**: last bullet of #plans. +30. **Stage-one cost model: additive medians only.** **Reworded**: the balancing bullet states it as an accepted limitation. +31. **The 2× overestimate and 24-second observations.** **Dropped**: measurements behind the advisory statement. +32. **Revision triggers before any shard-count increase.** **Dropped**: a plan for future work (spec-policy, "What the product is now": a spec is not a backlog). +33. **Each GitHub job computes the plan; digest check; no mutable inputs.** **Reworded**: the determinism bullet of #plans. +34. **Digests do not prove cross-job agreement; summary fields; the join's checks.** **Reworded**: the determinism bullet, and github-ci.md#required-checks. +35. **Blaxel computes the plan once; workers validate it.** **Reworded**: the determinism bullet. +36. **No native `--shard` in planned runs.** **Derivable**: from "Each shard runs exactly its assigned files" (#plans). +37. **Vitest absolute paths and substring ambiguity; the collection check; the combined `unit:workspaces` process.** **Reworded** in part: the substring trap is stated once, in test-runner.md#commands, and the collection check in #plans. The combined process is dropped as internal. +38. **Playwright `--test-list`; custom commands forward it.** **Dropped**: mechanism. Wrapper commands applying remotely is in blaxel-testshards.md#runs. +39. **Generated lists and configs retained.** **Dropped**: retention of generated inputs is left to the implementation. reporting.md#run-directory names `provider/` but does not require these files (per-file review round 1). +40. **A collection mismatch fails the surface as incomplete, with lists kept.** **Reworded** in part: the second bullet of #plans keeps the mismatch failing the surface as an infrastructure problem. Keeping the lists is dropped and left to the implementation (per-file review round 1). +41. **`not-assigned` surfaces skip everything; empty groups skipped; aggregates not skipped.** **Reworded** in part: second bullet of #plans. The aggregation rule is dropped as internal. +42. **`--skip-pre-commands`: caller-supplied artifacts, no fallback build.** **Moved** in part: github-ci.md#build-once (shards skip build steps; a missing archive fails, with no fallback). The option names are dropped. +43. **Planned execution is an exception to native sharding; native `--shard` remains for diagnostics.** **Dropped**: describes mechanisms outside any contract. +44. **Explicit shard counts; reviewed settings; a run never changes its own demand.** **Reworded**: last paragraph of #duration-baseline. Cited by `scripts/test/cli.mjs` (`#Shard-count scaling`). +45. **The recommendation algorithm, its 120-second target and its printed report.** **Reworded** in part: the two-minute target, the search up to the pool's size, and the pool's size when no count meets the target (per-file review round 1). The report's contents are dropped. +46. **Capacity owned by Blaxel; recommendations never override; GitHub's count is owned by github-ci.md.** **Reworded** in part: "A person changes the configured shard counts". The rest is derivable from blaxel-testshards.md#pool and github-ci.md#jobs. +47. **Blaxel and CI both run `scripts/run-test-shard.mjs`.** **Reworded**: the introduction. +48. **`--plan` input; grouping; the combined process.** **Dropped**: internal options. Cited by proofs (`#Shared shard worker`). +49. **`--test-retries` for non-unit surfaces; `FLAKY_TEST_RETRIES` injection.** **Moved**: test-runner.md#retries (the budget and annotation) and github-ci.md#jobs (shards pass the budget explicitly). +50. **Child environment composition; removing `FORCE_COLOR` and `NO_COLOR`.** **Dropped**: implementation. +51. **`--skip-pre-commands` (worker side).** **Moved**: as item 42. +52. **`--fail-fast`.** **Dropped**: nothing in the repository passes it (review round 1, finding 8). test-runner.md#retries now speaks only of ordinary runs. +53. **The worker writes `shard-.json` and exits nonzero on failure.** **Dropped**: internal. +54. **On Blaxel the worker preserves the outer token and `TMPDIR`; sidecars travel with artifacts.** **Moved**: blaxel-testshards.md#leases (the token), #temporary-files (`TMPDIR`) and reporting.md#resource-diagnostics. +55. **Blaxel provider report collection; GitHub retains summaries.** **Derivable**: from reporting.md#run-directory and github-ci.md#run-artifacts. +56. **Provider report types (`ProviderTask`, `ProviderShard`, `AttemptReport`, `CombinedReport`).** **Dropped**: the attempt and combined reports are produced and read by the coordinator. The shard summary a worker returns crosses revisions under blaxel-testshards.md#runs, with its shape left to code by the exception stated there (per-file review rounds 1 and 2). Cited by proofs (`#Blaxel provider report shape`). +57. **`NormalizedProcessLeak` is reporting's; generated declarations keep the unions.** **Dropped**: internal. +58. **Shard status values; completion decides first.** **Reworded**: sharded-execution.md#shard-outcomes. +59. **`infra-failed` meaning.** **Reworded**: first outcome bullet. +60. **A test failure is a completed shard and is not retried.** **Reworded**: second outcome bullet. +61. **A contradictory passed summary.** **Reworded**: second outcome bullet. +62. **A process leak is a completed lifecycle failure.** **Reworded**: second outcome bullet. reporting.md#honest-status covers its incompleteness. +63. **An unreadable summary.** **Reworded**: first outcome bullet. +64. **Observations kept from an infrastructure-failed shard do not settle the run.** **Reworded**: the paragraph after the bullets. +65. **Blaxel classifies per its spec; GitHub receives exits directly.** **Reworded**: the last sentence of #shard-outcomes. +66. **Each attempt's report files; an unreadable summary becomes infrastructure-failed.** **Derivable**: from the first outcome bullet and reporting.md#run-directory. +67. **The retry condition is owned by test-runner.md; the `stale-owner` exception; no GitHub retry coordinator.** **Reworded**: #shard-outcomes and blaxel-testshards.md#runs. +68. **Attempt history; the latest attempt wins; missing results; equal plans across attempts.** **Reworded** in part: "The run's result takes each shard from its latest attempt". The missing-result and plan-equality rules are derivable from the outcome rules and the fixed plan. +69. **The combined report's shard set and failed-task list; superseded observations; passing rule.** **Reworded**: the paragraph after the bullets. +70. **Normalization by surface; combined counts; deduplicated leaks; `not-assigned` and fail-fast parts.** **Dropped**: internal aggregation. +71. **Provider failure records; contradictions; `provider:` when no evidence exists.** **Derivable**: from reporting.md#honest-status. +72. **The timing hierarchy from provider results; links only to retained files.** **Derivable**: from reporting.md#timing-records and #run-directory. +73. **Testing: platform coverage for `resolveTimingProvider`.** **Dropped**: the identity scheme is no longer specified, and the platform rule is the testing policy's. +74. **Testing: seams launching installed runners for exact assignment, the combined config and `--test-list`.** **Derivable**: from the testing policy's seam rule and the collection check in #plans. +75. **Testing: the Electron wrapper seam.** **Derivable**: from the testing policy (each call site is its own seam). +76. **Testing: a live planned Blaxel run proves plan transfer.** **Reworded**: sharded-execution.md#testing-guidance. +77. **Testing: live GitHub evidence of plan agreement.** **Reworded**: same section. Cited by proofs (`#testing`). + +## test-registry.md + +The file is removed. Its rules now live in test-runner.md#the-test-registry. Cited by `specs/arch/desktop/e2e-harness.md`, `specs/arch/explainer-desktop-app.md`, `specs/arch/licensing.md` and `test/repo/test-runner-guardrails.test.ts` (the file); `specs/terms.md` defines *test surface*, *suite* and *execution group* against it. + +1. **Opening paragraph and ownership.** **Moved**: test-runner.md#the-test-registry and its introduction. +2. **Definition of a *test surface*; package association; "surface" is unrelated to artifacts.** **Moved** in part: the definition. Package association and the naming note are dropped as derivable. +3. **`test.config.mjs` is the registry; nothing is inferred from names.** **Moved**: test-runner.md#the-test-registry. Cited by proofs (`#source-of-truth`). +4. **Grouped versus top-level declarations; flattening; path normalization; `cwd` rules.** **Dropped**: loader implementation. +5. **`TestSurface` type.** **Dropped**: internal; the registry and its loader change together in one tree. +6. **Declared versus normalized surfaces.** **Dropped**: internal. +7. **The field list, including `isolated-github-only` and `supports`.** **Moved** in part: test-runner.md#the-test-registry names what a surface declares, and the `isolated-github-only` placement paragraph moved there. The note that targeted Blaxel runs skip the `supports` check is dropped as code. +8. **`TestService` type and the `publishUrlEnv` pattern.** **Dropped**: internal declaration format. +9. **Service lifetime, order, earlier URLs, full URLs, scoped and restored variables, the same declarations on every path.** **Moved** in part: test-runner.md#network-addresses-in-tests (a registry-declared service's URL arrives in the variable the registry names; no address derived from another), and blaxel-testshards.md#runs (the same declarations remotely). Order and restoration are derivable. Cited by `specs/arch/desktop/e2e-harness.md` (`#Surface services`). +10. **The suite record type and the include/exclude rule semantics.** **Dropped**: loader semantics. +11. **The registered suites.** **Reworded** in part: test-runner.md#the-test-registry defines suites and `all`, and names the excluded classes. The complete list is in `help` and the registry. +12. **`all` composition; explicit selection; lifecycle-fault and dynamic-services experiments; PostHog; daemon; build-config.** **Moved**: test-runner.md#the-test-registry. The dynamic-services and build-config notes are derivable from `all` being unit plus end-to-end. +13. **The Blaxel harness surfaces (`^blaxel-harness-surfaces`).** **Moved**: blaxel-testshards.md#testing-guidance. The anchor is gone. Cited by proofs. +14. **`DeclaredExecutionGroup`; flattening; ordering; `unit:build-config` as a separate surface.** **Dropped**: internal scheduling. +15. **`TestConfig` type.** **Dropped**: internal. +16. **The `.d.mts` conflation note.** **Dropped**: an implementation note. +17. **`SelectionOptions` and `selectSurfaces` semantics.** **Moved** in part: test-runner.md#commands (selectors). Tag AND-composition and the co-selector conflict error are dropped as code. +18. **`owningSurfaces`; path containment.** **Derivable**: from the code. +19. **Every test file has exactly one owner; `excludeRoots`.** **Moved**: test-runner.md#the-test-registry. +20. **`CANONICAL_TEST_INCLUDE_ROOTS`; `loadTestConfig` and `validateRegistry` signatures.** **Dropped**: internal. Cited by proofs (`#Discovery and validation`). +21. **Only `packages/` and `test/` are scanned.** **Moved**: test-runner.md#the-test-registry ("under `packages/` and `test/`"). +22. **Validation errors.** **Moved** in part: an undeclared `vitest.config.ts`, `vitest.e2e.config.ts` or `playwright.config.ts` is a registry error. The recognized names are kept, because other configs are deliberately unregistered (per-file review round 1). The other errors are derivable from the code. + +## test-runner.md + +1. **Authority paragraph listing what the spec owns and its companions.** **Reworded**: test-runner.md's second paragraph. +2. **Also authoritative for addresses, Vite responses and process lifecycle.** **Reworded**: same paragraph. +3. **"Why this exists": one runner for anyone, especially agents; the grammar is the contract.** **Reworded**: the opening paragraph. +4. **Without one runner, invocations drift and agents emit ignored arguments.** **Reworded**: the opening paragraph. +5. **The runner guides; named overrides; not a security boundary; recognizable departures only.** **Reworded**: test-runner.md#guidance-not-enforcement. +6. **The consolidation pays off three ways (bulleted).** **Dropped**: motivation that adds nothing to items 3–4 and changes no derivation. +7. **Provider definitions.** **Reworded**: test-runner.md#providers. +8. **GitHub Actions is not a selectable provider.** **Reworded**: #providers. +9. **No default provider for shortcuts; remote runs test a committed revision from `origin`; only verify resolves automatically.** **Reworded**: #providers and #^remote-revision. +10. **Commands code block.** **Reworded**: the commands table and #verify. +11. **`npm test -- verify` is equivalent to `npm run verify`.** **Derivable**: from the code and `package.json`. +12. **Command table.** **Reworded**: the table in test-runner.md#commands. +13. **Package-level scripts are not supported entrypoints.** **Reworded**: the opening paragraph. +14. **Native entrypoints refuse; every host; launch paths carry context; unauthenticated; the non-plan preflight exception.** **Reworded**: test-runner.md#native-test-entrypoints. The preflight-scope exception now sits in #processes-started-by-tests. +15. **The refusal's content on marked and unmarked hosts; fixed examples; registered entrypoints only.** **Reworded**: #native-test-entrypoints. +16. **The direct-run override, exactly `1`; what it bypasses; help; testing-policy permission.** **Reworded**: #native-test-entrypoints and the policy pointer in #shared-development-hosts. +17. **Selector definition and table.** **Reworded**: the selector bullets in #commands. Cited by proofs (`#selectors`). +18. **The local provider accepts every suite subject to guardrails; Blaxel accepts only `all`, `unit` and `e2e`, with targeted and planned paths.** **Moved**: blaxel-testshards.md#runs. +19. **`--file` resolves to one owning surface; the local `supports` check; unknown positional arguments; unknown `--against-test-guidance` options.** **Reworded** in part: the `--file` bullet in #commands, which keeps the rule that matched files belong to one selected surface (per-file review round 1). The `supports` asymmetry and option-parsing rules are dropped as code. +20. **Publication-qualifying test execution (`^test-publication-qualifying`).** **Moved**: the Blaxel source bullet in attestation.md#who-may-write. Nothing outside the area cited the anchor. +21. **A publication-qualifying verify; selection options count.** **Moved**: attestation.md#who-may-write ("no narrowing selector"). +22. **No opt-in; `--no-publish`; `--publish` is a usage error.** **Moved** in part: attestation.md#who-may-write and test-runner.md#verify. The `--publish` usage error is dropped as code. +23. **Two attestation source classes; `attest-write`; local never qualifies.** **Moved**: attestation.md#who-may-write. +24. **`--run-dir-output`: an atomic write right after creation; not inherited; orchestration validates the path.** **Moved**: reporting.md#run-directory. +25. **Broad runs are refused and directed to verify.** **Reworded**: test-runner.md#the-full-suite-shortcut. +26. **PostHog, daemon and harness surfaces are not broad verification.** **Moved**: test-runner.md#the-test-registry (outside `all`) and blaxel-testshards.md#testing-guidance (the harness). +27. **What trips the guardrail; other selections remain subject to local guardrails.** **Reworded**: #the-full-suite-shortcut. +28. **`--grep` alone; `--force`; verify passes `--force`; the broad-local override does not lift it.** **Reworded**: #the-full-suite-shortcut. +29. **Host marker (`^blaxel-host-marker`).** **Reworded**: test-runner.md#marked-hosts, anchor kept. It now says the marker is found through `HOME`, as `os.homedir()` reads it. +30. **One local file on marked hosts (`^local-one-file`).** **Reworded**: #one-local-file-on-marked-hosts, anchor kept. +31. **`--allow-extreme-inefficiency` is refused, naming its replacement; the shared phrase.** **Dropped**: the rename is history. The shared phrase moved to #guidance-not-enforcement, scoped to the shared-host and retry overrides (review round 1). +32. **The file boundary is set before preflight; native filters; uncommitted files; directories refused; `--force`.** **Reworded**: ^local-one-file and the native-filter paragraph in #commands. +33. **What the refusal says.** **Reworded**: the paragraph after ^local-one-file. +34. **Local-only placement takes precedence; agent tests.** **Reworded**: the same paragraph, which now also covers the harness driver (merge fix `8e6cfcb2`). +35. **The mutex across checkouts and its compatibility contract (`^local-test-mutex`).** **Reworded**: #one-local-run-at-a-time, anchor kept. The location and record are now pinned. +36. **When it is taken; verify's handoff; protection if the verify parent exits; commands that take none.** **Reworded**: ^local-test-mutex. Protection after the parent exits is derivable from staleness requiring every recorded process to be gone. +37. **The contention refusal's content.** **Reworded**: the choices list. +38. **The wait option; validation before waiting; cancellation statuses; combining with the bypass is an error; help documents it.** **Reworded**: the wait bullet and the paragraph after the list. Validation order and help are derivable. +39. **Held until processes exit and cleanup ends; recovery by identity; elapsed time never; stale processes fail the run.** **Reworded**: the staleness paragraph. +40. **The contention bypass and the independence of the overrides.** **Reworded**: the paragraph after the choices list. +41. **The self-test lock-location hook and the dry run.** **Dropped**: test seams for the proof and the code. +42. **Help exposes the paths and overrides.** **Derivable**: from the commands table ("Usage, including the guardrails and overrides below"). +43. **Verify runs phases and stops at the first failure.** **Reworded**: test-runner.md#verify. Cited by `specs/arch/testing-policy.md` and proofs (`#Verify orchestration`). +44. **The marker controls resolution; local verify follows the guardrails.** **Reworded**: #verify. +45. **Provider resolution rules; positional and `--provider` errors.** **Reworded**: #verify. The argument errors are dropped as code. +46. **Phases for each provider; where remote preflight runs; the double preflight under `auto`; retry validation; the success line; attestation.** **Reworded**: #verify. The double-preflight explanation is derivable, and the tested revision is now "selected, `HEAD` by default" (review round 1). +47. **The vibe-mode check (`^verify-vibe-mode`).** **Reworded**: #verify, anchor kept. It now says what vibe mode is, because no other spec defines it. +48. **`--plan`.** **Reworded**: #verify and the exit-code list. It runs no phases; choosing the provider on a marked host still runs the Blaxel checks (per-file review round 1). +49. **The build-config integrity surface.** **Dropped**: an ordinary unit surface in `all`. The build configuration it checks is owned by `telemetry/sink.md`, which cites it. +50. **PostHog roundtrip suite selection and command.** **Dropped**: owned by `telemetry/sink.md#real-posthog-integration-test-surface`, which has the command. test-runner.md keeps its exclusion from `all` and links there. +51. **PostHog suite exclusion, local-only placement, the key and the preflight.** **Moved**: the PostHog bullet in test-runner.md#the-test-registry. The key details are owned by the sink spec. +52. **Daemon acceptance selection and command.** **Reworded** in part: the daemon bullet in #the-test-registry. The command follows from the one-file rule and the variable. +53. **Daemon suite exclusion; a designated host; the exact acknowledgement; the test repeats the check; not an authorization boundary.** **Reworded**: the daemon bullet ("refuses to start unless `TV_DAEMON_TEST_HOST=1`"). +54. **What the daemon surface does (build, pack, install, bulldozer, asserted path).** **Dropped**: the test's own walk, owned by the test and the CLI specs. +55. **Persisted serve rejects port `0`; the temporary-home procedure; `TV_DAEMON_TEST_DIRTY_START`.** **Reworded** in part: the exception paragraph in #network-addresses-in-tests. The dirty-start mode is dropped as a test option that the code governs. +56. **Blaxel harness qualification (`^blaxel-harness-placement`).** **Moved**: blaxel-testshards.md#testing-guidance, with the anchor. Cited by proofs, which link the old file. +57. **The driver's fixture selections, committed registry, deliberate failures and cleanup.** **Moved** in part: blaxel-testshards.md#testing-guidance. The selection details belong to the driver and its README. +58. **The default budget; positive `--retries`; the unit notice; the per-test annotation pointer.** **Reworded**: the runner-budget bullet of #retries. The unit notice is dropped as code. +59. **Plain `--retries 0` for one file; resolution against what runs; native filters; `--grep` alone.** **Reworded**: the zero-retries bullet. +60. **Broader zero refused; the override's semantics; verify; it lifts nothing else; help follows the policy.** **Reworded**: the zero-retries bullet. +61. **Zero forms leave unit defaults, annotations and infrastructure retries alone.** **Reworded**: the zero-retries bullet. +62. **No automatic `maxFailures` cap.** **Reworded**: the paragraph closing #retries, scoped to ordinary runs (review round 1). +63. **Blaxel infrastructure retries, including the `stale-owner` exception.** **Moved**: the infrastructure-retries bullet of #retries, sharded-execution.md#shard-outcomes and blaxel-testshards.md#runs. +64. **CI is part of the retry contract; `--test-retries 2`; the desktop job inherits; change together.** **Moved**: github-ci.md#jobs (shards pass the budget explicitly). The change-together obligation is dropped as a maintenance rule that a competent change already follows. +65. **A local run writes a run directory and exits `0` or `1`.** **Derivable**: from reporting.md#run-directory and the exit-code list. Cited by proofs (`#local-execution`). +66. **Grouped execution; the combined `unit:workspaces` process; result splitting.** **Dropped**: internal scheduling. +67. **`--file` and `--grep` selections run surfaces one at a time.** **Dropped**: internal scheduling. +68. **Each surface task runs in its owned scope, with services and teardown before the next.** **Moved**: test-runner.md#processes-started-by-tests. +69. **Surfaces may declare a build, services or a command (registry-owned).** **Reworded**: test-runner.md#the-test-registry. +70. **`SKIP_ELECTRON_E2E=1` reclassifies desktop as skipped.** **Dropped**: code. attestation.md#who-may-write keeps the consequence that such a run cannot write. +71. **Test infrastructure binds port `0` (`^test-dynamic-ports`).** **Reworded**: test-runner.md#network-addresses-in-tests, anchor kept. Cited by proofs in product specs (`#Test infrastructure addresses`). +72. **Consumers receive full URLs; no numeric literals or fallbacks.** **Reworded**: same paragraph. +73. **Registry-declared service mechanics (own listener, Vite's port `0`, pre-transforms off, URL variables, order, missing variables fail).** **Reworded** in part: the Vite paragraph. Order and the missing-variable failure are derivable. +74. **Helpers return the bound URL; readiness messages.** **Reworded**: ^test-dynamic-ports. +75. **The `tv serve` test recipe (temporary home, port `0`, `--print-links`, `--port`).** **Derivable**: from the port rule together with the CLI specs that own the home, the hidden `--print-links` flag and the client-port rule (`product/cli.md`, `arch/cli/index.md`). +76. **Playwright configs read the published URL; no `webServer`; no fallback.** **Derivable**: from the full-URL rule. +77. **Static fixtures use relative URLs; injected service URLs; alternate hostnames.** **Derivable**: from the rule against numeric literals and derived addresses. +78. **Each URL is independent; the artifact host and view.** **Reworded**: "Nothing derives one service's address from another's port". +79. **Inert numeric ports are allowed.** **Reworded**: last sentence of ^test-dynamic-ports. +80. **A stable front listener across backend restarts.** **Derivable**: a technique that follows from the ban on reserve-then-release. +81. **Vite services never answer `304` (`^test-service-full-responses`).** **Reworded**: anchor kept. +82. **The Chromium shared-memory reason.** **Reworded**: same paragraph. +83. **Destructive fixture placement (`isolated-github-only`).** **Moved**: test-runner.md#the-test-registry. "This restriction is permanent" is dropped as emphasis without content. +84. **Lifecycle constants (`TEST_SURFACE_OWNER_ENV`, a two-second grace).** **Reworded** in part: the variable name is kept. The two-second value is dropped as tuning; the teardown bullet calls the wait between `SIGTERM` and `SIGKILL` "a short interval" (per-file review round 1). Cited by proofs (`#Surface process lifecycle`). +85. **Surface process scope and its owner (`^test-surface-owner`).** **Reworded**: #processes-started-by-tests, anchor kept. +86. **Token contents; opaque to tests; process groups; helpers preserve it; detached children.** **Reworded**: the token bullet and the pinned `SurfaceOwner`, which is now specified as a cross-revision format. Process-group mechanics are dropped. +87. **The scope's five-step order.** **Reworded** in part: the teardown bullet. The graceful interval after stopping services is not a separate step before the scan: the owner scans as soon as services and remaining commands stop, and a process that exits before any signal is not a leak (per-file review round 1). The rule that a build step must exit without live descendants is dropped as detail. +88. **Teardown after any exit; `SIGTERM` then `SIGKILL`; the two-second reason; idempotence; no process outlives the scope.** **Reworded**: the teardown bullet. +89. **A killed owner cannot clean up; ephemeral hosts; persistent hosts reap later.** **Reworded**: the stale-owner bullet. +90. **The combined Vitest projects; one scope; a leak fails every member.** **Dropped**: internal. A comma-separated `surfaceId` for multi-surface scopes is noted in the pinned type. +91. **Persistent workers inspect tokens; staleness by supervisor; reap and fail.** **Reworded**: the stale-owner bullet. +92. **Blaxel reserves the whole worker; recovery may stop abandoned supervisors; outer attribution; resource evidence.** **Moved**: blaxel-testshards.md#leases and #abandoned-runs, and reporting.md#resource-diagnostics. +93. **End-of-surface leak detection (`^test-process-leak`).** **Reworded**: the teardown bullet, anchor kept. The diagnostic field list is reduced to PID, command and sockets, plus the pinned summary fields. +94. **Linux and macOS discovery; never by port or name.** **Reworded**: the discovery bullet. +95. **A leak fails its surface even when tests passed; results stay visible; combined members all fail.** **Reworded**: the teardown bullet and reporting.md#honest-status. +96. **Exit-code list.** **Reworded**: #commands. Cited by proofs (`#exit-codes`). +97. **A remote command exits `0` only with a passed report.** **Reworded**: #commands. +98. **The Cursor agent environment workaround.** **Reworded**: test-runner.md#cursor-agent-environment. The constants block is dropped. +99. **Testing guidance: spawn the CLI; native refusals through the native tool; self-test seams.** **Derivable**: from the testing policy's acceptance shape and mocking policy. +100. **Testing guidance: surface services coverage.** **Derivable**: from the testing policy's seam rule. +101. **Testing guidance: compositional provider selection; the Git seam; the live planned run.** **Derivable**: from the testing policy. The live run is in blaxel-testshards.md#testing-guidance and sharded-execution.md#testing-guidance. +102. **Testing guidance: reading `ci.yml` proves only its contents.** **Derivable**: from the testing policy's honesty rule, with the live requirement in github-ci.md#testing-guidance. + +## Statements with no old passage + +These new statements have no predecessor. The ledger gives each one's reason. + +- The pinned `LocalTestMutexRecord`, `SurfaceOwner` and `WorkerLeaseRecord` shapes (test-runner.md, blaxel-testshards.md). The old specs called these contracts without giving their shape. +- The rule that what a Blaxel run's coordinator and workers exchange crosses revisions when they differ, as with `--commit` (blaxel-testshards.md#runs, per-file review round 1). +- The pinned `RunSummary` and `TimingEvent` fields, and the rule for changing the event stream (reporting.md, review rounds 1 and 2). +- That Blaxel runs, and so Blaxel attestations, cover Chromium only (test-runner.md, attestation.md). +- The marker being found through `HOME` while the mutex ignores it (test-runner.md). +- The required check names as they stand in the live ruleset (github-ci.md#required-checks). +- The testing guidance that destructive lifecycle tests belong in `isolated-github-only` surfaces (test-runner.md#testing-guidance). + +## Counts + +Each passage is counted once, under its primary outcome. 77 of the reworded or moved passages also drop a detail, which the entry names, so "Reworded" and "Moved" overstate how much survived word for word. + +| Old spec | Passages | Kept | Reworded | Moved | Derivable | Dropped | +|---|---|---|---|---|---|---| +| `attestation.md` | 52 | 1 | 48 | 1 | 1 | 1 | +| `blaxel-testshards.md` | 50 | 1 | 38 | 3 | 1 | 7 | +| `flaky-tests.md` | 16 | 0 | 0 | 10 | 2 | 4 | +| `github-ci.md` | 39 | 0 | 27 | 2 | 2 | 8 | +| `preflight.md` | 25 | 0 | 0 | 12 | 6 | 7 | +| `reporting.md` | 47 | 0 | 34 | 2 | 4 | 7 | +| `sharded-execution.md` | 77 | 0 | 43 | 4 | 7 | 23 | +| `test-registry.md` | 22 | 0 | 1 | 11 | 1 | 9 | +| `test-runner.md` | 102 | 0 | 66 | 15 | 11 | 10 | +| **All** | **430** | **2** | **257** | **60** | **35** | **76** | + +## Where a reviewer might start + +These are the judgment calls a human is most likely to disagree with, in rough order of consequence: + +- **Contracts the old specs named but did not shape, now pinned.** The mutex record and owner token (test-runner.md), the worker lease (blaxel-testshards.md) and the summary and event fields (reporting.md) are new authority. Each is pinned only as far as another revision or reader relies on it. +- **Internal types and algorithms left to code.** The normalized result and provider report types (reporting 9, 13; sharded-execution 56), the baseline file's format (sharded-execution 4–6), the planning arithmetic and LPT steps (sharded-execution 25–27) and the coordinator's option table (blaxel-testshards 39). The baseline, plan and shard summary cross revisions when a run's coordinator and workers differ; blaxel-testshards.md#runs states the rule for changing them, and their shapes stay in code. +- **Measured settings dropped with their measurements.** The 36-shard figures (blaxel-testshards 9), timeout profiles (blaxel-testshards 40), the `e2e:browser-app` two-worker pin (github-ci 34) and the cost-model revision triggers (sharded-execution 31–32). The settings themselves remain in code. +- **Content left to other owners.** Electron preflight and runtime preparation (`desktop/e2e-harness.md`: preflight 10, github-ci 16), the PostHog key (`telemetry/sink.md`: preflight 12, test-runner 50), flake discipline and network resets (testing policy: flaky-tests 15), and the `tv serve` test recipe (CLI specs: test-runner 75). +- **Testing guidance removed as derivable from the testing policy.** These are the derivable entries under each Testing heading (for example test-runner 99–102 and sharded-execution 73–75). The method's test is whether removal changes a correct proof derivation; the proofs, once re-derived, will show it. +- **Small rules dropped as unused or history.** The quarantine-file allowance (flaky-tests 11), `--fail-fast` (sharded-execution 52), the renamed `--allow-extreme-inefficiency` refusal (test-runner 31), `--audit-stale-owners` (blaxel-testshards 25) and the Blacksmith retirement record (github-ci 39). +- **Citations from outside the area.** `AGENTS.md`, `specs/spec-workflow.md`, `specs/arch/testing-policy.md` and `developer-skills/tvdev-setup/SKILL.md` point at `preflight.md`. `specs/terms.md` names `preflight.md`, `flaky-tests.md` and `test-registry.md` as owners. These are what readers meet first. All are listed below. + +## Citations of removed passages + +References outside the rewritten files whose target file, heading or block anchor no longer exists at `92574976`. They come from a scan of `specs/`, `proofs/`, `test/`, `scripts/`, `packages/`, `developer-skills/`, `.github/`, `AGENTS.md` and the guides. Citations whose targets survive, including `^test-dynamic-ports`, `^local-one-file`, `^local-test-mutex`, `^blaxel-host-marker`, `^verify-vibe-mode`, `^test-service-full-responses`, `^gha-prebuilt-partition`, `^blaxel-scratch` and `^resource-diagnostics`, are not listed. + +Two rows were already broken before the rewrite: the `#test-assertions` headings in attestation.md and github-ci.md did not exist in the old specs either. Test comments in `test/node/cli-assets.test.ts` and `test/repo/test-runner-prebuilt.test.ts` cite `github-ci.md#^gha-prebuilt-cli-outputs`, a proof anchor in `proofs/arch/cli/index.md`; that citation was also broken before the rewrite. + +| Old target | Now | Cited by | +|---|---|---| +| `attestation.md` `#activation-policy` | attestation.md#policy | `test/repo/test-runner-attestation.test.ts` | +| `attestation.md` `#prune-and-retention` | attestation.md#revoking-and-pruning | `scripts/test/cli.mjs`, `test/repo/test-runner-attestation.test.ts`, `test/repo/test-runner-guardrails.test.ts` | +| `attestation.md` `#skip-semantics-in-ci` | attestation.md#skipping-in-ci | `.github/workflows/ci.yml`, `test/repo/test-runner-attestation.test.ts` | +| `attestation.md` `#test-assertions` | no such heading in the old spec either; now attestation.md#testing-guidance | `test/repo/test-runner-attestation.test.ts`, `test/repo/test-runner-guardrails.test.ts` | +| `attestation.md` `#who-writes-and-when` | attestation.md#who-may-write | `.github/workflows/ci.yml`, `proofs/arch/test-runner/attestation.md`, `scripts/test/cli.mjs` | +| `blaxel-testshards.md` `#Dependency-hash caching` | dropped; reuse rule in blaxel-testshards.md#repository-and-credentials | `proofs/arch/test-runner/blaxel-testshards.md` | +| `blaxel-testshards.md` `#Local setup` | blaxel-testshards.md#setup | `specs/arch/testing-policy.md` | +| `blaxel-testshards.md` `#Locking` | blaxel-testshards.md#leases | `proofs/arch/test-runner/blaxel-testshards.md` | +| `blaxel-testshards.md` `#Older runners and pool operations` | blaxel-testshards.md#older-runners and #deleting-workers | `proofs/arch/test-runner/blaxel-testshards.md` | +| `blaxel-testshards.md` `#Options` | dropped; user-facing options named in their sections | `proofs/arch/test-runner/blaxel-testshards.md` | +| `blaxel-testshards.md` `#Pool provisioning` | blaxel-testshards.md#setup | `proofs/arch/test-runner/blaxel-testshards.md` | +| `blaxel-testshards.md` `#Repository` | blaxel-testshards.md#repository-and-credentials | `proofs/arch/test-runner/blaxel-testshards.md`, `scripts/test/blaxel-repository.mjs` | +| `blaxel-testshards.md` `#Shard lifecycle` | blaxel-testshards.md#runs (in part) | `proofs/arch/test-runner/blaxel-testshards.md` | +| `blaxel-testshards.md` `#^blaxel-known-leftovers` | blaxel-testshards.md#temporary-files | `proofs/arch/test-runner/blaxel-testshards.md` | +| `blaxel-testshards.md` `#^blaxel-recovery` | blaxel-testshards.md#abandoned-runs | `proofs/arch/test-runner/blaxel-testshards.md` | +| `blaxel-testshards.md` `#^blaxel-release` | blaxel-testshards.md#leases | `proofs/arch/test-runner/blaxel-testshards.md` | +| `blaxel-testshards.md` `#^blaxel-reservation` | blaxel-testshards.md#leases | `proofs/arch/test-runner/blaxel-testshards.md` | +| `flaky-tests.md` (file) | test-runner.md#retries | `proofs/arch/test-runner/flaky-tests.md`, `specs/arch/testing-policy.md` | +| `github-ci.md` `#Testing` | github-ci.md#testing-guidance | `proofs/arch/test-runner/github-ci.md` | +| `github-ci.md` `#^gha-pr-concurrency` | github-ci.md#when-it-runs | `proofs/arch/test-runner/github-ci.md` | +| `github-ci.md` `#^gha-prebuilt-tracked-rule` | github-ci.md#build-once | `proofs/arch/test-runner/github-ci.md` | +| `github-ci.md` `#browser-environment` | github-ci.md#dependency-and-browser-caches (probes) | `.github/workflows/ci.yml` | +| `github-ci.md` `#build-once-fan-out` | github-ci.md#build-once | `.github/workflows/ci.yml`, `scripts/run-test-shard.mjs`, `scripts/test/prebuilt.mjs` | +| `github-ci.md` `#ci-worker-counts` | github-ci.md#jobs and #browser-worker-counts | `packages/web/playwright.config.ts`, `packages/web/playwright.real-stack.config.ts`, `test/repo/test-runner-guardrails.test.ts` | +| `github-ci.md` `#dependency-install` | github-ci.md#dependency-and-browser-caches | `.github/actions/setup-workspace/action.yml`, `test/repo/test-runner-guardrails.test.ts` | +| `github-ci.md` `#desktop-runtime-preparation` | dropped; owned by desktop/e2e-harness.md | `specs/arch/explainer-desktop-app.md` | +| `github-ci.md` `#retirement-record` | dropped (history) | `test/repo/test-runner-guardrails.test.ts` | +| `github-ci.md` `#test-assertions` | no such heading in the old spec either; now github-ci.md#testing-guidance | `test/repo/test-runner-guardrails.test.ts` | +| `github-ci.md` `#tool-prerequisites-on-ci-workers` | derivable from test-runner.md#processes-started-by-tests | `.github/workflows/ci.yml` | +| `preflight.md` `#Contributor branches and remote revisions` | test-runner.md#^remote-revision | `AGENTS.md`, `developer-skills/tvdev-setup/SKILL.md`, `proofs/arch/test-runner/preflight.md`, `proofs/arch/test-runner/test-runner.md`, `specs/arch/desktop/e2e-harness.md`, `specs/arch/desktop/index.md`, `specs/arch/desktop/runtime.md`, `specs/arch/node-versions.md`, `specs/arch/telemetry/sink.md`, `specs/arch/testing-policy.md` | +| `preflight.md` `#^preflight-electron-runtime` | dropped; owned by desktop/e2e-harness.md | `specs/arch/explainer-desktop-app.md` | +| `preflight.md` `#contributor-branches-and-remote-revisions` | test-runner.md#^remote-revision | `specs/spec-workflow.md` | +| `preflight.md` `#testing-guidance` | blaxel-testshards.md#testing-guidance (live evidence) | `proofs/arch/test-runner/test-runner.md`, `proofs/product/desktop-app.md`, `specs/product/desktop-app.md` | +| `reporting.md` `#Normalized surface result` | reporting.md#the-compact-summary (pinned RunSummary) | `proofs/arch/test-runner/reporting.md` | +| `reporting.md` `#^resource-diagnostic-bounds` | reporting.md#resource-diagnostics | `proofs/arch/test-runner/reporting.md` | +| `reporting.md` `#^vitest-attempt-recording` | reporting.md#honest-status | `proofs/arch/test-runner/sharded-execution.md` | +| `sharded-execution.md` `#Blaxel provider report shape` | dropped (internal types) | `proofs/arch/test-runner/sharded-execution.md` | +| `sharded-execution.md` `#Explicit file execution` | sharded-execution.md#plans | `proofs/arch/test-runner/sharded-execution.md` | +| `sharded-execution.md` `#Manual` | sharded-execution.md#duration-baseline | `scripts/test/timing-baseline.mjs` | +| `sharded-execution.md` `#Provider` | sharded-execution.md#plans and #duration-baseline | `scripts/test/shard-plan.mjs` | +| `sharded-execution.md` `#Shard-count` | sharded-execution.md#duration-baseline | `scripts/test/cli.mjs` | +| `sharded-execution.md` `#Shared shard worker` | dropped in part; see account items 47-54 | `proofs/arch/test-runner/sharded-execution.md` | +| `sharded-execution.md` `#testing` | sharded-execution.md#testing-guidance | `proofs/arch/test-runner/test-runner.md` | +| `test-registry.md` (file) | test-runner.md#the-test-registry | `proofs/arch/test-runner/test-registry.md`, `specs/arch/desktop/e2e-harness.md`, `specs/arch/explainer-desktop-app.md`, `specs/arch/licensing.md`, `test/repo/test-runner-guardrails.test.ts` | +| `test-registry.md` `#Discovery and validation` | test-runner.md#the-test-registry | `proofs/arch/test-runner/test-registry.md` | +| `test-registry.md` `#Surface services` | test-runner.md#network-addresses-in-tests | `specs/arch/desktop/e2e-harness.md` | +| `test-registry.md` `#^blaxel-harness-surfaces` | blaxel-testshards.md#testing-guidance | `proofs/arch/test-runner/blaxel-testshards.md`, `proofs/arch/test-runner/test-registry.md` | +| `test-registry.md` `#source-of-truth` | test-runner.md#the-test-registry | `proofs/arch/test-runner/test-registry.md` | +| `test-runner.md` `#Surface process lifecycle` | test-runner.md#processes-started-by-tests | `proofs/arch/test-runner/sharded-execution.md`, `proofs/arch/test-runner/test-runner.md` | +| `test-runner.md` `#Test infrastructure addresses` | test-runner.md#network-addresses-in-tests | `proofs/product/cli.md`, `proofs/product/onboarding/onboarding-channels.md`, `proofs/product/resources/resources.md` | +| `test-runner.md` `#Verify orchestration` | test-runner.md#verify | `proofs/arch/test-runner/test-runner.md`, `specs/arch/testing-policy.md` | +| `test-runner.md` `#^blaxel-harness-placement` | blaxel-testshards.md#^blaxel-harness-placement | `proofs/arch/test-runner/blaxel-testshards.md`, `proofs/arch/test-runner/test-registry.md`, `proofs/arch/test-runner/test-runner.md` | +| `test-runner.md` `#exit-codes` | test-runner.md#commands | `proofs/arch/test-runner/test-runner.md` | +| `test-runner.md` `#local-execution` | derivable; see account item 65 | `proofs/arch/test-runner/test-runner.md` | +| `test-runner.md` `#local-guardrails` | test-runner.md#shared-development-hosts | `packages/artifact/README.md`, `proofs/arch/test-runner/test-runner.md` | +| `test-runner.md` `#selectors` | test-runner.md#commands | `proofs/arch/test-runner/test-runner.md` | +| `test-runner.md` `#verify-orchestration` | test-runner.md#verify | `proofs/arch/test-runner/test-runner.md` | diff --git a/frames/vitest.config.ts b/frames/vitest.config.ts index 6a6fc9e5..e3db306a 100644 --- a/frames/vitest.config.ts +++ b/frames/vitest.config.ts @@ -1,3 +1,4 @@ +import "../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; import { liquidTemplates } from "../config/liquid/plugin.ts"; diff --git a/packages/artifact/README.md b/packages/artifact/README.md index c46e0aae..9b5042b5 100644 --- a/packages/artifact/README.md +++ b/packages/artifact/README.md @@ -24,11 +24,13 @@ See [content messages](src/artifact-view-protocol.ts) and the [browser view clie ## Tests - `packages/artifact/test/unit/` — vitest. Type guards and disposal / error-path behavior. -- `packages/artifact/test/e2e/` — Playwright. Two Vite harness apps on ports `5174` (host) and `5175` (view) prove the classes work across real origins. +- `packages/artifact/test/e2e/` — Playwright. Two Vite harness apps, each bound to an available port by the runner, prove the classes work across real origins. Run through the root canonical runner: ```bash -npm run test:run -- run --provider local --package @telepath-computer/television-artifact -npm run test:run -- run --provider local --surface e2e:artifact +npm test -- local --file packages/artifact/test/unit/artifact-view-protocol.test.ts +npm test -- local --file packages/artifact/test/e2e/scroll-bridge.spec.ts ``` + +For broader validation on a host with `~/.tvdev-use-blaxel`, commit and push first, then run `npm test -- blaxel --surface e2e:artifact` or `npm test -- blaxel --package @telepath-computer/television-artifact`. Keep default retries for validation. Unmarked hosts can use these broader selectors with `local` without Blaxel access. Local runs share the [canonical runner's mutex](../../specs/arch/test-runner/test-runner.md#^local-test-mutex). diff --git a/packages/artifact/test/e2e/playwright.config.ts b/packages/artifact/test/e2e/playwright.config.ts index 3ba14672..8b438b90 100644 --- a/packages/artifact/test/e2e/playwright.config.ts +++ b/packages/artifact/test/e2e/playwright.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { defineConfig } from "@playwright/test"; diff --git a/packages/artifact/vitest.config.ts b/packages/artifact/vitest.config.ts index 137868cd..56d723c0 100644 --- a/packages/artifact/vitest.config.ts +++ b/packages/artifact/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/packages/canonical/vitest.config.ts b/packages/canonical/vitest.config.ts index 66a42618..da6a7f19 100644 --- a/packages/canonical/vitest.config.ts +++ b/packages/canonical/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 68650bc0..424bd4ba 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/packages/desktop/README.md b/packages/desktop/README.md index 308ff279..f219353d 100644 --- a/packages/desktop/README.md +++ b/packages/desktop/README.md @@ -104,7 +104,7 @@ Record each run in the pull request, with the readings above. ### Local verification -The pull request that moves the app to ToDesktop also runs full local verification on a Mac, in a checkout prepared with [tvdev-setup](../../developer-skills/tvdev-setup/SKILL.md). If `~/.tvdev-use-blaxel` exists, add `--allow-extreme-inefficiency`: +The pull request that moves the app to ToDesktop also runs full local verification on a Mac, in a checkout prepared with [tvdev-setup](../../developer-skills/tvdev-setup/SKILL.md). If `~/.tvdev-use-blaxel` exists, add `--against-test-guidance-broad-local-run`: ```sh npm run verify -- local diff --git a/packages/desktop/playwright.config.ts b/packages/desktop/playwright.config.ts index fb654572..ee7a8aba 100644 --- a/packages/desktop/playwright.config.ts +++ b/packages/desktop/playwright.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "@playwright/test"; requiredURL("TV_DESKTOP_E2E_URL"); diff --git a/packages/desktop/vitest.config.ts b/packages/desktop/vitest.config.ts index b9de8f2a..8ab088fc 100644 --- a/packages/desktop/vitest.config.ts +++ b/packages/desktop/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/packages/server/playwright.config.ts b/packages/server/playwright.config.ts index c87039bb..b5ac10c8 100644 --- a/packages/server/playwright.config.ts +++ b/packages/server/playwright.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig, devices } from "@playwright/test"; // Real-browser surface for the server's read-only Markdown styling. Unlike the diff --git a/packages/server/test/telemetry-posthog.integration.test.ts b/packages/server/test/telemetry-posthog.integration.test.ts index fe03a41c..e1d89299 100644 --- a/packages/server/test/telemetry-posthog.integration.test.ts +++ b/packages/server/test/telemetry-posthog.integration.test.ts @@ -3,7 +3,7 @@ * * The persisted-daemon cases register and remove a uniquely named real user * service through launchd or systemd. Run them only through - * `npm test -- local --suite telemetry-posthog-roundtrip`, with the PostHog + * `npm test -- local --file packages/server/test/telemetry-posthog.integration.test.ts`, with the PostHog * test read key available, on a host where real service installation is safe. * The asserted teardown path is `tv stop`; direct Daemon.uninstall() remains * only as failure cleanup so an interrupted assertion cannot leak a service. @@ -110,7 +110,7 @@ function stripEnvQuotes(value: string): string { function requirePostHogTestReadKey(): string { const value = process.env.TV_POSTHOG_TEST_READ_KEY?.trim(); if (value?.startsWith("phx_")) return value; - throw new Error("TV_POSTHOG_TEST_READ_KEY must be set to the PostHog test project read key. Run via `npm test -- local --suite telemetry-posthog-roundtrip` so the posthog-test-key preflight checks this before Vitest starts."); + throw new Error("TV_POSTHOG_TEST_READ_KEY must be set to the PostHog test project read key. Run via `npm test -- local --file packages/server/test/telemetry-posthog.integration.test.ts` so the posthog-test-key preflight checks this before Vitest starts."); } function builtEvent( diff --git a/packages/server/vitest.config.ts b/packages/server/vitest.config.ts index 581c602b..a919a6ce 100644 --- a/packages/server/vitest.config.ts +++ b/packages/server/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/packages/server/vitest.telemetry.config.ts b/packages/server/vitest.telemetry.config.ts index 87fce79b..24cd2d01 100644 --- a/packages/server/vitest.telemetry.config.ts +++ b/packages/server/vitest.telemetry.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/packages/shared/vitest.config.ts b/packages/shared/vitest.config.ts index 66a42618..da6a7f19 100644 --- a/packages/shared/vitest.config.ts +++ b/packages/shared/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/packages/skills/skills/tv-calendar/playwright.config.ts b/packages/skills/skills/tv-calendar/playwright.config.ts index 0d3f7206..5b1dd0e6 100644 --- a/packages/skills/skills/tv-calendar/playwright.config.ts +++ b/packages/skills/skills/tv-calendar/playwright.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { defineConfig } from "@playwright/test"; diff --git a/packages/skills/skills/tv-tasks/playwright.config.ts b/packages/skills/skills/tv-tasks/playwright.config.ts index e4e8b6da..8adb06f7 100644 --- a/packages/skills/skills/tv-tasks/playwright.config.ts +++ b/packages/skills/skills/tv-tasks/playwright.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { defineConfig } from "@playwright/test"; diff --git a/packages/skills/vitest.config.ts b/packages/skills/vitest.config.ts index 698fedcd..6c18d91c 100644 --- a/packages/skills/vitest.config.ts +++ b/packages/skills/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import path from "node:path"; import { fileURLToPath } from "node:url"; diff --git a/packages/utils/vitest.config.ts b/packages/utils/vitest.config.ts index b7008879..8d0404ee 100644 --- a/packages/utils/vitest.config.ts +++ b/packages/utils/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/packages/view-markdown/playwright.config.ts b/packages/view-markdown/playwright.config.ts index 33aa3f9e..e9b6d159 100644 --- a/packages/view-markdown/playwright.config.ts +++ b/packages/view-markdown/playwright.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig, devices } from "@playwright/test"; const BASE_URL = requiredURL("TV_VIEW_MARKDOWN_E2E_URL"); diff --git a/packages/view-markdown/vitest.config.ts b/packages/view-markdown/vitest.config.ts index 68650bc0..424bd4ba 100644 --- a/packages/view-markdown/vitest.config.ts +++ b/packages/view-markdown/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/packages/web/playwright.config.ts b/packages/web/playwright.config.ts index 705df73b..c528e1ee 100644 --- a/packages/web/playwright.config.ts +++ b/packages/web/playwright.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import fs from "node:fs"; import { defineConfig, firefox } from "@playwright/test"; @@ -33,7 +34,6 @@ export default defineConfig({ // single worker there, running files serially. Pin to 2 so each sharded job // uses both cores. Local runs keep the default (undefined). // TV_PW_WORKERS allows controlled CI worker-count measurements. - // (specs/arch/test-runner/github-ci.md#ci-worker-counts). workers: process.env.CI ? Number(process.env.TV_PW_WORKERS ?? "2") : undefined, retries: 0, reporter: "list", diff --git a/packages/web/playwright.real-stack.config.ts b/packages/web/playwright.real-stack.config.ts index cb3b916f..e44a19cb 100644 --- a/packages/web/playwright.real-stack.config.ts +++ b/packages/web/playwright.real-stack.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import fs from "node:fs"; import { defineConfig, firefox } from "@playwright/test"; @@ -15,7 +16,7 @@ if (!firefoxAvailable) { // The e2e:browser-app-real-stack surface serializes browser acceptances that // boot product servers; those servers starve when the files share a two-worker // invocation on 2-vCPU CI runners -// (specs/arch/test-runner/github-ci.md#ci-worker-counts). +// (specs/arch/test-runner/github-ci.md#browser-worker-counts). export default defineConfig({ testDir: "./test/e2e", testMatch: [ diff --git a/packages/web/test/e2e/browser-demo-mode.test.ts b/packages/web/test/e2e/browser-demo-mode.test.ts index 47e0e344..520e6006 100644 --- a/packages/web/test/e2e/browser-demo-mode.test.ts +++ b/packages/web/test/e2e/browser-demo-mode.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { test, expect } from "../../../../test/helpers/playwright.ts"; import { launchProductServer, type ProductServer } from "../../../../test/helpers/product-server.ts"; +import { recordApplicationRequests } from "../../../../test/helpers/application-requests.ts"; import { APPLICATION_SHELL_STATES, waitForApplicationRender } from "../../../../test/helpers/application-readiness.ts"; import { configureTestMotion, pickChannelByName } from "./helpers.ts"; @@ -72,11 +73,10 @@ async function createURLArtifact( return ((await response.json()) as { artifact: { id: string } }).artifact.id; } -async function openApp(page: Page, server: ProductServer, baseURL: string | undefined): Promise { - if (!baseURL) throw new Error("Expected Playwright baseURL"); - const appBaseURL = await server.appURL(baseURL); - await page.goto(`${appBaseURL}/packages/web/src/index.html`); +async function openApp(page: Page, server: ProductServer, requests: ReturnType): Promise { + await page.goto(server.serverURL); await waitForApp(page); + requests.expectBuilt(); } async function waitForApp(page: Page): Promise { @@ -106,8 +106,10 @@ test.describe("browser demo mode", () => { let home: string; let site: Awaited>; let server: ProductServer | null = null; + let requests: ReturnType; - test.beforeEach(async () => { + test.beforeEach(async ({ page }) => { + requests = recordApplicationRequests([page]); home = mkdtempSync(path.join(os.tmpdir(), "television-demo-home-")); site = await startExternalSite(); server = await launchProductServer({ @@ -117,6 +119,7 @@ test.describe("browser demo mode", () => { }); test.afterEach(async () => { + await requests.attach(test.info()); await server?.dispose(); server = null; await site.close(); @@ -124,7 +127,7 @@ test.describe("browser demo mode", () => { }); // spec: proofs/product/artifacts.md#^af-ac-demo-mode - test("the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)", async ({ page, baseURL }) => { + test("the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)", async ({ page }) => { if (!server) throw new Error("Product server was not started"); const [channel] = await channels(server); if (!channel) throw new Error("Expected a default channel"); @@ -132,7 +135,7 @@ test.describe("browser demo mode", () => { await createURLArtifact(server, pageURL, channel.id); const returnButton = page.getByRole("button", { name: RETURN_LABEL }); - await openApp(page, server, baseURL); + await openApp(page, server, requests); await expect(artifactFrame(page)).toHaveAttribute("src", /\/views\/url-unsupported\//); await expect(returnButton).toHaveCount(0); @@ -152,7 +155,7 @@ test.describe("browser demo mode", () => { }); // spec: proofs/product/artifacts.md#^af-ac-demo-return - test("the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)", async ({ page, baseURL }) => { + test("the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)", async ({ page }) => { if (!server) throw new Error("Product server was not started"); writeFileSync(path.join(home, MARKER), ""); const [channel] = await channels(server); @@ -160,7 +163,7 @@ test.describe("browser demo mode", () => { await createChannel(server, "Elsewhere"); await createURLArtifact(server, `${site.origin}/one`, channel.id); - await openApp(page, server, baseURL); + await openApp(page, server, requests); await expect(frameHeading(page)).toHaveText("Fixture page one"); await followNext(page); @@ -181,7 +184,7 @@ test.describe("browser demo mode", () => { }); // spec: proofs/arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-seam - test("bridge-shaped messages from a demo-mode page change nothing (^ab-ac-demo-inert-seam)", async ({ page, baseURL }) => { + test("bridge-shaped messages from a demo-mode page change nothing (^ab-ac-demo-inert-seam)", async ({ page }) => { if (!server) throw new Error("Product server was not started"); writeFileSync(path.join(home, MARKER), ""); const [channel] = await channels(server); @@ -191,7 +194,7 @@ test.describe("browser demo mode", () => { await createURLArtifact(server, pageURL, channel.id); await createURLArtifact(server, `${site.origin}/two`, channel.id, "Second page"); - await openApp(page, server, baseURL); + await openApp(page, server, requests); const selected = page.getByRole("option", { name: channel.name, exact: true }); await expect(selected).toHaveAttribute("aria-selected", "true"); const tabs = page.locator(".top-bar .tab"); diff --git a/packages/web/test/e2e/gate-boot-barrier.spec.ts b/packages/web/test/e2e/gate-boot-barrier.spec.ts index 6759b9b8..82dacf48 100644 --- a/packages/web/test/e2e/gate-boot-barrier.spec.ts +++ b/packages/web/test/e2e/gate-boot-barrier.spec.ts @@ -1,4 +1,5 @@ -import { expect, test, type Page } from "@playwright/test"; +import { type Page } from "@playwright/test"; +import { expect, test } from "../../../../test/helpers/playwright.ts"; import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/packages/web/test/e2e/theme-product-acceptance.test.ts b/packages/web/test/e2e/theme-product-acceptance.test.ts index f3180668..071fffe8 100644 --- a/packages/web/test/e2e/theme-product-acceptance.test.ts +++ b/packages/web/test/e2e/theme-product-acceptance.test.ts @@ -13,6 +13,8 @@ import { TelevisionClient } from "@telepath-computer/television-shared"; import { test, expect } from "../../../../test/helpers/playwright.ts"; import { launchProductServer, type ProductServer } from "../../../../test/helpers/product-server.ts"; import { seedThemePackage } from "../../../../test/helpers/theme-package.ts"; +import { recordApplicationRequests } from "../../../../test/helpers/application-requests.ts"; +import { startStableFrontProxy } from "../../../../test/helpers/stable-front-proxy.ts"; import { waitForApplicationShell } from "./helpers.ts"; import { APPLICATION_SHELL_STATES, @@ -611,12 +613,12 @@ test("Firefox refreshes a ready Markdown editor from a fixed-dark theme to light // proofs/arch/themes/delivery.md#^theme-delivery-t-active-package-refresh test("theme settings persist rendered app and artifact outcomes through a built-server restart", async ({ browser, - baseURL, }) => { - if (!baseURL) throw new Error("Expected Playwright baseURL"); const context = await browser.newContext({ colorScheme: null }); const page = await context.newPage(); + const requests = recordApplicationRequests([page]); const product = await launchProductServer(); + const front = await startStableFrontProxy(product.serverURL); try { const themeAPath = seedThemePackage( product.home, @@ -691,9 +693,10 @@ test("theme settings persist rendered app and artifact outcomes through a built- "https://example.com/appearance", ); - const appURL = await product.appURL(baseURL); - await page.goto(`${appURL}/packages/web/src/index.html`); + const appURL = front.url; + await page.goto(appURL); await waitForApplicationShell(page); + requests.expectBuilt(); await openSettings(page); await page.getByRole("button", { name: "Refresh themes" }).click(); @@ -843,6 +846,7 @@ test("theme settings persist rendered app and artifact outcomes through a built- )).toHaveCount(0); await product.restart(); + front.setTarget(product.serverURL); await waitForApplicationShell(page); await expectAppearance("dark", FIXED_DARK); const persistedDisplay = await fetch(new URL("/display", product.serverURL)); @@ -868,6 +872,8 @@ test("theme settings persist rendered app and artifact outcomes through a built- expect((await expectSandboxFrame(page, "overlay")).instance) .not.toBe(beforeAdaptiveOverlay.instance); } finally { + await requests.attach(test.info()); + await front.dispose(); await product.dispose(); await context.close(); } @@ -880,7 +886,6 @@ test("theme settings persist rendered app and artifact outcomes through a built- // proofs/arch/themes/delivery.md#^theme-delivery-t-frame-appearance-browser test("executable themes run only in the Chromium application lifecycle and isolate failures", async ({ page, - baseURL, browserName, }) => { test.skip(browserName !== "chromium", "Chromium owns executable-theme browser acceptance"); @@ -890,11 +895,12 @@ test("executable themes run only in the Chromium application lifecycle and isola const reconnectBackoffBudget = 30_000; const themeResetNavigationBudget = 30_000; test.setTimeout(test.info().timeout + reconnectBackoffBudget + 2 * themeResetNavigationBudget); - if (!baseURL) throw new Error("Expected Playwright baseURL"); const product = await launchProductServer(); + const front = await startStableFrontProxy(product.serverURL); const secondPage = await page.context().newPage(); const applicationPages = [page, secondPage] as const; + const requests = recordApplicationRequests(applicationPages); const selectReloadingTheme = async (value: string): Promise => { // A consented script is cleared by replacing both application documents. // Observe that navigation before applying ordinary post-render assertions. @@ -1062,13 +1068,14 @@ document.querySelector("#artifact-probe").addEventListener("click", () => { "Executable Theme Artifact", artifactPath, ); - const appURL = await product.appURL(baseURL); + const appURL = front.url; await Promise.all(applicationPages.map(async (applicationPage) => { await applicationPage.goto( - `${appURL}/packages/web/src/index.html`, + appURL, ); await waitForApplicationShell(applicationPage); })); + requests.expectBuilt(); await openSettings(page); await page.getByRole("button", { name: "Refresh themes" }).click(); const themeSelect = page.locator('tv-select[name="theme"]'); @@ -1783,6 +1790,7 @@ document.querySelector("#artifact-probe").addEventListener("click", () => { await expectExecutableRuns(applicationPages, 3); await product.restart(); + front.setTarget(product.serverURL); // Allow one capped retry interval plus the usual 5s shell-ready budget. await Promise.all(applicationPages.map((applicationPage) => waitForApplicationRender( @@ -1845,6 +1853,8 @@ document.querySelector("#artifact-probe").addEventListener("click", () => { expect(scriptRequests.length).toBeGreaterThan(0); expect(scriptRequests.every((owner) => owner === "application")).toBe(true); } finally { + await requests.attach(test.info()); + await front.dispose(); await product.dispose(); await secondPage.close(); } diff --git a/packages/web/test/e2e/update-journey.spec.ts b/packages/web/test/e2e/update-journey.spec.ts index 42db665c..5ad0664c 100644 --- a/packages/web/test/e2e/update-journey.spec.ts +++ b/packages/web/test/e2e/update-journey.spec.ts @@ -1,4 +1,5 @@ -import { expect, test, type Page } from "@playwright/test"; +import { type Page } from "@playwright/test"; +import { expect, test } from "../../../../test/helpers/playwright.ts"; import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/packages/web/test/e2e/update-notification.01.spec.ts b/packages/web/test/e2e/update-notification.01.spec.ts index 2efacd40..f7841cc8 100644 --- a/packages/web/test/e2e/update-notification.01.spec.ts +++ b/packages/web/test/e2e/update-notification.01.spec.ts @@ -1,4 +1,5 @@ -import { expect, test, type Locator, type Page } from "@playwright/test"; +import { type Locator, type Page } from "@playwright/test"; +import { expect, test } from "../../../../test/helpers/playwright.ts"; import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/packages/web/test/e2e/update-notification.02.spec.ts b/packages/web/test/e2e/update-notification.02.spec.ts index a5cbbc2c..32f84770 100644 --- a/packages/web/test/e2e/update-notification.02.spec.ts +++ b/packages/web/test/e2e/update-notification.02.spec.ts @@ -1,4 +1,5 @@ -import { expect, test, type Page } from "@playwright/test"; +import { type Page } from "@playwright/test"; +import { expect, test } from "../../../../test/helpers/playwright.ts"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/packages/web/test/e2e/update-reload.spec.ts b/packages/web/test/e2e/update-reload.spec.ts index 8df87161..63589ffc 100644 --- a/packages/web/test/e2e/update-reload.spec.ts +++ b/packages/web/test/e2e/update-reload.spec.ts @@ -1,4 +1,5 @@ -import { expect, test, type Page } from "@playwright/test"; +import { type Page } from "@playwright/test"; +import { expect, test } from "../../../../test/helpers/playwright.ts"; import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; diff --git a/packages/web/test/foundation-copy.test.ts b/packages/web/test/foundation-copy.test.ts index 0fb0a976..ca180f14 100644 --- a/packages/web/test/foundation-copy.test.ts +++ b/packages/web/test/foundation-copy.test.ts @@ -1,6 +1,7 @@ import { globSync, readFileSync, statSync } from "node:fs"; import path from "node:path"; -import { describe, expect, test } from "vitest"; +import { describe, expect } from "vitest"; +import { test } from "../../../test/helpers/vitest.ts"; import { enumerateTrackedPaths } from "../../../scripts/test/file-inventory.mjs"; import { buildVersionedWebBundle } from "../../../test/helpers/versioned-web-bundle.ts"; import { diff --git a/packages/web/vitest.config.ts b/packages/web/vitest.config.ts index e1364a47..0fca70fa 100644 --- a/packages/web/vitest.config.ts +++ b/packages/web/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; import { liquidTemplates } from "../../config/liquid/plugin.ts"; import { cssModuleScripts } from "../../config/css-module-scripts.ts"; diff --git a/proofs/arch/artifact-frame/artifact-bridge.md b/proofs/arch/artifact-frame/artifact-bridge.md index 2f1af862..ec5e3e85 100644 --- a/proofs/arch/artifact-frame/artifact-bridge.md +++ b/proofs/arch/artifact-frame/artifact-bridge.md @@ -79,7 +79,7 @@ Readiness assertions observe lifecycle state, never shielding, pointer suspensio - **Contract** (browser-bridge contract harness; the parent is mocked; authored platform identities cover macOS and Windows/Linux): the bridge captures option+arrow on macOS and ctrl+arrow on Windows/Linux for all four arrows, including in text-editing targets and with `repeat`; calls `preventDefault`; and posts `navigation-key`. Shift, every other extra modifier, composition, and non-arrow keys do not forward, while a descendant handler's `stopPropagation()` cannot starve the window capture listener — *(covered by tests in `packages/artifact/test/unit/artifact-bridge.test.ts`: "matches the platform navigation chord for every arrow", "rejects composition, non-arrows, missing chords, and every extra modifier", "captures matching chords in editing targets without stopping propagation", "leaves unmatched keydown behavior untouched", and "produces runnable source that behaves like installBridge when eval'd in a fresh window")*. ^ab-ac-navigation-key - **Contract** (browser-bridge and preload contract harnesses; parent `postMessage` and Electron IPC peers are mocked, forfeiting real handoff to the acceptance assertions below): each producer captures trusted pointer move, down, up, cancel, and click events and emits one `artifact-pointer` carrying the raw guest-viewport coordinates and button fields without preventing default, stopping propagation, or taking pointer capture; untrusted events are ignored — *(covered by `packages/artifact/test/unit/artifact-bridge.test.ts` “reports trusted artifact pointer input without interfering” and `packages/desktop/test/webview-bridge-preload.test.ts` “reports trusted webview pointer input without interfering”)*. ^ab-ac-pointer-producers - **Contract** (host contract harness plus production theme frames in a browser DOM; source windows, frame geometry, and pointer notifications are authored inputs, while browsing-context delivery is forfeited to the acceptance assertions): the current iframe or subscribed webview's source-valid `artifact-pointer` is accepted without readiness, mapped from guest coordinates through the frame's rendered rectangle and layout dimensions, normalized by the theme owner, and delivered once to each current theme frame. A zero layout dimension drops it, and the same payload from another window or replaced webview is ignored — *(covered by `packages/web/test/artifact-view.test.ts` “relays only current artifact pointer input to theme frames in application coordinates”)*. ^ab-ac-pointer-host -- **Acceptance** (outer boundary: physical pointer input in a real Chromium artifact iframe against a running production server and real theme package; the artifact fixture records its trusted event coordinates without altering input or delivery, and no mocks or production test hooks are used): move, down, up, and click cross the injected bridge and trusted channel to both theme frames in application-viewport coordinates, mapped from the coordinates actually received by the artifact (including Chromium's integer click coordinates), while the artifact's own click action remains effective — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^ab-ac-pointer-browser +- **Acceptance** (outer boundary: physical pointer input in a real Chromium artifact iframe within the built application, served through the real stable-front proxy as harness transport, against a running production server and real theme package; the artifact fixture records its trusted event coordinates without altering input or delivery, and no mocks or production test hooks are used): move, down, up, and click cross the injected bridge and trusted channel to both theme frames in application-viewport coordinates, mapped from the coordinates actually received by the artifact (including Chromium's integer click coordinates), while the artifact's own click action remains effective — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^ab-ac-pointer-browser - **Acceptance** (outer boundary: native mouse input in an ordinary HTML artifact webview in the real Electron app against a running production server and real theme package; no mocks or test hooks): the preload and subscribed IPC path deliver move, down, up, and click to both theme frames in application-viewport coordinates while the artifact's own click action remains effective — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “executes an executable theme in Electron and resets on remote consent and selection changes”)*. ^ab-ac-pointer-electron - **Seam** (browser bridge → host navigation handler, crossed by real `postMessage` in the browser seam harness; an authored document and tab/channel state are fixtures, with no mock on the handoff): each of the four forwarded key values reaches the app's navigation handling while focus is inside the document — *(covered by test: `packages/web/test/e2e/artifact-view.test.ts` "forwards every browser navigation arrow from a focused artifact document")*; product outcomes compose with [product/keyboard-navigation.md](../../../specs/product/keyboard-navigation.md). ^ab-ac-navigation-key-seam - **Contract** (desktop shell side of the native-key delivery seam; authored native key events and platform identities cover macOS and Windows/Linux; the renderer navigation receiver is mocked, forfeiting the real handoff to [#^ab-ac-electron-key-seam](#^ab-ac-electron-key-seam)): the matcher implements the same modifier, arrow, repeat, and composition matrix as [#^ab-ac-navigation-key](#^ab-ac-navigation-key), consumes a match, and delivers its arrow value once to app navigation — *(covered by tests: all cases under `packages/desktop/test/native-navigation-key.test.ts` "Electron native navigation chord", plus `packages/desktop/test/main.test.ts` "consumes a matching webview chord and sends its arrow once to the embedding renderer" and `packages/web/test/renderer.test.ts` "delivers Electron native arrows to the existing application navigation edge")*. ^ab-ac-electron-key-contract @@ -116,6 +116,6 @@ The recorder's two branches hand their reports to the host through one function, Demo-mode frames are not bridge participants ([^ab-demo-frame-inert](../../../specs/arch/artifact-frame/artifact-bridge.md#^ab-demo-frame-inert)): -- **Seam** (Chromium against a really-running server under the server-home hook and cross-origin fixture server declared in [product/artifacts.md](../../product/artifacts.md#test-hooks); two external page artifacts on one channel and a second, newer channel are fixtures created through the production API; code evaluated inside the fixture page's frame posts bridge-shaped `bridge-ready`, a horizontal and a vertical `navigation-key` each chosen to have somewhere to move, `navigation-request`, and `artifact-pointer` messages to its parent through real `postMessage`; the default motion override is the only mock): after the page posts them, the focused channel and selected tab are unchanged, the fixture's first page is still framed at its own URL, and Back and Forward remain absent. Theme pointer delivery is not observed here; [#^ab-ac-demo-inert-host](#^ab-ac-demo-inert-host) carries it — *(covered by test: `packages/web/test/e2e/browser-demo-mode.test.ts` "bridge-shaped messages from a demo-mode page change nothing (^ab-ac-demo-inert-seam)")*. ^ab-ac-demo-inert-seam +- **Seam** (the built application in Chromium against a really-running server under the server-home hook and cross-origin fixture server declared in [product/artifacts.md](../../product/artifacts.md#test-hooks); two external page artifacts on one channel and a second, newer channel are fixtures created through the production API; code evaluated inside the fixture page's frame posts bridge-shaped `bridge-ready`, a horizontal and a vertical `navigation-key` each chosen to have somewhere to move, `navigation-request`, and `artifact-pointer` messages to its parent through real `postMessage`; the default motion override is the only mock): after the page posts them, the focused channel and selected tab are unchanged, the fixture's first page is still framed at its own URL, and Back and Forward remain absent. Theme pointer delivery is not observed here; [#^ab-ac-demo-inert-host](#^ab-ac-demo-inert-host) carries it — *(policy-grade test: `packages/web/test/e2e/browser-demo-mode.test.ts` "bridge-shaped messages from a demo-mode page change nothing (^ab-ac-demo-inert-seam)")*. ^ab-ac-demo-inert-seam - **Contract** (host contract harness; an authored external page artifact posed under browser demo mode, with synthetic `MessageEvent`s whose source is the frame's own window; the transport mock forfeits the real crossing to [#^ab-ac-demo-inert-seam](#^ab-ac-demo-inert-seam)): every message type in the [message protocol](../../../specs/arch/artifact-frame/artifact-bridge.md#message-protocol) produces no navigation-handler, pointer-forwarder, navigation-state, reload, or status-reply call and leaves the frame's trusted identity null — *(covered by test: `packages/web/test/artifact-view.test.ts` "browser demo mode artifact frame > acts on no bridge message from a demo-mode frame (^ab-ac-demo-inert-host)")*. ^ab-ac-demo-inert-host diff --git a/proofs/arch/desktop/e2e-harness.md b/proofs/arch/desktop/e2e-harness.md index 4d2dcd45..ee3d7d5b 100644 --- a/proofs/arch/desktop/e2e-harness.md +++ b/proofs/arch/desktop/e2e-harness.md @@ -6,7 +6,7 @@ Proves [specs/arch/desktop/e2e-harness.md](../../../specs/arch/desktop/e2e-harne ## Coverage model -Coverage declarations are carried inside the migrated assertion blocks below. +Coverage declarations are carried inside the migrated assertion blocks below. The Operations section’s local-file, full-surface, and marked-host routes are subject to the [runner’s admission and diagnostic assertions](../test-runner/test-runner.md#^t-runner-file-admission). Command guidance is reviewed against that owner; no desktop-specific mutex or admission mechanism is introduced. A Linux Blaxel run exercises Linux Electron behavior and cannot establish macOS default application-data identity, whose existing real-Electron proof is cited below. ## Assertions diff --git a/proofs/arch/desktop/runtime.md b/proofs/arch/desktop/runtime.md index ec28bb10..4af334a2 100644 --- a/proofs/arch/desktop/runtime.md +++ b/proofs/arch/desktop/runtime.md @@ -8,7 +8,7 @@ Proves [specs/arch/desktop/runtime.md](../../../specs/arch/desktop/runtime.md). Repository checks read the root and desktop workspace manifests, the lockfile and the installed `electron` package, and prove one exact Electron version with no extractor chain. The ToDesktop build's generated manifest carries the same version, which the [upload-directory contract](./distribution.md#^desktop-dist-t-upload) proves. -Runtime validity is proven at the contract tier. The runtime inspection that the e2e harness and preflight share classifies authored package and runtime trees, with the platform supplied as an input. Supplying the platform forfeits real host detection, which the [harness's runtime-setup seam](./e2e-harness.md#^desktop-t-e2e-runtime-handoff) crosses on the CI host; that seam also hands Playwright the executable path that validation builds. [Preflight's planner seam](../test-runner/preflight.md#^preflight-t-electron-planner-handoff) shows that classifying the real installed package leaves the runtime tree unchanged, so validation does not start an installation. +Runtime validity is proven at the contract tier. The runtime inspection that the e2e harness and preflight share classifies authored package and runtime trees, with the platform supplied as an input. Supplying the platform forfeits real host detection, which the [harness's runtime-setup seam](./e2e-harness.md#^desktop-t-e2e-runtime-handoff) crosses on the CI host; that seam also hands Playwright the executable path that validation builds. [Preflight's planner seam](../test-runner/test-runner.md#^preflight-t-electron-planner-handoff) shows that classifying the real installed package leaves the runtime tree unchanged, so validation does not start an installation. A real cold installation under the affected Node 26 release proves that Electron's installer leaves a runtime satisfying the validity predicate, which the extractor boundary protects. Global setup's use of the installer is crossed by the harness seam above, and Blaxel's repair path by [Blaxel shards](../test-runner/blaxel-testshards.md). A development run's installation through Electron's resolver, the browser build target, and the major-upgrade procedure order no test; the procedure's real-host step points to the [product's checks](../../product/desktop-app.md). diff --git a/proofs/arch/telemetry/sink.md b/proofs/arch/telemetry/sink.md index eae56547..99d5b0ea 100644 --- a/proofs/arch/telemetry/sink.md +++ b/proofs/arch/telemetry/sink.md @@ -6,7 +6,7 @@ Proves [specs/arch/telemetry/sink.md](../../../specs/arch/telemetry/sink.md). ## Coverage model -Ordinary and isolated release builds cross the real bundler and compiled skills-install action with real copied skill files. Their analytics transport is intercepted before any network access, so production routing is observed without sending to production. The live test-project suite proves the actual delivery and stored privacy outcome. Request contracts cover both collectors, and the other buffer/lifecycle seams below retain their declared boundaries. No test changes PostHog settings or historical data. +Ordinary and isolated release builds cross the real bundler and compiled skills-install action with real copied skill files. Their analytics transport is intercepted before any network access, so production routing is observed without sending to production. The live test-project suite proves the actual delivery and stored privacy outcome. Request contracts cover both collectors, and the other buffer/lifecycle seams below retain their declared boundaries. No test changes PostHog settings or historical data. The documented one-file entrypoint uses the existing live suite; file admission and the refusal’s local secret/setup route are covered by [the runner proof](../test-runner/test-runner.md#^t-runner-local-redirection), not by duplicating the PostHog delivery assertions. ## Test hooks diff --git a/proofs/arch/test-runner/attestation.md b/proofs/arch/test-runner/attestation.md index f9e04b46..ace60eaf 100644 --- a/proofs/arch/test-runner/attestation.md +++ b/proofs/arch/test-runner/attestation.md @@ -28,7 +28,8 @@ Shapes per [testing-policy.md](../../../specs/arch/testing-policy.md). - The skip decision affirms only when all of: policy flag true, version valid and matching, ref exists, object is a commit, name↔tree invariant holds, age within `[0, 7d]`; each single negation yields no-skip, and unreachable-origin and malformed-policy inputs yield no-skip. Missing, non-integer, out-of-domain, and malformed `testpassVersion` values each disable both the skip decision and the writer. - The writer's version bracket: start-read and completion-read must both succeed, be enabled, and agree; a mid-run flag disable or version change makes the run ineligible (no publish), covered as an explicit permutation. - The age computation uses the committer timestamp; unparseable, missing, and **future** timestamps reject (no-skip); both bounds are inclusive and both endpoints are asserted exactly: age exactly 0 skips, age exactly 7d skips, and the surrounding values are covered on both sides (6d23h skips; 7d1h does not, −1s does not). -- The eligibility predicate rejects each clause independently: a local or narrowed developer verify, phase/tree mismatch (`--commit` ≠ HEAD), dirty-at-start, changed-before-end (bracket check), `--ignore-uncommitted`, shard subsets, a skipped gate surface, nonstandard retries, attested-skip runs, disabled policy, wrong version. `--no-publish` bypasses the writer in the canonical CLI's verify orchestration before eligibility is evaluated; no committed test runs that orchestration to completion (boundary gaps below). +- The eligibility predicate rejects each clause independently: a local or narrowed developer verify, a tested commit whose tree differs from the checkout's (`--commit` naming a different tree), dirty-at-start, changed-before-end (bracket check), `--ignore-uncommitted`, shard subsets, a skipped gate surface, nonstandard retries, attested-skip runs, disabled policy, wrong version. `--no-publish` bypasses the writer in the canonical CLI's verify orchestration before eligibility is evaluated; no committed test runs that orchestration to completion (boundary gaps below). +- **Contract — retry options into eligibility facts.** Production verify fact construction classifies `--against-test-guidance-turn-flakes-into-failures-on-broad-runs` as nonstandard both alone and with `--retries 0`; the eligibility predicate consequently rejects publication. Exercise the production conversion from parsed options, not only a fixture with `canonicalRetries: false` already supplied. Default/explicit standard budgets remain canonical, and nonstandard numeric/per-test budgets retain their existing rejection. This proves the [standard-retry condition](../../../specs/arch/test-runner/attestation.md#who-may-write), using authored option/run facts and no remote writer. Canonical retry parsing and execution are covered by [the runner](test-runner.md#^t-runner-retry-admission); complete verify-to-publication remains the live boundary below. Evidence: `test/repo/test-runner-attestation.test.ts`, “parsed retry departures become noncanonical eligibility facts” and “canonical retries require the runner default AND the default flaky budget”. ^attest-retry-options - The production writer accepts only source `blaxel-verify` or `github-ci`. The developer verify handoff supplies `blaxel-verify` only after its provider qualification passes. The CI `write` command refuses outside GitHub Actions workflow `CI` / job `attest-write`, refuses without `TV_TEST_REF_PUBLICATION=required`, and supplies `github-ci` only after those checks. A full passing local-provider fact set calls no publisher; a direct local invocation of the CI writer writes no ref. - The writer's create-only path: pushes with the empty-lease CAS; on rejection validates the existing ref — valid ⇒ idempotent success without pushing, malformed ⇒ loud non-success. - The name↔tree invariant checker flags a ref whose object is not a commit or whose commit tree differs from the ref name. diff --git a/proofs/arch/test-runner/blaxel-testshards.md b/proofs/arch/test-runner/blaxel-testshards.md index 58b93ef0..25708751 100644 --- a/proofs/arch/test-runner/blaxel-testshards.md +++ b/proofs/arch/test-runner/blaxel-testshards.md @@ -6,13 +6,17 @@ Proves [specs/arch/test-runner/blaxel-testshards.md](../../../specs/arch/test-ru ## Coverage model -The host marker’s provider-selection behavior is covered by [the runner proof](test-runner.md#^t-blaxel-host-marker). The corporate credential location is setup information checked by human review, not a new token-source mechanism. The existing token-source assertions remain unchanged; no test reads the vault or asserts its item name. +The host marker’s provider-selection behavior and one-file local limit are covered by [the runner marker proof](test-runner.md#^t-blaxel-host-marker) and [local admission proof](test-runner.md#^t-runner-file-admission). The setup sentence directs callers to those rules rather than defining a second marker mechanism. The runner’s [live remote handoff](test-runner.md#^t-runner-remote-handoff) adds native-entrypoint and retry-policy evidence on the real planned and targeted paths; this proof retains ownership of Blaxel transport, leasing, and setup. The corporate credential location is setup information checked by human review, not a new token-source mechanism. The existing token-source assertions remain unchanged; no test reads the vault or asserts its item name. -Coverage declarations are carried inside the migrated assertion blocks below. +Lease decisions and failure ordering are checked cheaply at the worker/coordinator contracts. Real Linux processes and files cross the guard, owner-registration, and deletion seams. These tests do not claim Blaxel transport coverage. -`test/repo/blaxel-stale-owner-audit.test.ts` checks diagnostic construction and coordinator ordering on every host. Its live diagnostic cases require Linux procfs and run only on Linux; they establish no macOS backend coverage. `test/repo/test-runner-provider.test.ts` exercises shell activation ordering with an authored nvm replacement in a shell isolated from host startup files. That contract check does not install nvm or establish real worker activation. +The live acceptances below enter through the canonical CLI on a dedicated pool: normal retrieval, refusal while another coordinator holds the worker, recovery after coordinator death, and pool creation/deletion. They share a small committed fixture selection and fresh workers rather than repeating a full suite for every failure permutation. Both planned and targeted execution paths are crossed. Resource retention composes with [reporting's diagnostics assertions](reporting.md#Resource diagnostics); helper teardown belongs to [the testing-policy proof](../testing-policy.md#Worker-owned builds). -Repository selection composes several pieces across clean seams. A contract test proves URL conversion. A seam test runs real Git against a temporary checkout's `origin` to prove selection, and a process-boundary test runs the real coordinator to prove that an unusable selection stops it before dispatch. A further seam test executes the worker's generated `checkout` script under real `bash` and Git against two local bare repositories with shared history, proving that one persistent checkout moves between repositories in both directions without re-cloning or deleting ignored dependency files. That test reaches the repositories through local paths, so it forfeits GitHub HTTPS transport and token authentication. The coordinator's use of the selector and the generated script is a source assertion. The live runs that the spec's [Testing](../../../specs/arch/test-runner/blaxel-testshards.md#Testing) directive orders for a repository-selection change carry the GitHub transport, token, and real worker handoff: they run from checkouts of two repositories on the same pool, including workers whose checkout last served the other repository. When the two runs' commits have identical dependency declarations, those workers' logs must show no clone and a `deps` cache hit, which is the live evidence that switching repositories keeps installed dependencies. A focused test runs the worker's generated `deps` script after real checkouts of both repositories, with a stand-in `npm`. It proves that the cache decision follows the dependency declarations and ignores which repository supplied them. +The spec's [testing guidance](../../../specs/arch/test-runner/blaxel-testshards.md#testing-guidance) require real API/transport, fresh provisioning, and observed Node/Chromium temporary paths. Private workers and bounded disposable processes supply those observations; no deliberately exhausted host or shared-pool contamination is needed. The live driver and its native fixtures live in `experiment:blaxel-harness` and `experiment:blaxel-harness-fixtures`, excluded from `all` and ordinary CI by their experiment kind. Mocks below forfeit those crossings to the named live assertions. The live driver shares these observations within one fresh private pool; focused contracts carry uncommon failure permutations without repeating paid-service runs. + +The lease contracts and filesystem seams run only on Linux because their production helpers inspect procfs and mount records. This does not establish macOS worker support. `test/repo/blaxel-stale-owner-audit.test.ts` checks diagnostic construction and coordinator ordering on every host. Its live diagnostic cases require Linux procfs and run only on Linux; they establish no macOS backend coverage. `test/repo/test-runner-provider.test.ts` exercises shell activation ordering with an authored nvm replacement in a shell isolated from host startup files. That contract check does not install nvm or establish real worker activation. + +Repository selection composes several pieces across clean seams. A contract test proves URL conversion. A seam test runs real Git against a temporary checkout's `origin` to prove selection, and a process-boundary test runs the real coordinator to prove that an unusable selection stops it before dispatch. A further seam test executes the worker's generated `checkout` script under real `bash` and Git against two local bare repositories with shared history, proving that one persistent checkout moves between repositories in both directions without re-cloning or deleting ignored dependency files. That test reaches the repositories through local paths, so it forfeits GitHub HTTPS transport and token authentication. The coordinator's use of the selector and the generated script is a source assertion. The live runs that the spec's [testing guidance](../../../specs/arch/test-runner/blaxel-testshards.md#testing-guidance) orders for a repository-selection change carry the GitHub transport, token, and real worker handoff: they run from checkouts of two repositories on the same pool, including workers whose checkout last served the other repository. When the two runs' commits have identical dependency declarations, those workers' logs must show no clone and a `deps` cache hit, which is the live evidence that switching repositories keeps installed dependencies. A focused test runs the worker's generated `deps` script after real checkouts of both repositories, with a stand-in `npm`. It proves that the cache decision follows the dependency declarations and ignores which repository supplied them. Pool discovery also has a contract test that runs the shared discovery helper used by the pool manager and coordinator with the installed Blaxel SDK against a loopback HTTP server returning authored paginated sandbox responses. The SDK builds requests and handles pages through its native-fetch path over loopback HTTP. This proves page traversal, label filtering and deployed-only filtering; production HTTP/2 transport, real authentication and service compatibility are covered by live evidence: a read-only multi-page listing and the required live Blaxel run. @@ -27,34 +31,71 @@ Pool discovery also has a contract test that runs the shared discovery helper us The SDK reads `BL_API_URL`, `BL_API_KEY`, and `BL_WORKSPACE`; the discovery contract test supplies a loopback API and fixture credentials in an isolated working directory and home, with an explicit child environment that preserves the test owner token and disables SDK telemetry and HTTP/2. Production uses the configured Blaxel API and credentials. +The worker lock and temporary-cleanup helpers accept filesystem roots for disposable seam fixtures; production uses the specified worker paths. The lease fixture supplies `auditPids`, its explicit process set, before procfs inspection. Production supplies no process set and enumerates the whole worker. This substitutes enumeration only and forfeits whole-worker coverage to the live acceptance; identity/token reads and signal revalidation remain real. Their decision layer accepts a clock, owner-audit/signal operations, and filesystem probes for contract failure injection; production uses worker time, procfs, signals, and the filesystem. + +A coordinator milestone hook, enabled only by `TV_TEST_RUNNER_SELFTEST`, can wait on a fixture-controlled rendezvous immediately before report download while renewal continues; ordinary runs do not wait. It changes scheduling, not lease state, process results, or report bytes. Live recovery uses the existing `--lock-ttl-seconds` override to shorten the abandonment wait, not a fake clock or altered retry budget. + +`retainAttemptEvidence` accepts the lease’s SDK filesystem and a coordinator download directory. The prepared-but-undispatched contract substitutes reads backed by real fixture files; worker inventory creation, teardown and local writes remain real. Production reads the worker results tree through the installed SDK. + +Pool-management contract tests may supply recording/failing SDK operations; ordinary pool commands use the installed SDK. These substitutions forfeit provider behavior to the live pool acceptance below. + ## Assertions ### Test assertions -- Pool discovery includes matching workers on later API pages and excludes workers with other project, purpose, pool, or architecture labels, as required by the spec's [Pool](../../../specs/arch/test-runner/blaxel-testshards.md#Pool) contract. **Contract:** the consumer is the shared discovery helper and installed SDK, called in a child process; the service is the HTTP mock described above. The helper collects the unfiltered listing before filtering locally. Both the manager mode (all matching workers) and coordinator mode (only matching `DEPLOYED` workers) must traverse all pages, including an earlier page containing no matching workers. A page-fetch failure rejects discovery rather than returning a partial pool. Fixture sandbox records omit a region so discovery needs no data-plane connection. (policy-grade test: `test/repo/blaxel-pool-discovery.test.ts`) ^blaxel-pool-pagination +- Pool discovery includes matching workers on later API pages and excludes workers with other project, purpose, pool, or architecture labels, as required by the spec's [Pool](../../../specs/arch/test-runner/blaxel-testshards.md#pool) contract. **Contract:** the consumer is the shared discovery helper and installed SDK, called in a child process; the service is the HTTP mock described above. The helper collects the unfiltered listing before filtering locally. Both manager and coordinator modes must traverse all pages, including an earlier page containing no matching workers. The coordinator sees only published, matching `DEPLOYED` workers; the manager also retains the pool's unpublished and partially provisioned workers. A page-fetch failure rejects discovery rather than returning a partial pool. Fixture sandbox records omit a region so discovery needs no data-plane connection. (policy-grade test: `test/repo/blaxel-pool-discovery.test.ts`, including unpublished and partially provisioned worker records) ^blaxel-pool-pagination The repository-selection assertions: -- **Contract — URL conversion.** `githubRepositoryHttpsUrl` converts GitHub HTTPS URLs with or without `.git` or a trailing slash, HTTPS URLs carrying embedded credentials, `git@github.com:/` URLs, and `ssh://git@github.com/` URLs to `https://github.com//.git`, as the spec's [Repository](../../../specs/arch/test-runner/blaxel-testshards.md#Repository) section requires. No converted URL contains the embedded credential, and a rejection diagnostic never repeats it, including for userinfo containing `@`, a scp-style `user:secret@host:path` remote, or a malformed scheme separator. It rejects URLs on other hosts, local paths and `file:` URLs, and GitHub URLs that do not name exactly an owner and a repository. (policy-grade test: `test/repo/blaxel-repository.test.ts` “converts GitHub remote URLs to credential-free HTTPS repository URLs”) ^blaxel-repo-url-conversion -- **Seam — selection from the checkout.** Real Git runs in a temporary repository; no part of Git is replaced. With `origin` set to an SSH GitHub URL and no explicit URL, `resolveBlaxelRepositoryUrl` returns that repository's HTTPS URL. A credential-bearing explicit URL is converted and selected even when `origin` is a non-GitHub URL, which is not consulted. A non-GitHub `origin` with no explicit URL fails, as does a repository with no `origin`. This proves the selection rules in the spec's [Repository](../../../specs/arch/test-runner/blaxel-testshards.md#Repository) section. (policy-grade test: `test/repo/blaxel-repository.test.ts` “selects the repository from the checkout's origin unless --repo-url is given”) ^blaxel-repo-origin-selection -- **Acceptance — rejection before dispatch.** The real coordinator runs as a child process with real argv in a temporary Git repository whose `origin` is not on GitHub. The fixtures are that repository and a placeholder token in `BLAXEL_TV_GH_TOKEN`, so the run gets past the token check without a real credential. The coordinator exits `2` with the conversion diagnostic and creates no output directory, so it wrote no request and contacted no worker. This proves the spec's [Repository](../../../specs/arch/test-runner/blaxel-testshards.md#Repository) promise that an unusable selection fails before dispatch. (policy-grade test: `test/repo/blaxel-repository.test.ts` “stops the coordinator before dispatch when origin is not a GitHub repository”) ^blaxel-repo-reject-before-dispatch -- **Seam — one checkout moves between repositories.** The handoff is from the coordinator's generated `checkout` script to Git on a worker, crossed with real `bash` and Git. The fixtures are two local bare repositories that share a base commit, each with commits the other lacks, reached through local paths instead of GitHub HTTPS (forfeit above). Running the script for repository A, then B, then A again on one checkout root clones only the first time. The B run fetches a commit only B holds. The final A run fetches an A-only commit created after the first clone, so neither switch can succeed from objects the checkout already has. A sentinel inside `.git` and an ignored `node_modules` sentinel both survive every switch. After each run, `HEAD` is the requested commit and `origin` points at that run's repository. This proves the checkout behavior in the spec's [Repository](../../../specs/arch/test-runner/blaxel-testshards.md#Repository) section and the [Shard lifecycle](../../../specs/arch/test-runner/blaxel-testshards.md#Shard lifecycle) `checkout` step. (policy-grade test: `test/repo/blaxel-repository.test.ts` “moves one persistent checkout between repositories without re-cloning or deleting ignored dependencies”) ^blaxel-repo-checkout-switch -- **Contract — dependency reuse across repositories.** The generated `checkout` and `deps` scripts run under real `bash`, Git, and `sha256sum` on one checkout root, using the two-repository fixtures above. A stand-in `npm` on `PATH` replaces `npm ci`: it records each call and creates `node_modules`. This forfeits real installation, which the live runs carry. The first `deps` run is a miss and calls `npm`. A second run is a hit. After the checkout moves to the other repository's commit with identical declarations, `deps` is still a hit and calls nothing. After it moves to a commit that changes `package-lock.json`, `deps` is a miss and calls `npm` again. This proves [Dependency-hash caching](../../../specs/arch/test-runner/blaxel-testshards.md#Dependency-hash caching) and the dependency-reuse promise in [Repository](../../../specs/arch/test-runner/blaxel-testshards.md#Repository). (policy-grade test: `test/repo/blaxel-repository.test.ts` “reuses installed dependencies across repositories until their declarations change”) ^blaxel-repo-deps-reuse -- **Contract — coordinator wiring (source).** The coordinator selects its repository through `resolveBlaxelRepositoryUrl` with the `--repo-url` option, carries no hard-coded repository URL, builds the worker's `checkout` step from `blaxelCheckoutScript` with the selected URL and resolved commit, and builds its `deps` step from `blaxelDependencyScript`. The live runs above carry the real handoff. This supports the spec's [Repository](../../../specs/arch/test-runner/blaxel-testshards.md#Repository) section and its [Options](../../../specs/arch/test-runner/blaxel-testshards.md#Options) default for `--repo-url`. (policy-grade test: `test/repo/blaxel-repository.test.ts` “selects the coordinator repository and worker checkout through the shared repository module”) ^blaxel-repo-coordinator-wiring +- **Contract — URL conversion.** `githubRepositoryHttpsUrl` converts GitHub HTTPS URLs with or without `.git` or a trailing slash, HTTPS URLs carrying embedded credentials, `git@github.com:/` URLs, and `ssh://git@github.com/` URLs to `https://github.com//.git`, as the spec's [Repository and credentials](../../../specs/arch/test-runner/blaxel-testshards.md#repository-and-credentials) section requires. No converted URL contains the embedded credential, and a rejection diagnostic never repeats it, including for userinfo containing `@`, a scp-style `user:secret@host:path` remote, or a malformed scheme separator. It rejects URLs on other hosts, local paths and `file:` URLs, and GitHub URLs that do not name exactly an owner and a repository. (policy-grade test: `test/repo/blaxel-repository.test.ts` “converts GitHub remote URLs to credential-free HTTPS repository URLs”) ^blaxel-repo-url-conversion +- **Seam — selection from the checkout.** Real Git runs in a temporary repository; no part of Git is replaced. With `origin` set to an SSH GitHub URL and no explicit URL, `resolveBlaxelRepositoryUrl` returns that repository's HTTPS URL. A credential-bearing explicit URL is converted and selected even when `origin` is a non-GitHub URL, which is not consulted. A non-GitHub `origin` with no explicit URL fails, as does a repository with no `origin`. *Superfluous in part:* the explicit URL is the coordinator script's `--repo-url` option, which the spec leaves to the code. This proves the selection rules in the spec's [Repository and credentials](../../../specs/arch/test-runner/blaxel-testshards.md#repository-and-credentials) section. (policy-grade test: `test/repo/blaxel-repository.test.ts` “selects the repository from the checkout's origin unless --repo-url is given”) ^blaxel-repo-origin-selection +- **Acceptance — rejection before dispatch.** The real coordinator runs as a child process with real argv in a temporary Git repository whose `origin` is not on GitHub. The fixtures are that repository and a placeholder token in `BLAXEL_TV_GH_TOKEN`, so the run gets past the token check without a real credential. The coordinator exits `2` with the conversion diagnostic and creates no output directory, so it wrote no request and contacted no worker. This proves the spec's [Repository and credentials](../../../specs/arch/test-runner/blaxel-testshards.md#repository-and-credentials) promise that an unusable selection fails before dispatch. (policy-grade test: `test/repo/blaxel-repository.test.ts` “stops the coordinator before dispatch when origin is not a GitHub repository”) ^blaxel-repo-reject-before-dispatch +- **Seam — one checkout moves between repositories.** The handoff is from the coordinator's generated `checkout` script to Git on a worker, crossed with real `bash` and Git. The fixtures are two local bare repositories that share a base commit, each with commits the other lacks, reached through local paths instead of GitHub HTTPS (forfeit above). Running the script for repository A, then B, then A again on one checkout root clones only the first time. The B run fetches a commit only B holds. The final A run fetches an A-only commit created after the first clone, so neither switch can succeed from objects the checkout already has. A sentinel inside `.git` and an ignored `node_modules` sentinel both survive every switch. After each run, `HEAD` is the requested commit and `origin` points at that run's repository. This proves the checkout behavior in the spec's [Repository and credentials](../../../specs/arch/test-runner/blaxel-testshards.md#repository-and-credentials) section and the worker checkout. (policy-grade test: `test/repo/blaxel-repository.test.ts` “moves one persistent checkout between repositories without re-cloning or deleting ignored dependencies”) ^blaxel-repo-checkout-switch +- **Contract — dependency reuse across repositories.** The generated `checkout` and `deps` scripts run under real `bash`, Git, and `sha256sum` on one checkout root, using the two-repository fixtures above. A stand-in `npm` on `PATH` replaces `npm ci`: it records each call and creates `node_modules`. This forfeits real installation, which the live runs carry. The first `deps` run is a miss and calls `npm`. A second run is a hit. After the checkout moves to the other repository's commit with identical declarations, `deps` is still a hit and calls nothing. After it moves to a commit that changes `package-lock.json`, `deps` is a miss and calls `npm` again. This proves the dependency-reuse promise in [Repository and credentials](../../../specs/arch/test-runner/blaxel-testshards.md#repository-and-credentials). (policy-grade test: `test/repo/blaxel-repository.test.ts` “reuses installed dependencies across repositories until their declarations change”) ^blaxel-repo-deps-reuse +- **Contract — coordinator wiring (source).** The coordinator selects its repository through `resolveBlaxelRepositoryUrl` with the `--repo-url` option, carries no hard-coded repository URL, builds the worker's `checkout` step from `blaxelCheckoutScript` with the selected URL and resolved commit, and builds its `deps` step from `blaxelDependencyScript`. The live runs above carry the real handoff. This supports the spec's [Repository and credentials](../../../specs/arch/test-runner/blaxel-testshards.md#repository-and-credentials) section. *Superfluous in part:* the spec states only that workers use the checkout's `origin`; the coordinator script's `--repo-url` option is the code's. (policy-grade test: `test/repo/blaxel-repository.test.ts` “selects the coordinator repository and worker checkout through the shared repository module”) ^blaxel-repo-coordinator-wiring The remaining coverage lives in `test/repo/test-runner-provider.test.ts`. -- The canonical recommended shard count is 36, and the pool default remains 72 workers using `blaxel/playwright-chromium:latest`, 8192 MB, and the existing labels and naming scheme. -- Pool-manager source assertions require pinned nvm `v0.40.3`, `.nvmrc` selection, fresh-worker provisioning before architecture verification, and no alternate image or size defaults. +- The explicit recovery-interval forwarding contract also pins the coordinator's 1200-second default in every timeout profile. This source check proves the configured values; the lease seam and live driver carry actual renewal behavior. +- The canonical recommended shard count is 36, and the pool default remains 72 workers using `blaxel/playwright-chromium:latest`, 8192 MB, and the existing labels and naming scheme. *Superfluous in part:* the image, memory size and naming scheme are the code's. +- *Superfluous in part:* the spec does not pin the nvm version. Pool-manager source assertions require pinned nvm `v0.40.3`, `.nvmrc` selection, fresh-worker provisioning before architecture verification, and no alternate image or size defaults. - Coordinator source assertions require nvm activation after checkout and before dependency/test work, and the generated `deps` script includes `.nvmrc` and the root manifest in its dependency cache identity. -- The `electron-repair` source assertion requires Electron's own installer with the worker-local archive cache and rejects a separate `@electron/get`, unzip, or runtime-deletion path. +- *Superfluous:* the spec leaves worker Electron repair to the code, and Electron runtime preparation to [the e2e harness](../../../specs/arch/desktop/e2e-harness.md). The `electron-repair` source assertion requires Electron's own installer with the worker-local archive cache and rejects a separate `@electron/get`, unzip, or runtime-deletion path. - `classifyBlaxelShardStatus` returns `infra-failed` for a setup-step failure before `test-run`, an absent summary, an unconfirmed worker exit, or a summary that declares `infra-failed`; it returns `failed` for a failed summary or a passed summary contradicted by a known nonzero exit, and `passed` only for a passing summary with exit `0`. -- `normalizeBlaxelSurfaces` normalizes a passing Blaxel report into the surface's `NormalizedSurfaceResult` ([sharded-execution.md](../../../specs/arch/test-runner/sharded-execution.md)), including repository-relative conversion of absolute worker-native paths. +- `normalizeBlaxelSurfaces` normalizes a passing Blaxel report into the surface's `NormalizedSurfaceResult` ([sharded-execution.md](../../../specs/arch/test-runner/sharded-execution.md)), including repository-relative conversion of absolute worker-native paths. *Superfluous in part:* the normalized result type is the code's; the fields readers use are pinned in [the compact summary](../../../specs/arch/test-runner/reporting.md#the-compact-summary). - A targeted command runs under the generic surface supervisor with the selected surface owner token, and a failed target still writes a shard summary instead of being retried as infrastructure. - Provider summaries and normalized run events preserve each shard's restore/wake, checkout, dependency cache outcome, readiness, surface compute, and report-download phases without parsing human logs. - The normal-lease acquisition scheduler never exceeds its concurrency bound, attempts each requested shard exactly once, and preserves input order in its outcomes. A partial batch writes a provider report containing acquired-but-not-run, unavailable, and thrown-error shard outcomes before releasing every partial success. - Signal shutdown writes completed, running, and pending shard outcomes before lease cleanup; a report-write failure does not suppress cleanup. -- Unconfirmed ordinary execution cleanup orders named stop, owner-token reaping, confirmed zero-owner state, then lock release; failed owner confirmation retains the lease until TTL. Completed and undispatched paths do not run descendant cleanup. +- Ordinary cleanup/release coverage is superseded by [the lease ordering contract](#^blaxel-t-lease-order) and [the live retrieval acceptance](#^blaxel-t-retrieval); shell completion alone no longer permits release. - A stale-owner setup failure records the contaminated sandbox and its lifecycle evidence, excludes that sandbox from subsequent attempts, and permits a configured infrastructure retry only on a distinct sandbox that independently passes lease acquisition and owner audit. -**Boundary gaps — stated honestly per [testing-policy.md](../../../specs/arch/testing-policy.md):** the classifier, normalization, bounded acquisition scheduler, and provisioning/activation/cache command shapes are covered without calling Blaxel. The scheduler test uses an injected acquisition callback and therefore does not cross the Blaxel API boundary. The code-only checks cannot prove nvm installation, selected Node/npm versions, or cache behavior inside the base image. After review, deleting and recreating `poc` and running an operator-approved targeted shard provide that seam evidence along with the real coordinator path: leasing sandboxes, lock protocol, architecture validation, checkout/runtime/dependency setup, streaming logs, and result download. There is no recorded-fixture stand-in for that external boundary today. Repository switching on real workers has no committed test; the live runs from two repositories' checkouts described in the coverage model provide that evidence for a repository-selection change. +**Boundary gaps — stated honestly per [testing-policy.md](../../../specs/arch/testing-policy.md):** the classifier, normalization, bounded acquisition scheduler, and provisioning/activation/cache command shapes are covered without calling Blaxel. The scheduler test uses an injected acquisition callback and therefore does not cross the Blaxel API boundary. Code-only checks cannot prove installation or operation inside the base image. The live assertions below supply that provider evidence. Repository switching on real workers has no committed test; the live runs from two repositories' checkouts described in the coverage model remain required for a repository-selection change. + +### Lease, recovery, and scratch + +- **Contract — lease decisions.** The shared lock decision layer consumes authored current, expired, legacy, incomplete, and unreadable records with a supplied worker clock and audit/signal results. Under [Leases](../../../specs/arch/test-runner/blaxel-testshards.md#leases) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#abandoned-runs), renewal preserves ownership only for the matching unexpired lease; the default recovery interval is 1200 seconds regardless of execution timeout. Expiry permits cleanup, not dispatch: both quiet and active abandoned runs reach a new lease only after confirmed cleanup and a clean audit. Unknown identity or surviving owners hold the worker. Incomplete/unreadable records have the same clean-audit requirement for acquisition and deletion. Legacy positive TTLs are respected, live legacy owners are skipped, and new records use `expiresAt: 0`. After a deletion reservation, an ordinary acquisition is refused. Audit mode never recovers or deletes residue. Mock clock and audit/signal operations forfeit real serialization and termination to the guard seam and live recovery below. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “expired active and quiet leases use the same cleanup; uncertain owners hold”, “legacy, incomplete, unreadable and audit leases fail closed without clean ownership”, and “cleanup failures keep the old identity, while explicit deletion reserves an expired worker”; `test/repo/blaxel-worker-lease.test.ts` renewable-default and old-identity cases) ^blaxel-t-lease-decisions + +- **Contract — coordinator ordering and failures.** The coordinator's lease lifecycle uses recording/failing transport, download, and cleanup callbacks. Under [release](../../../specs/arch/test-runner/blaxel-testshards.md#leases), renewal spans acquisition, execution, cleanup, and retrieval. A successful shell exit still requires descendant confirmation; unknown dispatch or failed cleanup cannot release. Successful retrieval and identity validation precede release; a download retry neither reruns tests nor resets worker results. An unused lease waits for pending writes and retains produced evidence. On interruption, the authoritative interrupted report precedes cleanup/retrieval and cannot become an ordinary completion report; cleanup still runs after a report-write error. A coordinator that loses ownership stops issuing mutations and reports infrastructure failure. These callbacks forfeit API, signals, and downloads to the live retrieval/recovery assertions; existing signal-report fixtures supply breadth without another native suite. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “retrieval retries after teardown without rerunning work; uncertainty never releases”, “renewal continues while retrieval waits and stops on ownership loss”, and “preparation evidence is downloaded before an undispatched attempt releases”; `test/repo/test-runner-provider.test.ts` interrupted-report and explicit TTL-forwarding cases; `test/repo/blaxel-worker-lease.test.ts` delayed-entry case) ^blaxel-t-lease-order + +- **Seam — guarded registration and ownership.** Production worker-entry and lock helpers cross real Linux process/filesystem boundaries in a private fixture root as an ordinary repository test on Linux. It requires no Blaxel API or credentials; it runs only on Linux because it uses procfs, signals and the OS guard. Two competing acquisitions yield only one owner; killing a process holding the short guard releases the OS guard without deleting another lease. A request paused before entry registration, then resumed after ownership changes, cannot touch a checkout sentinel. Old-ID renewal/release cannot change the new record. Registered setup descendants preserve the outer token when a nested surface replaces its own token; recovery stops only the attributed PID/start identities, including a descendant that ignores TERM and needs KILL. A separate live fixture owner remains untouched and prevents reuse. The fixtures are small processes, records produced by the real helper, and rendezvous files. The supplied process set includes the registered supervisor, its reported descendant and the unrelated-owner sentinel. This mocks whole-worker enumeration and forfeits that protection to live recovery; procfs inspection, identity revalidation, signals and filesystem operations remain real. This proves [registration](../../../specs/arch/test-runner/blaxel-testshards.md#leases) and [recovery](../../../specs/arch/test-runner/blaxel-testshards.md#abandoned-runs), not provider dispatch. (policy-grade tests: `test/repo/blaxel-worker-lease.test.ts` competing acquisition, delayed entry, old-identity and registered-descendant recovery cases) ^blaxel-t-worker-guard + +- **Seam — bounded filesystem cleanup.** The production cleaner crosses the real filesystem in a disposable parent containing both allowed legacy prefixes, stale scratch with complete and missing metadata, the current scratch, unrelated files/directories, and symlinks to outside sentinels. With a lease and clean-audit precondition supplied as a mock, it removes exactly the permitted real directories, leaves the current directory and outside sentinels intact, rejects symlink roots, and records counts, allocated bytes and free-space observations under [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#temporary-files). The audit mock forfeits real disuse confirmation to the live retrieval/recovery cases. No assertion equates allocated bytes with the filesystem's free-space increase. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “only known real directories are removed, with allocated-byte evidence and outside links preserved”; `test/repo/blaxel-worker-lease.test.ts` “attempt identity and recovery evidence survive checkout resetting the result tree”) ^blaxel-t-scratch-cleanup + +- **Contract — cleanup uncertainty.** The cleaner's decision layer consumes authored filesystem-probe results and removal errors. An unexpected mount boundary is skipped, and an incomplete removal is reported without claiming the whole directory was reclaimed; neither permits new test dispatch under [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#Temporary files). These mocked conditions forfeit actual filesystem operations to the cleanup seam and live retrieval assertion; a real mount or full disk adds no necessary coverage. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “mount boundaries and incomplete removal remain reported failures without reclaimed counts” and “cleanup failures keep the old identity, while explicit deletion reserves an expired worker”) ^blaxel-t-scratch-errors + +- **Seam — unreadable process ownership.** The generated lock command runs without root privileges against real Linux procfs. On a root test worker the fixture drops the command to UID/GID 65534; otherwise it uses the current user. Its explicit process set permits acquisition without reading unrelated processes' environments, including a same-user nondumpable process on a root test host. The unscoped production audit refuses preparation when a process environment cannot be read, preserves the lease and known leftovers, and reports the PID and permission failure without environment contents. A root-only case changes a child's credentials after exec to make that same-user process nondumpable: the audit must still refuse it when included in the supplied process set. This checks [release](../../../specs/arch/test-runner/blaxel-testshards.md#leases)'s unreadable-audit rule. Credential changes and procfs reads are real; the root-only case is skipped on unprivileged CI and runs on the private Blaxel worker. (policy-grade tests: `test/repo/blaxel-worker-lease.test.ts` “fixture audit runs without root access to unrelated process environments”, “an unreadable whole-worker audit preserves the lease and its evidence”, and “fixture scope still refuses an unreadable process in its process set”) ^blaxel-t-unreadable-audit + +Shared live declaration: the following acceptances run in `experiment:blaxel-harness` via `npm test -- local --file test/runner-fixtures/blaxel-harness/driver/harness.test.ts` on a dedicated credentialed Blaxel coordinator sandbox outside the worker pool, from a committed and pushed checkout, never a shared developer host or a leased worker, under [harness placement](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-harness-placement). The caller supplies `TV_BLAXEL_HARNESS_COORDINATOR` and `TV_BLAXEL_HARNESS_POOL` as described in the [harness instructions](../../../test/runner-fixtures/blaxel-harness/README.md); no personal resource name is prescribed. They spawn the canonical CLI with real argv and inspect its exit and retained run directory. A fresh dedicated pool, at least two committed native fixture files in `experiment:blaxel-harness-fixtures`, known leftover directories and unrelated sentinels are fixtures. The two files permit the planned run's file-atomic two-shard assignment. Under the [harness surfaces](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-harness-placement), `npm test -- blaxel --tag blaxel-harness-fixture --pool --shards 2 --allow-shard-count-override 1` delegates `--suite selection --surfaces experiment:blaxel-harness-fixtures`; `npm test -- blaxel --surface experiment:blaxel-harness-fixtures --pool ` takes the targeted path. Workers read the committed registry. The coordinator, SDK, worker scripts, native runners, transport, ownership audits and filesystem operations are real. The report-download rendezvous above is the only scheduling hook; no mechanism is mocked. A bounded test driver owns and finally cleans up its coordinators and private workers. These tests do not invoke fixtures restricted to isolated GitHub hosts. + +- **Acceptance — reports before reuse.** A small planned run with at least two shards traverses real acquisition, architecture verification, checkout, runtime/dependency setup, execution and log/report transport under [Runs](../../../specs/arch/test-runner/blaxel-testshards.md#runs) and [Repository and credentials](../../../specs/arch/test-runner/blaxel-testshards.md#repository-and-credentials). Each uploaded script path lies inside its current shard attempt directory. The retained setup phases include `temporary-cleanup`, with cleanup counts and free-space evidence. While it waits at the download rendezvous, observed renewals keep the leases valid; a competing canonical run cannot acquire the occupied capacity or alter its result files. After retrieval, the original native reports, setup/cleanup evidence and diagnostics have the original identities and readable local paths, scratch is gone, and the workers become available to a targeted run. Legacy-prefix and stale-scratch fixtures disappear, unrelated sentinels and persistent caches survive, and the next run does not accumulate an older results tree. The targeted run crosses its distinct dispatch path too. This proves [release](../../../specs/arch/test-runner/blaxel-testshards.md#leases) and [scratch lifetime](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch); the shared diagnostics observations are owned by [reporting](reporting.md#^reporting-t-resource-live). (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, with `fixtures/outcomes.test.ts` and `fixtures/temporary.test.ts` under the same harness directory) ^blaxel-t-retrieval + +- **Acceptance — abandoned run recovery.** In two small scenarios, kill the real coordinator: once while a shard supervisor and a token-bearing child remain, and once after the shard stops but before download. The next canonical run, after the shortened configured recovery interval, stops the old owned processes where present, confirms a clean worker, clears allowed residue, and runs on that worker. Its downloaded recovery receipt retains the old identity, renewal/deadline, cleanup result and discarded-report flag across results reset. No old report becomes a pass. This proves the same [automatic recovery path](../../../specs/arch/test-runner/blaxel-testshards.md#abandoned-runs) works for active and quiet orphans without waiting for their voluntary exit. Failure permutations are carried by the contracts, not more live crash runs. (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, with `fixtures/outcomes.test.ts` and `fixtures/temporary.test.ts` under the same harness directory) ^blaxel-t-recovery-live + +- **Seam — temporary-directory inheritance.** In the live planned and targeted fixture runs above, the shard environment reaches real Node and Chromium: Node's `os.tmpdir()` and created files are inside the current `tmp/`, and actual Chromium shared-memory descriptor paths resolve there, including unlinked files. Inspect the launched browser's descriptors on the worker; an environment-variable echo alone is insufficient. Nested surface tokens must not erase the outer ownership token or `TMPDIR`. After process teardown, current scratch is absent while retained sidecars are readable. This honors [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-scratch) and [testing guidance](../../../specs/arch/test-runner/blaxel-testshards.md#testing-guidance). A small HTML page is the browser fixture; no browser or path mechanism is mocked, and no memory-pressure or exact byte-saving claim is made. (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, with `fixtures/outcomes.test.ts` and `fixtures/temporary.test.ts` under the same harness directory) ^blaxel-t-tmpdir-live + +### Pool mutation + +- **Contract — publication and deletion ordering.** The pool manager uses authored sandbox states and recording/failing SDK operations. Under [Pool](../../../specs/arch/test-runner/blaxel-testshards.md#pool) and [Deleting workers](../../../specs/arch/test-runner/blaxel-testshards.md#deleting-workers), it publishes eligibility only after successful provisioning/verification, refuses to refill partial pools, refuses deletion of valid leases, and unpublishes before deleting an idle or expired worker. An expired lease can be deleted despite failed process cleanup; a failed deletion stays unavailable and visible. Unreadable records follow the common contract above; command failure on a previously eligible worker holds/reports it, while a fresh worker never published may be deleted without a lease. SDK substitutions forfeit actual metadata changes and deletion to the live pool acceptance. (policy-grade tests: `test/repo/blaxel-lease-contracts.test.ts` “pool eligibility follows verification, and deletion reserves then unpublishes” and “failed deletion of never-published workers remains retryable without their runtime”) ^blaxel-t-pool-order + +- **Acceptance — fresh pool and safe deletion.** Through canonical pool commands and the real API, create the dedicated pool used by the live assertions, observe workers become eligible only after runtime/architecture verification, and run the fixtures on them. Attempt deletion while the retrieval gate holds valid leases: the command reports refusals and exits nonzero, and the original run still completes. After release, delete the pool and confirm its workers are absent; a new coordinator cannot dispatch there. This proves the [pool mutation](../../../specs/arch/test-runner/blaxel-testshards.md#deleting-workers) and [setup](../../../specs/arch/test-runner/blaxel-testshards.md#setup) spine through actual Blaxel label changes and deletion. These observations share the retrieval scenario rather than provisioning another pool. (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, with `fixtures/outcomes.test.ts` and `fixtures/temporary.test.ts` under the same harness directory) ^blaxel-t-pool-live diff --git a/proofs/arch/test-runner/flaky-tests.md b/proofs/arch/test-runner/flaky-tests.md deleted file mode 100644 index f0c5bcaf..00000000 --- a/proofs/arch/test-runner/flaky-tests.md +++ /dev/null @@ -1,16 +0,0 @@ -*How the promises in Flaky Tests are proven.* - -# Flaky Tests — proof - -Proves [specs/arch/test-runner/flaky-tests.md](../../../specs/arch/test-runner/flaky-tests.md). - -## Coverage model - -Coverage declarations are carried inside the migrated assertion blocks below. - -## Assertions - -### Test assertions - -This spec owns conventions whose enforcement is review-judged rather than gate-enforced ([testing-policy.md](../../../specs/arch/testing-policy.md)); it adds no test assertions of its own. The recovered-flake normalization those conventions rely on is asserted in [reporting.md](../../../specs/arch/test-runner/reporting.md). The `FLAKY_TEST_RETRIES` resolver has no dedicated unit test today. No dedicated test enforces the annotation patterns. - diff --git a/proofs/arch/test-runner/github-ci.md b/proofs/arch/test-runner/github-ci.md index ae0a54f4..b24aa107 100644 --- a/proofs/arch/test-runner/github-ci.md +++ b/proofs/arch/test-runner/github-ci.md @@ -11,7 +11,7 @@ Coverage declarations are carried inside the migrated assertion blocks below. PR ## Test hooks -`setup-workspace`'s `cache-namespace` input and `setup-shard-readiness`'s `cache-namespace` input place cache entries from a validation run ([#Testing](../../../specs/arch/test-runner/github-ci.md#Testing)) under scratch prefixes. `setup-shard-readiness`'s `cache-key-suffix` input forces an exact-key miss without requiring a source or dependency change. The production workflow passes none of these inputs and therefore uses the `nm` workspace namespace, the `shard-readiness-v2` readiness namespace, and no suffix. +`setup-workspace`'s `cache-namespace` input and `setup-shard-readiness`'s `cache-namespace` input place cache entries from a validation run ([testing guidance](../../../specs/arch/test-runner/github-ci.md#testing-guidance)) under scratch prefixes. `setup-shard-readiness`'s `cache-key-suffix` input forces an exact-key miss without requiring a source or dependency change. The production workflow passes none of these inputs and therefore uses the `nm` workspace namespace, the `shard-readiness-v2` readiness namespace, and no suffix. `checkManifestCoversRegistry` accepts the registry and manifest to compare. `runRecipes` accepts `root` (the workspace where recipes run) and `manifest` (the recipes to run). `packUntrackedFiles` accepts `root` (the Git workspace to enumerate) and `archivePath` (the archive to write); the helper CLI exposes the same workspace substitution as `--root`. Production uses the repository root and the committed registry and manifest. @@ -23,21 +23,21 @@ Shapes per [testing-policy.md](../../../specs/arch/testing-policy.md). Workflow- **Contract — the committed workflow encodes this spec** (the YAML is the producer-side artifact under test): -- **Contract** (committed CI YAML parsed directly; no mocks or hooks): workflow concurrency is scoped by workflow, event and PR number, using a unique run ID for non-PR events; cancellation is enabled only for PR events. This proves the declared [superseded PR policy](../../../specs/arch/test-runner/github-ci.md#^gha-pr-concurrency), preserving independent push/manual runs and different PRs. Hosted cancellation is GitHub behavior, not a mocked acceptance path — *(covered by test: `test/repo/public-workflow-boundaries.test.ts` “cancels only superseded runs of the same PR while keeping other CI runs independent”)*. ^gha-t-pr-concurrency +- **Contract** (committed CI YAML parsed directly; no mocks or hooks): workflow concurrency is scoped by workflow, event and PR number, using a unique run ID for non-PR events; cancellation is enabled only for PR events. This proves the declared [superseded PR policy](../../../specs/arch/test-runner/github-ci.md#when-it-runs), preserving independent push/manual runs and different PRs. Hosted cancellation is GitHub behavior, not a mocked acceptance path — *(covered by test: `test/repo/public-workflow-boundaries.test.ts` “cancels only superseded runs of the same PR while keeping other CI runs independent”)*. ^gha-t-pr-concurrency - The pull-request and push triggers both target `main` and `integration/**`, so nested integration branch names are included. — *(covered by test: `test/repo/test-runner-guardrails.test.ts` “CI validates pushes and pull requests for main and nested integration branches”)*. ^gha-ci-branch-targets - Both setup composites read `node-version-file: .nvmrc`, expose no independent Node-version input or default, and select Node before resolving cache identity. Every CI job that executes a repository command has exactly one direct or composite selector; the three direct sites precede the attestation decision, shard-plan agreement validator, and attestation writer respectively. `setup-playwright` carries no selector and every call follows an owning setup composite. - The ordinary dependency cache step uses an exact key with no `restore-keys`; the key string carries the OS, CPU-architecture, Node major resolved from the selected runtime, and execution-image identity components; its `hashFiles` list includes `package-lock.json` and the conservative `packages/skills/package.json` input; and the cached path is `node_modules`. - `setup-shard-readiness` selects Node before resolving its key, derives the key's Node-major label from that runtime, requires non-empty `ImageOS` and `ImageVersion`, carries the full input hash, uses the `shard-readiness-v2` payload namespace, and restores one archive containing the ordinary cached paths plus `~/.cache/ms-playwright`, with no fallback key. The build's `npm ci`, browser preparation, and save are miss-only. A shard miss cannot fail at restore: it emits a warning, self-seeds dependencies plus Chromium and Firefox without saving the combined cache, and records a cache-miss fallback phase. -- The build and every e2e shard execute real Chromium and Firefox launch probes. `e2e:desktop` keeps the `setup-playwright` composite action; an e2e shard invokes it only while self-seeding a miss. No job declares a container (containerization deferred; see Browser environment). +- The build and every e2e shard execute real Chromium and Firefox launch probes. `e2e:desktop` keeps the `setup-playwright` composite action; an e2e shard invokes it only while self-seeding a miss. No job declares a container. - Every job that reaches the runner's owner-lifecycle paths (the e2e shard jobs, `unit`, and `e2e:desktop`) asserts-or-installs the inspection tool set (`ss`, readable `/proc`). - The build job runs the `test.prebuilt.mjs` recipes and uploads the artifact under the manifest-derived name (`prebuilt.mjs artifact-name` — no hardcoded duplicate) with `retention-days: 1`; the e2e shard jobs download and extract it with stored permissions and set `TEST_SHARD_SKIP_PRECOMMANDS=1`; the `e2e:desktop` job does neither (fan-out scope above); the join job checks out the repository. **The workflow contains no hand-maintained output list**, and no second artifact carries source to a consumer. -- **Contract.** The packed set equals exactly the untracked files and symlinks git reports for a workspace, minus those under `node_modules`, `.git`, and `.test-runs` — compared against git's own listing, with no expected-path list in the test. Fixtures: a real git repository built by the packer's own recipe runner, seeded with the shapes that distinguish this contract — an executable file, a symlink, a zero-byte file, an empty directory, an output no ignore rule covers, generated files under both root and nested `node_modules`, and test scratch under `.test-runs`. The real repository's build is not packed in this test; two full product builds to prove a packer property would cost more than the property is worth, and live CI exercises that path on every run. That is what makes a shard with skipped pre-commands able to run the surfaces that would have followed a `preCommand` build: the licensing gate and its inventories, tarball and desktop assertions, asset-copy checks, and development-layout lookups all read files the build wrote, and every such file ships. A second assertion covers [the one rule](../../../specs/arch/test-runner/github-ci.md#^gha-prebuilt-tracked-rule): after the recipes no tracked file differs from the commit, and a fixture recipe that leaves one modified fails the build job. ^gha-prebuilt-cli-outputs +- **Contract.** The packed set equals exactly the untracked files and symlinks git reports for a workspace, minus those under `node_modules`, `.git`, and `.test-runs` — compared against git's own listing, with no expected-path list in the test. Fixtures: a real git repository built by the packer's own recipe runner, seeded with the shapes that distinguish this contract — an executable file, a symlink, a zero-byte file, an empty directory, an output no ignore rule covers, generated files under both root and nested `node_modules`, and test scratch under `.test-runs`. The real repository's build is not packed in this test; two full product builds to prove a packer property would cost more than the property is worth, and live CI exercises that path on every run. That is what makes a shard with skipped pre-commands able to run the surfaces that would have followed a `preCommand` build: the licensing gate and its inventories, tarball and desktop assertions, asset-copy checks, and development-layout lookups all read files the build wrote, and every such file ships. A second assertion covers [the tracked-file rule](../../../specs/arch/test-runner/github-ci.md#build-once): after the recipes no tracked file differs from the commit, and a fixture recipe that leaves one modified fails the build job. ^gha-prebuilt-cli-outputs - The manifest's `artifactName` is the literal `"prebuilt-dists"` (typed literal, asserted). - The shard worker suppresses `preCommand` under both knob forms while leaving the surface command unchanged, proven by executing a minimal plan file through the real worker process. - The join (`e2e-required`) depends on the build job and the e2e matrix. Its job-level condition combines `always()` with the shared draft guard: drafts report neutral/skipped without entering the join script, while every non-draft run executes the strict path. That path passes only when build and e2e both report success and fails for missing, skipped, cancelled, or failed dependencies. It checks out the repository and runs its validator from that checkout; no artifact carries source to it. It also depends on the `vibe-mode` job, which checks out the repository, selects Node once, and runs the vibe-mode check; a vibe-mode result other than success fails the join before the attested-skip path and the build and e2e checks, and the job takes no attestation dependency. -- The registry/manifest consistency check holds: every `preCommand`-declaring surface is covered by exactly one manifest recipe (this replaces the former prose-only "manifest covers every surface" line with the mechanical check the manifest shape enables). +- The registry/manifest consistency check holds: every `preCommand` declared by a surface in the `all` suite is covered by exactly one manifest recipe, as [build once](../../../specs/arch/test-runner/github-ci.md#build-once) requires. -- `packages/web/playwright.config.ts` pins CI workers to 2. +- `packages/web/playwright.config.ts` pins CI workers to 2. *Superfluous:* the spec states only the `e2e:browser-app-real-stack` worker count; this pin is the code's. **Contract — worker and validator behavior** (each side proven in isolation): @@ -48,6 +48,6 @@ Shapes per [testing-policy.md](../../../specs/arch/testing-policy.md). Workflow- - Direct local test jobs resolve their artifact from `--run-dir-output`, validate that exact path under the workspace `.test-runs/` root, and never require the root to contain only one directory. - A missing artifact or a failed extraction fails the consuming job; no path is inspected and no rebuild fallback exists. -**Acceptance — live CI as the outermost boundary:** static YAML checks cannot prove that a live job restores an equivalent tree or that artifact fan-out delivers working builds. When required by the spec’s [Testing](../../../specs/arch/test-runner/github-ci.md#Testing) section, live evidence must include a fully green run of an isolated copy of the complete `ci.yml` job graph, triggered only from its implementation branch. Its jobs must show (a) an image-bound combined-cache **hit** with `npm ci` and browser preparation skipped, every shard's launch check green, and the full suite green; (b) `preCommand`s skipped while surfaces consume the artifact; and (c) the join validating exact plan agreement from its own checkout. A focused isolated run must force a shard-key miss and prove that the warning, local dependency/browser setup, cache-miss fallback metric, both browser launches, tests, and join remain green. An induced build failure must prove that the join cannot report a false pass. Because `ci.yml` triggers only on pushes to `main` or branches under `integration/` and pull requests targeting those branches, an isolated validation copy must use one of those trigger paths to provide this evidence before promotion. Required-check context wiring under the production names remains for the operator to verify on the first production run. A miss-path run is required to produce the cache entry used by the hit-path run. Every subsequent CI run exercises the promoted paths. +**Acceptance — live CI as the outermost boundary:** static YAML checks cannot prove that a live job restores an equivalent tree or that artifact fan-out delivers working builds. When required by the spec’s [testing guidance](../../../specs/arch/test-runner/github-ci.md#testing-guidance), live evidence must include a fully green run of an isolated copy of the complete `ci.yml` job graph, triggered only from its implementation branch. Its jobs must show (a) an image-bound combined-cache **hit** with `npm ci` and browser preparation skipped, every shard's launch check green, and the full suite green; (b) `preCommand`s skipped while surfaces consume the artifact; and (c) the join validating exact plan agreement from its own checkout. A focused isolated run must force a shard-key miss and prove that the warning, local dependency/browser setup, cache-miss fallback metric, both browser launches, tests, and join remain green. An induced build failure must prove that the join cannot report a false pass. Because `ci.yml` triggers only on pushes to `main` or branches under `integration/` and pull requests targeting those branches, an isolated validation copy must use one of those trigger paths to provide this evidence before promotion. Required-check context wiring under the production names remains for the operator to verify on the first production run. A miss-path run is required to produce the cache entry used by the hit-path run. Every subsequent CI run exercises the promoted paths. **Boundary gaps — stated honestly per [testing-policy.md](../../../specs/arch/testing-policy.md):** key construction enforces that changing a prepare input causes a cache miss. This was verified by changing those inputs in an isolated cache run, but no committed test simulates the Actions cache service. diff --git a/proofs/arch/test-runner/preflight.md b/proofs/arch/test-runner/preflight.md deleted file mode 100644 index 618c6690..00000000 --- a/proofs/arch/test-runner/preflight.md +++ /dev/null @@ -1,42 +0,0 @@ -*How the promises in Preflight are proven.* - -# Preflight — proof - -Proves [specs/arch/test-runner/preflight.md](../../../specs/arch/test-runner/preflight.md). - -## Coverage model - -Contributor branch ownership and commit/push practice are reviewed as development procedure. The existing assertions below cover runtime preflight; the branch guidance changes no runtime check and orders no additional tests. - -Coverage declarations are carried inside the migrated assertion blocks below. - -## Test hooks - - -`TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT` supplies one failed local result as `:`. `TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT` supplies one failed remote result as `::`, or a passing result as `:passed`. Passing results are permitted only for automatic provider resolution in `verify --plan`; other callers refuse them before dispatch. `runRemotePreflight` accepts `allowPassingSelftest` for that diagnostic caller, defaulting to false. Both environment controls require `TV_TEST_RUNNER_SELFTEST=1`, as defined by [test-runner.md's Test hooks](./test-runner.md#Test hooks); otherwise the runner rejects them. These controls replace check execution, so they prove only how the runner handles preflight outcomes. Production sets none of them. - -The `electron` option to `runPreflights` accepts `root` to select the package and planner workspace and `plan` to supply an `ElectronE2EPlan`. The `plan` override replaces the shared environment planner, so it covers only how the Electron check handles planner results. Production supplies neither override and uses the current workspace and the real planner. `readPostHogTestReadKey` accepts `env` to supply the environment source and `root` to select the `.env` lookup directory; production calls it without either override. - -`validateProcessLifecycleHost` accepts `platform` to select the host contract, `procfsReadable` to state whether Linux process identity is readable, and `commandAvailable` to report the required command capabilities. `runProcessLifecycleProbe` accepts `platform` to select the backend and `runCommand` to supply its command runner. The `runCommand` override replaces command execution, so it covers probe-result handling rather than host capability. Production calls both functions with the detected platform and the host's actual process-inspection capabilities. - -## Assertions - -### Test assertions - -Coverage spans `test/repo/node-toolchain.test.ts`, `test/repo/test-runner-preflight.test.ts`, `test/repo/process-lifecycle.test.ts`, and `test/repo/test-runner-shard-worker.test.ts`. The successful `process-lifecycle` assertion is a seam test against a real disposable child and the host process/socket tables; injected missing-capability cases are contract tests of host validation. - -- `requiredPreflights` supports local execution with `["node", "process-lifecycle", ...selected surface checks]` and the Blaxel coordinator with `["node", "blaxel-github-token", "blaxel-auth"]`. -- The `node` check retains that name, reports the running Node/npm versions when they satisfy repository policy, and returns the shared toolchain diagnostic when either version is unsupported; canonical local execution stops before a selected test after this failure. That short-circuit assertion invokes its nested canonical CLI with `--no-publish`, keeping optional ref transport outside the preflight boundary under test. -- The planned shard worker records successful `process-lifecycle` and `stale-owner-cleanup` entries before its selected tasks. -- The Linux `process-lifecycle` check uses a real disposable child and the host's real `/proc` and `ss` boundaries; it passes only when the child can be attributed. Failure fixtures cover a missing `ss`, unreadable procfs, and unsupported platform without replacing the successful real-host seam test. -- The daemon-host acknowledgement check fails unless `TV_DAEMON_TEST_HOST` is exactly `1`, names the required value in its diagnostic, and records the environment-variable name when it passes. -- PostHog key-source tests cover environment and repository-root `.env` lookup plus the missing-key result from the reader. -- **Contract — Electron pre-setup states** (planner results and a temporary resolvable-package fixture; planner behavior is a declared mock forfeited to the seam below): with skip unset, a missing Electron package fails, an absent or valid result with no planner failures passes, and an invalid result fails. The preflight path invokes no installer and performs no runtime mutation. With `SKIP_ELECTRON_E2E=1`, it reports skipped. — *(covered by test: `test/repo/test-runner-preflight.test.ts` “checks Electron package presence and consumes every planner state without installing”)*. ^preflight-t-electron-states -- **Seam — Electron preflight-to-planner handoff** (the real `runPreflights` dispatcher, Electron check, shared environment planner, installed declared Electron package, package filesystem, and Linux host; no mocks): selecting the Electron check against the package's actual acceptable pre-setup state consumes the real planner result, passes, and leaves the runtime tree unchanged. The contract assertion carries the other planner-result permutations. — *(covered by test: `test/repo/test-runner-preflight.test.ts` “hands the real installed Electron package to the shared planner without mutation”)*. ^preflight-t-electron-planner-handoff - -**Boundary gaps — stated honestly per [testing-policy.md](../../../specs/arch/testing-policy.md):** - -- Apart from the `node` toolchain and Linux `process-lifecycle` seams above, check *execution* — `playwright-chromium`/`playwright-firefox`, the git-safety sequence, and the provider-auth checks — has no committed boundary test. In the guardrail tests ([test-runner.md](../../../specs/arch/test-runner/test-runner.md)) remote preflight is replaced by a self-test seam, so the real `git status` / `git rev-parse` / `git fetch` / `git branch -r --contains` chain and the `@blaxel/core` calls are not exercised here. The git-safety checks are reachable with a real temporary git repository and no external credentials; the provider-auth checks require real Blaxel credentials. -- No committed test forces the planned worker's `process-lifecycle` preflight to fail and proves the resulting infrastructure summary, and no test invokes `posthog-test-key` through `runPreflights` to assert the missing-key or invalid-`phx_` diagnostic. The current tests cover the successful worker record, key lookup, and missing reader result. -- The macOS `process-lifecycle` backend requires a real macOS host for its `ps` environment/identity and `lsof` boundary test and is not proven by the Linux gate. - diff --git a/proofs/arch/test-runner/reporting.md b/proofs/arch/test-runner/reporting.md index a214d39b..392f774a 100644 --- a/proofs/arch/test-runner/reporting.md +++ b/proofs/arch/test-runner/reporting.md @@ -6,7 +6,13 @@ Proves [specs/arch/test-runner/reporting.md](../../../specs/arch/test-runner/rep ## Coverage model -Coverage declarations are carried inside the migrated assertion blocks below. +Coverage declarations are carried inside the migrated assertion blocks below. Real nested canonical CLI cases use the runner’s [gated private lock-location hook](test-runner.md#test-hooks), with a fresh path per independent scenario. This keeps them separate from the outer run and parallel workers while retaining real acquisition and cleanup; it forfeits production-location coverage to [the runner’s location seam](test-runner.md#^t-runner-lock-location). Their native success/failure cases also establish [canonical exit propagation](test-runner.md#^t-runner-native-exit). + +Native-report contracts carry normalization breadth; real native-runner and CLI crossings carry attempt capture and final output. + +Resource diagnostics compose a bounded writer contract, a real Linux probe/parent-lifecycle seam, and the dedicated Blaxel acceptances owned by [the provider proof](blaxel-testshards.md#Lease, recovery, and scratch). A small native failure and a recovered retry establish retained evidence; they do not reproduce resource exhaustion. Authored unavailable/reset/oversized probe inputs cover uncommon diagnostic failures without filling a disk or causing an OOM kill. Those mocks forfeit kernel reading and transport to the real seams. + +Planned and targeted execution share the collector; the parent seams cover capture and the dedicated Blaxel acceptances cover download. The same sidecars pass through provider normalization and attempt history; a recovered infrastructure attempt must not replace earlier evidence. Recovery receipts are observed in the provider's crash acceptance, not a duplicate crash here. No test claims start/end samples measure a peak or that an unavailable kernel log excludes OOM. The dedicated live acceptance checks download, reuse and recovery receipts alongside the focused bounds, parent capture and retained-link assertions. ## Test hooks @@ -15,6 +21,8 @@ Coverage declarations are carried inside the migrated assertion blocks below. `TV_VITEST_ATTEMPT_FILE` supplies the NDJSON output path to the production Vitest attempt recorder. The canonical runner and the planned shard worker set it to the current native run's attempt-result path. A Vitest invocation that configures the recorder outside those entrypoints must set it; the recorder fails without it. +The resource collector accepts a probe reader and clock; contract tests provide authored readings/errors and times, while production reads Linux counters, memory information and filesystem statistics at the current time. The writer accepts an output root and write operation; tests use disposable directories and may inject a write failure, while production writes into the current results tree. Local parent/child seam fixtures use the runner’s existing guarded registry hook, not a replacement reporter or fabricated native result. That hook is not forwarded to Blaxel workers. + ## Assertions ### Test assertions @@ -39,4 +47,17 @@ The event-normalization assertions are **contract tests** over recorded native-r - The spawned CLI writes summary, results, and events files for a passing run, a native test failure, and an early preflight failure, with the matching run status. - A planned shard worker running under GitHub Actions finalizes its own run directory and retains its shard summary, plan, native results, and logs inside it. - A repository workflow assertion pins full run-directory artifact upload on success and failure and a 30-day artifact retention period. +- **Contract — summary fields.** A built summary carries the fields the spec pins for readers of earlier runs: `schemaVersion: 1`; the `run` block's identity and three statuses; each listed test's surface, file, line, title, attempt count and annotation flag; and each leak's PID, executable, owning surfaces and listening sockets. Passing tests are not listed. Proves [the compact summary](../../../specs/arch/test-runner/reporting.md#the-compact-summary). *(covered by `test/repo/test-runner-reporting.test.ts` “normalizes Vitest failures without enumerating passed tests in summary” and “canonicalizes identities and deduplicates process leaks without losing surface failures”)* ^reporting-t-summary-fields +- **Contract — events read across revisions.** Baseline update refuses a Blaxel full-suite stream with an event removed, so that sequence numbers no longer match line positions, and writes nothing. This exercises one rule of the [envelope the update checks](../../../specs/arch/test-runner/reporting.md#timing-records); the shared `runId`, the status values and the Blaxel timing-provider identity go through the same validator without cases of their own. *(covered by `test/repo/test-runner-timing-baseline.test.ts` “aborts without a partial write for unreadable runs, invalid Blaxel events, and owner disagreement”)* ^reporting-t-event-envelope +- **Acceptance — run-directory identity.** The real CLI given `--run-dir-output` writes the absolute path of its own run directory without inspecting sibling run directories, and the identity reader accepts only a direct child of the checkout's `.test-runs/`. Proves [`--run-dir-output`](../../../specs/arch/test-runner/reporting.md#run-directory). *(covered by `test/repo/test-runner-reporting.test.ts` “writes its run directory identity without inspecting sibling run directories” and “accepts a run identity only for a direct child of this checkout's run root”)* ^reporting-t-run-dir-output +- **Acceptance — refusals leave no run directory.** In a disposable checkout with the production CLI and an authored two-surface registry, committed so that Blaxel selection reads it, the real CLI refuses a local full-suite shortcut, a Blaxel selection that resolves to two surfaces, an unsupported Blaxel suite and an empty `--run-dir-output` with exit `2`. After each refusal the checkout has no `.test-runs/` directory, and no run-directory identity was written. The declared failing remote-preflight hook is a tripwire: reaching remote preflight would change the diagnostic, so the refusals are shown to come first. Proves that [a refusal leaves no run directory](../../../specs/arch/test-runner/reporting.md#run-directory). *(covered by `test/repo/test-runner-guardrails.test.ts` “refuses %j before remote preflight and creates no run directory”, for each case, and “refuses an empty --run-dir-output before remote preflight and creates no run directory”)* ^reporting-t-refusal-no-run-dir + +### Resource diagnostics + +- **Contract — bounded, honest observations.** The collector/writer consumes authored start/end memory, filesystem, OOM-counter and kernel-line readings, including unavailable probes, a counter reset and over-budget excerpts, and writes real files in a temporary results root. Under [resource diagnostics](../../../specs/arch/test-runner/reporting.md#^resource-diagnostics) and their [bounds](../../../specs/arch/test-runner/reporting.md#resource-diagnostics), values keep units/time, unknowns keep reasons rather than zero, and resets produce no negative OOM delta. A failed native attempt, even if later recovered, or incomplete execution requests the interval's kernel evidence. Authored timestamped lines prove that a whole-second read is filtered to the exact observation interval and untimestamped OOM lines are counted as omitted. The bounded reader/writer retains at most 64 KiB of kernel lines and 128 KiB per sidecar, and 8 MiB total per shard attempt including receipts (*superfluous in part:* the spec requires bounds per surface and per shard attempt, and the sizes are the code's); compact start/end evidence survives before optional excerpts, with explicit truncation. Only allowed fields are serialized: supplied credential, query/body, environment and argument sentinels do not appear. Injected probe and write failures warn without changing the original native outcome. Mock readings/write failures forfeit OS reading to the parent seam and actual download to the live assertion below. ([`test/repo/resource-diagnostics.test.ts`](../../../test/repo/resource-diagnostics.test.ts)) ^reporting-t-resource-bounds + +- **Seam — parent capture survives the child.** The production surface parent crosses real Linux procfs/filesystem probes and an installed native runner's child-process boundary in a disposable results root. Authored tiny native fixture tests produce a failure/retry and an exit before native output; both planned-worker and targeted execution wiring are exercised. The retained baseline exists before preparation begins and the final observation follows teardown; a terminated child cannot erase the baseline. A separate interrupted-parent fixture leaves the baseline with an explicitly missing final observation rather than invented zeroes. Real probe values are checked for their source/units and availability reasons, not exact memory totals. No kernel OOM is manufactured; inaccessible kernel evidence must remain unknown. No mechanisms are mocked. This proves the capture boundary in [resource diagnostics](../../../specs/arch/test-runner/reporting.md#^resource-diagnostics); the live assertion carries remote retrieval. ([`test/repo/test-runner-resource-parent.test.ts`](../../../test/repo/test-runner-resource-parent.test.ts); combined-project member attribution in [`test/repo/test-runner-shard-worker.test.ts`](../../../test/repo/test-runner-shard-worker.test.ts), “runs exact files from two Vitest projects in one process”) ^reporting-t-resource-parent + +- **Contract — retained links and attempt history.** Provider normalization and infrastructure-attempt aggregation consume authored shard/task envelopes referring to real temporary sidecar files, with recorded native failed/recovered fixtures. Under [the run directory's normalized results](../../../specs/arch/test-runner/reporting.md#run-directory) and [resource diagnostics](../../../specs/arch/test-runner/reporting.md#Resource diagnostics), each advertised `resourceDiagnosticsPaths` entry exists relative to the final run directory; earlier attempt evidence survives a later successful infrastructure attempt. Missing native output remains incomplete and cannot manufacture a selected-surface result. Samples and kernel excerpts appear in neither compact console output nor timing events. Mocked provider envelopes forfeit remote creation/download to the live assertion; recorded native inputs forfeit attempt capture to the real parent seam and existing native-recorder assertions above. ([`test/repo/resource-diagnostics-links.test.ts`](../../../test/repo/resource-diagnostics-links.test.ts)) ^reporting-t-resource-links +- **Acceptance — evidence reaches the caller.** Share the dedicated-pool canonical CLI runs in [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval), under its shared `experiment:blaxel-harness` declaration and dedicated coordinator placement. Its `experiment:blaxel-harness-fixtures` selection includes a committed fail-once native fixture that recovers under the default retry budget and a fixture that fails all attempts. Planned and targeted runs retain the native outcomes and their start/end sidecars after worker scratch is removed; every normalized diagnostic link opens a downloaded file with the correct run/shard/surface identity and native-attempt references. The failed run still fails, the recovered case retains both attempts, and any inaccessible kernel evidence says so. The provider's [crash acceptance](blaxel-testshards.md#^blaxel-t-recovery-live) supplies the downloaded recovery receipt and its disclosed report loss. Boundary: real CLI, Blaxel API, native runners and report files, with the provider proof's scheduling hook and no mocks. This proves [resource retention](../../../specs/arch/test-runner/reporting.md#Resource diagnostics) end to end without a resource-exhaustion experiment. (policy-grade test: `test/runner-fixtures/blaxel-harness/driver/harness.test.ts` “qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle”, using the harness’s real failed/recovered native fixtures) ^reporting-t-resource-live diff --git a/proofs/arch/test-runner/sharded-execution.md b/proofs/arch/test-runner/sharded-execution.md index 8f289374..a159db77 100644 --- a/proofs/arch/test-runner/sharded-execution.md +++ b/proofs/arch/test-runner/sharded-execution.md @@ -6,7 +6,9 @@ Proves [specs/arch/test-runner/sharded-execution.md](../../../specs/arch/test-ru ## Coverage model -Coverage declarations are carried inside the migrated assertion blocks below. +Coverage declarations are carried inside the migrated assertion blocks below. Assertions marked *superfluous* cover behavior the current spec leaves to the code, such as the assignment algorithm and the weighting constants; they are kept as coverage of that code. + +The [Blaxel guard seam](blaxel-testshards.md#^blaxel-t-worker-guard) and [live temporary-directory seam](blaxel-testshards.md#^blaxel-t-tmpdir-live) prove that the outer shard attempt's ownership and `TMPDIR` survive the shared worker and nested surface environments required by [Leases](../../../specs/arch/test-runner/blaxel-testshards.md#leases) and [Temporary files](../../../specs/arch/test-runner/blaxel-testshards.md#temporary-files). The [planned live acceptance](blaxel-testshards.md#^blaxel-t-retrieval) also supplies real plan transfer. Diagnostic artifact fields and the attempt history kept under [shard outcomes](../../../specs/arch/test-runner/sharded-execution.md#shard-outcomes) compose with [reporting's link contract](reporting.md#^reporting-t-resource-links) and [live retrieval](reporting.md#^reporting-t-resource-live). Those assertions have passed; this proof relies on their retained evidence without duplicate provider runs. ## Test hooks @@ -31,11 +33,11 @@ Planner and baseline assertions are **contract tests** over temporary registries - A GitHub CI process invoking the local command records the workflow-supplied GitHub timing provider; a missing or `local-*` timing-provider input fails initialization. - An unreadable run event, a Blaxel `all` candidate whose event stream fails validation, and a baseline path whose current owner disagrees all fail without a partial write. - The same inventory, baseline, timing provider, commit/tree, and shard count produce byte-identical plans and the same plan ID across repeated processes. -- LPT assignment follows the descending-weight, least-loaded-bin, and tie-break rules exactly; every selected file appears once, every selected surface appears in each shard, and predicted totals equal assigned weights. -- A plan for any timing provider weighs a known file by its Blaxel median, and a baseline entry keyed by another timing provider is rejected. Unknown files receive the exact p90-based or 60-second fallback with their source recorded. +- *Superfluous in part:* the spec balances by expected duration and leaves the assignment algorithm to the code. LPT assignment follows the descending-weight, least-loaded-bin, and tie-break rules exactly; every selected file appears once, every selected surface appears in each shard, and predicted totals equal assigned weights. +- A plan for any timing provider weighs a known file by its Blaxel median, and a baseline entry keyed by another timing provider is rejected. Unknown files receive the exact p90-based or 60-second fallback with their source recorded. *Superfluous in part:* the spec requires only that an unknown file be weighted as expensive; the constants are the code's. - Inventory enumeration over a checked-out target and a temporary index loaded from that same target produces identical normalized paths. Canonical test files inside registered roots with zero or multiple registry owners fail planning; untracked files and files outside registered roots or selected owners are absent. - Real Vitest and Playwright invocations collect every assigned path exactly once and no other path; config rejection, substring broadening, and normalized-identity mismatch fail the shard. -- A real combined Vitest invocation runs only two explicitly assigned files owned by different unit surfaces and reports both files. The shared `vitestProjectsArgs` handoff writes its `test.projects` config and builds the Vitest command prefix for both local and sharded runs. The fixture configs omit their own root, so collection depends on the generated project roots. Both projects record attempts, including two ordered attempts for a deliberate fail-then-pass case enabled for this run; normalization attributes one recovered flake to its owning surface and none to the other surface. This is a seam through the real generated config, installed Vitest and per-project recorder, with no mocks. A fixture whose path makes another positional filter ambiguous is rejected before Vitest starts. Proves [combined execution](../../../specs/arch/test-runner/test-runner.md#Surface process lifecycle), [attempt recording](../../../specs/arch/test-runner/reporting.md#^vitest-attempt-recording) and [explicit file execution](../../../specs/arch/test-runner/sharded-execution.md#Explicit file execution). Evidence: `test/repo/test-runner-shard-worker.test.ts`. ^combined-vitest-projects +- A real combined Vitest invocation runs only two explicitly assigned files owned by different unit surfaces and reports both files. The shared `vitestProjectsArgs` handoff writes its `test.projects` config and builds the Vitest command prefix for both local and sharded runs. The fixture configs omit their own root, so collection depends on the generated project roots. Both projects record attempts, including two ordered attempts for a deliberate fail-then-pass case enabled for this run; normalization attributes one recovered flake to its owning surface and none to the other surface. This is a seam through the real generated config, installed Vitest and per-project recorder, with no mocks. A fixture whose path makes another positional filter ambiguous is rejected before Vitest starts. Proves [attempt recording](../../../specs/arch/test-runner/reporting.md#honest-status) and [explicit file execution](../../../specs/arch/test-runner/sharded-execution.md#plans). *Superfluous in part:* combining surfaces into one Vitest invocation is the code's. Evidence: `test/repo/test-runner-shard-worker.test.ts`. ^combined-vitest-projects - A real Playwright invocation using a generated `--test-list` runs all tests in two assigned files, including a fixture path containing regular-expression punctuation, and runs no unassigned file. - Planned-worker command construction appends the generated Playwright test list unchanged to a registry-declared Playwright command. The committed real-runner seam covers the default Playwright command; invoking the Electron wrapper with a generated list remains a boundary gap. - A shard with no assigned files for a surface does not execute its pre-command or runner; a real sentinel-writing pre-command proves the process boundary was not crossed. @@ -43,10 +45,10 @@ Planner and baseline assertions are **contract tests** over temporary registries - A collected-file mismatch is infrastructure-incomplete and cannot report a passing empty shard. - Fail-fast preserves every assigned-but-not-run task as skipped through the shard summary and provider normalization. - Infrastructure retries preserve the plan ID and assignments; combining attempts rejects a changed plan. +- **Contract — plan validation.** The worker-side check rejects a plan whose ID is not the digest of its content, or whose timing provider, tested commit, tested tree, total or index differs from the expected values, and returns a valid plan unchanged. Proves the worker's [digest check](../../../specs/arch/test-runner/blaxel-testshards.md#runs) on a plan that may cross runner revisions. *(covered by `test/repo/test-runner-shard-plan.test.ts` “applies global LPT and materializes every surface on every shard”)* ^shard-t-plan-validation - The GitHub join accepts one complete set of shard summaries with equal plan ID, timing provider, tested tree, and total; it rejects a changed plan, duplicate or missing index, and assigned-versus-collected mismatch even when every shard process exited zero. - A shard task carrying a process leak remains a non-retryable failed shard, propagates the complete leak record to each owning normalized surface, and emits one durable process-leak event without dropping native failures or recovered flakes. - Provider normalization merges assigned parts, ignores `not-assigned` parts for counts, durations, and leaks, and emits the complete timing-event hierarchy. -- Shard-count recommendations select the smallest simulated count under 120 seconds, obey the supplied capacity, and flag an atomic file over target. The baseline update reports only the Blaxel recommendation, simulated up to the Blaxel pool size. +- Shard-count recommendations select the smallest simulated count whose busiest shard has at most 120 seconds of predicted work, fall back to the supplied capacity when none does, and flag an atomic file over target. The baseline update reports only the Blaxel recommendation, simulated up to the Blaxel pool size. **Boundary statement:** real local runner processes prove explicit file selection for Vitest and the default Playwright command, report shape, and join logic. They do not prove that the Electron wrapper consumes a generated Playwright list, that a Blaxel worker accepts the transferred plan, or that independently started GitHub matrix jobs derive the same plan on hosted machines. Live Electron, Blaxel, and GitHub runs must validate those execution boundaries before activation. - diff --git a/proofs/arch/test-runner/test-registry.md b/proofs/arch/test-runner/test-registry.md deleted file mode 100644 index ff3af3aa..00000000 --- a/proofs/arch/test-runner/test-registry.md +++ /dev/null @@ -1,38 +0,0 @@ -*How the promises in Test Registry are proven.* - -# Test Registry — proof - -Proves [specs/arch/test-runner/test-registry.md](../../../specs/arch/test-runner/test-registry.md). - -## Coverage model - -Coverage declarations are carried inside the migrated assertion blocks below. - -## Assertions - -### Test assertions - -These run in-process as pure functions over the loaded registry, plus real filesystem reads (discovery walks the real repository tree, and validation cases write throwaway configs into real temp directories). There is no subprocess, network, or remote boundary in the registry, so no boundary is left unproven by mocking — the gap class that `testing-policy.md` warns about does not arise here. Current coverage lives in `test/repo/test-runner-config.test.ts`. - -- `validateRegistry` returns no errors for the repository's own registry. -- `selectSurfaces` with `suite: "e2e"` returns only `kind === "e2e"` surfaces. -- `selectSurfaces` with `package: "@telepath-computer/television-web"` returns exactly `unit:browser-app` and `e2e:browser-app`. -- `selectSurfaces` with `package: "@telepath-computer/canonical"` returns exactly `unit:canonical`. -- `owningSurfaces` maps `packages/canonical/test/build-canonical.test.ts` to `unit:canonical`. -- `selectSurfaces` with `surface: "e2e:browser-app"` returns exactly that surface. -- Every `.test.ts` / `.spec.ts` file discovered under the canonical include roots has at least one owning surface; none is unowned. -- No discovered test file is owned by more than one surface. -- `owningSurfaces` respects `excludeRoots`: a file under an excluded sub-tree resolves to the sibling surface that owns that sub-tree, not the surface that excluded it. -- `selectSurfaces` throws when `--file` is combined with a selector whose surfaces do not own the file, and the error names the owning surface(s). -- `validateRegistry` reports an unregistered runner config placed under a canonical include root. -- **Contract:** `validateRegistry` accepts a surface with two valid Vite service declarations and reports each invalid service declaration listed under [#Discovery and validation](../../../specs/arch/test-runner/test-registry.md#Discovery and validation). The test uses temporary config-file fixtures and no mocked mechanism. -- **Contract:** loading the repository registry preserves the dynamic-service fixture's ordered declarations unchanged. -- The artifact e2e surface declares two independently published services for its view and host origins, with no deterministic port-window helper. -- The desktop e2e surface declares its fixture Vite service through `TV_DESKTOP_E2E_URL`, and the desktop config and tests contain no fixed fixture port. -- `validateRegistry` ignores a runner config placed outside the canonical include roots (e.g. under `prototypes/` or `experiments/`). -- The `all` suite resolves to the unit and non-agent e2e execution groups, in `order`, includes `unit:build-config`, and the `unit:workspaces` group contains the package unit surfaces in declared order. -- Every lifecycle fault surface retains `kind: "experiment"` and `tag:isolated-github-only`; none is selected by `all`. -- The `unit:build-config` surface owns `test/repo/build-config-integrity.test.ts`; `unit:root` excludes that file. -- The `telemetry-posthog-roundtrip` suite resolves to the `telemetry-posthog-roundtrip:integration` surface, and that surface owns `packages/server/test/telemetry-posthog.integration.test.ts`; `unit:server` excludes that file. -- The `daemon-acceptance` suite resolves to `daemon-acceptance:cli`, and that surface exclusively owns `test/node/daemon-acceptance.test.ts`; `e2e:node` excludes that file. The surface declares the `daemon-test-host` preflight and is absent from `all`. - diff --git a/proofs/arch/test-runner/test-runner.md b/proofs/arch/test-runner/test-runner.md index 44eef1fc..e6241374 100644 --- a/proofs/arch/test-runner/test-runner.md +++ b/proofs/arch/test-runner/test-runner.md @@ -1,4 +1,4 @@ -*How the promises in Test Runner are proven.* +*Canonical CLI guardrails, local ownership, native entrypoints, retry policy, the test registry, and preflight checks proven through real process boundaries and declared downstream substitutions.* # Test Runner — proof @@ -6,28 +6,55 @@ Proves [specs/arch/test-runner/test-runner.md](../../../specs/arch/test-runner/t ## Coverage model -Coverage declarations are carried inside the migrated assertion blocks below. Provider-selection assertions honor the spec’s directive to spawn the canonical CLI through real subprocesses and filesystem marker lookup: unmarked selection never reaches remote preflight, while marked selection consumes declared preflight results. +Canonical command decisions are exercised by spawning the real CLI, observing exit status and output, and using real temporary home markers. Dry-run and remote-preflight hooks replace only downstream execution or external checks; they prove neither native execution nor Blaxel access. Unimplemented committed-test obligations are marked *(test to be written)*. Live-run obligations describe the required evidence; each affected contribution records its runs and reports. -The [combined-project seam](sharded-execution.md#^combined-vitest-projects) crosses the shared `vitestProjectsArgs` handoff that writes the combined config and selects it for the Vitest command used by local and sharded runs. It exercises project roots, exact file selection and attempt recording. The local wrapper's pre-commands, execution without an explicit file list and result splitting remain within the real local end-to-end boundary gap below; the shared-helper seam does not claim that whole path. +Local file eligibility uses working-tree fixtures, including untracked files; remote eligibility uses real Git commit inventories. Separate real Vitest and Playwright handoffs establish that an admitted file does not widen during native collection. The marked-host local limit and the zero-retry rule use the same one-file check. -Two test-infrastructure rules concern the product's `tv serve` and are proven where the product is. The recipe for a test that spawns `tv serve` gives the server a temporary [Television home](../../../specs/product/cli.md#^cli-home) whose config file sets port `0`, passes that home with `--home` to the server and every client command, and passes the acquired port as client `--port`. It is a way of writing tests, honored by the design of the tests that spawn the server: the [product CLI proof](../../product/cli.md#Acceptance criteria: CLI spine) declares it for its acceptances, and review checks it elsewhere. No separate assertion is ordered for it; the repository scan below guards only numeric test-service URLs. The daemon-acceptance surface's form of installation (a temporary home, a released stable port written with `tv config set port`, then `tv --home serve --persist`) is carried by the same proof's [home-based persisted-install assertion](../../product/cli.md#^cli-ac-persist-home-install). +Nested canonical fixture runs supply a gated private lock path by default; the read-only production-location and hook-validation cases supply the specific environment under assertion. Mutex acceptance tests run competing canonical processes at private paths; acquisition, process identity, sharing, release, and refusal are real. The location seam observes the real CLI's selected acquisition path without taking the host lock. No test claims coordination with runners predating the mutex or with another operating-system user. Ordinary fixtures are small test programs that signal readiness and wait until the test releases them. Routine coverage includes reclaiming a private record naming an already-exited process with no survivors. Destructive orphan, leak, and forced-death scenarios run only in the isolated GitHub lifecycle surface, with its existing cleanup requirements, as the spec's testing guidance directs. Each ordinary mutex test file supports validation through a one-file local run; destructive cases remain separate. Linux evidence is not evidence for the macOS process backend. -## Test hooks +Verify orchestration uses a small temporary checkout containing the production runner and authored registry/test fixtures. Lightweight phase programs may replace downstream lint, type-check, package-manifest, and final vibe checks so tests can observe mutex lifetime without nesting the full gate. Those replacements forfeit validation of the replaced tools; phase selection, the verify process, its canonical test-phase child, and native fixture execution remain real. A fixture verify is never reported as repository verification. + +Native refusal tests load registered configs through the installed native tools. Each way the runner starts a native tool—local, planned, targeted, and non-plan diagnostic—is checked to pass the context the config check looks for. Recorded live Blaxel runs cover remote launch, file collection, and retry handoffs; these are validation evidence for changes affecting those paths, not tests that dispatch Blaxel from inside the regular suite. Attestation's [retry eligibility proof](attestation.md#^attest-retry-options) owns conversion of the same selected policy into publication facts. + +Written guidance about file loops, safe pushes, override judgment, and host-health priorities is checked by contribution review under the [testing-policy proof](../testing-policy.md). Help and actionable refusal output are runtime behavior and have CLI assertions below. + +The [combined-project seam](sharded-execution.md#^combined-vitest-projects) crosses the shared `vitestProjectsArgs` handoff that writes the combined config and selects it for the Vitest command used by local and sharded runs. It exercises project roots, exact file selection and attempt recording. The local wrapper's pre-commands, execution without an explicit file list, and result splitting are not established by that seam or by the single-file CLI finalization evidence below. + +The Blaxel exception to the ordinary live-owner protection of [processes started by tests](../../../specs/arch/test-runner/test-runner.md#processes-started-by-tests), [abandoned-run recovery](../../../specs/arch/test-runner/blaxel-testshards.md#abandoned-runs), is proven by [the worker guard seam](blaxel-testshards.md#^blaxel-t-worker-guard) and [automatic recovery acceptance](blaxel-testshards.md#^blaxel-t-recovery-live). The same provider proof carries preservation of the outer shard attempt token through nested surface scopes. These assertions do not weaken or relocate the existing ordinary-owner cleanup tests, including fixtures restricted to isolated GitHub execution. +Two test-infrastructure rules concern the product's `tv serve` and are proven where the product is. *Superfluous in part:* the spec states only the daemon-acceptance port exception; the recipe below is kept as the product CLI proof's test-writing convention. The recipe for a test that spawns `tv serve` gives the server a temporary [Television home](../../../specs/product/cli.md#^cli-home) whose config file sets port `0`, passes that home with `--home` to the server and every client command, requests startup links with `--print-links`, and passes the acquired port as client `--port`. It is a way of writing tests, honored by the design of the tests that spawn the server: the [product CLI proof](../../product/cli.md#Acceptance criteria: CLI spine) declares it for its acceptances, and review checks it elsewhere. No separate assertion is ordered for it; the repository scan below guards only numeric test-service URLs. The daemon-acceptance surface's form of installation (a temporary home, a released stable port written with `tv config set port`, then `tv --home serve --persist`) is carried by the same proof's [home-based persisted-install assertion](../../product/cli.md#^cli-ac-persist-home-install). + +The test registry, preflight checks and per-test flaky annotations are proven under [Test registry](#test-registry), [Preflight checks](#preflight-checks) and [Retry selection and execution](#retry-selection-and-execution). Registry assertions are pure functions over the loaded registry with real filesystem and Git reads. Preflight assertions are contract tests of host validation plus seams against a real disposable child and the host's process tables. + +Assertions marked *superfluous* cover behavior the current spec leaves to the code. They are kept as coverage of that code and order nothing further. + +## Test hooks `TV_TEST_RUNNER_SELFTEST=1` permits the guarded self-test seams that separately check for it; the variable changes no behavior by itself. Every other value leaves those seams disabled. A seam input supplied without it is ignored or refused as the seam's owning spec states. Production leaves it unset. -`TV_TEST_RUNNER_DRY_RUN=1` lets a provider shortcut print its resolved provider, suite, surfaces, and files after selection and guardrail checks without running preflight or tests. It requires `TV_TEST_RUNNER_SELFTEST=1`, and `verify` rejects it because verification requires a finalized run. Production leaves it unset. +`TV_TEST_RUNNER_DRY_RUN=1` lets a provider shortcut print its resolved provider, suite, surfaces, and files after selection and guardrail checks without running preflight or tests. For local execution it also reports the resolved path that would be passed to mutex acquisition, stopping before acquisition. It requires `TV_TEST_RUNNER_SELFTEST=1`, and `verify` rejects it because verification requires a finalized run. Production leaves it unset. + +`TV_TEST_RUNNER_LOCK_PATH` supplies a private lock location only with `TV_TEST_RUNNER_SELFTEST=1`; production resolves the location from operating-system identity outside checkouts. Independent scenarios use separate paths, while deliberate contenders share one. This substitutes location only, never the acquisition or lifetime mechanism. The CLI warns when this hook is active and refuses an ungated hook. + +`TV_TEST_RUNNER_CONTEXT=1` is the invocation signal production launchers emit for registered native configs; declared native fixtures may supply it. Direct-invocation fixtures remove it from their child environment. Launcher assertions start without an inherited signal and preserve what the real launcher emits, so the outer test run cannot supply the answer. This signal grants no canonical mutex exemption and is not an authentication mechanism. + +Temporary registry entries, test files, Vite plugin lifecycle callbacks, and cooperative readiness/release files are authored fixtures. Verify's downstream phase substitutes are declared in the coverage model; they do not replace the canonical test-phase command or invent a passing full-gate result. `isCursorAgentRuntime` accepts `env` to supply the environment used for Cursor-agent classification. `stripCursorAgentEnv` accepts `env` to supply the child environment it copies and filters. Production classifies `process.env` at canonical CLI startup, and the Electron wrapper filters its inherited child environment. `createSurfaceSupervisor` accepts `termGraceMs` to set the wait after cleanup signals for an active surface scope. `reapStaleOwnerProcesses` accepts `termGraceMs` to set the wait while removing abandoned surface scopes. Production omits both controls and uses `TEST_PROCESS_TERM_GRACE_MS`. +`TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT` supplies one failed local result as `:`. `TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT` supplies one failed remote result as `::`, or a passing result as `:passed`. Passing results are permitted only for automatic provider resolution in `verify --plan`; other callers refuse them before dispatch. `runRemotePreflight` accepts `allowPassingSelftest` for that diagnostic caller, defaulting to false. Both environment controls require `TV_TEST_RUNNER_SELFTEST=1`; otherwise the runner rejects them. These controls replace check execution, so they prove only how the runner handles preflight outcomes. Production sets none of them. + +The `electron` option to `runPreflights` accepts `root` to select the package and planner workspace and `plan` to supply an `ElectronE2EPlan`. The `plan` override replaces the shared environment planner, so it covers only how the Electron check handles planner results. Production supplies neither override and uses the current workspace and the real planner. `readPostHogTestReadKey` accepts `env` to supply the environment source and `root` to select the `.env` lookup directory; production calls it without either override. + +`validateProcessLifecycleHost` accepts `platform` to select the host contract, `procfsReadable` to state whether Linux process identity is readable, and `commandAvailable` to report the required command capabilities. `runProcessLifecycleProbe` accepts `platform` to select the backend and `runCommand` to supply its command runner. The `runCommand` override replaces command execution, so it covers probe-result handling rather than host capability. Production calls both functions with the detected platform and the host's actual process-inspection capabilities. + ## Assertions ### Test assertions -The runner's own tests run the **real `scripts/test/cli.mjs` process** with `spawnSync` and assert exit codes and stdout/stderr — the subprocess/CLI boundary that [testing-policy.md](../../../specs/arch/testing-policy.md) requires for process behavior. The guardrail and provider-resolution decision chains execute for real in that process; self-test-gated env seams stand in only for the expensive downstream that sits *outside* the asserted decision (actually launching a suite, or running the real remote preflight), never inside the chain under assertion. Current coverage lives in `test/repo/test-runner-guardrails.test.ts`. +The runner's own tests run the **real `scripts/test/cli.mjs` process** and assert exit codes and stdout/stderr — the subprocess/CLI boundary that [testing-policy.md](../../../specs/arch/testing-policy.md) requires for process behavior. The guardrail and provider-resolution decision chains execute for real in that process; self-test-gated env seams stand in only for the expensive downstream that sits *outside* the asserted decision (actually launching a suite, or running the real remote preflight), never inside the chain under assertion. Standing coverage lives in `test/repo/test-runner-guardrails.test.ts`. The shortcut-admission bullets below use unmarked homes; they exercise the full-`all` shortcut guardrail, not marked-host file admission. - A bare provider shortcut (`local` | `blaxel`) is refused with exit `2` and the broad-run message. - An explicit `--suite all` shortcut is refused with exit `2`. @@ -37,12 +64,13 @@ The runner's own tests run the **real `scripts/test/cli.mjs` process** with `spa - Targeted selectors (`--file`, `--surface`, `--package`, `--runner`, `--tag`) and the `--suite unit` / `--suite e2e` / `--suite telemetry-posthog-roundtrip` / `--suite daemon-acceptance` selections pass the guardrail unless a selected surface carries an execution-placement tag. - `daemon-acceptance` fails preflight before its build when `TV_DAEMON_TEST_HOST` is not exactly `1`, and passes that check when the designated-host acknowledgement is present. - `isolated-github-only` surfaces are rejected before preflight for developer local and every Blaxel path; the exact dedicated GitHub workflow identity plus explicit opt-in permits only its local execution path. +- **Acceptance — harness placement configuration.** The real canonical CLI accepts explicitly named coordinator and private-pool environment values for local harness selection without prescribing personal resource names. Missing configuration, an inherited shard owner, and remote driver selection are refused before preflight. Authored environment values and the existing guarded dry-run hook replace only downstream execution; this proves the CLI decision, not actual sandbox placement. The [live harness](blaxel-testshards.md#^blaxel-t-retrieval) checks coordinator labels and crosses the configured private pool. Proves [harness placement](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-harness-placement). (policy-grade tests: `test/repo/test-runner-guardrails.test.ts` “accepts a configured Blaxel harness coordinator and private pool without fixed names” and “refuses missing Blaxel harness configuration and leased or remote placement before preflight”) ^t-blaxel-harness-placement - `--suite all` includes `unit:build-config`, which validates the ordinary CLI output and an isolated release-configured `--outfile` bundle without network or secrets, as proven by [the build seam](../telemetry/sink.md#^t-build-config-integrity). -- **Acceptance** (real canonical CLI process, real temporary home with no marker): default `verify` and explicit `--provider auto` select local phases without bypass or Blaxel preflight/fallback diagnostics. The [preflight hook](preflight.md#Test hooks) `TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT` is supplied with `TV_TEST_RUNNER_SELFTEST` explicitly removed from the child environment and must remain untouched: reaching remote preflight would fail the command, so a successful local plan proves it was not consulted. The same unauthorized input on a marked home must fail with the remote hook’s authorization error, confirming the detector is active. No remote mechanism is mocked on the unmarked path; `--plan` proves selection and phase construction, not suite execution. Proves [provider resolution](../../../specs/arch/test-runner/test-runner.md#Verify orchestration). Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-local-verify-default -- **Acceptance** (real CLI process and temporary marked home; remote checks replaced by the declared preflight hook): a passing remote result makes default `verify --plan` select Blaxel phases. This proves provider selection, forfeiting real service and Git checks to the [preflight proof](preflight.md#Assertions), which records their committed boundary-test gap. The [live-evidence directive](../../../specs/arch/test-runner/preflight.md#Testing) governs live validation. Passing self-test results are refused outside `verify --plan`, before any remote dispatch. Proves [provider resolution](../../../specs/arch/test-runner/test-runner.md#Verify orchestration). Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-marked-verify-default -- **Acceptance** (real CLI process and temporary marked home; declared failed remote-preflight result, with the same forfeit as above): failed remote preflight refuses verification with exit `2`, the failing check, remediation, and the explicit `local --allow-extreme-inefficiency` escape hatch. Passing the bypass flag without selecting `local` does not enable fallback. Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-marked-verify-refusal -- `verify local` (positional or `--provider local`) when `~/.tvdev-use-blaxel` exists is refused with exit `2` unless `--allow-extreme-inefficiency` is passed. -- **Acceptance** (real CLI process, real temporary home directories and marker files): marker presence enables protection regardless of contents, marker absence permits explicit local verification and explicit Blaxel plans, and `AGENT_BLAXEL_GUARDRAILS` does not affect protection. The default-provider cases are covered by [unmarked selection](#^t-local-verify-default) and [marked selection](#^t-marked-verify-default). The same marker leaves targeted local commands available. Proves [host marker](../../../specs/arch/test-runner/test-runner.md#^blaxel-host-marker). Remote preflight is supplied through the existing self-test seam, forfeiting real service access as documented below; marker lookup and provider selection are real. Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-blaxel-host-marker +- **Acceptance** (real canonical CLI process, real temporary home with no marker): default `verify` and explicit `--provider auto` select local phases without bypass or Blaxel preflight/fallback diagnostics. The [preflight hook](#test-hooks) `TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT` is supplied with `TV_TEST_RUNNER_SELFTEST` explicitly removed from the child environment and must remain untouched: reaching remote preflight would fail the command, so a successful local plan proves it was not consulted. The same unauthorized input on a marked home must fail with the remote hook’s authorization error, confirming the detector is active. No remote mechanism is mocked on the unmarked path; `--plan` proves selection and phase construction, not suite execution. Proves [provider resolution](../../../specs/arch/test-runner/test-runner.md#verify). Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-local-verify-default +- **Acceptance** (real CLI process and temporary marked home; remote checks replaced by the declared preflight hook): a passing remote result makes default `verify --plan` select Blaxel phases. This proves provider selection, forfeiting real service and Git checks to [preflight checks](#preflight-checks), which records their committed boundary-test gap. The [live-evidence directive](../../../specs/arch/test-runner/blaxel-testshards.md#testing-guidance) governs live validation. Passing self-test results are refused outside `verify --plan`, before any remote dispatch. Proves [provider resolution](../../../specs/arch/test-runner/test-runner.md#verify). Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-marked-verify-default +- **Acceptance** (real CLI process and temporary marked home; declared failed remote-preflight result, with the same forfeit as above): failed remote preflight refuses verification with exit `2`, the failing check, remediation, and the explicit `local --against-test-guidance-broad-local-run` escape hatch. Passing the bypass flag without selecting `local` does not enable fallback. Proves [provider resolution](../../../specs/arch/test-runner/test-runner.md#verify). Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-marked-verify-refusal +- **Acceptance** (real CLI, real marked home): explicit local verify, positional or `--provider local`, refuses with exit `2` unless `--against-test-guidance-broad-local-run` is passed; an admitted plan forwards the option to its broad child. Proves [local verify](../../../specs/arch/test-runner/test-runner.md#verify). Plans forfeit actual lock ownership to [verify lifetime](#^t-runner-verify-lock). Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-marked-local-verify +- **Acceptance** (real CLI process, real temporary home directories and marker files): marker presence enables protection regardless of contents, marker absence permits explicit local verification and explicit Blaxel plans, and `AGENT_BLAXEL_GUARDRAILS` does not affect protection. The default-provider cases are covered by [unmarked selection](#^t-local-verify-default) and [marked selection](#^t-marked-verify-default). A marked home admits only explicit one-file local selections without the broad-local override; an unmarked home retains wider local selections. Proves [host marker](../../../specs/arch/test-runner/test-runner.md#^blaxel-host-marker). Remote preflight is supplied through the existing self-test seam, forfeiting real service access as documented below; marker lookup and provider selection are real. Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-blaxel-host-marker - **Acceptance** (real script process in a temporary directory): the vibe-mode check exits nonzero and names vibe mode when `specs/vibe-waiver.md` exists, and exits zero otherwise; `verify --plan` lists it as the last phase for both providers. Proves [vibe-mode check](../../../specs/arch/test-runner/test-runner.md#^verify-vibe-mode). A full verify on a vibe branch is not exercised; the phase runner is the same one every other phase uses. Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-verify-vibe-mode - `verify` rejects a positional provider given together with `--provider` (exit `2`). - `verify` rejects an invalid positional provider (exit `2`); `gha` is invalid everywhere a provider is named — shortcut, positional, and `--provider` (exit `2`). Acceptance-shaped: the real `scripts/test/cli.mjs` process, real argv, asserted exit codes and stderr. @@ -52,6 +80,58 @@ The runner's own tests run the **real `scripts/test/cli.mjs` process** with `spa - The CLI help documents `--no-publish` as the attestation opt-out for a qualifying Blaxel verify. The removed opt-in spelling is rejected as a usage error rather than silently accepted. - A repository test reads .github/workflows/ci.yml and asserts that the pull request end-to-end shard step invokes scripts/run-test-shard.mjs with --test-retries 2. +### File selection and guidance at the command boundary + +These assertions derive from [local file admission](../../../specs/arch/test-runner/test-runner.md#^local-one-file), [selectors](../../../specs/arch/test-runner/test-runner.md#commands), and [retries](../../../specs/arch/test-runner/test-runner.md#retries). CLI assertions spawn the production entrypoint. A dry run may stop after the decision; it never substitutes the decision or supplies a fabricated file count. + +- **Contract — file boundary.** The resolver reads real authored working-tree files and real Git commit inventories. It distinguishes an explicit one-file selection from directories and filters matching several files, respects ownership/exclusions and compatible co-selectors, and retains errors for empty, unowned, or conflicting selections. Local input includes newly created/untracked files; remote input uses the selected commit even when the working tree differs. Real CLI checks show that `list` and standalone preflight use the same file resolution as execution. A filename that is also a substring or regular-expression match for another file must not be mistaken for a safe native filter. This proves inventory/selection breadth, not native collection, which is covered by [the native-file seam](#^t-runner-exact-native-file). Evidence: `test/repo/test-runner-file-selection.test.ts` and `test/repo/test-runner-guardrails.test.ts`. ^t-runner-file-boundary +- **Acceptance — marked local admission.** A real marked home permits `local --file ` with optional grep and compatible co-selectors. Surface, suite, package, runner, and tag selections without `--file` refuse with exit `2`, including a one-file surface; a multi-file file filter also refuses before preflight/test work. The same wider selections remain available on an unmarked home. Real temporary marker files vary presence and contents; dry-run only replaces admitted downstream execution. Ordinary invalid selections still produce selection errors. Evidence: `test/repo/test-runner-guardrails.test.ts` and `test/repo/test-runner-file-selection.test.ts`. ^t-runner-file-admission +- **Seam — admitted file to native collection.** The real canonical CLI launches installed Vitest and Playwright with an admitted file from an authored fixture selection, including an adversarial sibling path. Native results show exactly the admitted file and optional title subset; no sibling test executes. The registry, filesystem, command construction, native config, and runner are real. This crosses the local handoff omitted by the file-boundary contract. Planned-worker file collection is covered by [sharded execution](sharded-execution.md#^combined-vitest-projects); targeted remote collection is covered by the adversarial sibling in the [live targeted run](#^t-runner-remote-handoff). Evidence: `test/repo/test-runner-file-selection.test.ts`. ^t-runner-exact-native-file +- **Acceptance — actionable refusal.** A marked-host broad-local refusal gives a shell-usable equivalent Blaxel command when that selection is supported, preserving selectors and grep, and states commit/push first. Unsupported combinations explain the limitation without inventing equivalence. Full verify points to remote verify. PostHog, daemon, and agent selections instead give their local prerequisite and a supported one-file form or broad-local override, regardless of how the surface was selected. Output discourages serial file loops and leaves Git and remote execution untouched. Fixtures include shell metacharacters in arguments; a shell with an inert argv-recording command may check quoting, forfeiting remote execution to the live provider evidence. Evidence: `test/repo/test-runner-guardrails.test.ts`. ^t-runner-local-redirection +- **Acceptance — independent options.** Real CLI decisions show that the broad-local flag lifts only the marked-host file limit; the contention flag lifts only an occupied mutex; the retry flag lifts only broader zero-retry refusal; `--force` lifts only the full-`all` shortcut guardrail. A full-`all` marked local shortcut needs `--force` and the broad-local flag, and an occupied mutex additionally needs its own bypass or a wait. Use private real ownership for contention cases and dry-run for selection-only cases. No Cartesian product or broad suite execution is needed. Covered by `test/repo/test-runner-guardrails.test.ts` and `test/repo/test-runner-mutex.test.ts`. ^t-runner-independent-options +- **Acceptance — usage and help.** The real CLI refuses `--allow-extreme-inefficiency` with exit `2` and names its replacement; an unknown `--against-test-guidance` suffix also fails visibly. Values attached to the broad-local and broad-zero flags are refused rather than treating `=false` as consent. Help gives the one-file path, pushed remote path, wait option, the independent overrides, the narrow/broad zero-retry distinction, and the testing policy's permission distinction. Assert the actionable choices rather than a prose snapshot. *Superfluous in part:* the spec does not name `--allow-extreme-inefficiency`; its refusal is kept as coverage of the code. Proves [local help](../../../specs/arch/test-runner/test-runner.md#shared-development-hosts) and [native help](../../../specs/arch/test-runner/test-runner.md#native-test-entrypoints). Covered by `test/repo/test-runner-guardrails.test.ts`, `test/repo/test-runner-mutex.test.ts`, and `test/repo/test-runner-native-entrypoints.test.ts`. ^t-runner-guidance-usage + +### Local mutex + +These assertions prove [same-user host coordination](../../../specs/arch/test-runner/test-runner.md#^local-test-mutex). Except for production-location resolution, they use the gated private lock path. Cooperative fixtures expose readiness and wait for an explicit release, so observing a contender exit while the holder is still blocked proves nonblocking refusal without a fragile elapsed-time threshold. + +- **Seam — production location.** Fresh canonical CLI processes run local dry-run selections with no private-path hook, from different working directories and with ordinary HOME/TMP overrides. They report the path selected for acquisition through the real CLI's production location resolution. That path stays the same, follows the pinned location convention, and is outside those checkout directories. The named handoff is CLI location resolution to mutex acquisition: dry-run reports the same resolved acquisition argument that live execution passes unchanged, not a separately computed diagnostic. The private-path lifetime tests exercise acquisition at that handoff. No process in this read-only check acquires or writes the live host lock. Covered by `test/repo/test-runner-mutex.test.ts`. ^t-runner-lock-location +- **Contract — compatible ownership records.** A fixture pins the ownership representation written and read by production lock instances, so incompatible changes between participating checkouts are visible. It supplies authored record fields, not a fabricated process-liveness result. Real cross-process reading and identity checks are exercised by contention and recovery below; the contract alone claims no exclusion or cleanup. Covered by `test/repo/test-runner-mutex.test.ts`. ^t-runner-lock-record +- **Acceptance — dead owner without survivors.** A private lock record names the captured identity of a real fixture process that has exited normally and left no owned processes. The next canonical local run checks real process liveness, performs the real stale-owner scan, reclaims the lock, and executes its small native fixture. An invocation using the wait option can likewise acquire this abandoned lock. The authored record uses the representation pinned above; it replaces neither liveness nor recovery. This runs in the regular suite and creates no orphan or deliberately stale test process. Covered by `test/repo/test-runner-mutex.test.ts`. ^t-runner-lock-dead-owner +- **Acceptance — contention.** One canonical local fixture holds the lock while another exits `2`, before its preflight or test sentinel runs. The message identifies the real holder PID/process identity, command, and checkout, mentions waiting and the named contention bypass, explains the risk, and cautions against stopping the holder. Marked and unmarked diagnostics provide their applicable next steps. Simultaneous acquisition attempts admit only one holder. All lock/process mechanisms are real; workload is a small cooperative native fixture. Covered by `test/repo/test-runner-mutex.test.ts`. ^t-runner-lock-contention +- **Acceptance — ordinary lifetime and release.** A local fixture remains protected through test completion and registered service teardown. An authored Vite plugin's normal close callback runs a contender and records its refusal before returning; no artificial shutdown delay or resistant process is needed. After success, native failure, and preflight failure, a later clean invocation acquires the same lock without stale-process failure. Real fixture events identify the execution/teardown boundary; native success/failure can reuse the reporting cases cited below. The declared failed-preflight hook may supply that downstream failure, forfeiting capability checks to the preflight proof. This proves ownership lifetime, not merely creation of a holder file. Covered by `test/repo/test-runner-mutex.test.ts` and `test/repo/test-runner-verify-mutex.test.ts`. ^t-runner-lock-release +- **Acceptance — verify ownership.** The real verify process takes the private lock before its first phase and retains it until its final phase and cleanup. Its actual canonical test-phase child runs without self-contention; an unrelated or arbitrarily nested canonical run sharing the path still refuses. On a marked fixture home, verify passes the broad-local and applicable contention options to its child. Small authored checkout/registry fixtures and downstream phase substitutes are declared in the coverage model; the lock and direct test-child handoff are never replaced. Proves [verify](../../../specs/arch/test-runner/test-runner.md#verify) as well as the mutex. Covered by `test/repo/test-runner-verify-mutex.test.ts`. ^t-runner-verify-lock +- **Acceptance — waiting.** A local run with `--wait-for-local-test-mutex` identifies the holder and cancellation route, remains blocked with no preflight/tests started, then acquires after release and completes under the ordinary lifetime rules. The same path in local verify starts no phase until acquisition. Usage errors, broader marked selections, placement failures, and a broad zero-retry request without its override refuse before waiting. Combining wait and the contention bypass exits `2`. No test assumes FIFO order or an implicit timeout. Covered by `test/repo/test-runner-mutex.test.ts` and `test/repo/test-runner-verify-mutex.test.ts`. ^t-runner-lock-wait +- **Acceptance — cancelled wait.** Sending SIGINT or SIGTERM to the waiting canonical process exits `130` or `143`, respectively, starts none of its phases/tests, and leaves the cooperative holder alive and protected. The test owns both processes and releases the holder normally; it creates no orphaned test process or resistant child. Proves waiting and [signal exit codes](../../../specs/arch/test-runner/test-runner.md#commands). Covered by `test/repo/test-runner-mutex.test.ts` and `test/repo/test-runner-verify-mutex.test.ts`. ^t-runner-wait-cancel +- **Acceptance — contention bypass.** With a free mutex the bypassing run still takes it and excludes an ordinary contender. With an occupied mutex it runs concurrently without replacing or releasing the original ownership; after it exits, an ordinary contender remains refused until the original holder finishes. Small cooperative native fixtures make the overlap observable without broad host load. Covered by `test/repo/test-runner-mutex.test.ts`. ^t-runner-lock-bypass +- **Acceptance — informational and remote paths.** While a private mutex is occupied, `list`, help, local standalone preflight, `verify --plan`, and self-test dry-run do not contend for it; dry-run still enforces selection rules. On a marked home, a broader `list` selection succeeds and a broader standalone preflight reaches its checks without a one-file refusal. A real `blaxel` invocation with dry-run unset reaches remote preflight while the private lock is held. The declared failed local/remote preflight hooks stop these commands there, forfeiting actual capability checks to [preflight checks](#preflight-checks) and dispatch to the live remote evidence below. The asserted failed-preflight diagnostic distinguishes reaching that boundary from a mutex or selection refusal. This regular-suite assertion proves the submitting-host mutex exemption at the admission boundary. Covered by `test/repo/test-runner-mutex.test.ts`. ^t-runner-lock-exemptions +- **Acceptance — isolated test locations.** A hook without the exact self-test gate exits `2`; the gated hook warns and uses the private path with real locking. Independent fixture locations do not contend, and intentional contenders on one path do. Nested reporting, toolchain, and lifecycle CLI tests explicitly use this hook and still reach their asserted native/preflight/lifecycle outcomes. This proves fixture isolation under an outer local run or worker context, not an ancestry exemption. Covered by `test/repo/test-runner-mutex.test.ts`, `test/repo/test-runner-reporting.test.ts`, `test/repo/node-toolchain.test.ts`, and the isolated lifecycle orchestrator. ^t-runner-lock-selftest +- **Seam — interruption and stale recovery, isolated GitHub only.** The canonical mutex handoff to the existing process-lifecycle machinery uses real processes, owner tokens, process groups, signals, process inspection, and listeners. An interrupted active run stays protected through descendant cleanup. If a verify parent dies while its direct test-phase supervisor lives, a contender still refuses; a waiter remains blocked. If ownership is abandoned, a later invocation establishes process exit or reaps surviving owned processes before admission. Finding stale test processes cleans them and fails that invocation, including a waiter whose holder died; a later clean invocation proceeds. A live identity is not reclaimed because its record is old. Reuse the lifecycle fault surface, with independent cleanup and timeout; this assertion runs neither locally on a developer host nor on Blaxel. Covered by `test/runner-fixtures/process-lifecycle/orchestrator/mutex.test.ts` on the Linux backend in the authorized isolated GitHub workflow; the contribution validation record retains the run and artifacts. No macOS execution is claimed. ^t-runner-lock-faults + +### Native entrypoints + +These assertions prove [the native-entrypoint contract](../../../specs/arch/test-runner/test-runner.md#native-test-entrypoints). Registered configurations come from the registry, including entries outside the canonical discovery roots. No assertion attempts to authenticate invocation context or intercept arbitrary unregistered native commands. + +- **Acceptance — native refusal.** Real Vitest and Playwright processes loading registered configs without runner context refuse before any test body starts, on marked and unmarked fixture homes. Their diagnostic gives the canonical one-file form, applicable broader remote/push guidance, and the exact direct-native override with the protections it skips. A config-loading sweep through the native tools confirms that every separately wired registered entrypoint reaches the shared check, before unrelated service-setup errors. The sweep uses a files-only listing or verified no-match file filter that loads each real config without executing tests even when the refusal is absent; a red assertion or regression cannot launch the registered suites. Keep the sweep in one independently runnable test file and measure its duration during implementation; host-marker message variants need only one config per native tool. No browser or full suite is needed for refusal. Covered by `test/repo/test-runner-native-entrypoints.test.ts`. ^t-runner-native-refusal +- **Acceptance — deliberate native use.** The exact `TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN=1` permits real native fixture execution for each tool; absent, `0`, or `true` do not. The fixture supplies its required configuration/services. A missing required service setting still fails after the entrypoint check is lifted. Direct execution takes no canonical mutex or marked-host file protection; a small multi-file fixture and occupied private canonical lock make this departure observable. This proves the documented override, not permission to use it for broad local task validation. Covered by `test/repo/test-runner-native-entrypoints.test.ts`. ^t-runner-native-override +- **Seam — launch context.** The real local canonical launcher, planned shard worker, targeted supervised-command path, and non-plan diagnostic shard path each pass their own invocation context to a guarded native config and reach a small test fixture without the direct-native override. Declared nested runner fixtures remain admitted. Test each separately wired launcher; context-only unit tests cannot replace those crossings. Planned and targeted remote deployment are additionally covered by the live handoff below; the non-plan path retains its preflight exception. Covered by `test/repo/test-runner-native-entrypoints.test.ts`. ^t-runner-native-context + +### Retry selection and execution + +These assertions prove [retry policy](../../../specs/arch/test-runner/test-runner.md#retries); they share the file-boundary contract rather than inventing another meaning of one file. + +The per-test annotation's conventions, investigated flakiness only and on the one offending case, are checked by review rather than by a test. Its separate retry behavior is part of the attempts seam below, and the recovered-flake normalization it relies on is proven in [reporting](reporting.md). The `FLAKY_TEST_RETRIES` resolver, including its default of 5, has no dedicated unit test. + +- **Acceptance — retry admission.** The real CLI accepts plain `--retries 0` for one explicit file with optional grep, on either host kind and provider; file eligibility is evaluated against the actual local working tree or selected remote commit. It refuses zero for a surface or wider selection, multi-file filters, grep without a file, and verify, unless `--against-test-guidance-turn-flakes-into-failures-on-broad-runs` is present. The refusal recommends the default budget, explains the one-file form, and names the override. The rule also applies to unit-only selections despite their default zero budget. The override works alone or with explicit zero, conflicts with a positive count, and lifts no other guardrail. Negative/non-integer counts refuse. Positive counts remain allowed; unit-only selections report that they are ignored. Verify validates before phases and forwards defaults, positive overrides, or the deliberate zero policy. Real fixture files/commits and CLI dry-run or verify plans may bound downstream work; actual retry behavior is proven by the following seams. Evidence: `test/repo/test-runner-guardrails.test.ts` and `test/repo/test-runner-file-selection.test.ts`. ^t-runner-retry-admission +- **Seam — resolved budget to native attempts.** The canonical local path and worker adapters pass resolved budgets to real Vitest and Playwright fixtures. Fail-then-pass cases show default non-unit retries and recovered-flake reporting, while either permitted zero form makes the same first failure final. A positive override reaches the native runner, unit defaults remain zero, and per-test flaky annotations retain their separate retry behavior even with blanket zero. This uses authored small tests and real native attempts, not a mocked successful command; normalization composes with [reporting](reporting.md) and infrastructure retry behavior with [sharded execution](sharded-execution.md). Evidence: `test/repo/test-runner-file-selection.test.ts`. ^t-runner-retry-attempts +- **Seam — live canonical Blaxel handoff (recorded run evidence).** Changes affecting these paths are validated on a committed, pushed revision through the real Blaxel API and worker transport, under the existing [live provider](../../../specs/arch/test-runner/blaxel-testshards.md#testing-guidance) and [plan-transfer](../../../specs/arch/test-runner/sharded-execution.md#testing-guidance) directives. The required passing full Blaxel verify supplies planned-worker native-context evidence. Small targeted and planned selections establish plain one-file zero and deliberate broader zero, respectively, from actual native attempt counts and coordinator/worker inputs retaining numeric zero and the independent infrastructure retry budget. The targeted fixture includes an adversarial sibling file that a native substring or regular-expression filter could collect; native results must show only the selected file. Reuse the committed native fixtures where possible. No process-fault experiment runs on Blaxel. These are recorded validation runs, not a new test surface that dispatches Blaxel: the contribution's validation record cites the tested commit, exact commands, results, and retained reports. ^t-runner-remote-handoff + +### Standing local execution evidence + +- **Acceptance — native exit propagation.** The spawned canonical local CLI runs a real Vitest fixture to success, then runs its intentional native failure and exits `1` with a failed normalized result. A separate preflight-failure case uses the declared preflight hook and proves only that failure path. Proves [exit codes](../../../specs/arch/test-runner/test-runner.md#commands). *(covered by `test/repo/test-runner-reporting.test.ts`, “the real CLI finalizes all top-level files for success, native failure, and preflight failure”; its nested commands use gated private mutex locations)*. ^t-runner-native-exit + +### Standing lifecycle evidence + The lifecycle assertions are seam tests that run real subprocesses and listeners through the canonical runner boundary; no process or socket mechanism under assertion is mocked: - **Seam:** the surface-service manager starts two real Vite services in declaration order and passes both independently bound full URLs across the real process environment to a real Playwright child. The later Vite config observes the earlier service URL, both servers answer real HTTP requests, teardown restores prior parent values on success and later-service startup failure, and both ports can be rebound afterward. The handoff is registry declaration → service manager → process environment → Playwright config; there is no mock on that path. @@ -67,9 +147,61 @@ The lifecycle assertions are seam tests that run real subprocesses and listeners - A clean fixture surface reports no lifecycle failure. - **Contract:** lifecycle cleanup omits a candidate whose revalidation reports `already-exited` before either signal, while retaining an `already-exited` candidate that had already required `SIGTERM` or `SIGKILL`; the real listener leak assertions prove the retained record crosses the supervisor and reporting seam. - A stale owner token found at worker startup is reported as an infrastructure preflight failure and is reaped before any selected surface starts. +- **Contract — owner token format.** An encoded token starts `tv1.` and decodes to the owner fields it was made from; a different version prefix, malformed base64url, or an owner with an invalid supervisor PID decodes to nothing. Proves the [token format shared across revisions](../../../specs/arch/test-runner/test-runner.md#processes-started-by-tests). *(covered by `test/repo/process-lifecycle.test.ts` “round-trips only versioned, valid owner tokens”)* ^t-runner-owner-token-format **Boundary gaps — stated honestly per [testing-policy.md](../../../specs/arch/testing-policy.md):** -- The guardrail/verify tests stub the downstream with self-test seams, so no test here pins a real local end-to-end run — a real suite executing and the runner propagating a non-zero child exit code to `1`. -- On marked homes, the tests replace the Blaxel/`origin` checks with passing or failing results; the real remote-preflight boundary it depends on is owned and assessed by [preflight.md](../../../specs/arch/test-runner/preflight.md), and the real-provider report boundary by [sharded-execution.md](../../../specs/arch/test-runner/sharded-execution.md). +- The reporting suite already runs a real one-file CLI success and native failure, as cited above; dry-run tests do not add execution evidence. Mutex assertions cite the ordinary tests and isolated Linux workflow evidence above. Native-entrypoint and whole-option/help assertions cite their implemented tests above. File admission and retry assertions cite their implemented tests above. Full repository verify remains external validation, not a nested unit-test fixture. +- In marked-home provider-resolution cases, the tests replace the Blaxel/`origin` checks with passing or failing results; the real remote-preflight boundary it depends on is the [remote-revision check](../../../specs/arch/test-runner/test-runner.md#^remote-revision), assessed under [preflight checks](#preflight-checks), and the real-provider report boundary by [sharded-execution.md](../../../specs/arch/test-runner/sharded-execution.md). - The workflow-file assertion reads the committed YAML file. It catches removal of the retry argument from the pull request CI workflow, but it does not prove that a live GitHub Actions job starts correctly or reaches the test runner. Live GitHub Actions execution is still confirmed by CI itself and by review. + +### Test registry + +These assertions prove [the test registry](../../../specs/arch/test-runner/test-runner.md#the-test-registry). They run in-process as pure functions over the loaded registry, plus real filesystem reads: discovery walks the real repository tree, and validation cases write throwaway configs into real temporary directories. The committed-snapshot assertion also reads real Git objects; none of these tests uses a network or remote runner. Current coverage lives in `test/repo/test-runner-config.test.ts`. + +The registry identifies candidate surfaces; it does not establish that a native file filter executes exactly one file. That is the [file-boundary proof](#^t-runner-file-boundary), with [marked-host admission](#^t-runner-file-admission) and [local-only refusal guidance](#^t-runner-local-redirection). Explicit suite or owned-file selection uses the standing ownership and exclusion cases. + +- `validateRegistry` returns no errors for the repository's own registry. +- `selectSurfaces` with `suite: "e2e"` returns only `kind === "e2e"` surfaces. +- `selectSurfaces` with `package: "@telepath-computer/television-web"` returns exactly `unit:browser-app` and `e2e:browser-app`. +- `selectSurfaces` with `package: "@telepath-computer/canonical"` returns exactly `unit:canonical`. +- `owningSurfaces` maps `packages/canonical/test/build-canonical.test.ts` to `unit:canonical`. +- `selectSurfaces` with `surface: "e2e:browser-app"` returns exactly that surface. +- Every `.test.ts` / `.spec.ts` file discovered under the canonical include roots has at least one owning surface; none is unowned. +- No discovered test file is owned by more than one surface. +- `owningSurfaces` respects `excludeRoots`: a file under an excluded sub-tree resolves to the sibling surface that owns that sub-tree, not the surface that excluded it. +- `selectSurfaces` throws when `--file` is combined with a selector whose surfaces do not own the file, and the error names the owning surface(s). +- `validateRegistry` reports an unregistered runner config placed under a canonical include root. +- **Contract:** `validateRegistry` accepts a surface with two valid Vite service declarations and reports each invalid service declaration. The test uses temporary config-file fixtures and no mocked mechanism. *Superfluous in part:* the spec lists the services a surface may declare, not the invalid forms, which the code owns. +- **Contract:** loading the repository registry preserves the dynamic-service fixture's ordered declarations unchanged. +- **Contract — committed normalization.** The working-tree and committed loaders produce identical normalized surfaces for the same registry and package manifests, including a config nested below its package directory. A real Git fixture moves a package after a commit and proves that the snapshot still derives its working directory from the selected commit's manifests. This supports [planning from the tested commit](../../../specs/arch/test-runner/sharded-execution.md#plans) with real filesystem and Git reads, not native execution. Covered by `test/repo/test-runner-file-selection.test.ts`, “normalizes committed and local registry surfaces identically” and “derives package directories from the selected commit's manifests”. ^registry-committed-normalization +- The artifact e2e surface declares two independently published services for its view and host origins, with no deterministic port-window helper. +- The desktop e2e surface declares its fixture Vite service through `TV_DESKTOP_E2E_URL`, and the desktop config and tests contain no fixed fixture port. +- `validateRegistry` ignores a runner config placed outside the canonical include roots (e.g. under `prototypes/` or `experiments/`). +- The `all` suite resolves to the unit and non-agent e2e execution groups, in `order`, includes `unit:build-config`, and the `unit:workspaces` group contains the package unit surfaces in declared order. *Superfluous in part:* the spec states only that `all` is the unit and end-to-end surfaces; execution groups and their order are the code's. +- Every lifecycle fault surface retains `kind: "experiment"` and `tag:isolated-github-only`; none is selected by `all`. +- **Contract — live harness selection.** Load the committed registry with the real selector and inspect the committed driver's dispatch selections. Both [Blaxel harness surfaces](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-harness-placement) have `kind: "experiment"`; `all` and `e2e-ci` select neither, and tag `blaxel-harness-fixture` selects only `experiment:blaxel-harness-fixtures`. The driver's planned and targeted selections name only that fixture surface, never the driver, as [harness placement](../../../specs/arch/test-runner/blaxel-testshards.md#^blaxel-harness-placement) requires. This is a source-level contract with no mocks or provider calls; real dispatch is carried by [the provider retrieval acceptance](blaxel-testshards.md#^blaxel-t-retrieval). (policy-grade test: `test/repo/test-runner-config.test.ts` “keeps Blaxel harness fixtures separate from the driver and ordinary suites”) ^registry-t-blaxel-harness-selection +- The `unit:build-config` surface owns `test/repo/build-config-integrity.test.ts`; `unit:root` excludes that file. +- The `telemetry-posthog-roundtrip` suite resolves to the `telemetry-posthog-roundtrip:integration` surface, and that surface owns `packages/server/test/telemetry-posthog.integration.test.ts`; `unit:server` excludes that file. +- The `daemon-acceptance` suite resolves to `daemon-acceptance:cli`, and that surface exclusively owns `test/node/daemon-acceptance.test.ts`; `e2e:node` excludes that file. The surface declares the `daemon-test-host` preflight and is absent from `all`. + +### Preflight checks + +These assertions prove the checks that precede tests: the [remote-revision check](../../../specs/arch/test-runner/test-runner.md#^remote-revision), the process-backend check in [processes started by tests](../../../specs/arch/test-runner/test-runner.md#processes-started-by-tests), the daemon-host acknowledgement in [the test registry](../../../specs/arch/test-runner/test-runner.md#the-test-registry), and two checks other specs own: the Electron check that shares the [e2e harness's environment plan](../../../specs/arch/desktop/e2e-harness.md#^desktop-e2e-environment-plan) and the [PostHog key](../../../specs/arch/telemetry/sink.md#real-posthog-integration-test-surface). Contributor branch ownership and commit and push practice are reviewed as development procedure; they change no runtime check and order no tests. + +Coverage spans `test/repo/node-toolchain.test.ts`, `test/repo/test-runner-preflight.test.ts`, `test/repo/process-lifecycle.test.ts`, and `test/repo/test-runner-shard-worker.test.ts`. The successful `process-lifecycle` assertion is a seam test against a real disposable child and the host process and socket tables; injected missing-capability cases are contract tests of host validation. + +- `requiredPreflights` supports local execution with `["node", "process-lifecycle", ...selected surface checks]` and the Blaxel coordinator with `["node", "blaxel-github-token", "blaxel-auth"]`. +- The `node` check retains that name, reports the running Node/npm versions when they satisfy repository policy, and returns the shared toolchain diagnostic when either version is unsupported; canonical local execution stops before a selected test after this failure. That short-circuit assertion invokes its nested canonical CLI with `--no-publish`, keeping optional ref transport outside the preflight boundary under test. *Superfluous in part:* the check's name is the code's, and the version policy is [node-versions.md](../../../specs/arch/node-versions.md)'s. +- The planned shard worker records successful `process-lifecycle` and `stale-owner-cleanup` entries before its selected tasks. +- The Linux `process-lifecycle` check uses a real disposable child and the host's real `/proc` and `ss` boundaries; it passes only when the child can be attributed. Failure fixtures cover a missing `ss`, unreadable procfs, and unsupported platform without replacing the successful real-host seam test. +- The daemon-host acknowledgement check fails unless `TV_DAEMON_TEST_HOST` is exactly `1`, names the required value in its diagnostic, and records the environment-variable name when it passes. +- PostHog key-source tests cover environment and repository-root `.env` lookup plus the missing-key result from the reader. +- **Contract — Electron pre-setup states** (planner results and a temporary resolvable-package fixture; planner behavior is a declared mock forfeited to the seam below): with skip unset, a missing Electron package fails, an absent or valid result with no planner failures passes, and an invalid result fails. The preflight path invokes no installer and performs no runtime mutation. With `SKIP_ELECTRON_E2E=1`, it reports skipped. — *(covered by test: `test/repo/test-runner-preflight.test.ts` “checks Electron package presence and consumes every planner state without installing”)*. ^preflight-t-electron-states +- **Seam — Electron preflight-to-planner handoff** (the real `runPreflights` dispatcher, Electron check, shared environment planner, installed declared Electron package, package filesystem, and Linux host; no mocks): selecting the Electron check against the package's actual acceptable pre-setup state consumes the real planner result, passes, and leaves the runtime tree unchanged. The contract assertion carries the other planner-result permutations. — *(covered by test: `test/repo/test-runner-preflight.test.ts` “hands the real installed Electron package to the shared planner without mutation”)*. ^preflight-t-electron-planner-handoff + +**Boundary gaps — stated honestly per [testing-policy.md](../../../specs/arch/testing-policy.md):** + +- Apart from the `node` toolchain and Linux `process-lifecycle` seams above, check *execution* — `playwright-chromium`/`playwright-firefox`, the git-safety sequence, and the provider-auth checks — has no committed boundary test. In the guardrail tests remote preflight is replaced by a self-test seam, so the real `git status` / `git rev-parse` / `git fetch` / `git branch -r --contains` chain and the `@blaxel/core` calls are not exercised here. The git-safety checks are reachable with a real temporary git repository and no external credentials; the provider-auth checks require real Blaxel credentials. +- No committed test forces the planned worker's `process-lifecycle` preflight to fail and proves the resulting infrastructure summary, and no test invokes `posthog-test-key` through `runPreflights` to assert the missing-key or invalid-`phx_` diagnostic. The current tests cover the successful worker record, key lookup, and missing reader result. +- The macOS `process-lifecycle` backend requires a real macOS host for its `ps` environment/identity and `lsof` boundary test and is not proven by the Linux gate. diff --git a/proofs/arch/testing-policy.md b/proofs/arch/testing-policy.md index deca7180..0c17f73c 100644 --- a/proofs/arch/testing-policy.md +++ b/proofs/arch/testing-policy.md @@ -1,7 +1,25 @@ -*How the testing policy’s development discipline is checked.* +*How worker-owned test builds are proven, and how testing discipline is reviewed.* # Testing policy — proof Proves [specs/arch/testing-policy.md](../../specs/arch/testing-policy.md). -The policy governs authoring and execution discipline, which is checked by review of the contribution and its recorded validation. It orders no tests of its wording or of an agent’s compliance. Runtime enforcement belongs to the test-runner specs and their proofs; its pointer to the shared-branch workflow moves no runtime promise into this spec and orders no additional tests. +## Coverage model + +The policy governs authoring, focused iteration, verification, and agent permissions, which are checked by review of the contribution and its recorded validation. That review includes the instruction path from AGENTS.md to the policy: narrow local feedback, shared-host risk, waiting and override judgment, pushed Blaxel revisions for broader team-host work, no serial local evasion, and the distinction between ordinary one-file zero retries and broader validation with defaults. Development-branch commits, pushes, and coordinator holds are owned by the linked workflow and checked the same way. This policy orders no tests of its wording or of an agent’s compliance. Runtime admission, diagnostics, lock lifetime, and retry behavior are proven by [the runner proof](test-runner/test-runner.md); the guidance creates no second runtime test suite. + +Reusable helper builds are runtime behavior owned here. A small contract covers cache publication and failed preparation; real Playwright and Vitest worker lifecycles prove build reuse and final disposal. These are helper tests, not additional product browser walks. Real builder output must be exercised at least once; injected builders earn only lifecycle coverage. + +The built-application default is honored by the setup of the product tests that need it. [Themes](../product/themes-and-appearance.md) and [demo mode](../product/artifacts.md#^af-ac-demo-mode) retain their product assertions while changing their asset-loading boundary. No source-text test asserts that every acceptance uses a particular helper. + +Killed workers cannot promise teardown. The [Blaxel scratch/recovery assertions](test-runner/blaxel-testshards.md#Lease, recovery, and scratch) carry the backstop rather than duplicating a crash experiment here. + +## Test hooks + +The shared build-lifetime helper accepts a temporary parent and preparation callback; contract tests may supply a builder that writes fixture files, waits on a rendezvous, or rejects, while real callers run the source-CLI or versioned-web preparation. Its teardown entry point is the same one the runner-worker adapters call; exposing it for a contract does not stand in for testing those adapters. + +## Worker-owned builds + +- **Contract — publish successful builds and dispose owned work.** With a real temporary filesystem and recording preparation callbacks, concurrent requests in one worker reuse a pending/successful build for the same key, while distinct version keys remain distinct. A failed preparation removes its partial output and is not published; a later request may prepare it again. Teardown waits for in-flight preparation to settle and removes all owned outputs, including one that failed after creating files. Per-case server disposal is covered by the real worker seam below. These callbacks mock build/asset preparation and forfeit actual executable output to the worker seam below. This proves [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime). (policy-grade tests: `test/repo/worker-builds.test.ts`, all four cases) ^testing-t-build-lifetime + +- **Seam — real runner workers own their builds.** The installed Playwright and Vitest runners each execute a small authored fixture through the actual shared helper/adaptor path, using a private temporary root. Two cases assigned to the same worker reuse the source-CLI preparation and a same-version web bundle; a later worker owns a separate lifetime. Source CLI output starts a real product server, and the real web output is served/read, establishing that published results are usable. Within this same fixture, a bundle path naming an absent child makes the real copy fail after allocation of a per-launch runtime/home; those private outputs are removed, and a subsequent valid launch still uses the worker build. The missing path is fixture data. After per-case server cleanup the shared outputs still exist; after normal worker teardown they do not. No runner, builder, filesystem or server mechanism is mocked. This crosses each framework's teardown boundary under [worker build lifetime](../../specs/arch/testing-policy.md#^worker-build-lifetime), without a browser or an extra product assertion. [The real version-stamp seam](updates/version-advertisement.md#^t-web-stamp) supplies version breadth; this seam need not rebuild a matrix of versions. (policy-grade tests: `test/node/worker-build-lifetime.e2e.test.ts`, the Playwright and Vitest cases with their authored native fixtures in `test/fixtures/worker-builds/`) ^testing-t-build-worker-seam diff --git a/proofs/arch/themes/delivery.md b/proofs/arch/themes/delivery.md index 96b5843d..9a468515 100644 --- a/proofs/arch/themes/delivery.md +++ b/proofs/arch/themes/delivery.md @@ -6,6 +6,8 @@ Proves [specs/arch/themes/delivery.md](../../../specs/arch/themes/delivery.md). ## Coverage model +The persistence and executable-lifecycle browser walks share the built-application setup declared in [the product coverage model](../../product/themes-and-appearance.md#Coverage model). Their retained request inventories establish that loading boundary; these are the same product walks, with no duplicate browser launch here. + Coverage declarations are carried inside the migrated assertion blocks below. The active-route acceptance crosses a really-running server and real package folders for ordinary files, escapes, and validators. JavaScript-route breadth covers all three reserved entries at that same public HTTP boundary: main-entry consent matches and mismatches exact IDs, each manifest declaration is independently enabled, disabled, or absent, and invalid syntax remains fixture bytes rather than being executed. A sibling JavaScript file remains an ordinary static asset. One missing-root recovery walk removes a package with every entry enabled and main consent granted, then proves empty CSS and JavaScript entries, unavailable ordinary assets, and exact restored bytes and validators. The package-root `README.md` exclusion retains its own assertion. Main-script coverage composes a mocked-DOM contract for the resource owner's element mechanics with application-composition contracts for confirmed consent, connection and event wiring, forced reload, and one-shot Settings restoration. DOM, service, history, and navigation mocks forfeit loading, execution, transport, and actual browser reload to [the product executable-theme browser acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-browser) and [reset acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-reset); they buy deterministic lifecycle breadth only. The reset contract deliberately follows installed include presence rather than inferred execution, so it also covers the harmless conservative reload when consent installed an include whose server-side manifest gate returned empty JavaScript. A same-ID manifest refresh remains an in-place include replacement even when it closes that gate. The same product proof carries the required real-Electron execution path. Static ownership and exclusion from standalone entry graphs live in [UI architecture](../ui/index.md#^ui-t-theme-resources). @@ -58,7 +60,7 @@ The [visible application-theme seam](#^theme-delivery-t-visible-app) is covered - **Acceptance** (outer boundary: the built application in Chromium and Firefox against a real server and scratch data directory; fixture: an installed theme importing a nested stylesheet with relative day/night SVG URLs; no mocks or test hooks): selecting the theme resolves its registered `--app-wallpaper-image` relative to the declaring stylesheet, loads the correct asset through the application's composed `--app-wallpaper`, changes the image with appearance, and clears it when selecting None — *(covered by test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme-relative wallpaper URLs render through the built app token in Chromium and Firefox”)*. ^theme-delivery-t-wallpaper-url -- **Acceptance** (outer boundary: a real Chromium browser loads production shell modules and a live-canonical HTML artifact against a really-running server; a selected package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and temporary data directory are fixtures; no mocks or test hooks): changing only the nested stylesheet crosses the real `theme-changed` transport and application-link replacement, then changes imported computed presentation in both document classes without registry refresh — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-delivery-t-active-package-refresh +- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a really-running server; a selected package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and temporary data directory are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): changing only the nested stylesheet crosses the real `theme-changed` transport and application-link replacement, then changes imported computed presentation in both document classes without registry refresh — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-delivery-t-active-package-refresh - **Contract** (application theme-resource owner against a mocked DOM, forfeiting browser loading and execution to [the product browser acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-browser)): a refresh creates one cache-busted classic external script in the document head at the connected server's absolute `/theme/main.js`; replacement detaches the prior element before appending the new one, clear removes it, and main-script operations neither replace nor reorder the stylesheet link or sandboxed frames — *(covered by `packages/web/test/theme.test.ts` “installs one cache-busted classic script from the connected server”, “removes the prior script before appending its replacement and clears it”, and “leaves stylesheet and sandboxed frame identity and ordering unchanged”)*. ^theme-delivery-t-app-script-module - **Contract** (renderer composition in a mocked DOM, with connection and application services replaced by controllable boundary mocks, forfeiting real connection event generation and browser resource loading to the browser and Electron assertions below): connection installs the application theme link before display state is ready; a selection change or same-ID resource-refreshing `theme-changed` event replaces it; a consent-membership-only event preserves it; and disconnect removes it — *(covered by test: `packages/web/test/renderer.test.ts` “ties the theme link to connection and theme events before display readiness” and “ties the theme script to confirmed consent, refresh, and connection lifecycle”)*. ^theme-delivery-t-app-link-renderer - **Contract** (application composition against controllable connection and application-service boundaries in a mocked DOM, forfeiting real event transport and script execution to [the product browser acceptance](../../product/themes-and-appearance.md#^theme-ac-executable-browser)): connection leaves the main script absent before confirmed display state; confirmed matching active-ID consent appends the first include only after first-party runtime setup; reconnect reevaluates confirmed state; opt-in injects without replacing the active stylesheet link or frame elements; active-package and registry events with unchanged selection and consent replace the stylesheet and main include; unrelated consent membership preserves all active resource identities; effective appearance changes preserve the stylesheet link and main include while frame replacement follows [the frame-appearance lifecycle](#^theme-delivery-t-frame-appearance-lifecycle); and disconnect removes them. Each path calls the one resource owner rather than using Settings registry state as an execution gate — *(covered by `packages/web/test/renderer.test.ts` “ties the theme script to confirmed consent, refresh, and connection lifecycle” and “reconciles theme frames across confirmed setup, refresh, selection, and connection state”)*. ^theme-delivery-t-app-script-renderer @@ -71,8 +73,8 @@ The [visible application-theme seam](#^theme-delivery-t-visible-app) is covered - **Contract** (application composition against controllable connection, display-state, and registry boundaries in a mocked DOM; deferred registry results and failures are authored outcomes; the mocks forfeit real transport and iframe replacement to [#^theme-delivery-t-frame-browser](#^theme-delivery-t-frame-browser)): selection change and same-ID active-package or registry refresh destroy both prior frames before resolving the current registry, then recreate only enabled destination frames. If initial or replacement lookup fails, both frames remain absent rather than retaining stale executable content; that failure does not interrupt the stylesheet, main-script, or application paths, and reconnect or a later selection, active-package, or registry trigger retries. A resolution or rejection from a superseded connection, theme ID, or lookup can neither install a frame nor remove frames installed by the current lookup. Consent-only events preserve both identities. Null selection and disconnect remove both frames; a registry refresh that disables either declaration leaves that frame absent while independently recreating the other enabled frame; and frame creation, replacement, or removal never requests a full application reload — *(covered by `packages/web/test/renderer.test.ts` “installs ready theme frames after deferred application mount”, “reconciles theme frames across confirmed setup, refresh, selection, and connection state”, “fails theme frame lookup closed while stylesheet, main script, and application continue”, “discards superseded theme frame resolutions and rejections”, and `packages/web/test/theme.test.ts` “remembers ordinary application focus and restores accidental current-frame focus”)*. ^theme-delivery-t-frame-lifecycle - **Contract** (the production application appearance-to-frame composition path in a mocked DOM; an effective root `data-theme` mutation, enabled registry snapshots, deferred fresh lookups, lookup failure, and rapid superseding mutations are authored outcomes; posing the root mutation and replacing the registry and DOM boundaries are mocks that forfeit the real appearance producer and event transport, script execution, and browsing-context replacement to [#^theme-delivery-t-frame-appearance-browser](#^theme-delivery-t-frame-appearance-browser)): under the [sandboxed-frame lifecycle](../../../specs/arch/themes/delivery.md#sandboxed-frames), observing an effective appearance change synchronously removes both current frames in the observer's callback turn before beginning the fresh registry lookup, preserves the active stylesheet and main-script elements without requesting application reload, then recreates every enabled frame for the new value. A failed initial or replacement appearance lookup leaves both absent; a later effective appearance change performs another fresh lookup and can restore enabled frames. A result superseded by a later appearance cannot install stale frames — *(covered by `packages/web/test/renderer.test.ts` “reconciles theme frames across confirmed setup, refresh, selection, and connection state”, “fails theme frame lookup closed while stylesheet, main script, and application continue”, and “discards superseded theme frame resolutions and rejections”)*. ^theme-delivery-t-frame-appearance-lifecycle - **Contract** (production host pointer forwarder against current frame-window and application-event boundary mocks in a browser DOM; the frame windows record `postMessage` and the event methods record interference, forfeiting real dispatch and hit testing to [#^theme-delivery-t-frame-browser](#^theme-delivery-t-frame-browser)): capture-phase listeners forward each trusted application-document `pointermove`, `pointerdown`, `pointerup`, `pointercancel`, and `click` once to each currently installed frame and never to a removed frame. The same internal forwarder accepts a source-authorized artifact input after the artifact owner maps it to application coordinates. Both paths map respectively to `television-theme-pointer-move`, `television-theme-pointer-down`, `television-theme-pointer-up`, `television-theme-pointer-cancel`, and `television-theme-pointer-click`; every message has only `type`, `clientX`, `clientY`, `button`, and `buttons`. Move sends `button: -1` and the input's current `buttons`; down sends the changed input `button` and post-transition `buttons` including the new press; up sends the changed input `button` and post-transition `buttons` excluding the release; click sends the clicked input `button` and `buttons: 0`; and cancel sends `button: -1` and `buttons: 0`, including for an authored cancel input reporting nonzero `buttons`. Each message uses the target frame's current `contentWindow` with destination `"*"`. Untrusted application events are ignored, messages are neither queued nor replayed, and forwarding never prevents default, stops propagation, or takes pointer capture. Any `message` event sourced from either current frame produces no application operation or reply regardless of payload, including an artifact-pointer lookalike — *(covered by `packages/web/test/theme.test.ts` “forwards the closed trusted pointer protocol without interfering or replaying” and “accepts no frame-originated command or reply path”, plus `packages/web/test/artifact-view.test.ts` “relays only current artifact pointer input to theme frames in application coordinates”)*. ^theme-delivery-t-frame-pointer -- **Acceptance** (outer boundary: the built application in Chromium against a really-running server and real package folders; current registry state, production host elements, public entry routes, the injected artifact bridge, and both theme iframe windows remain on the path; no mocks or test hooks): independently and jointly enabled entries execute automatically without main consent in separate `allow-scripts` opaque-origin documents, expose their cache-busted route through `document.currentScript.src`, cannot inspect application DOM, and contain load, syntax, and runtime failures. Real pointer move, down, up, cancel, and click input in the application document reaches both frames with the exact viewport coordinates and button fields while the same application sequence and click remain effective; a drag that produces no click forwards only its down, motion, and up transitions. Real move, down, up, and click input in a bridge-capable artifact also reaches both frames in application-viewport coordinates while the artifact's click remains effective. Ordinary application focus changes remain untouched. When either frame unexpectedly acquires focus, focus returns without scrolling to the last connected application element or, when unavailable, to `#app`; the production Settings popover remains operable in its document layer and an application modal dialog remains operable in the browser top layer — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. This is the same browser walk as [product acceptance](../../product/themes-and-appearance.md#^theme-ac-sandboxed-frames), not a second launch. ^theme-delivery-t-frame-browser -- **Acceptance** (outer boundary: the production Settings appearance control in the built Chromium application against a really-running server; a real package enabling both sandbox entries, entry-script startup probes, application-color probes visible through transparent regions of both frames, and a temporary data directory are fixtures; no mocks or test hooks): under the [sandboxed-frame appearance contract](../../../specs/arch/themes/delivery.md#sandboxed-frames), an effective light-to-dark appearance change records both old frame removals before either replacement, leaves no old-appearance frame, and reruns both entries. Each replacement document root carries `data-theme="dark"`; its used root scheme and its host element's used scheme are both dark. Rendered-pixel probes behind otherwise empty portions of the background and overlay frames remain visible, proving transparent iframe canvas composition rather than only transparent computed backgrounds — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. This is the same browser walk as [product frame-appearance acceptance](../../product/themes-and-appearance.md#^theme-ac-frame-appearance), not a second launch. ^theme-delivery-t-frame-appearance-browser +- **Acceptance** (outer boundary: the built application in Chromium against a really-running server and real package folders; current registry state, production host elements, public entry routes, the injected artifact bridge, and both theme iframe windows remain on the path; the real stable-front proxy supplies harness transport; no mocks or test hooks): independently and jointly enabled entries execute automatically without main consent in separate `allow-scripts` opaque-origin documents, expose their cache-busted route through `document.currentScript.src`, cannot inspect application DOM, and contain load, syntax, and runtime failures. Real pointer move, down, up, cancel, and click input in the application document reaches both frames with the exact viewport coordinates and button fields while the same application sequence and click remain effective; a drag that produces no click forwards only its down, motion, and up transitions. Real move, down, up, and click input in a bridge-capable artifact also reaches both frames in application-viewport coordinates while the artifact's click remains effective. Ordinary application focus changes remain untouched. When either frame unexpectedly acquires focus, focus returns without scrolling to the last connected application element or, when unavailable, to `#app`; the production Settings popover remains operable in its document layer and an application modal dialog remains operable in the browser top layer — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. This is the same browser walk as [product acceptance](../../product/themes-and-appearance.md#^theme-ac-sandboxed-frames), not a second launch. ^theme-delivery-t-frame-browser +- **Acceptance** (outer boundary: the production Settings appearance control in the built Chromium application against a really-running server; a real package enabling both sandbox entries, entry-script startup probes, application-color probes visible through transparent regions of both frames, and a temporary data directory are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the [sandboxed-frame appearance contract](../../../specs/arch/themes/delivery.md#sandboxed-frames), an effective light-to-dark appearance change records both old frame removals before either replacement, leaves no old-appearance frame, and reruns both entries. Each replacement document root carries `data-theme="dark"`; its used root scheme and its host element's used scheme are both dark. Rendered-pixel probes behind otherwise empty portions of the background and overlay frames remain visible, proving transparent iframe canvas composition rather than only transparent computed backgrounds — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. This is the same browser walk as [product frame-appearance acceptance](../../product/themes-and-appearance.md#^theme-ac-frame-appearance), not a second launch. ^theme-delivery-t-frame-appearance-browser - **Acceptance** (outer boundary: the real Electron app connected to a really-running server; an installed theme, its exact computed application-shell color, and an isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): selecting the installed theme renders its application style, and quitting and relaunching with the saved connection renders the same persisted theme without another selection — *(covered by test: `packages/desktop/test/e2e/webview-event-parity.test.ts` “renders the persisted app theme after quitting and relaunching”)*. ^theme-delivery-t-app-link-electron-startup - **Seam** (the production application theme-link module in Chromium against a really-running server on the app's origin and authored theme package resources; no mocks or test hooks): setup, selection, and watched-file refresh keep one cache-busted link last; each replaced link is detached; nested imports, font, and image resources load from the stable path; and appearance changes alter neither link identity nor request count — *(covered by test: `packages/web/test/e2e/appearance-delivery.test.ts` “theme link stays final and mode changes reuse its nested resources”)*. ^theme-delivery-t-app-link - **Acceptance** (outer boundary: the built application and a canonical-v2-styled HTML artifact in Chromium against a really-running server and its installed Clouds package; no mocks or test hooks): the marked app document receives the Clouds wallpaper through the stable route, the artifact keeps its canonical background without requesting or painting that wallpaper, and selecting `None` restores both bare baselines through the null theme while `/theme/theme.css` is empty — *(covered by `packages/web/test/e2e/appearance-delivery.test.ts` “Clouds targets the marked app without painting canonical artifacts”)*. ^theme-delivery-t-app-document diff --git a/proofs/index.md b/proofs/index.md index 1c3fbb7d..1e3d39db 100644 --- a/proofs/index.md +++ b/proofs/index.md @@ -14,7 +14,7 @@ Generated from the first-line description of every proof by `scripts/specs-index - `licensing.md` (33 lines) — *How Television's licensing promises are proven where users receive them: the license gate over the real tree, the packed CLI tarball, the desktop upload directory, the served browser notices, and Electron's license files in the installed desktop app.* - `tab-pages.md` (30 lines) — *How the promises in Tab pages are proven.* - `telemetry.md` (89 lines) — *How the promises in Telemetry are proven.* - - `themes-and-appearance.md` (55 lines) — *How the promises in Themes and appearance are proven.* + - `themes-and-appearance.md` (57 lines) — *How the promises in Themes and appearance are proven.* - `update-notifications.md` (88 lines) — *How Television's update promises are proven: real browser and Electron journeys for the server-update notice, auto-reload, the desktop self-update notice, the desktop recommendation and the gate, and the desktop product's update check on a candidate build for the desktop app's own updates.* - `versioning.md` (27 lines) — *How Television's release versions are proven where users receive them: the packed CLI package and the desktop upload directory built from the same commit, the built `tv` process and running server, update details, and the native About panel in the desktop product's Mac install check.* - **onboarding/** @@ -28,7 +28,7 @@ Generated from the first-line description of every proof by `scripts/specs-index - `licensing.md` (24 lines) — *How the licensing machinery is proven: generator, gate and loader contracts over authored inputs, and seams through every real shipping build, the suite's inventory handoff and the committed asset manifest.* - `making-skills.md` (18 lines) — *How the promises in Making skills are proven.* - `node-versions.md` (18 lines) — *How the Node version contracts are proven: repository declarations, the advisory floor in the packed CLI package, and the toolchain checker at its two developer entry points.* - - `testing-policy.md` (7 lines) — *How the testing policy’s development discipline is checked.* + - `testing-policy.md` (25 lines) — *How worker-owned test builds are proven, and how testing discipline is reviewed.* - **artifact-frame/** - `artifact-bridge.md` (121 lines) — *How the promises in Artifact bridge are proven.* - `index.md` (24 lines) — *How the promises in Artifact frame (architecture) are proven.* @@ -77,19 +77,16 @@ Generated from the first-line description of every proof by `scripts/specs-index - `sessions.md` (31 lines) — *How the promises in Telemetry sessions are proven.* - `sink.md` (58 lines) — *How the promises in Telemetry sink are proven.* - **test-runner/** - - `attestation.md` (66 lines) — *How the promises in Test Attestation are proven.* - - `blaxel-testshards.md` (60 lines) — *How the promises in Blaxel Test Shards are proven.* - - `flaky-tests.md` (16 lines) — *How the promises in Flaky Tests are proven.* + - `attestation.md` (67 lines) — *How the promises in Test Attestation are proven.* + - `blaxel-testshards.md` (101 lines) — *How the promises in Blaxel Test Shards are proven.* - `github-ci.md` (53 lines) — *How the promises in GitHub Actions CI are proven.* - - `preflight.md` (42 lines) — *How the promises in Preflight are proven.* - - `reporting.md` (42 lines) — *How the promises in Reporting are proven.* - - `sharded-execution.md` (52 lines) — *How the promises in Sharded Execution are proven.* - - `test-registry.md` (38 lines) — *How the promises in Test Registry are proven.* - - `test-runner.md` (75 lines) — *How the promises in Test Runner are proven.* + - `reporting.md` (63 lines) — *How the promises in Reporting are proven.* + - `sharded-execution.md` (54 lines) — *How the promises in Sharded Execution are proven.* + - `test-runner.md` (207 lines) — *Canonical CLI guardrails, local ownership, native entrypoints, retry policy, the test registry, and preflight checks proven through real process boundaries and declared downstream substitutions.* - **themes/** - `authoring.md` (35 lines) — *How theme guidance, generated vocabulary and the authored application reference are proven at the built `television` skill boundary.* - `bundled-installation.md` (42 lines) — *How the promises in Bundled theme installation are proven.* - - `delivery.md` (87 lines) — *How the promises in Theme delivery are proven.* + - `delivery.md` (89 lines) — *How the promises in Theme delivery are proven.* - `index.md` (63 lines) — *How the promises in Theme architecture are proven.* - **ui/** - `conformance.md` (15 lines) — *How production UI conformance is checked while automated markup comparison is scheduled for TV-649.* diff --git a/proofs/product/artifacts.md b/proofs/product/artifacts.md index e402e310..17124fc6 100644 --- a/proofs/product/artifacts.md +++ b/proofs/product/artifacts.md @@ -6,7 +6,7 @@ Proves [specs/product/artifacts.md](../../specs/product/artifacts.md). ## Coverage model -Coverage declarations are carried inside the migrated assertion blocks below. +Coverage declarations are carried inside the migrated assertion blocks below. The two demo-mode acceptances and the bridge non-participation seam share the built application and built unsupported-URL view from the real product-server helper. Each walk retains a request-event inventory of its built document and assets; these observations use the same browser session as the product assertions. ## Test hooks @@ -49,12 +49,12 @@ Coverage relationship: serving an artifact's content at `/artifact//` to a b Coverage relationship: fresh loading after a channel switch is crossed by [arch/artifact-frame/reload-navigation.md#^rn-ac-persist-browser](../arch/artifact-frame/reload-navigation.md#^rn-ac-persist-browser), whose real-browser case removes every frame while another channel is current and observes a newly loaded frame on return. Its history restoration is reload-navigation's promise; this spec adds no second channel-switch walk. Rearrangement continuity is deliberately not promised, so it creates no assertion. -Browser demo mode ([#^af-demo-mode](../../specs/product/artifacts.md#^af-demo-mode)) is proven by two Chromium acceptance walks, with a routing contract for the breadth of which artifacts it reaches. Shared declaration for the two walks: the outer boundary is Chromium driving the production app against a really-running server launched with the server-home hook above; the external page is a two-page HTML fixture served by a local HTTP server on an origin other than the Television server's, never a public site; the external page artifact is created through the production API; the default motion override is the only mock. Playwright reads the cross-origin frame's document to observe which fixture page is shown. +Browser demo mode ([#^af-demo-mode](../../specs/product/artifacts.md#^af-demo-mode)) is proven by two Chromium acceptance walks, with a routing contract for the breadth of which artifacts it reaches. Shared declaration for the two walks: the outer boundary is Chromium driving the built application and built unsupported-URL view against a really-running server launched with the server-home hook above; the external page is a two-page HTML fixture served by a local HTTP server on an origin other than the Television server's, never a public site; the external page artifact is created through the production API; the default motion override is the only mock. Playwright reads the cross-origin frame's document to observe which fixture page is shown. -- **Acceptance** (shared demo-mode walk): with no marker file, the external page artifact's frame shows the unsupported-URL error page. After the marker file is created and the app is reloaded, the frame's document is the fixture's first page at its own URL, the frame shows no Back or Forward control, and the artifact icon is a button named by [content.yml#return_aria_label](../../specs/ui/app/artifact-frame/content.yml#return_aria_label). After the marker file is deleted and the app is reloaded again, the frame shows the unsupported-URL error page and the icon is not a button — *(covered by test: `packages/web/test/e2e/browser-demo-mode.test.ts` "the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)")*. ^af-ac-demo-mode -- **Acceptance** (shared demo-mode walk, marker present): following the fixture's link shows its second page; pressing the artifact icon shows the first page again. Following the link again and reloading the app shows the first page. Following the link once more, switching to another channel, and returning shows the first page. Back and Forward remain absent throughout — *(covered by test: `packages/web/test/e2e/browser-demo-mode.test.ts` "the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)")*. ^af-ac-demo-return +- **Acceptance** (shared demo-mode walk): with no marker file, the external page artifact's frame shows the unsupported-URL error page. After the marker file is created and the app is reloaded, the frame's document is the fixture's first page at its own URL, the frame shows no Back or Forward control, and the artifact icon is a button named by [content.yml#return_aria_label](../../specs/ui/app/artifact-frame/content.yml#return_aria_label). After the marker file is deleted and the app is reloaded again, the frame shows the unsupported-URL error page and the icon is not a button — *(policy-grade test: `packages/web/test/e2e/browser-demo-mode.test.ts` "the home-directory marker turns direct external pages on and off at app load (^af-ac-demo-mode)")*. ^af-ac-demo-mode +- **Acceptance** (shared demo-mode walk, marker present): following the fixture's link shows its second page; pressing the artifact icon shows the first page again. Following the link again and reloading the app shows the first page. Following the link once more, switching to another channel, and returning shows the first page. Back and Forward remain absent throughout — *(policy-grade test: `packages/web/test/e2e/browser-demo-mode.test.ts` "the artifact icon, an app reload, and a channel round trip each return to the original page (^af-ac-demo-return)")*. ^af-ac-demo-return - **Contract** (the production artifact-frame view under jsdom with an authored external page artifact and a fake application service posing demo mode on — mocks that forfeit real document loading to [#^af-ac-demo-return](#^af-ac-demo-return); theme-change and artifact-content-change events are injected on that service): neither event reassigns the demo page's frame `src`, which stays the artifact's own URL — *(covered by test: `packages/web/test/artifact-view.test.ts` "browser demo mode artifact frame > leaves a demo-mode page's URL untouched through theme and content change events (^af-ac-demo-untouched)")*. ^af-ac-demo-untouched - **Contract** (the production routing functions `artifactRenderRoute` and `artifactDocumentRoute` over authored artifact records and recorded current URLs; no mocks): with demo mode on in a browser, `http:` and `https:` artifact URLs that are not Television artifact addresses route to a direct iframe at the artifact's own URL, including a URL on the Television host itself; a Television artifact address keeps the shared-artifact route both on another host and on the connected Television host; a non-web URL keeps the unsupported-URL page. Path and Markdown artifacts, a Television-served artifact whose recorded current URL is an external web page, and every artifact in the desktop app route identically with demo mode on and off — *(covered by tests: `packages/web/src/artifact-dispatcher.test.ts` "browser demo mode routing", all four cases)*. ^af-ac-demo-scope -Coverage relationship: the two walks cross the real server's marker-file check, its delivery to the browser, routing, and frame rendering, and compose with [#^af-ac-demo-scope](#^af-ac-demo-scope), which carries the classification and routing breadth off the walked path, and [#^af-ac-demo-untouched](#^af-ac-demo-untouched), which keeps Television's own reload triggers away from the page. The walks reach the server through the development proxy, so app and server share an origin; that the flag route sends no CORS headers is [artifact isolation's route contract](../arch/artifact-frame/isolation.md#^iso-t-routes). That demo pages cannot forward the navigation chord or feed theme pointer effects is carried by the bridge's non-participation assertions [arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-seam](../arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-seam) and [arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-host](../arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-host). A site that refuses framing is outside Television's behavior, since Television only leaves the browser's own outcome in place, and orders no test. Which views show the return button, and what pressing it requests, are the frame's contracts at [ui/app/artifact-frame/index.md#^af-ui-ac-return-scope](../ui/app/artifact-frame/index.md#^af-ui-ac-return-scope) and [ui/app/artifact-frame/index.md#^af-ui-ac-frame-actions](../ui/app/artifact-frame/index.md#^af-ui-ac-frame-actions). +Coverage relationship: the two walks cross the real server's marker-file check, its delivery to the browser, routing, and frame rendering, and compose with [#^af-ac-demo-scope](#^af-ac-demo-scope), which carries the classification and routing breadth off the walked path, and [#^af-ac-demo-untouched](#^af-ac-demo-untouched), which keeps Television's own reload triggers away from the page. The built application and server share an origin; that the flag route sends no CORS headers is [artifact isolation's route contract](../arch/artifact-frame/isolation.md#^iso-t-routes). That demo pages cannot forward the navigation chord or feed theme pointer effects is carried by the bridge's non-participation assertions [arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-seam](../arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-seam) and [arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-host](../arch/artifact-frame/artifact-bridge.md#^ab-ac-demo-inert-host). A site that refuses framing is outside Television's behavior, since Television only leaves the browser's own outcome in place, and orders no test. Which views show the return button, and what pressing it requests, are the frame's contracts at [ui/app/artifact-frame/index.md#^af-ui-ac-return-scope](../ui/app/artifact-frame/index.md#^af-ui-ac-return-scope) and [ui/app/artifact-frame/index.md#^af-ui-ac-frame-actions](../ui/app/artifact-frame/index.md#^af-ui-ac-frame-actions). diff --git a/proofs/product/cli.md b/proofs/product/cli.md index 808e55e7..7cbeaf1f 100644 --- a/proofs/product/cli.md +++ b/proofs/product/cli.md @@ -45,7 +45,7 @@ The connect-link walks add no production hooks: they use the built executable, r ### Acceptance criteria: CLI spine -The CLI acceptance boundary is a spawned `tv` process receiving real argument tokens and reporting its real exit code, stdout, and stderr. These tests use temporary homes, config files, token, content, and skills-directory fixtures. A foreground server follows [the test-runner recipe for spawned `tv serve`](../../specs/arch/test-runner/test-runner.md#Test infrastructure addresses): its home's config file sets port `0`, the server and every client command receive that home with `--home`, the server publishes its acquired port through startup output, and client commands pass that port with `--port`. Unless an assertion says otherwise, the config file sets only `port`, so the server runs with its default authentication. Every assertion labeled **Acceptance** below replaces no production mechanism. +The CLI acceptance boundary is a spawned `tv` process receiving real argument tokens and reporting its real exit code, stdout, and stderr. These tests use temporary homes, config files, token, content, and skills-directory fixtures. A foreground server follows [the test-runner recipe for spawned `tv serve`](../../specs/arch/test-runner/test-runner.md#network-addresses-in-tests): its home's config file sets port `0`, the server and every client command receive that home with `--home`, the server publishes its acquired port through startup output, and client commands pass that port with `--port`. Unless an assertion says otherwise, the config file sets only `port`, so the server runs with its default authentication. Every assertion labeled **Acceptance** below replaces no production mechanism. The single dispatch seam is **argument tokens → command dispatch → command action**. Commander parses the process arguments and hands the resulting values to one registered action. Each implemented principal outcome below has one spawned acceptance test; the in-process contract tests in [arch/cli/index.md](../../specs/arch/cli/index.md) cover command and option variations through that same dispatcher, while the named seam tests there prove the real server, filesystem, and build handoffs. Both direct-install acceptances use the production packaged collection; the stale-cleanup path begins with the retired directory present in its real temporary destination. Interactive installer delegation retains its owned exception. @@ -87,7 +87,7 @@ The single dispatch seam is **argument tokens → command dispatch → command a - **Acceptance.** An installation whose server never answers crosses the same packed-package, installed-binary, process, production daemon-construction, host service-manager, and log-file boundaries. After the stop phase, a new temporary home's config file holds a stable nonzero port and `listen` `192.0.2.1`, written with the installed `tv --home config set port listen 192.0.2.1`. `192.0.2.1` is in TEST-NET-1 and assigned to no interface, so every boot of the service's server fails to bind and exits. Installed `tv --home serve --persist` exits `1`, no sooner than 15 seconds after the process starts, writes nothing to stdout, and writes exactly the [product's timeout error](../../specs/product/cli.md#^cli-persist-health-timeout) to stderr, naming `http://localhost:` and `/logs/tv.log` for that absolute home. Afterwards the service remains installed according to both `Daemon.status()` and the presence of its definition, and `/logs/tv.log` holds at least one `server startup failed` record from the service's boots showing `192.0.2.1` failed, and the command's `persisted service did not respond` record. The elapsed-time bound shows that the real process waited rather than failing early; the exact deadline is proven against a fake clock by the [architecture timeout contract](../arch/cli/index.md#^cli-persist-health-timeout-contract). The temporary home, the TEST-NET-1 address, and a `DO_NOT_TRACK` environment value, which keeps the telemetry notice out of stderr, are declared fixtures; no mechanism is replaced — *(policy-grade test: the unanswered-install phase of `test/node/daemon-acceptance.test.ts` "installs, refreshes, boots, stops, and persist-uninstalls the production service")*. ^cli-ac-persist-unanswered - **Acceptance.** The distinct persisted-uninstall spelling continues from a real production-identity installation that replaces the unanswered service and invokes installed `tv --home serve --persist-uninstall`. It exits `0`, prints exactly `{ "status": "stopped" }`, and leaves the service absent and inactive — *(policy-grade test: the persist-uninstall phase of `test/node/daemon-acceptance.test.ts` "installs, refreshes, boots, stops, and persist-uninstalls the production service")*. ^cli-ac-persist-uninstall-alias -These six persisted-service assertions live in the manually selected `daemon-acceptance` suite. It is excluded from CI, ordinary `verify`, and `--suite all`; an operator runs `TV_DAEMON_TEST_HOST=1 npm test -- local --suite daemon-acceptance` only on a designated developer host where replacing the globally installed `tv` package and the real `com.television.server` user service is acceptable. Before the assertions, the harness removes any production-named service state; every installation uses its own temporary home. For each installation it obtains a stable nonzero port by binding loopback port `0` and releasing it, writes that port with `tv --home config set port `, and installs immediately, because persisted serve refuses port `0`; this is the declared exception to [continuous test-port reservation](../../specs/arch/test-runner/test-runner.md#^test-dynamic-ports). A freshly packed `tv` remains installed in the active npm prefix after the suite; the service is always removed. +These six persisted-service assertions live in the manually selected `daemon-acceptance` suite. It is excluded from CI, ordinary `verify`, and `--suite all`; an operator runs `TV_DAEMON_TEST_HOST=1 npm test -- local --file test/node/daemon-acceptance.test.ts` only on a designated developer host where replacing the globally installed `tv` package and the real `com.television.server` user service is acceptable. Before the assertions, the harness removes any production-named service state; every installation uses its own temporary home. For each installation it obtains a stable nonzero port by binding loopback port `0` and releasing it, writes that port with `tv --home config set port `, and installs immediately, because persisted serve refuses port `0`; this is the declared exception to [continuous test-port reservation](../../specs/arch/test-runner/test-runner.md#^test-dynamic-ports). A freshly packed `tv` remains installed in the active npm prefix after the suite; the service is always removed. #### Owned exception: interactive external installer diff --git a/proofs/product/desktop-app.md b/proofs/product/desktop-app.md index 81fc357d..079ffe2d 100644 --- a/proofs/product/desktop-app.md +++ b/proofs/product/desktop-app.md @@ -14,7 +14,7 @@ The implementation pull request ran the three Mac checks on two builds it releas The architecture proofs carry the breadth. The [desktop root](../arch/desktop/index.md) proves the window title, icon and About-panel values Television supplies to Electron, and that naming the app fixes where its data lives; the Mac install check is where those reach the native Dock, menu, About panel and saved connection. The [ToDesktop build proof](../arch/desktop/distribution.md) proves, with automated tests, what the upload directory and configuration contain. The Mac install check observes what the built app makes of them: the effect of the product name, the installed bundle's identifier, its version fields and macOS's assessment of its signature, which the operator reads by hand on the real Mac. No check starts the built app with `ELECTRON_RUN_AS_NODE` set, so the effect of the `runAsNode` fuse setting is not observed; the [configuration contract](../arch/desktop/distribution.md#^desktop-dist-t-config) shows that the configuration sets it. -The implementation pull request also records a run of `npm run verify -- local` on a supported Mac, the evidence [preflight](../../specs/arch/test-runner/preflight.md#Testing) requires from a real macOS host. +The implementation pull request also records a run of `npm run verify -- local` on a supported Mac, which exercises the test runner's macOS process backend ([processes started by tests](../../specs/arch/test-runner/test-runner.md#processes-started-by-tests)). Some promises order no test ([spec purposes](../../specs/arch/testing-policy.md#^spec-purposes)). That no build exists for any computer but an Apple Silicon Mac, and that the download link keeps its address across releases, follow from the build target chosen in the ToDesktop dashboard and from ToDesktop's hosting, which no Television test observes. That wording about the desktop app names the Mac where it names a platform ([^desktop-mac-wording](../../specs/product/desktop-app.md#^desktop-mac-wording)) is copy, which review checks. That the administrator guide gives the user the link and the installation steps is guide content, which [the guide's spec](../../specs/arch/cli/admin-guide.md#Testing) leaves to review. That the checks run before a change reaches `main` is a procedure that review keeps, and which checks a change needs is the judgment of its author and reviewers; no test decides either. The npm package's deprecation message and its installable published versions are npm registry state, which no repository test observes. The private desktop workspace, proven by the [ToDesktop build proof](../arch/desktop/distribution.md#^desktop-dist-t-workspace), is what keeps npm from publishing a new version of it. diff --git a/proofs/product/onboarding/onboarding-channels.md b/proofs/product/onboarding/onboarding-channels.md index c3d6fe56..fa9ce311 100644 --- a/proofs/product/onboarding/onboarding-channels.md +++ b/proofs/product/onboarding/onboarding-channels.md @@ -49,7 +49,7 @@ connect to the server directly. ### Acceptance test criteria -The built-process criteria below are **acceptance** shape ([testing-policy.md](../../../specs/arch/testing-policy.md)): the boundary is the built `tv` CLI, spawned as a real process against a real temporary home and observed through spawned `tv` commands and real HTTP. The server and every command receive the home with `--home`; its config file sets port `0`, and commands pass the acquired port with `--port`, following [the test-runner recipe](../../../specs/arch/test-runner/test-runner.md#Test infrastructure addresses). Bundle variation uses the production resolution mechanism — the built binary is copied beside a fixture bundle, or beside none. Setup reaches starting states through real boots and real `tv` commands. Fixtures are named per assertion. No mocks or test hooks are used. The tests derive the designated focus slug from the package; [arch/onboarding/content.md#^t-production-tree](../../arch/onboarding/content.md#^t-production-tree) independently pins the production value, so the acceptance walk does not duplicate package data. +The built-process criteria below are **acceptance** shape ([testing-policy.md](../../../specs/arch/testing-policy.md)): the boundary is the built `tv` CLI, spawned as a real process against a real temporary home and observed through spawned `tv` commands and real HTTP. The server and every command receive the home with `--home`; its config file sets port `0`, and commands pass the acquired port with `--port`, following [the test-runner recipe](../../../specs/arch/test-runner/test-runner.md#network-addresses-in-tests). Bundle variation uses the production resolution mechanism — the built binary is copied beside a fixture bundle, or beside none. Setup reaches starting states through real boots and real `tv` commands. Fixtures are named per assertion. No mocks or test hooks are used. The tests derive the designated focus slug from the package; [arch/onboarding/content.md#^t-production-tree](../../arch/onboarding/content.md#^t-production-tree) independently pins the production value, so the acceptance walk does not duplicate package data. - **Fresh install.** First serve against a new home creates every bundled onboarding channel with its configured name and one configured-or-default tab page per configured artifact in artifact-list order; all channels are unpinned and the display-active channel is the designated channel from the bundle. Every entry document served over HTTP carries the corresponding packaged content, and every directory artifact serves its sibling files byte-identical to the package. Expectations come from the shipped config and sources at runtime. Fixture: the shipped production bundle. TV Guide uses the same source-derived check for its single-file welcome document, with no separate welcome-image fetch; the task and calendar directories retain real sibling-file serving coverage. Every installed artifact's ID, as `tv` reports it, is in the generated form rather than the earlier predictable form, and the store of each artifact whose shipped config entry declares one holds its declared starting value with each date moved from the declared `shiftDatesFrom` to the installation day, as `tv resource json get --artifact` reports; today that is Company To-dos, holding the starting tasks due around the day of the walk. This proves [fresh installation](../../../specs/product/onboarding/onboarding-channels.md#^fresh-install), [generated IDs](../../../specs/product/onboarding/onboarding-channels.md#^onboarding-artifact-ids) and [stores that start with data](../../../specs/product/onboarding/onboarding-channels.md#^onboarding-stores) — *(policy-grade test: `test/node/fresh-install-onboarding.test.ts` “Fresh install creates every bundled channel and focuses the designated channel”, which finds artifacts through their channels' pages and derives the declared stores from the shipped config)*. ^ac-fresh-install - **A config-only home is brand-new.** The fresh-install walk's first serve runs over a home that holds only its config file and installs every bundled channel, so a config file does not make a home count as already served. This proves the config-only case of [fresh installation](../../../specs/product/onboarding/onboarding-channels.md#^fresh-install) — *(policy-grade test: `test/node/fresh-install-onboarding.test.ts` "Fresh install creates every bundled channel and focuses the designated channel")*. ^ac-config-only-home diff --git a/proofs/product/resources/resources.md b/proofs/product/resources/resources.md index d7b71006..bbcd7c2e 100644 --- a/proofs/product/resources/resources.md +++ b/proofs/product/resources/resources.md @@ -27,7 +27,7 @@ Not tested: [the model's limits](../../../specs/product/resources/resources.md#^ ### Acceptance criteria -Shared declaration: each server is the built CLI's `tv --home serve` over a temporary home whose config file sets port `0`, following [the test-runner recipe](../../../specs/arch/test-runner/test-runner.md#Test infrastructure addresses); every command is a spawned built `tv` process given that home and port. Authentication is on unless an assertion says otherwise. HTML artifacts are authored fixture files registered through `tv create-path-artifact`, and the viewer adds shared artifacts through `tv create-url-artifact`; stores are written only through pages and `tv resource json` commands, and share links are made only through `tv share-artifact`. Browser walks use the host-name mapping hook above and the testing policy's default CSS-motion override, which forfeits only motion that no assertion here claims. No other mock or test hook is used. +Shared declaration: each server is the built CLI's `tv --home serve` over a temporary home whose config file sets port `0`, following [the test-runner recipe](../../../specs/arch/test-runner/test-runner.md#network-addresses-in-tests); every command is a spawned built `tv` process given that home and port. Authentication is on unless an assertion says otherwise. HTML artifacts are authored fixture files registered through `tv create-path-artifact`, and the viewer adds shared artifacts through `tv create-url-artifact`; stores are written only through pages and `tv resource json` commands, and share links are made only through `tv share-artifact`. Browser walks use the host-name mapping hook above and the testing policy's default CSS-motion override, which forfeits only motion that no assertion here claims. No other mock or test hook is used. - **Acceptance — a page uses its artifact's store.** In a real browser, the Television app is loaded from the server at a mapped plain-HTTP host name, and shows an artifact whose source imports `/sdk/v1/resources.js` and calls `getStore()`, with no artifact ID or token in the source. Before anything writes to the store, `tv resource json get --artifact ` prints `{"exists":false}` and the artifact's record has no store; the page's `getAccess` resolves `read-write`, its `onAccessChanged` callback hears `read-write`, and its `onValue` listener on the whole value hears no value. The page writes, the listener hears the write, and `tv resource json get --artifact ` then shows it; the listener, registered before the first write, was called for no other reason. A `tv resource json set --artifact ` from the shell reaches the page's listener without a reload, and the page's `onResourcesChanged` callback receives `changed` with the path written, and no resource ID or artifact ID. A handle from `getStore(resourceID)`, given the store's own resource ID as the record shows it, fails its first operation with `not-enabled` while the artifact's store keeps working. The walk runs in Chromium and in Firefox. This proves [an artifact's store](../../../specs/product/resources/resources.md#^rs-artifact-store), [the SDK's use of the page's own address](../../../specs/product/resources/resources.md#^rs-sdk-own-address), [handles](../../../specs/product/resources/resources.md#^rs-sdk-handles), the artifact's own ID carrying `read-write` under [levels](../../../specs/product/resources/resources.md#^rs-id-levels), the page's half of [events](../../../specs/product/resources/resources.md#^rs-events) and [plain HTTP](../../../specs/product/resources/resources.md#^rs-plain-http) — *(policy-grade test: `test/node/resource-page-acceptance.test.ts` "uses its artifact's store before and after the first write, hears shell changes, and finds stores by resource ID not enabled", in Chromium and Firefox)*. ^rs-ac-page-spine - **Acceptance — pages behind an HTTPS front.** In Chromium, the Television app is loaded at [the test front's](#Test hooks) `https:` address, and shows an artifact whose source imports `/sdk/v1/resources.js` and calls `getStore()`. The artifact's frame is a secure context at an address on the front's origin, although its document's own origin is opaque, and the SDK opens its page connection at `wss://` on the front's host. The page's `getAccess` resolves `read-write`; its write reaches `tv resource json get --artifact `; and a `tv resource json set --artifact ` from the shell reaches its listener without a reload. The artifact's share link at `read`, opened at the front's address, connects at `wss://` too, and its page's `getAccess` resolves `read`, its listener hears the store's value, and its write is refused with `read-only`, leaving the store unchanged. This proves [HTTPS fronts](../../../specs/product/resources/resources.md#^rs-https-front) — *(policy-grade test: `test/node/resource-https-front-acceptance.test.ts` "uses its artifact's store and a share link's level through a front that terminates TLS")*. ^rs-ac-https-front diff --git a/proofs/product/themes-and-appearance.md b/proofs/product/themes-and-appearance.md index 210a5ae0..605d4336 100644 --- a/proofs/product/themes-and-appearance.md +++ b/proofs/product/themes-and-appearance.md @@ -6,6 +6,8 @@ Proves [specs/product/themes-and-appearance.md](../../specs/product/themes-and-a ## Coverage model +The persistence and executable-lifecycle walks in `theme-product-acceptance.test.ts` load the built application through the real stable-front proxy, keeping the browser origin and authentication across backends that bind port `0` on restart. Request-event inventories retained by both walks establish the built document and asset paths across their navigations. Their product outcomes, fixtures and motion declarations stay the same. The separate `settings.test.ts` registry/control walk keeps its existing development-module boundary. Reset already uses the built application and is unchanged. + Coverage declarations are carried inside the migrated assertion blocks below. The executable-theme browser spine uses two connected application documents in one real Chromium session against a running production server. Real package fixtures drive main-enabled, unflagged, iframe-enabled, syntax-error, and uncaught-exception paths through Settings, persisted display state, the three public script routes, the production application resource owner, two real iframe boundaries, and an artifact document. Theme-owned markers, execution counters, package assets, and frame messages make absence before main consent, main execution in both application documents, automatic sandbox execution, repeat execution, document-lifetime effects, reload reset, classic-script-relative URL resolution, and frame replacement observable without a test hook or mock. Browser error events and ordinary application controls establish failure isolation. @@ -39,16 +41,16 @@ Clouds' association with Television 1.3 composes with [product versioning's rele ### Acceptance criteria -- **Acceptance** (outer boundaries: real Settings controls over HTTP and websocket connections; the persistence matrix loads production shell modules through the test Vite server against a built server, while the cross-browser `None`/Clouds case loads that server's built shell; valid, invalid, and dot-prefixed theme folders and real temporary homes are fixtures; no mocks or test hooks): a fresh installation offers None plus the themes in [the bundled inventory](../../specs/ui/themes/bundled.yml); refresh shows valid packages and readable invalid-folder errors while every dot-prefixed folder is absent from options and errors; selection propagates to a second client; `None` selects the null theme and clears the active theme ID; the selected theme renders in Chromium and Firefox; and the server-confirmed selection survives a server restart and reconnection in the same browser page — *(covered by `packages/web/test/e2e/settings.test.ts` “settings controls cross the registry and confirmed display state” and `packages/web/test/e2e/theme-product-acceptance.test.ts` “the real dropdown renders its selected built-shell theme in Chromium and Firefox” and “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-selection +- **Acceptance** (outer boundaries: real Settings controls over HTTP and websocket connections; the persistence and cross-browser `None`/Clouds walks load the built application, while the separate registry/control walk in `settings.test.ts` retains its development-module boundary; valid, invalid, and dot-prefixed theme folders and real temporary homes are fixtures; the persistence walk uses the real stable-front proxy as harness transport; no mocks or test hooks): a fresh installation offers None plus the themes in [the bundled inventory](../../specs/ui/themes/bundled.yml); refresh shows valid packages and readable invalid-folder errors while every dot-prefixed folder is absent from options and errors; selection propagates to a second client; `None` selects the null theme and clears the active theme ID; the selected theme renders in Chromium and Firefox; and the server-confirmed selection survives a server restart and reconnection in the same browser page — *(policy-grade tests: `packages/web/test/e2e/settings.test.ts` “settings controls cross the registry and confirmed display state” and `packages/web/test/e2e/theme-product-acceptance.test.ts` “the real dropdown renders its selected built-shell theme in Chromium and Firefox” and “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-selection - **Acceptance** (outer boundary: Firefox drives the real Settings controls in the built application against a really-running production server and its real event transport; fixed-dark and light theme packages, a ready Markdown editor with authored contents, and a temporary home are fixtures; no mocks or test hooks): selecting the light theme changes both the shell and the same loaded Markdown editor from the fixed-dark presentation to the light presentation without replacing the editor document or losing its contents — *(covered by test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “Firefox refreshes a ready Markdown editor from a fixed-dark theme to light”)*. The same browser walk supplies the delivery evidence at [theme-delivery-t-markdown-refresh-firefox](../arch/themes/delivery.md#^theme-delivery-t-markdown-refresh-firefox). ^theme-ac-markdown-refresh-firefox -- **Acceptance** (outer boundary: two real Chromium application documents drive the built application's Settings controls against a running production server and its real event transport; main-enabled, unflagged, syntax-error, and uncaught-exception theme packages, a package-relative asset, an artifact document, and a temporary home are fixtures; no mocks or test hooks): before consent, the selected `enableMainJS: true` theme has no `main.js` include or request in either application document. Granting exact-ID consent runs its classic main script in both documents after the application runtime is available, exposes `document.currentScript.src`, resolves the package asset from that URL, survives a built-server restart, and leaves the artifact script-free. Appearance changes do not rerun it. An unrelated active-package edit, restoration after complete package-root loss, and server reconnection each rerun it; the missing-root interval executes nothing. An unflagged `main.js` executes nowhere. Separately consented syntax-error and uncaught-exception main scripts report browser errors and leave an ordinary Settings control operational — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-executable-browser +- **Acceptance** (outer boundary: two real Chromium application documents drive the built application's Settings controls against a running production server and its real event transport; main-enabled, unflagged, syntax-error, and uncaught-exception theme packages, a package-relative asset, an artifact document, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): before consent, the selected `enableMainJS: true` theme has no `main.js` include or request in either application document. Granting exact-ID consent runs its classic main script in both documents after the application runtime is available, exposes `document.currentScript.src`, resolves the package asset from that URL, survives a built-server restart, and leaves the artifact script-free. Appearance changes do not rerun it. An unrelated active-package edit, restoration after complete package-root loss, and server reconnection each rerun it; the missing-root interval executes nothing. An unflagged `main.js` executes nowhere. Separately consented syntax-error and uncaught-exception main scripts report browser errors and leave an ordinary Settings control operational — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-executable-browser - **Acceptance** (outer boundary: the same two real Chromium application documents and running production server; a main-enabled theme whose script installs document-lifetime probes, a second theme, and persisted Settings-open state are fixtures; no mocks or test hooks): while the main include is active, withdrawing consent, selecting the second theme, and selecting `None` each reload both connected application documents, clear the prior script's document-lifetime effects, and start under the confirmed destination state. Only a document whose Settings popover was open reopens it; its URL marker is consumed before reopening and causes no later reload. After selecting the second theme, selecting the still-consented main-enabled theme again reinstalls its include and executes it without another consent grant. Changing themes while no main include is installed remains in place — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable theme reset reloads connected Chromium documents under confirmed state”)*. ^theme-ac-executable-reset -- **Acceptance** (outer boundary: the same real Chromium application and running production server; a package independently enabling both sandbox entries, a destination theme, syntax-error and uncaught-exception iframe entries, Settings and application modal-dialog controls, and pointer-observation probes are fixtures; no mocks or test hooks): without main-page consent, the background and foreground entries execute in separate opaque-origin `allow-scripts` frames at their specified layers and cannot access the application DOM. Theme selection and an unrelated active-package edit replace both frame documents; package-root loss replaces them with empty entries, restoration executes fresh frames, and registry refresh that disables either declaration removes only that frame. Syntax and uncaught runtime errors stay in their affected frames while the application operates. Trusted pointer move, down, up, cancel, and click events in the application document reach both frames with viewport coordinates and button state while the same underlying application sequence, including click, remains effective; a drag that produces no click forwards its down, movement, and up but no click. Pointer move, down, up, and click inside the bridge-capable artifact reach both frames in application-viewport coordinates while the artifact's own click remains effective. Unexpected focus returns first to the last connected application element and then to `#app` when that element is unavailable. The foreground layers do not prevent using the production Settings popover in its document layer or an application modal dialog in the browser top layer — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-sandboxed-frames -- **Acceptance** (outer boundary: the production Settings appearance control in a real Chromium application against a running production server; a real active package enabling both sandbox entries, entry-script startup probes, visible application probes behind transparent regions of both frames, and a temporary home are fixtures; no mocks or test hooks): under the [frame lifecycle](../../specs/product/themes-and-appearance.md#^theme-iframe-lifecycle), an effective light-to-dark appearance change removes both old frame documents before installing replacements, leaves no old-appearance frame, and reruns both entries in documents whose root `data-theme` is `dark`. Each replacement host and document root has a used dark scheme, and the application probes remain visibly unchanged through the frames' transparent regions — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-frame-appearance -- **Acceptance** (outer boundary: a real Chromium browser loads production shell modules and a live-canonical HTML artifact against a running built server and its real `/events` transport; a real active package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and a temporary home are fixtures; no mocks or test hooks): changing only the nested stylesheet updates the application and artifact presentation; moving the complete active package away produces their foundation and canonical baselines while the exact selected theme ID remains confirmed; restoring it at the same path restores both presentations and keeps that ID, without registry refresh or another selection — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-active-package +- **Acceptance** (outer boundary: the same real Chromium application and running production server; a package independently enabling both sandbox entries, a destination theme, syntax-error and uncaught-exception iframe entries, Settings and application modal-dialog controls, and pointer-observation probes are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): without main-page consent, the background and foreground entries execute in separate opaque-origin `allow-scripts` frames at their specified layers and cannot access the application DOM. Theme selection and an unrelated active-package edit replace both frame documents; package-root loss replaces them with empty entries, restoration executes fresh frames, and registry refresh that disables either declaration removes only that frame. Syntax and uncaught runtime errors stay in their affected frames while the application operates. Trusted pointer move, down, up, cancel, and click events in the application document reach both frames with viewport coordinates and button state while the same underlying application sequence, including click, remains effective; a drag that produces no click forwards its down, movement, and up but no click. Pointer move, down, up, and click inside the bridge-capable artifact reach both frames in application-viewport coordinates while the artifact's own click remains effective. Unexpected focus returns first to the last connected application element and then to `#app` when that element is unavailable. The foreground layers do not prevent using the production Settings popover in its document layer or an application modal dialog in the browser top layer — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-sandboxed-frames +- **Acceptance** (outer boundary: the production Settings appearance control in a real Chromium application against a running production server; a real active package enabling both sandbox entries, entry-script startup probes, visible application probes behind transparent regions of both frames, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the [frame lifecycle](../../specs/product/themes-and-appearance.md#^theme-iframe-lifecycle), an effective light-to-dark appearance change removes both old frame documents before installing replacements, leaves no old-appearance frame, and reruns both entries in documents whose root `data-theme` is `dark`. Each replacement host and document root has a used dark scheme, and the application probes remain visibly unchanged through the frames' transparent regions — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-frame-appearance +- **Acceptance** (outer boundary: a real Chromium browser loads the built application and a live-canonical HTML artifact against a running built server and its real `/events` transport; a real active package with a byte-stable entry stylesheet, nested imported stylesheet, computed-presentation probes, and a temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): changing only the nested stylesheet updates the application and artifact presentation; moving the complete active package away produces their foundation and canonical baselines while the exact selected theme ID remains confirmed; restoring it at the same path restores both presentations and keeps that ID, without registry refresh or another selection — *(policy-grade test: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart”)*. ^theme-ac-active-package - **Acceptance** (outer boundary: the real Electron app and another client connected to a running server; a theme enabling main, background, and foreground entries, a second theme, an artifact webview, and an isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): before consent the main script does not execute, while both sandboxed frames are created and execute in the application. Pointer move, down, up, and click inside the ordinary HTML artifact webview reach both theme frames in application-viewport coordinates while the artifact's click remains effective. Granting consent through Settings executes `main.js` in the application renderer but no theme JavaScript in the artifact webview, and the renderer exposes the same Node-facing globals and preload bridge before and after activation. Consent survives application restart; remote main opt-out and remote selection of the second theme each reload the application document, clear its document-lifetime probe, and apply confirmed state. A remote same-theme active-package event replaces both frame documents without itself causing a full application reload — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “executes an executable theme in Electron and resets on remote consent and selection changes”)*. ^theme-ac-executable-electron - **Acceptance** (outer boundary: the real Electron app connected to a running server; an installed theme and isolated Electron profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): the selected application theme renders again after quitting and relaunching through the saved connection — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “renders the persisted app theme after quitting and relaunching”)*. ^theme-ac-selection-electron -- **Acceptance** (outer boundary: the built application in a real Chromium browser against a really-running server; one adaptive package and one fixed-dark package, local Television-managed HTML and markdown artifacts, a third-party URL artifact, both sandboxed entries with startup probes, dedicated computed-style probes, controlled system appearance, and a real temporary home are fixtures; no mocks or test hooks): under the adaptive theme, `Adapt to system`, `Light`, and `Dark` produce their expected effective appearance in the application and applicable artifacts. Under the fixed-dark theme, selecting and persisting an opposing Light preference leaves the application root, foundation mode values and native scheme, both artifact classes, and both frame host/document pairs dark; it does not reload app or artifacts, and it keeps the existing frame documents because effective appearance did not change. The same fixed results survive a built-server restart. Returning to the adaptive theme applies the latest Light preference; that effective change replaces and reruns both frames while leaving their transparent composition intact — *(covered by `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart” and “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-appearance +- **Acceptance** (outer boundary: the built application in a real Chromium browser against a really-running server; one adaptive package and one fixed-dark package, local Television-managed HTML and markdown artifacts, a third-party URL artifact, both sandboxed entries with startup probes, dedicated computed-style probes, controlled system appearance, and a real temporary home are fixtures; the real stable-front proxy supplies harness transport; no mocks or test hooks): under the adaptive theme, `Adapt to system`, `Light`, and `Dark` produce their expected effective appearance in the application and applicable artifacts. Under the fixed-dark theme, selecting and persisting an opposing Light preference leaves the application root, foundation mode values and native scheme, both artifact classes, and both frame host/document pairs dark; it does not reload app or artifacts, and it keeps the existing frame documents because effective appearance did not change. The same fixed results survive a built-server restart. Returning to the adaptive theme applies the latest Light preference; that effective change replaces and reruns both frames while leaving their transparent composition intact — *(policy-grade tests: `packages/web/test/e2e/theme-product-acceptance.test.ts` “theme settings persist rendered app and artifact outcomes through a built-server restart” and “executable themes run only in the Chromium application lifecycle and isolate failures”)*. ^theme-ac-appearance - **Acceptance** (outer boundary: the real Electron app connected to a really-running server; one adaptive and one fixed-dark installed package, an existing Television-managed artifact webview, a third-party URL artifact webview, native-appearance observation, computed-style probes, and an isolated profile are fixtures; the declared desktop shell-version test hook forfeits only installed-version provenance to upgrade-gate coverage): the adaptive theme carries `system`, `light`, and `dark` through `nativeTheme` to both retained webviews and their media-query results. Under the fixed-dark theme, an opposing Light preference persists while the application, `nativeTheme`, native-surface appearance source, and both current webviews remain dark and loaded. Returning to the adaptive theme applies that latest Light preference; the third-party URL webview stays loaded, while the Television-managed artifact follows its ordinary theme-selection reload lifecycle — *(covered by `packages/desktop/test/e2e/webview-event-parity.test.ts` “applies manifest-resolved appearance to Electron and retained webviews”)*. This is the same Electron walk as [desktop appearance acceptance](../arch/desktop/appearance.md#^desktop-appearance-t-electron), not a second launch. ^theme-ac-appearance-electron - **Acceptance** (outer boundary: a real browser loads artifacts from a really running production `Server`; the committed old-style v1 document, an authored v2 control document, fresh production canonical tree, temporary storage, and installed loud unscoped theme are fixtures; no mocks or test hooks): the v1 artifact loads its wrapper, base, components, and font but makes no theme request, renders the fixed light canonical probe, and does not render the loud theme value; the v2 control requests the theme and renders its loud value and dark scheme — *(covered by `packages/server/test/e2e/canonical.spec.ts` “frozen v1 stays fixed light while live v2 receives the active theme”)*. ^theme-ac-frozen-v1 - **Acceptance** (outer boundary: a spawned built CLI server against a real fresh temporary home; the packaged bundled-theme tree is the fixture; no mocks or test hooks): first boot installs every bundled theme byte-for-byte, selects Clouds from an unset theme, exposes its exact stylesheet and wallpaper assets, and records the one-time decision — *(covered by `test/node/theme-packaging.test.ts` “Built CLI installs every packaged theme and initially selects Clouds”)*. ^theme-ac-bundled diff --git a/scripts/manage-blaxel-testshards.mjs b/scripts/manage-blaxel-testshards.mjs index 58996712..4c41f705 100644 --- a/scripts/manage-blaxel-testshards.mjs +++ b/scripts/manage-blaxel-testshards.mjs @@ -2,6 +2,9 @@ import { SandboxInstance } from "@blaxel/core"; import { readFileSync } from "node:fs"; import process from "node:process"; +import { randomUUID } from "node:crypto"; +import { workerLeaseCommand } from "./test/blaxel-worker-lease.mjs"; +import { publishWorker, deleteWorker } from "./test/blaxel-pool-lifecycle.mjs"; import { listBlaxelPoolSandboxes } from "./test/blaxel-pool.mjs"; import { DEFAULT_BLAXEL_POOL_SIZE } from "./testshard-constants.mjs"; @@ -33,7 +36,7 @@ if (command === "ensure" && !/^\d+$/.test(nodeSelector)) fail(".nvmrc must conta if (command === "list") { const sandboxes = await listPoolSandboxes(); for (const sandbox of sandboxes) { - console.log(`${sandbox.metadata?.name}\t${sandbox.status}\t${sandbox.spec?.runtime?.memory ?? "?"}MB\t${sandbox.spec?.image ?? "?"}`); + console.log(`${sandbox.metadata?.name}\t${sandbox.status}\t${sandbox.metadata?.labels?.purpose}\t${sandbox.spec?.runtime?.memory ?? "?"}MB\t${sandbox.spec?.image ?? "?"}`); } process.exit(0); } @@ -56,14 +59,12 @@ if (command === "ensure") { region, labels: { project: "television", - purpose: "testshards", + purpose: "testshards-provisioning", pool, arch, }, }); - await sandbox.wait(); - await provisionSandboxRuntime(sandbox); - await verifySandboxArchitecture(sandbox); + await publishWorker(sandbox, { provision: provisionSandboxRuntime, verify: verifySandboxArchitecture, update: SandboxInstance.updateMetadata }); console.log(` ${sandbox.metadata?.name}: ${sandbox.status}`); } process.exit(0); @@ -71,11 +72,20 @@ if (command === "ensure") { if (command === "delete") { const sandboxes = await listPoolSandboxes(); + let refused = false; for (const sandbox of sandboxes) { - console.log(`deleting ${sandbox.metadata?.name}`); - await SandboxInstance.delete(sandbox.metadata.name); + try { + await deleteWorker(sandbox, { reserve: reserveDeletion, update: SandboxInstance.updateMetadata, remove: SandboxInstance.delete }); + console.log(`deleted ${sandbox.metadata.name}`); + } catch (error) { refused = true; console.error(`held ${sandbox.metadata.name}: ${error.message}`); } } - process.exit(0); + process.exit(refused ? 1 : 0); +} + +async function reserveDeletion(sandbox) { + const script = ["set -eo pipefail", 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', '. "$NVM_DIR/nvm.sh"', "nvm use --silent default", "set -u", workerLeaseCommand({ operation: "delete", lockId: `delete-${randomUUID()}` })].join("; "); + const result = await sandbox.process.exec({ name: `tv-delete-reservation-${randomUUID()}`, command: `bash -lc ${shellQuote(script)}`, waitForCompletion: true, timeout: 30 }); + if (result.exitCode !== 0) throw new Error(`ownership check refused deletion: ${result.logs ?? result.exitCode}`); } async function provisionSandboxRuntime(sandbox) { diff --git a/scripts/run-blaxel-testshards.mjs b/scripts/run-blaxel-testshards.mjs index c0d6dfbe..76c0f9d8 100644 --- a/scripts/run-blaxel-testshards.mjs +++ b/scripts/run-blaxel-testshards.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { targetCommand } from "./test/target-command.mjs"; import { listBlaxelPoolSandboxes } from "./test/blaxel-pool.mjs"; import { RECOMMENDED_TEST_SHARD_COUNT } from "./testshard-constants.mjs"; import { combineAttemptReports, failedTasksFromShards, formatShardLogs, runInfrastructureAttempts, summarizeInfrastructureAttempts, writeProviderReports } from "./test/provider-outputs.mjs"; @@ -10,9 +11,13 @@ import { blaxelCheckoutScript, blaxelDependencyScript, resolveBlaxelRepositoryUr import { transferShardInputs } from "./test/blaxel-shard-dispatch.mjs"; import { buildLinuxStaleOwnerAuditInvocation, parseLinuxStaleOwnerAuditOutput } from "./test/blaxel-stale-owner-audit.mjs"; import { normalizeSetupTimingStatus, normalizeTimingPhase } from "./test/phase-metrics.mjs"; +import { RESOURCE_SIDECAR_BYTES, RESOURCE_ATTEMPT_BYTES, writeResourceDiagnostic } from "./test/resource-diagnostics.mjs"; import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; +import { randomUUID } from "node:crypto"; +import { renewLease, finishLease, retainAttemptEvidence } from "./test/blaxel-lease-lifecycle.mjs"; +import { workerLeaseCommand, workerEntryCommand, shardScratchPath } from "./test/blaxel-worker-lease.mjs"; import path from "node:path"; import process from "node:process"; @@ -39,6 +44,7 @@ const timeouts = resolveTimeouts(timeoutProfile, options); const playwrightBrowsersPath = options["playwright-browsers-path"] ?? "/home/playwright/.cache/ms-playwright"; const runId = `${new Date().toISOString().replace(/[:.]/g, "-")}-${process.pid}`; const activeLeases = new Map(); +const pendingAcquisitions = new Set(); const attempts = []; let finalReport = null; let activeAttempt = null; @@ -136,6 +142,7 @@ async function runStaleOwnerAudit() { const sandboxes = rotatePool(await listPoolSandboxes(), `${runId}:stale-owner-audit`, 1); if (sandboxes.length === 0) fail(`No deployed x64 Blaxel worker is available in pool ${pool}.`); const lease = await acquireAny(sandboxes, { shardIndex: 1, slot: 0, requestedWorkers: 1, verifyArchitecture: false }); + if (shuttingDown) await shutdownPromise; if (!lease) fail(`No available Blaxel worker in pool ${pool} for the read-only stale-owner audit.`); const { command, args } = buildLinuxStaleOwnerAuditInvocation(); @@ -143,15 +150,17 @@ async function runStaleOwnerAudit() { lease.processName = processName; try { console.log(`acquired stale-owner audit lease: ${lease.sandbox.metadata.name}`); - const setup = await runSystemDependencySetup(lease); - lease.processName = processName; - console.log(`read-only diagnostic command: ${[command, ...args].map(shellQuote).join(" ")}`); - const proc = await lease.sandbox.process.exec({ - name: processName, - command: [command, ...args].map(shellQuote).join(" "), - waitForCompletion: true, - timeout: 30, - }); + const remoteScript = `${lease.input.scratch}/audit.sh`; + lease.pendingWrites = lease.sandbox.fs.write(remoteScript, `#!/usr/bin/env bash\nset -e\n${systemDependenciesCommand()}\n${[command, ...args].map(shellQuote).join(" ")}\n`); + await lease.pendingWrites; + await leaseOperation(lease, "prepare"); + await leaseOperation(lease, "dispatch", { processName }); + lease.executionRequested = true; + const setup = { name: "system-deps" }; + const proc = await (lease.dispatchRequest = lease.sandbox.process.exec({ + name: processName, command: workerEntryCommand({ ...lease.input, processName }, remoteScript), + waitForCompletion: true, timeout: timeouts.workerTimeoutSeconds, + })); const rawLogs = await lease.sandbox.process.logs(processName, "all").catch(() => proc.logs ?? ""); const logs = chunkToText(rawLogs); if (proc.exitCode !== 0) throw new Error(`read-only stale-owner diagnostic exited ${proc.exitCode}: ${logs.trim()}`); @@ -170,6 +179,7 @@ async function runStaleOwnerAudit() { ...audit, }; console.log(`${JSON.stringify(result, null, 2)}`); + lease.retrieved = true; if (!audit.capabilities.procfs.readable || audit.processScan.status !== "ok") return 2; if (!audit.capabilities.ss.available || !audit.capabilities.ss.usable) return 2; if (staleOwnerCount > 0) return 3; @@ -181,24 +191,6 @@ async function runStaleOwnerAudit() { } } -async function runSystemDependencySetup(lease) { - const args = ["-lc", systemDependenciesCommand()]; - const processName = `tv-system-deps-${runId}`.replace(/[^a-zA-Z0-9-]/g, "-"); - lease.processName = processName; - console.log(`system dependency setup: ${["bash", ...args].map(shellQuote).join(" ")}`); - const proc = await lease.sandbox.process.exec({ - name: processName, - command: ["bash", ...args].map(shellQuote).join(" "), - waitForCompletion: true, - timeout: timeouts.workerTimeoutSeconds, - }); - const rawLogs = await lease.sandbox.process.logs(processName, "all").catch(() => proc.logs ?? ""); - const logs = chunkToText(rawLogs); - if (logs.trim()) console.log(logs.trimEnd()); - if (proc.exitCode !== 0) throw new Error(`Blaxel system dependency setup exited ${proc.exitCode}: ${logs.trim()}`); - return { name: "system-deps", command: ["bash", ...args], exitCode: proc.exitCode }; -} - function validateStaleOwnerAuditOptions(parsed) { const allowed = new Set(["audit-stale-owners", "pool", "arch"]); const unexpected = Object.keys(parsed).filter((key) => !allowed.has(key)); @@ -238,7 +230,7 @@ async function runAttempt({ attempt, shardIndices, excludedSandboxes = [], outpu maxConcurrency: MAX_PARALLEL_LEASE_ACQUISITIONS, acquire: async ({ shardIndex, slot }) => { attemptState.shardStartedAt.set(shardIndex, Date.now()); - const lease = await acquireAny(sandboxes, { shardIndex, slot, requestedWorkers: shardIndices.length }); + const lease = await acquireAny(sandboxes, { shardIndex, slot, requestedWorkers: shardIndices.length, attempt: attempt + 1 }); if (lease) attemptState.leases.set(shardIndex, lease); return lease; }, @@ -254,6 +246,7 @@ async function runAttempt({ attempt, shardIndices, excludedSandboxes = [], outpu attemptState.leases.set(outcome.shardIndex, outcome.lease); acquired.push(outcome.lease); } + if (shuttingDown) await shutdownPromise; const acquisitionIncomplete = acquisitionOutcomes.some((outcome) => !outcome.lease); let results; if (acquisitionIncomplete) { @@ -265,11 +258,12 @@ async function runAttempt({ attempt, shardIndices, excludedSandboxes = [], outpu console.log(`acquired: ${formatShardLeases(acquired)}`); console.log(`starting: ${formatShardLeases(acquired)}`); results = await Promise.all(acquired.map(async (lease) => { - const result = await runShard(lease, outputDir, { + const result = await (lease.workPromise = runShard(lease, outputDir, { + attempt, onExecutionRequested: () => attemptState.executionRequested.add(lease.shardIndex), onExecutionStarted: () => attemptState.executionStarted.add(lease.shardIndex), onResultProgress: (progress) => attemptState.partialResults.set(lease.shardIndex, progress), - }); + })); attemptState.partialResults.delete(lease.shardIndex); attemptState.completedResults.set(lease.shardIndex, result); return result; @@ -296,6 +290,7 @@ async function runAttempt({ attempt, shardIndices, excludedSandboxes = [], outpu durationMs: Date.now() - startedAt, shards: results.sort((a, b) => a.shardIndex - b.shardIndex), }; + if (shuttingDown) await shutdownPromise; writeProviderReports(outputDir, report); return report; } finally { @@ -331,33 +326,69 @@ function hashString(value) { return hash >>> 0; } -async function acquireAny(sandboxes, { shardIndex, slot, requestedWorkers, verifyArchitecture = true }) { - const lockId = `${runId}-shard-${shardIndex}`; - const expiresAt = Math.floor(Date.now() / 1000) + timeouts.lockTtlSeconds; +async function leaseOperation(lease, operation, extra = {}) { + if (lease.lost) throw lease.lost; + if (operation === "prepare") lease.preparationRequested = true; + const body = ["set -eo pipefail", 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', '. "$NVM_DIR/nvm.sh"', "nvm use --silent default", "set -u", workerLeaseCommand({ ...lease.input, operation, ...extra })].join("; "); + const result = await lease.sandbox.process.exec({ name: `tv-${operation}-${randomUUID()}`, command: `bash -lc ${shellQuote(body)}`, waitForCompletion: true, timeout: 30 }); + const lines = chunkToText(result.logs ?? "").trim().split("\n"); + let response; + try { response = JSON.parse(lines.at(-1)); } catch { + const details = chunkToText(await lease.sandbox.process.logs(result.name, "all")).slice(-4000); + throw new Error(`worker ${operation} returned no ownership evidence (exit ${result.exitCode}): ${details}`); + } + if (result.exitCode !== 0 || !response.ok) throw new Error(`worker ${operation}: ${response.reason ?? "failed"}`); + return response; +} + +function acquireAny(sandboxes, options) { + const pending = acquireWorker(sandboxes, options); + pendingAcquisitions.add(pending); + return pending.finally(() => pendingAcquisitions.delete(pending)); +} + +async function acquireWorker(sandboxes, { shardIndex, slot, requestedWorkers, verifyArchitecture = true, attempt = 1 }) { + const lockId = `${runId}-s${shardIndex}-a${attempt}-${randomUUID().slice(0, 8)}`; + const input = { lockId, runId, shardIndex, attempt, attemptId: lockId, owner: `${os.hostname()}:${process.pid}`, ttlSeconds: timeouts.lockTtlSeconds, audit: auditStaleOwners, summaryIdentity: shardSummaryIdentity(shardIndex) }; + input.scratch = shardScratchPath(input); for (const sandbox of acquisitionOrder(sandboxes, { slot, requestedWorkers })) { - if (sandbox.__leased) continue; - const command = `bash -lc ${shellQuote(lockCommand({ lockId, shardIndex, expiresAt }))}`; - const proc = await sandbox.process.exec({ name: `tv-lock-${lockId}`, command, waitForCompletion: true, timeout: 30 }); - if (proc.exitCode === 0) { - let retainLock = false; - try { - if (verifyArchitecture && !(await verifySandboxArchitecture(sandbox))) { - sandbox.__leased = true; - continue; - } - sandbox.__leased = true; - const lease = { sandbox, lockId, shardIndex, processName: null }; - activeLeases.set(lockId, lease); - retainLock = true; - return lease; - } finally { - if (!retainLock) await releaseLock(sandbox, lockId); - } + if (sandbox.__leased || shuttingDown) continue; + const lease = { sandbox, lockId, shardIndex, input: { ...input, worker: sandbox.metadata.name }, processName: null }; + try { + await leaseOperation(lease, "acquire"); + } catch (error) { + console.error(`Skipping ${sandbox.metadata.name}: ${error.message}`); + continue; + } + sandbox.__leased = true; + activeLeases.set(lockId, lease); + lease.stopRenewal = renewLease({ + intervalMs: Math.max(250, Math.min(30_000, timeouts.lockTtlSeconds * 1000 / 4)), + renew: () => leaseOperation(lease, "renew"), + onLost: (error) => { lease.lost = error; console.error(`Lease renewal lost on ${sandbox.metadata.name}: ${error.message}`); }, + }); + try { + if (!verifyArchitecture || await verifySandboxArchitecture(sandbox)) return lease; + } catch (error) { + console.error(`Architecture check failed on ${sandbox.metadata.name}: ${error.message}`); } + await stopAndReleaseLease(lease, { killProcess: false }); + activeLeases.delete(lockId); } return null; } +function shardSummaryIdentity(shardIndex) { + return { + shardIndex, shardTotal, + ...(shardPlan ? { + planId: shardPlan.planId, timingProvider: shardPlan.timingProvider, + testedCommit: shardPlan.testedCommit, testedTree: shardPlan.testedTree, + assignedFiles: shardPlan.shards[shardIndex - 1].surfaces.flatMap(surface => surface.files.map(file => file.path)).sort(), + } : {}), + }; +} + async function verifySandboxArchitecture(sandbox) { const check = [ "set -euo pipefail", @@ -393,24 +424,10 @@ function acquisitionOrder(sandboxes, { slot, requestedWorkers }) { return ordered; } -function lockCommand({ lockId, shardIndex, expiresAt }) { - const owner = `${os.hostname()}:${process.pid}`; - const freshJson = JSON.stringify({ lockId, runId, shard: `${shardIndex}/${shardTotal}`, owner, expiresAt }); - const reclaimedJson = JSON.stringify({ lockId, runId, shard: `${shardIndex}/${shardTotal}`, owner, expiresAt, reclaimed: true }); - return [ - "set -euo pipefail", - "lock=/tmp/tv-testshard.lock", - "now=$(date +%s)", - `if mkdir "$lock" 2>/dev/null; then node -e ${shellQuote(`const fs=require("fs"); const data=${JSON.stringify(freshJson)}; const obj=JSON.parse(data); obj.acquiredAt=Number(process.env.now||${Date.now()}); fs.writeFileSync("/tmp/tv-testshard.lock/owner.json", JSON.stringify(obj));`)}; exit 0; fi`, - "expires=0", - "if [ -f \"$lock/owner.json\" ]; then expires=$(node -e 'try{console.log(JSON.parse(require(\"fs\").readFileSync(process.argv[1],\"utf8\")).expiresAt||0)}catch{console.log(0)}' \"$lock/owner.json\"); fi", - `if [ "$expires" -gt 0 ] && [ "$expires" -lt "$now" ]; then rm -rf "$lock"; mkdir "$lock"; node -e ${shellQuote(`const fs=require("fs"); const data=${JSON.stringify(reclaimedJson)}; const obj=JSON.parse(data); obj.acquiredAt=Number(process.env.now||${Date.now()}); fs.writeFileSync("/tmp/tv-testshard.lock/owner.json", JSON.stringify(obj));`)}; exit 0; fi`, - "exit 42", - ].join("; "); -} async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onExecutionStarted = () => {}, onResultProgress = () => {} } = {}) { const { sandbox, shardIndex, lockId } = lease; + const diagnosticAttemptId = lease.input.attemptId; const startedAt = Date.now(); if (lease.restorePhase) { lease.restorePhase.completedAt = new Date(startedAt).toISOString(); @@ -419,50 +436,86 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE const overallStarted = lease.restorePhase ? Date.parse(lease.restorePhase.startedAt) : startedAt; const processName = `tv-testshard-${runId}-s${shardIndex}`.replace(/[^a-zA-Z0-9-]/g, "-"); lease.processName = processName; - const remoteScript = `/tmp/${processName}.sh`; + const remoteScript = `${lease.input.scratch}/worker.sh`; + lease.remoteScript = remoteScript; const logPath = path.join(outputDir, `shard-${shardIndex}.log`); lease.logPath = logPath; const summaryPath = path.join(outputDir, `shard-${shardIndex}.json`); - const planRemotePath = shardPlanRaw ? `/tmp/${processName}.plan.json` : null; - const dispatch = await transferShardInputs({ - write: (remotePath, contents) => sandbox.fs.write(remotePath, contents), + const planRemotePath = shardPlanRaw ? `${lease.input.scratch}/plan.json` : null; + const dispatch = await (lease.pendingWrites = transferShardInputs({ + write: (remotePath, contents) => { + if (lease.lost || shuttingDown) throw lease.lost ?? new Error("coordinator interrupted before upload"); + return sandbox.fs.write(remotePath, contents); + }, planRemotePath, planRaw: shardPlanRaw, remoteScript, - remoteScriptContents: shardScript({ shardIndex, lockId, planRemotePath }), - }); + remoteScriptContents: shardScript({ shardIndex, lockId, planRemotePath, diagnosticAttemptId, scratch: lease.input.scratch }), + })); if (!dispatch.ok) { appendLog(logPath, `\n[coordinator] ${dispatch.failureMessage}\n`); return dispatchFailureResult({ lease, outputDir, startedAt, logPath, dispatch }); } - const command = `bash ${remoteScript}`; + const command = workerEntryCommand({ ...lease.input, processName }, remoteScript); let stream; let retainedResultProgress = null; + let resourceDiagnosticsPaths = []; + const copiedArtifacts = new Map(); let failureStep = "worker-execution"; + let exitCode = null; + let executionError = null; try { + await leaseOperation(lease, "prepare"); + await leaseOperation(lease, "dispatch", { processName }); + if (lease.lost || shuttingDown) throw lease.lost ?? new Error("coordinator interrupted before dispatch"); lease.executionRequested = true; onExecutionRequested(); - const proc = await sandbox.process.exec({ - name: processName, - command, - env: { GH_TOKEN: blaxelGithubToken }, - timeout: timeouts.workerTimeoutSeconds, - keepAlive: true, - }); + const proc = await (lease.dispatchRequest = sandbox.process.exec({ name: processName, command, env: { GH_TOKEN: blaxelGithubToken }, timeout: timeouts.workerTimeoutSeconds, keepAlive: true })); onExecutionStarted(); stream = sandbox.process.streamLogs(processName, { - onStdout: (chunk) => appendLog(logPath, chunk), - onStderr: (chunk) => appendLog(logPath, chunk), + onStdout: (chunk) => appendLog(logPath, chunk), onStderr: (chunk) => appendLog(logPath, chunk), onError: (error) => console.error(`Log stream failed for shard ${shardIndex}: ${error.message}`), }); const completed = await sandbox.process.wait(processName, { maxWait: timeouts.workerTimeoutSeconds * 1000, interval: 5000 }); lease.executionCompletionConfirmed = true; - stream?.close?.(); - const logs = await sandbox.process.logs(processName, "all").catch(() => ""); - if (logs && !fs.existsSync(logPath)) fs.writeFileSync(logPath, logs); - failureStep = "report-download"; + exitCode = completed.exitCode ?? proc.exitCode ?? null; + } catch (error) { executionError = error; } + stream?.close?.(); + let result; + try { + await finishLease({ + settleWrites: async () => { await lease.pendingWrites; }, + assertOwned: () => { if (lease.lost) throw lease.lost; }, + teardown: () => confirmTeardown(lease), + download: async () => { + await beforeDownload(lease); + let logError = null; + if (lease.executionRequested) { + try { + const logs = await sandbox.process.logs(processName, "all"); + appendLog(logPath, `\n[coordinator] final worker log:\n${chunkToText(logs)}`); + } catch (error) { logError = error; } + } + failureStep = "report-download"; + result = await downloadResult(); + // Retain available artifacts even when a required log read failed. + if (logError) throw logError; + lease.retrieved = true; + }, + release: async () => { await leaseOperation(lease, "release", { retrieved: true }); lease.released = true; }, + }); + if (executionError) return { ...result, status: "infra-failed", ...classifyBlaxelCoordinatorFailure({ failureStep: "worker-execution", message: executionError.message }), failureMessage: executionError.message }; + return result; + } catch (error) { + appendLog(logPath, `\n[coordinator] ${error.stack || error.message}\n`); + return { ...retainedResultProgress, shardIndex, shardTotal, sandbox: sandbox.metadata.name, outputDir, status: "infra-failed", exitCode, startedAt: new Date(overallStarted).toISOString(), completedAt: new Date().toISOString(), durationMs: Date.now() - overallStarted, logPath, resourceDiagnosticsPaths, tasks: retainedResultProgress?.tasks ?? [], ...classifyBlaxelCoordinatorFailure({ failureStep, message: error.message }), failureMessage: error.message }; + } finally { await lease.stopRenewal?.(); } + + async function downloadResult() { const reportStarted = Date.now(); - const exitCode = completed.exitCode ?? proc.exitCode ?? null; + await leaseOperation(lease, "check"); + resourceDiagnosticsPaths = await copyResourceDiagnostics(sandbox, { outputDir, shardIndex, diagnosticAttemptId, copiedArtifacts }); + await retainAttemptEvidence(lease, outputDir); const plannedShard = shardPlan?.shards?.[shardIndex - 1]; let shardSummary = null; let shardSummaryError = null; @@ -484,6 +537,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE summaryPath: fs.existsSync(summaryPath) ? summaryPath : null, summaryError: shardSummaryError, nativeResultPath, + resourceDiagnosticsPaths: [...resourceDiagnosticsPaths], logPath: fs.existsSync(logPath) ? logPath : null, setupTimings, planId: shardSummary?.planId ?? shardPlan?.planId, @@ -510,8 +564,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE appendLog(logPath, `\n[coordinator] ${shardSummaryError}\n`); } } catch (error) { - shardSummaryError = `Failed to read shard summary: ${error.message}`; - appendLog(logPath, `\n[coordinator] ${shardSummaryError}\n`); + throw new Error(`Failed to read shard summary: ${error.message}`); } retainResultProgress(); const remoteShardNativePath = `/workspace/television/.testshards/results/native-shard-${shardIndex}.json`; @@ -525,9 +578,12 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE } } catch {} retainResultProgress(); - const copiedArtifacts = new Map(); for (const [taskIndex, task] of (shardSummary?.tasks ?? []).entries()) { const copiedTask = { ...tasks[taskIndex] }; + copiedTask.resourceDiagnosticsPaths = [ + ...(task.resourceDiagnosticsPaths ?? []).map((remotePath) => copiedArtifacts.get(remotePath)).filter(Boolean), + ...resourceDiagnosticsPaths.filter((file) => path.basename(file) === "recovery.json"), + ]; for (const field of ["nativeResultPath", "attemptResultPath", "workspaceNativeResultPath", "lifecycleResultPath", "processLeakResultPath"]) { if (!task[field]) continue; copiedTask[field] = await copyRemoteArtifact(sandbox, task[field], { outputDir, shardIndex, copiedArtifacts }); @@ -549,9 +605,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE const setupRaw = await sandbox.fs.read(`/workspace/television/.testshards/results/setup-shard-${shardIndex}.json`); fs.writeFileSync(path.join(outputDir, `setup-shard-${shardIndex}.json`), setupRaw.endsWith("\n") ? setupRaw : `${setupRaw}\n`); setupTimings = [lease.restorePhase, ...JSON.parse(setupRaw)].filter(Boolean); - } catch { - setupTimings = [lease.restorePhase].filter(Boolean); - } + } catch (error) { throw new Error(`Failed to retain setup timings: ${error.message}`); } retainResultProgress(); const identityError = shardPlan && shardSummary ? validateDownloadedShardSummary(shardSummary, shardPlan, shardIndex) : null; const resultError = downloadedResultError(shardSummary, exitCode, shardIndex); @@ -559,6 +613,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE shardSummaryError = identityError ?? resultError; appendLog(logPath, `\n[coordinator] ${shardSummaryError}\n`); } + if (!shardSummary || identityError) throw new Error(shardSummaryError ?? "missing shard summary"); const hasSummaryFile = fs.existsSync(summaryPath); const status = classifyDownloadedShardStatus({ identityError, shardSummary, exitCode, hasSummaryFile, setupTimings }); const completedAt = new Date().toISOString(); @@ -569,37 +624,7 @@ async function runShard(lease, outputDir, { onExecutionRequested = () => {}, onE failureStep: hasSummaryFile ? "summary-parse" : "summary-read", failureMessage: shardSummaryError ?? `Shard ${shardIndex}/${shardTotal} produced no readable summary.`, } : null; - return { shardIndex, shardTotal, shard: `${shardIndex}/${shardTotal}`, sandbox: sandbox.metadata.name, outputDir, status, exitCode, startedAt: new Date(overallStarted).toISOString(), completedAt, durationMs: Date.now() - overallStarted, summaryPath: fs.existsSync(summaryPath) ? summaryPath : null, summaryError: shardSummaryError, nativeResultPath, logPath, setupTimings, planId: shardSummary?.planId ?? shardPlan?.planId, timingProvider: shardSummary?.timingProvider ?? shardPlan?.timingProvider, testedTree: shardSummary?.testedTree ?? shardPlan?.testedTree, testedCommit: shardSummary?.testedCommit ?? shardPlan?.testedCommit, predictedDurationMs: shardSummary?.predictedDurationMs, phases: [...(shardSummary?.phases ?? []), reportPhase], assignedFiles: shardSummary?.assignedFiles ?? [], collectedFiles: shardSummary?.collectedFiles ?? [], failureKind: identityError ? "plan-validation" : resultError ? "result-report" : missingSummaryFailure?.failureKind ?? shardSummary?.failureKind ?? failedInfrastructureTask?.failureKind, failureStep: identityError ? "plan-validation" : resultError ? "summary-validation" : missingSummaryFailure?.failureStep ?? shardSummary?.failureStep ?? failedInfrastructureTask?.failureStep, failureMessage: identityError ?? resultError ?? missingSummaryFailure?.failureMessage ?? shardSummary?.failureMessage ?? failedInfrastructureTask?.failureMessage, processLeaks: shardSummary?.processLeaks ?? [], tasks }; - } catch (error) { - stream?.close?.(); - appendLog(logPath, `\n[coordinator] ${error.stack || error.message}\n`); - const completedAt = new Date().toISOString(); - const plannedShard = shardPlan?.shards?.[shardIndex - 1]; - const failure = classifyBlaxelCoordinatorFailure({ failureStep, message: error.message }); - return { - ...retainedResultProgress, - shardIndex, - shardTotal, - shard: `${shardIndex}/${shardTotal}`, - sandbox: sandbox.metadata.name, - outputDir, - status: "infra-failed", - error: error.message, - ...failure, - failureMessage: error.message, - planId: retainedResultProgress?.planId ?? shardPlan?.planId, - timingProvider: retainedResultProgress?.timingProvider ?? shardPlan?.timingProvider, - testedCommit: retainedResultProgress?.testedCommit ?? shardPlan?.testedCommit, - testedTree: retainedResultProgress?.testedTree ?? shardPlan?.testedTree, - predictedDurationMs: retainedResultProgress?.predictedDurationMs ?? plannedShard?.predictedDurationMs, - startedAt: retainedResultProgress?.startedAt ?? new Date(overallStarted).toISOString(), - completedAt, - durationMs: Date.now() - overallStarted, - logPath, - assignedFiles: retainedResultProgress?.assignedFiles ?? plannedShard?.surfaces?.flatMap((surface) => surface.files.map((file) => file.path)).sort() ?? [], - collectedFiles: retainedResultProgress?.collectedFiles ?? [], - tasks: retainedResultProgress?.tasks ?? [], - }; + return { shardIndex, shardTotal, shard: `${shardIndex}/${shardTotal}`, sandbox: sandbox.metadata.name, outputDir, status, exitCode, startedAt: new Date(overallStarted).toISOString(), completedAt, durationMs: Date.now() - overallStarted, summaryPath: fs.existsSync(summaryPath) ? summaryPath : null, summaryError: shardSummaryError, nativeResultPath, resourceDiagnosticsPaths, remoteScript, scratch: lease.input.scratch, logPath, setupTimings, planId: shardSummary?.planId ?? shardPlan?.planId, timingProvider: shardSummary?.timingProvider ?? shardPlan?.timingProvider, testedTree: shardSummary?.testedTree ?? shardPlan?.testedTree, testedCommit: shardSummary?.testedCommit ?? shardPlan?.testedCommit, predictedDurationMs: shardSummary?.predictedDurationMs, phases: [...(shardSummary?.phases ?? []), reportPhase], assignedFiles: shardSummary?.assignedFiles ?? [], collectedFiles: shardSummary?.collectedFiles ?? [], failureKind: identityError ? "plan-validation" : resultError ? "result-report" : missingSummaryFailure?.failureKind ?? shardSummary?.failureKind ?? failedInfrastructureTask?.failureKind, failureStep: identityError ? "plan-validation" : resultError ? "summary-validation" : missingSummaryFailure?.failureStep ?? shardSummary?.failureStep ?? failedInfrastructureTask?.failureStep, failureMessage: identityError ?? resultError ?? missingSummaryFailure?.failureMessage ?? shardSummary?.failureMessage ?? failedInfrastructureTask?.failureMessage, processLeaks: shardSummary?.processLeaks ?? [], tasks }; } } @@ -647,6 +672,41 @@ function retainTaskWithoutRemoteArtifacts(task) { if (retained[field]) retained[field] = null; } if (Array.isArray(retained.generatedInputPaths)) retained.generatedInputPaths = []; + retained.resourceDiagnosticsPaths = []; + return retained; +} + +async function copyResourceDiagnostics(sandbox, { outputDir, shardIndex, diagnosticAttemptId, copiedArtifacts }) { + const root = "/workspace/television/.testshards/results/resources"; + const retained = []; + let bytes = 0; + try { + const listing = await sandbox.fs.ls(root); + for (const entry of (listing.files ?? []).slice(0, 1024)) { + if (!/^[A-Za-z0-9_.-]+\.json$/.test(entry.name)) continue; + try { + if (entry.size > RESOURCE_SIDECAR_BYTES || bytes + entry.size > RESOURCE_ATTEMPT_BYTES) throw new Error("budget"); + const remotePath = `${root}/${entry.name}`; + const raw = await sandbox.fs.read(remotePath); + const size = Buffer.byteLength(raw); + if (size > RESOURCE_SIDECAR_BYTES || bytes + size > RESOURCE_ATTEMPT_BYTES) throw new Error("budget"); + const record = JSON.parse(raw); + if (record.schemaVersion !== 1 || record.identity?.runId !== runId || record.identity?.shardIndex !== shardIndex || record.identity?.attemptId !== diagnosticAttemptId) throw new Error("identity"); + const localPath = path.join(outputDir, "artifacts", `shard-${shardIndex}`, "resources", entry.name); + fs.mkdirSync(path.dirname(localPath), { recursive: true }); + if (record.kind === "recovery") { + // The bootstrap receipt must exist before checkout can supply this + // shared writer. Retention adopts its whitelist and attempt budget. + if (!writeResourceDiagnostic(path.dirname(path.dirname(localPath)), path.basename(entry.name, ".json"), record)) continue; + } else fs.writeFileSync(localPath, raw); + copiedArtifacts.set(remotePath, localPath); + retained.push(localPath); bytes += size; + } catch { console.error(`[resource diagnostics] shard ${shardIndex}: sidecar unavailable, invalid or over budget`); } + } + if ((listing.files?.length ?? 0) > 1024) console.error(`[resource diagnostics] shard ${shardIndex}: file listing truncated`); + } catch (error) { + if (error?.status !== 404 && error?.response?.status !== 404) console.error(`[resource diagnostics] shard ${shardIndex}: directory unavailable`); + } return retained; } @@ -680,6 +740,7 @@ function targetOptions() { config: options["target-config"] ?? null, cwd: options["target-cwd"] ?? ".", file: options["target-file"] ?? null, + files: parseTargetStringArray(options["target-files-json"], "--target-files-json"), grep: options["target-grep"] ?? null, retries: options["target-retries"] ?? "5", command: parseTargetCommand(options["target-command-json"]), @@ -688,53 +749,6 @@ function targetOptions() { }; } -function targetCommand() { - const target = targetOptions(); - if (!target) return ""; - if (target.command) { - if (target.runner !== "playwright") fail("--target-command-json currently supports Playwright surfaces only"); - const args = [ - "cd", shellQuote(target.cwd), "&&", - 'PLAYWRIGHT_JSON_OUTPUT_NAME="$TV_TARGET_NATIVE_RESULT"', - ...target.command.map(shellQuote), - ]; - if (target.file) args.push(shellQuote(target.file)); - if (target.grep) args.push("-g", shellQuote(target.grep)); - if (target.retries) args.push(`--retries=${target.retries}`); - args.push("--reporter=json"); - return args.join(" "); - } - if (target.runner) { - const args = ["cd", shellQuote(target.cwd), "&&"]; - if (target.runner === "vitest") { - args.push("npx", "vitest", "run", "--config", shellQuote(target.config), "--reporter=json", '--outputFile "$TV_TARGET_NATIVE_RESULT"'); - if (target.file) args.push(shellQuote(target.file)); - if (target.grep) args.push("-t", shellQuote(target.grep)); - if (target.retries) args.push(`--retry=${target.retries}`); - } else if (target.runner === "playwright") { - args.push('PLAYWRIGHT_JSON_OUTPUT_NAME="$TV_TARGET_NATIVE_RESULT"', "npx", "playwright", "test", "--config", shellQuote(target.config), "--reporter=json"); - if (target.file) args.push(shellQuote(target.file)); - if (target.grep) args.push("-g", shellQuote(target.grep)); - if (target.retries) args.push(`--retries=${target.retries}`); - } else { - fail("--target-runner must be vitest or playwright"); - } - return args.join(" "); - } - const args = [ - "npm", - "--workspace", - target.workspace, - "run", - "test:e2e", - "--if-present", - "--", - ]; - if (target.file) args.push(target.file); - if (target.grep) args.push("-g", target.grep); - args.push(`--retries=${target.retries}`, "--reporter=list"); - return args.map(shellQuote).join(" "); -} function parseTargetCommand(raw) { return parseTargetStringArray(raw, "--target-command-json"); @@ -766,14 +780,11 @@ function systemDependenciesCommand() { return 'if ! command -v python3 >/dev/null 2>&1 || ! command -v xauth >/dev/null 2>&1 || ! command -v ss >/dev/null 2>&1 || ! ldconfig -p 2>/dev/null | grep -q "libgtk-3.so.0" || ! ldconfig -p 2>/dev/null | grep -q "libXtst.so.6"; then apt-get update && apt-get install -y --no-install-recommends python3 xauth libgtk-3-0 libxtst6 iproute2; fi'; } -function shardScript({ shardIndex, lockId, planRemotePath }) { +function shardScript({ shardIndex, lockId, planRemotePath, diagnosticAttemptId, scratch }) { return `#!/usr/bin/env bash set -euo pipefail -lock=/tmp/tv-testshard.lock shard_plan=${shellQuote(planRemotePath ?? "")} -release_lock() { - if [ -f "$lock/owner.json" ] && grep -q '"lockId":"${lockId}"' "$lock/owner.json"; then rm -rf "$lock"; fi -} +export TMPDIR=${shellQuote(`${scratch}/tmp`)} export CI=1 export SKIP_ACP_TESTS=1 export NVM_DIR="\${NVM_DIR:-$HOME/.nvm}" @@ -783,11 +794,14 @@ export npm_config_fund=false export PLAYWRIGHT_BROWSERS_PATH=${playwrightBrowsersPath} export TEST_RETRIES=${testRetries} export TV_TEST_TIMING_PROVIDER=blaxel-playwright-x64-4vcpu +export TV_TEST_RUN_ID=${shellQuote(runId)} +export TV_TEST_SHARD_INDEX=${shardIndex} +export TV_TEST_SHARD_ATTEMPT=${shellQuote(diagnosticAttemptId)} mkdir -p /workspace /cache/npm if [ -z "\${GH_TOKEN:-}" ]; then echo "GH_TOKEN missing" >&2; exit 2; fi -setup_metrics=/tmp/tv-testshard-setup-${shardIndex}.jsonl -deps_cache_status=/tmp/tv-testshard-deps-cache-${shardIndex}.txt -owner_report=/tmp/tv-stale-owner-${shardIndex}.json +setup_metrics=${shellQuote(`${scratch}/setup.jsonl`)} +deps_cache_status=${shellQuote(`${scratch}/deps.txt`)} +owner_report=${shellQuote(`${scratch}/owners.json`)} : > "$setup_metrics" : > "$deps_cache_status" export TV_TEST_DEPS_CACHE_STATUS_FILE="$deps_cache_status" @@ -825,23 +839,32 @@ runtime_versions() { node --version npm --version } +restore_attempt_evidence() { + mkdir -p /workspace/television/.testshards/results + cp -R ${shellQuote(`${scratch}/evidence/.`)} /workspace/television/.testshards/results/ +} write_setup_metrics() { - mkdir -p /workspace/television/.testshards/results 2>/dev/null || true + restore_attempt_evidence node - "$setup_metrics" "/workspace/television/.testshards/results/setup-shard-${shardIndex}.json" "/workspace/television/.testshards/results/shard-${shardIndex}.json" "$deps_cache_status" <<'METRICS_NODE' const fs = require('fs'); const [input, setupOutput, shardOutput, depsCacheStatusFile] = process.argv.slice(2); const depsCacheStatus = fs.existsSync(depsCacheStatusFile) ? fs.readFileSync(depsCacheStatusFile, 'utf8').trim() : ''; const rows = (fs.existsSync(input) ? fs.readFileSync(input, 'utf8').split(/\\n/).filter(Boolean).map((line) => JSON.parse(line)) : []) .map((row) => row.name === 'deps' && ['hit', 'miss'].includes(depsCacheStatus) ? { ...row, cacheStatus: depsCacheStatus } : row); +const cleanupPath = '/workspace/television/.testshards/results/temporary-cleanup.json'; +if (fs.existsSync(cleanupPath)) rows.unshift({ name: 'temporary-cleanup', status: 0, durationMs: 0, cleanup: JSON.parse(fs.readFileSync(cleanupPath, 'utf8')) }); fs.writeFileSync(setupOutput, JSON.stringify(rows, null, 2) + "\\n"); if (!fs.existsSync(shardOutput)) { const failed = rows.find((row) => row.status && row.status !== 0); if (failed) { const summary = { + ...${JSON.stringify(shardSummaryIdentity(shardIndex))}, status: 'infra-failed', - failureKind: failed.name === 'checkout' ? 'checkout' : 'setup', + collectedFiles: [], + neverProduced: [failed.name === 'test-run' ? 'final shard summary' : 'test execution'], + failureKind: failed.name === 'checkout' ? 'checkout' : failed.name === 'test-run' ? 'worker-execution' : 'setup', failureStep: failed.name, - failureMessage: failed.name === 'checkout' ? 'Repository checkout failed on the Blaxel worker. Check GitHub token validity and repository access from the sandbox.' : 'Setup step ' + failed.name + ' failed before tests ran.', + failureMessage: failed.name === 'checkout' ? 'Repository checkout failed on the Blaxel worker. Check GitHub token validity and repository access from the sandbox.' : failed.name === 'test-run' ? 'Test execution ended without a finalized shard summary.' : 'Setup step ' + failed.name + ' failed before tests ran.', suite: ${JSON.stringify(suite)}, shardIndex: ${shardIndex}, shardTotal: ${shardTotal}, @@ -857,20 +880,20 @@ if (!fs.existsSync(shardOutput)) { } METRICS_NODE } -trap 'write_setup_metrics; rm -f "$owner_report" "$deps_cache_status"; if [ -n "$shard_plan" ]; then rm -f "$shard_plan"; fi; release_lock' EXIT -rm -rf /workspace/television/.testshards 2>/dev/null || true -cat > /tmp/tv-git-askpass.sh <<'ASKPASS' +trap 'write_setup_metrics' EXIT +cat > ${scratch}/git-askpass.sh <<'ASKPASS' #!/usr/bin/env bash case "$1" in *Username*) echo x-access-token ;; *) echo "$GH_TOKEN" ;; esac ASKPASS -chmod 700 /tmp/tv-git-askpass.sh -export GIT_ASKPASS=/tmp/tv-git-askpass.sh +chmod 700 ${scratch}/git-askpass.sh +export GIT_ASKPASS=${scratch}/git-askpass.sh export GIT_TERMINAL_PROMPT=0 run_step checkout bash -lc ${shellQuote(blaxelCheckoutScript({ repoUrl, commit }))} cd /workspace/television +restore_attempt_evidence run_step runtime-activation activate_runtime run_step runtime-versions runtime_versions run_step deps bash -c ${shellQuote(blaxelDependencyScript())} @@ -934,7 +957,7 @@ elif [ ${shellQuote(suite)} = target ]; then export TV_TARGET_RUN_ID=${shellQuote(`${runId}:${shardIndex}`)} export TV_TARGET_NAME=${shellQuote(`${targetOptions()?.workspace ?? "target"} ${targetOptions()?.file ?? ""} ${targetOptions()?.grep ?? ""}`)} export TV_TARGET_RUNNER=${shellQuote(targetOptions()?.runner ?? "target")} - export TV_TARGET_COMMAND=${shellQuote(targetCommand())} + export TV_TARGET_COMMAND=${shellQuote(targetCommand(targetOptions()))} export TV_TARGET_LIFECYCLE_RESULT="/workspace/television/.testshards/results/target-lifecycle-${shardIndex}.json" export TV_TARGET_NATIVE_RESULT="/workspace/television/.testshards/results/native-shard-${shardIndex}.json" run_step test-run node scripts/test/supervised-command.mjs --run-id "$TV_TARGET_RUN_ID" --surface "$TV_TARGET_SURFACE_ID" --result "$TV_TARGET_LIFECYCLE_RESULT" ${targetOptions()?.preCommand ? `--pre-command-json ${shellQuote(JSON.stringify(targetOptions().preCommand))}` : ""} ${targetOptions()?.services?.length ? `--services-json ${shellQuote(JSON.stringify(targetOptions().services))}` : ""} -- bash -c "$TV_TARGET_COMMAND" @@ -969,7 +992,10 @@ const task = { completedAt, durationMs: Date.now() - Date.parse(startedAt), nativeResultPath: fs.existsSync(process.env.TV_TARGET_NATIVE_RESULT) ? process.env.TV_TARGET_NATIVE_RESULT : null, + attemptResultPath: fs.existsSync(process.env.TV_TARGET_NATIVE_RESULT + ".attempts.ndjson") ? process.env.TV_TARGET_NATIVE_RESULT + ".attempts.ndjson" : null, + generatedInputPaths: fs.existsSync(process.env.TV_TARGET_NATIVE_RESULT + ".runner.mjs") ? [process.env.TV_TARGET_NATIVE_RESULT + ".runner.mjs"] : [], lifecycleResultPath: process.env.TV_TARGET_LIFECYCLE_RESULT, + resourceDiagnosticsPaths: lifecycle.resourceDiagnosticsPaths || [], processLeaks, failureKind: lifecycle.failureKind || null, failureStep: lifecycle.failureStep || null, @@ -997,63 +1023,47 @@ exit "$test_status" `; } -async function releaseLock(sandbox, lockId) { - const command = `bash -lc ${shellQuote(`lock=/tmp/tv-testshard.lock; if [ -f "$lock/owner.json" ] && grep -q '\"lockId\":\"${lockId}\"' "$lock/owner.json"; then rm -rf "$lock"; fi`)}`; - await sandbox.process.exec({ name: `tv-release-${lockId}`, command, waitForCompletion: true, timeout: 30 }).catch(() => {}); +function confirmTeardown(lease) { + lease.teardownPromise ??= (async () => { + const result = await leaseOperation(lease, "teardown"); + lease.teardownConfirmed = true; + if (lease.logPath) appendLog(lease.logPath, `\n[coordinator] owner teardown: ${JSON.stringify(result.record.teardown)}\n`); + })(); + return lease.teardownPromise; } -async function stopAndReleaseLease(lease, { killProcess }) { - if (killProcess && lease.processName) { - await lease.sandbox.process.kill(lease.processName).catch(() => {}); - try { - await reapUnconfirmedShardOwners(lease); - } catch (error) { - if (lease.logPath) appendLog(lease.logPath, `\n[coordinator] Owner cleanup was not confirmed; retaining lease ${lease.lockId} until TTL: ${error.stack || error.message}\n`); - throw error; - } + +async function beforeDownload(lease) { + if (process.env.TV_TEST_RUNNER_SELFTEST !== "1" || !process.env.TV_BLAXEL_DOWNLOAD_GATE) return; + const gate = process.env.TV_BLAXEL_DOWNLOAD_GATE; + fs.mkdirSync(gate, { recursive: true }); + fs.writeFileSync(path.join(gate, `shard-${lease.shardIndex}.json`), JSON.stringify({ pid: process.pid, lockId: lease.lockId, sandbox: lease.sandbox.metadata.name, scratch: lease.input.scratch })); + while (!shuttingDown && !fs.existsSync(path.join(gate, "continue"))) { + if (lease.lost) throw lease.lost; + await new Promise((resolve) => setTimeout(resolve, 100)); } - await releaseLock(lease.sandbox, lease.lockId); } -async function reapUnconfirmedShardOwners(lease) { - const processName = `tv-owner-cleanup-${lease.lockId}`.replace(/[^a-zA-Z0-9-]/g, "-"); - const reportPath = `/tmp/${processName}.json`; - const script = [ - "set -eo pipefail", - 'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"', - '. "$NVM_DIR/nvm.sh"', - "nvm use --silent default", - "set -u", - "cd /workspace/television", - "status=0", - `node scripts/test/stale-owner-reaper-cli.mjs --output ${shellQuote(reportPath)} || status=$?`, - `cat ${shellQuote(reportPath)}`, - 'exit "$status"', - ].join("; "); - const completed = await lease.sandbox.process.exec({ - name: processName, - command: `bash -lc ${shellQuote(script)}`, - waitForCompletion: true, - timeout: 30, - }); - const raw = await lease.sandbox.fs.read(reportPath); - let report; - try { - report = JSON.parse(raw); - } catch (error) { - throw new Error(`Unconfirmed shard owner cleanup returned invalid JSON: ${error.message}`); - } - const validExit = completed.exitCode === 0 || completed.exitCode === 1; - const consistentExit = completed.exitCode === (report.staleDetected ? 1 : 0); - const cleanupConfirmed = report.cleanupConfirmed === true && - Array.isArray(report.liveOwners) && report.liveOwners.length === 0 && - Array.isArray(report.remainingStaleOwners) && report.remainingStaleOwners.length === 0 && - Array.isArray(report.processLeaks); - if (!validExit || !consistentExit || !cleanupConfirmed) { - throw new Error(`Unconfirmed shard owner cleanup failed closed (exit=${completed.exitCode}, staleDetected=${report.staleDetected}, cleanupConfirmed=${report.cleanupConfirmed}, liveOwners=${report.liveOwners?.length ?? "invalid"}, remainingStaleOwners=${report.remainingStaleOwners?.length ?? "invalid"})`); - } - if (lease.logPath) appendLog(lease.logPath, `\n[coordinator] unconfirmed execution owner cleanup: ${JSON.stringify(report)}\n`); - return report; +async function stopAndReleaseLease(lease, { killProcess }) { + if (lease.released) return; + if (lease.finishing) return lease.finishing; + lease.finishing = (async () => { + try { + await lease.pendingWrites; + await lease.dispatchRequest?.catch(() => {}); + // Even a successful named process wait is not a descendant audit. + await confirmTeardown(lease); + // Teardown always stops owned processes. On interruption, also wait for + // the in-flight work promise to finish retaining its reports. + if (killProcess && lease.workPromise) await lease.workPromise; + if (lease.released) return; + if (!lease.preparationRequested && !lease.executionRequested) lease.retrieved = true; + if (!lease.retrieved) throw new Error("results not fully retained; keeping lease for recovery"); + await leaseOperation(lease, "release", { retrieved: true }); + lease.released = true; + } finally { await lease.stopRenewal?.(); } + })(); + return lease.finishing; } function buildInterruptedReportSnapshot(signal) { @@ -1114,9 +1124,10 @@ function installSignalHandlers({ killProcesses }) { process.exit(130); } shuttingDown = true; - const leases = [...activeLeases.values()]; - console.error(`Received ${signal}; preserving the interrupted attempt, then ${killProcesses ? "stopping remote processes and " : ""}releasing ${leases.length} Blaxel lease(s)...`); + console.error(`Received ${signal}; waiting for in-flight acquisitions, then retaining results and cleaning up owned workers...`); shutdownPromise = (async () => { + await Promise.allSettled([...pendingAcquisitions]); + const leases = [...activeLeases.values()]; let interruptedSnapshot = null; const { reportError, releaseError, finalizeError } = await finishInterruptedRun({ preserveReport: () => { @@ -1124,10 +1135,12 @@ function installSignalHandlers({ killProcesses }) { writeInterruptedReportSnapshot(interruptedSnapshot); }, releaseLeases: async () => { - await Promise.allSettled(leases.map((lease) => stopAndReleaseLease(lease, { killProcess: killProcesses }))); - for (const lease of leases) activeLeases.delete(lease.lockId); + const results = await Promise.allSettled(leases.map((lease) => stopAndReleaseLease(lease, { killProcess: killProcesses }))); + for (const lease of leases) if (lease.released) activeLeases.delete(lease.lockId); + const failures = results.flatMap((result, index) => result.status === "rejected" ? [`${leases[index].sandbox.metadata.name}: ${result.reason.message}`] : []); + if (failures.length) throw new Error(failures.join("; ")); }, - finalizeReport: () => writeInterruptedReportSnapshot(interruptedSnapshot), + finalizeReport: () => writeInterruptedReportSnapshot(buildInterruptedReportSnapshot(signal) ?? interruptedSnapshot), }); if (reportError) console.error(`Failed to preserve interrupted Blaxel report: ${reportError.stack || reportError.message || reportError}`); if (releaseError) console.error(`Blaxel shard cleanup failed: ${releaseError.stack || releaseError.message || releaseError}`); @@ -1146,13 +1159,14 @@ function listPoolSandboxes() { function resolveTimeouts(profile, overrides) { const profiles = { - normal: { workerTimeoutSeconds: 240, lockTtlSeconds: 900 }, + normal: { workerTimeoutSeconds: 240, lockTtlSeconds: 1200 }, cold: { workerTimeoutSeconds: 480, lockTtlSeconds: 1200 }, repair: { workerTimeoutSeconds: 480, lockTtlSeconds: 1200 }, }; const selected = { ...profiles[profile] }; if (overrides["worker-timeout-seconds"]) selected.workerTimeoutSeconds = Number.parseInt(overrides["worker-timeout-seconds"], 10); if (overrides["lock-ttl-seconds"]) selected.lockTtlSeconds = Number.parseInt(overrides["lock-ttl-seconds"], 10); + if (!Number.isSafeInteger(selected.lockTtlSeconds) || selected.lockTtlSeconds <= 0) fail("--lock-ttl-seconds must be a positive integer"); return selected; } diff --git a/scripts/run-test-shard.mjs b/scripts/run-test-shard.mjs index 7269eddf..39c9b6b5 100644 --- a/scripts/run-test-shard.mjs +++ b/scripts/run-test-shard.mjs @@ -253,7 +253,7 @@ function taskFromSurface(surface) { cwd, nativeResultPath, // --skip-pre-commands / TEST_SHARD_SKIP_PRECOMMANDS=1: the caller supplies - // prebuilt outputs (specs/arch/test-runner/github-ci.md#build-once-fan-out). + // prebuilt outputs (specs/arch/test-runner/github-ci.md#build-once). preCommand: surface.preCommand && !shouldSkipPreCommands(options) ? { command: surface.preCommand[0], args: surface.preCommand.slice(1), cwd: process.cwd() } : null, env: {}, executionGroup: surface.executionGroup ?? null, @@ -290,9 +290,9 @@ function taskFromSurface(surface) { } function testRetryArgs(surface) { - if (testRetries <= 0 || surface.kind === "unit") return []; - if (surface.runner === "playwright") return [`--retries=${testRetries}`]; - if (surface.runner === "vitest") return [`--retry=${testRetries}`]; + const retries = surface.kind === "unit" ? 0 : testRetries; + if (surface.runner === "playwright") return [`--retries=${retries}`]; + if (surface.runner === "vitest") return [`--retry=${retries}`]; return []; } diff --git a/scripts/test/attestation.d.mts b/scripts/test/attestation.d.mts index 63102db7..f5fbcfce 100644 --- a/scripts/test/attestation.d.mts +++ b/scripts/test/attestation.d.mts @@ -142,7 +142,7 @@ export function pruneTestpassRefs(options?: { fetch?: boolean; }): TestpassPruneResult; export function collectRunSurfaces(runDir: string, options?: { fs?: { readFileSync(path: string, encoding: string): string } }): Array<{ id: string; status: string }>; -export function canonicalRetryFacts(options?: { retriesOption?: string; env?: Record }): boolean; +export function canonicalRetryFacts(options?: { options?: Record; env?: Record }): boolean; export function maybePublishAttestation(options: { facts: Parameters[0]; commit: string; diff --git a/scripts/test/attestation.mjs b/scripts/test/attestation.mjs index c753db95..53bbfbd6 100644 --- a/scripts/test/attestation.mjs +++ b/scripts/test/attestation.mjs @@ -280,8 +280,8 @@ export function collectRunSurfaces(runDir, { fs: fsModule } = {}) { } /** Canonical retry facts: runner-level 2 and the default flaky budget. */ -export function canonicalRetryFacts({ retriesOption, env = process.env } = {}) { - return (retriesOption ?? "2") === "2" && (env.FLAKY_TEST_RETRIES ?? "5") === "5"; +export function canonicalRetryFacts({ options = {}, env = process.env } = {}) { + return !options["against-test-guidance-turn-flakes-into-failures-on-broad-runs"] && (options.retries ?? "2") === "2" && (env.FLAKY_TEST_RETRIES ?? "5") === "5"; } /** diff --git a/scripts/test/blaxel-classification.mjs b/scripts/test/blaxel-classification.mjs index 87ebbab3..9547c243 100644 --- a/scripts/test/blaxel-classification.mjs +++ b/scripts/test/blaxel-classification.mjs @@ -36,7 +36,12 @@ export function validateDownloadedShardSummary(summary, plan, shardIndex) { const assignedFiles = [...(summary.assignedFiles ?? [])].sort(); const collectedFiles = [...(summary.collectedFiles ?? [])].sort(); if (JSON.stringify(assignedFiles) !== JSON.stringify(expectedFiles)) return `downloaded shard ${shardIndex} assigned files disagree with its plan`; - if (summary.failureKind === "stale-owner" && collectedFiles.length === 0 && (summary.tasks ?? []).length === 0) return null; + if (collectedFiles.length === 0 && (summary.tasks ?? []).length === 0) { + if (summary.failureKind === "stale-owner") return null; + // A setup failure or killed shell can retain its assignment without ever + // producing native results. This remains infrastructure failure, not a pass. + if (summary.status === "infra-failed" && (summary.neverProduced ?? []).some(value => ["test execution", "final shard summary"].includes(value))) return null; + } const taskAssignedFiles = (summary.tasks ?? []).flatMap((task) => task.assignedFiles ?? []).sort(); const taskCollectedFiles = (summary.tasks ?? []).flatMap((task) => task.collectedFiles ?? []).sort(); if (JSON.stringify(taskAssignedFiles) !== JSON.stringify(expectedFiles)) return `downloaded shard ${shardIndex} task assignments disagree with its plan`; diff --git a/scripts/test/blaxel-lease-acquisition.d.mts b/scripts/test/blaxel-lease-acquisition.d.mts index 70b2c46a..103f5b23 100644 --- a/scripts/test/blaxel-lease-acquisition.d.mts +++ b/scripts/test/blaxel-lease-acquisition.d.mts @@ -22,3 +22,4 @@ export function acquireShardLeases(options: { maxConcurrency: number; acquire: (assignment: { shardIndex: number; slot: number }) => Promise; }): Promise>>; +export function leaseRecoveryArgs(options: Record): unknown[]; diff --git a/scripts/test/blaxel-lease-acquisition.mjs b/scripts/test/blaxel-lease-acquisition.mjs index 4f2d767e..6501d361 100644 --- a/scripts/test/blaxel-lease-acquisition.mjs +++ b/scripts/test/blaxel-lease-acquisition.mjs @@ -83,3 +83,7 @@ export async function acquireShardLeases({ shardIndices, maxConcurrency, acquire function errorMessage(error) { return error instanceof Error ? error.message : String(error); } + +export function leaseRecoveryArgs(options) { + return options["lock-ttl-seconds"] === undefined ? [] : ["--lock-ttl-seconds", options["lock-ttl-seconds"]]; +} diff --git a/scripts/test/blaxel-lease-lifecycle.d.mts b/scripts/test/blaxel-lease-lifecycle.d.mts new file mode 100644 index 00000000..e0c288a5 --- /dev/null +++ b/scripts/test/blaxel-lease-lifecycle.d.mts @@ -0,0 +1,3 @@ +export function renewLease(options: {renew: () => Promise; intervalMs: number; onLost: (error: Error) => void}): () => Promise; +export function finishLease(options: {settleWrites: () => Promise; teardown: () => Promise; download: () => Promise; release: () => Promise; assertOwned: () => void; attempts?: number}): Promise; +export function retainAttemptEvidence(lease: {preparationRequested?: boolean; executionRequested?: boolean; retrieved?: boolean; teardownConfirmed?: boolean; shardIndex: number; input: Record; sandbox: {fs: {read(file: string): Promise; ls(directory: string): Promise<{files?: Array<{name: string}>; subdirectories?: Array<{name: string}>}>}}}, directory: string): Promise; diff --git a/scripts/test/blaxel-lease-lifecycle.mjs b/scripts/test/blaxel-lease-lifecycle.mjs new file mode 100644 index 00000000..4c4980de --- /dev/null +++ b/scripts/test/blaxel-lease-lifecycle.mjs @@ -0,0 +1,63 @@ +import fs from "node:fs"; +import path from "node:path"; + +/** Coordinator-local lifecycle; transport operations are supplied at one seam. */ +export function renewLease({ renew, intervalMs, onLost }) { + let stopped = false; + let pending = Promise.resolve(); + const tick = () => { + if (stopped) return; + pending = renew().catch((error) => { stopped = true; clearInterval(timer); onLost(error); }); + pending.finally(() => { if (!stopped) timer = setTimeout(tick, intervalMs); }); + }; + let timer = setTimeout(tick, intervalMs); + return async () => { stopped = true; clearTimeout(timer); await pending; }; +} + +export async function finishLease({ settleWrites, teardown, download, release, assertOwned, attempts = 3 }) { + await settleWrites(); + assertOwned(); + // Failed teardown still allows evidence retrieval, but can never release. + let teardownError = null; + try { await teardown(); } catch (error) { teardownError = error; } + let retrievalError = null; + for (let attempt = 0; attempt < attempts; attempt++) { + assertOwned(); + try { await download(); retrievalError = null; break; } catch (error) { retrievalError = error; } + } + if (teardownError) throw teardownError; + if (retrievalError) throw retrievalError; + assertOwned(); + await release(); +} + +export async function retainAttemptEvidence(lease, directory) { + if (!lease.preparationRequested && !lease.executionRequested) { lease.retrieved = true; return; } + const root = "/workspace/television/.testshards/results"; + const identity = JSON.parse(await lease.sandbox.fs.read(`${root}/attempt.json`)); + for (const key of ["lockId", "runId", "shardIndex", "attemptId"]) if (identity[key] !== lease.input[key]) throw new Error(`attempt evidence identity mismatch: ${key}`); + let evidence; + try { evidence = JSON.parse(await lease.sandbox.fs.read(`${root}/evidence.json`)); } + catch (error) { + if (lease.teardownConfirmed || error?.status !== 404) throw error; + // Teardown uncertainty must not prevent retaining already-produced evidence. + const files = []; + const collect = async (directory) => { + const listing = await lease.sandbox.fs.ls(directory); + for (const file of listing.files ?? []) files.push(path.posix.relative(root, `${directory}/${file.name}`)); + for (const child of listing.subdirectories ?? []) await collect(`${directory}/${child.name}`); + }; + await collect(root); + evidence = { runId: lease.input.runId, attemptId: lease.input.attemptId, files }; + } + if (evidence.runId !== lease.input.runId || evidence.attemptId !== lease.input.attemptId || !Array.isArray(evidence.files)) throw new Error("invalid retained-evidence inventory"); + for (const name of evidence.files) { + // Optional diagnostic reads use their bounded, warning-only transport below. + // They must not make a native result fail or bypass its retention budget. + if (typeof name === "string" && name.startsWith("resources/")) continue; + if (typeof name !== "string" || name.startsWith("/") || name.split("/").includes("..")) throw new Error("invalid evidence path"); + const raw = await lease.sandbox.fs.read(`${root}/${name}`); + const target = path.join(directory, "artifacts", `shard-${lease.shardIndex}`, name); + fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, raw); + } +} diff --git a/scripts/test/blaxel-pool-lifecycle.d.mts b/scripts/test/blaxel-pool-lifecycle.d.mts new file mode 100644 index 00000000..7a4390db --- /dev/null +++ b/scripts/test/blaxel-pool-lifecycle.d.mts @@ -0,0 +1,3 @@ +export interface Worker {metadata: {name: string; labels: Record}; wait: () => Promise} +export function publishWorker(worker: Worker, operations: {provision: (worker: Worker) => Promise; verify: (worker: Worker) => Promise; update: (name: string, metadata: {labels: Record}) => Promise}): Promise; +export function deleteWorker(worker: Pick, operations: {reserve: (worker: Pick) => Promise; update: (name: string, metadata: {labels: Record}) => Promise; remove: (name: string) => Promise}): Promise; diff --git a/scripts/test/blaxel-pool-lifecycle.mjs b/scripts/test/blaxel-pool-lifecycle.mjs new file mode 100644 index 00000000..0f742989 --- /dev/null +++ b/scripts/test/blaxel-pool-lifecycle.mjs @@ -0,0 +1,18 @@ +/** The manager's SDK seam; fresh/unavailable states retain pool identity. */ +export async function publishWorker(sandbox, { provision, verify, update }) { + await sandbox.wait(); + await provision(sandbox); + await verify(sandbox); + return update(sandbox.metadata.name, { labels: { ...sandbox.metadata.labels, purpose: "testshards" } }); +} + +export async function deleteWorker(sandbox, { reserve, update, remove }) { + // A fresh sandbox was never eligible, so no coordinator can own its checkout. + if (sandbox.metadata.labels.purpose !== "testshards-provisioning") { + await reserve(sandbox); + await update(sandbox.metadata.name, { labels: { ...sandbox.metadata.labels, purpose: "testshards-unavailable" } }); + } + // Keep the never-published state if deletion fails: provisioning may have + // failed before installing the runtime needed by the ownership check. + await remove(sandbox.metadata.name); +} diff --git a/scripts/test/blaxel-pool.d.mts b/scripts/test/blaxel-pool.d.mts new file mode 100644 index 00000000..275c21ab --- /dev/null +++ b/scripts/test/blaxel-pool.d.mts @@ -0,0 +1,2 @@ +import type { SandboxInstance } from "@blaxel/core"; +export function listBlaxelPoolSandboxes(options: {pool: string; arch: string; deployedOnly?: boolean}): Promise; diff --git a/scripts/test/blaxel-pool.mjs b/scripts/test/blaxel-pool.mjs index ab385c43..cfedce7c 100644 --- a/scripts/test/blaxel-pool.mjs +++ b/scripts/test/blaxel-pool.mjs @@ -5,7 +5,7 @@ export async function listBlaxelPoolSandboxes({ pool, arch, deployedOnly = false return sandboxes.filter((sandbox) => { const labels = sandbox.metadata?.labels ?? {}; return labels.project === "television" - && labels.purpose === "testshards" + && (deployedOnly ? labels.purpose === "testshards" : ["testshards", "testshards-provisioning", "testshards-unavailable"].includes(labels.purpose)) && labels.pool === pool && labels.arch === arch && (!deployedOnly || sandbox.status === "DEPLOYED"); diff --git a/scripts/test/blaxel-repository.mjs b/scripts/test/blaxel-repository.mjs index 9f0005b8..c8e7f1a8 100644 --- a/scripts/test/blaxel-repository.mjs +++ b/scripts/test/blaxel-repository.mjs @@ -1,6 +1,6 @@ import { spawnSync } from "node:child_process"; -// specs/arch/test-runner/blaxel-testshards.md#Repository +// specs/arch/test-runner/blaxel-testshards.md#repository-and-credentials export const BLAXEL_WORKER_CHECKOUT = "/workspace/television"; export const BLAXEL_DEPENDENCY_CACHE = "/cache/blaxel-testshards"; diff --git a/scripts/test/blaxel-worker-lease.d.mts b/scripts/test/blaxel-worker-lease.d.mts new file mode 100644 index 00000000..794478b5 --- /dev/null +++ b/scripts/test/blaxel-worker-lease.d.mts @@ -0,0 +1,9 @@ +import type { Stats } from "node:fs"; +export interface LeaseInput { lockId: string; runId?: string; shardIndex?: number; shardTotal?: number; attempt?: number; attemptId?: string; owner?: string; ttlSeconds?: number; expiresAt?: number; tmpRoot?: string; resultsRoot?: string; scratch?: string; auditPids?: number[]; audit?: boolean; operation?: string; processName?: string; pid?: number; pidFromEnv?: boolean; retrieved?: boolean; worker?: string; summaryIdentity?: Record } +export interface CleanupReport { ok: boolean; before: {availableBytes: number; freeBytes: number}; after: {availableBytes: number; freeBytes: number}; removed: Record; skips: unknown[]; errors: unknown[] } +export interface LeaseRecord { lockId: string; runId: string; attemptId: string; recoverAfter: number; lastRenewal: number; expiresAt: number; recovery: {discardedReports: boolean} | null } +export function shardScratchPath(input: LeaseInput): string; +export function workerLeaseCommand(input: LeaseInput): string; +export function workerEntryCommand(input: LeaseInput, script: string): string; +export function workerLeaseOperation(input: LeaseInput, hooks?: {now?: () => number; scanOwners?: () => Promise>>; stopOwners?: (token: string) => Promise; cleanTemporary?: (input: {tmpRoot: string; current: string | null; onlyCurrent: boolean}) => Promise}): Promise<{ok: boolean; reason?: string; deleting?: boolean; record: LeaseRecord}>; +export function cleanWorkerTemporaryDirectories(input: {tmpRoot: string; current: string | null; onlyCurrent?: boolean}, probes?: {lstat?: (file: string) => Stats; remove?: (file: string) => void}): Promise; diff --git a/scripts/test/blaxel-worker-lease.mjs b/scripts/test/blaxel-worker-lease.mjs new file mode 100644 index 00000000..209ff0d5 --- /dev/null +++ b/scripts/test/blaxel-worker-lease.mjs @@ -0,0 +1,274 @@ +import { createHash } from "node:crypto"; +import { createRequire } from "node:module"; +const require = createRequire(import.meta.url); + +export function shardScratchPath({ tmpRoot = "/tmp", lockId, shardIndex, attempt = 1 }) { + const digest = createHash("sha256").update(lockId).digest("hex").slice(0, 20); + return `${tmpRoot}/tv-testshards/r-${digest}-s${shardIndex}-a${attempt}`; +} + +// This code runs before checkout, including after another revision ran here. +// Serialize the actual implementation; never read a helper from the worker clone. +export function workerLeaseCommand(input) { + const tmpRoot = input.tmpRoot ?? "/tmp"; + const source = `${cleanWorkerTemporaryDirectories.toString()}\n${workerLeaseOperation.toString()}\nworkerLeaseOperation(JSON.parse(Buffer.from(process.argv[1], 'base64url'))).then(result => { console.log(JSON.stringify(result)); if (!result.ok) process.exitCode = 42; }).catch(error => { console.error(error.message); process.exitCode = 2; });`; + return `flock -w 20 -F ${quote(`${tmpRoot}/tv-testshard.guard`)} node -e ${quote(source)} ${quote(Buffer.from(JSON.stringify({ ...input, scratch: input.scratch ?? shardScratchPath(input) })).toString("base64url"))}`; +} + +export function workerEntryCommand(input, script) { + // exec keeps the registered shell PID/start identity. The token exists at + // process creation, so procfs can observe it even before entry registration. + const register = workerLeaseCommand({ ...input, operation: "enter", pidFromEnv: true }); + return `env TV_TEST_SHARD_OWNER=${quote(input.lockId)} bash -c ${quote(`export TV_SHARD_ENTRY_PID=$$; ${register} && exec bash ${quote(script)}`)}`; +} + +function quote(value) { return `'${String(value).replaceAll("'", `'"'"'`)}'`; } + +/** Must run under the short flock guard. Hooks substitute only declared seams. */ +export async function workerLeaseOperation(input, hooks = {}) { + const fs = require("node:fs"); + const path = require("node:path"); + const now = hooks.now ?? Date.now; + const tmpRoot = input.tmpRoot ?? "/tmp"; + const resultsRoot = input.resultsRoot ?? "/workspace/television/.testshards/results"; + const lock = path.join(tmpRoot, "tv-testshard.lock"); + const ownerFile = path.join(lock, "owner.json"); + const operation = input.operation ?? "acquire"; + const ttlMs = (input.ttlSeconds ?? 1200) * 1000; + if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0) throw new Error("invalid recovery interval"); + if (!/^[A-Za-z0-9_.:-]+$/.test(input.lockId ?? "")) throw new Error("invalid lock identity"); + if (fs.lstatSync(tmpRoot).isSymbolicLink()) throw new Error("temporary root is a symlink"); + if (fs.existsSync(lock) && (!fs.lstatSync(lock).isDirectory() || fs.lstatSync(lock).isSymbolicLink())) throw new Error("unsafe lock root"); + let record = null; + const exists = fs.existsSync(lock); + try { record = JSON.parse(fs.readFileSync(ownerFile, "utf8")); } catch (error) { if (!exists && error.code !== "ENOENT") throw error; } + const valid = record && typeof record.lockId === "string" && typeof record.runId === "string" && typeof record.attemptId === "string" && Number.isFinite(record.lastRenewal) && Number.isFinite(record.recoverAfter); + const expired = valid && record.recoverAfter <= now(); + const held = valid ? !expired : Number.isFinite(record?.expiresAt) && record.expiresAt > now() / 1000; + const write = (value) => { + fs.mkdirSync(lock, { recursive: true }); + fs.writeFileSync(`${ownerFile}.next`, JSON.stringify(value)); + fs.renameSync(`${ownerFile}.next`, ownerFile); + }; + const denied = (reason) => ({ ok: false, reason }); + + // Inspect all tokens, including malformed tokens, conservatively. No arguments + // or environments enter the evidence. Kernel threads and zombies own no work. + function readProcess(pid) { + const base = `/proc/${pid}`; + const stat = fs.readFileSync(`${base}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 1).trim().split(/\s+/); + if (fields.length < 20) throw new Error("unreadable process identity"); + if (["Z", "X"].includes(fields[0])) return null; + const environment = fs.readFileSync(`${base}/environ`, "utf8").split("\0"); + const value = (key) => environment.find((entry) => entry.startsWith(`${key}=`))?.slice(key.length + 1); + const shardToken = value("TV_TEST_SHARD_OWNER"); + const surfaceToken = value("TV_TEST_SURFACE_OWNER"); + if (!shardToken && !surfaceToken) return null; + let command = "unavailable"; + try { command = path.basename(fs.readlinkSync(`${base}/exe`)); } catch (error) { if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; } + return { pid, startTime: fields[19], shardToken, surfaceToken, command }; + } + async function scan() { + const owners = []; + // Seam fixtures supply their process set before inspection. Production + // enumerates the whole worker and refuses any unreadable ownership. + const pids = input.auditPids ?? fs.readdirSync("/proc").filter((name) => /^\d+$/.test(name)).map(Number); + for (const pid of pids) { + try { const entry = readProcess(pid); if (entry) owners.push(entry); } + catch (error) { + if (["EACCES", "EPERM"].includes(error.code)) throw new Error(`worker owner audit unreadable at PID ${pid} (${error.code}); worker reuse refused`); + if (!["ENOENT", "ESRCH"].includes(error.code)) throw error; + } + } + return owners; + } + const scanOwners = hooks.scanOwners ?? scan; + const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + async function stop(token) { + const actions = new Map(); + const signal = async (name) => { + for (const candidate of (await scanOwners()).filter((entry) => entry.shardToken === token)) { + let current; + try { current = readProcess(candidate.pid); } catch (error) { if (["ENOENT", "ESRCH"].includes(error.code)) continue; throw error; } + if (!current || current.startTime !== candidate.startTime || current.shardToken !== token) continue; + try { process.kill(current.pid, name); } + catch (error) { if (error.code === "ESRCH") continue; throw error; } + actions.set(`${current.pid}:${current.startTime}`, { pid: current.pid, startTime: current.startTime, command: current.command, outcome: name }); + } + }; + await signal("SIGTERM"); + const deadline = Date.now() + 2000; + while ((await scanOwners()).some((entry) => entry.shardToken === token) && Date.now() < deadline) await pause(50); + await signal("SIGKILL"); + const killDeadline = Date.now() + 2000; + while ((await scanOwners()).some((entry) => entry.shardToken === token) && Date.now() < killDeadline) await pause(50); + if ((await scanOwners()).some((entry) => entry.shardToken === token)) throw new Error("owned processes survived cleanup"); + return [...actions.values()]; + } + const stopOwners = hooks.stopOwners ?? stop; + const cleanTemporary = hooks.cleanTemporary ?? cleanWorkerTemporaryDirectories; + const clean = async (current, onlyCurrent = false) => { + const result = await cleanTemporary({ tmpRoot, current, onlyCurrent }); + if (!result.ok) throw new Error(`temporary cleanup incomplete: ${JSON.stringify(result)}`); + return result; + }; + const auditClean = async () => (await scanOwners()).length === 0; + + if (["acquire", "delete"].includes(operation)) { + if (record?.deleting) return operation === "delete" ? { ok: true, deleting: true } : denied("worker reserved for deletion"); + if (exists && (held || input.audit)) return denied("worker lease held"); + let recovery = null; + let recoveryCleanup = null; + if (operation === "delete" && valid && expired) { + // Deleting the sandbox is itself cleanup; do not require successful signals. + } else if (valid && expired) { + const processes = await stopOwners(record.lockId); + if (!await auditClean()) return denied("other live or stale owners remain"); + recoveryCleanup = await clean(null); + recovery = { kind: "recovery", previous: { runId: record.runId, lockId: record.lockId, worker: input.worker ?? "unknown", lastRenewal: record.lastRenewal, recoverAfter: record.recoverAfter }, cleanup: "confirmed", discardedReports: true, processes: processes.slice(0, 128), omittedProcesses: Math.max(0, processes.length - 128) }; + } else if (!input.audit && !await auditClean()) return denied("live or stale owners remain"); + if (operation === "delete") { + write({ lockId: input.lockId, deleting: true }); + return { ok: true, deleting: true }; + } + if (recovery) fs.rmSync(resultsRoot, { recursive: true, force: true }); + const acquiredAt = now(); + const scratch = input.scratch; + if (!scratch || path.dirname(scratch) !== path.join(tmpRoot, "tv-testshards")) throw new Error("invalid scratch path"); + const parent = path.dirname(scratch); + if (fs.existsSync(parent) && fs.lstatSync(parent).isSymbolicLink()) throw new Error("scratch root is a symlink"); + fs.mkdirSync(path.join(scratch, "tmp"), { recursive: true, mode: 0o700 }); + record = { lockId: input.lockId, runId: input.runId, shardIndex: input.shardIndex, attemptId: input.attemptId, owner: input.owner, acquiredAt, lastRenewal: acquiredAt, recoverAfter: acquiredAt + ttlMs, expiresAt: 0, scratch, audit: Boolean(input.audit), dispatch: null, recovery, recoveryCleanup, summaryIdentity: input.summaryIdentity }; + write(record); + return { ok: true, record }; + } + + if (!valid || record.lockId !== input.lockId || expired || record.deleting) return denied("lease lost or expired"); + if (operation === "check") { + return { ok: true, record }; + } else if (operation === "renew") { + record.lastRenewal = now(); record.recoverAfter = record.lastRenewal + ttlMs; write(record); + } else if (operation === "prepare") { + if (record.dispatch || record.prepared) return denied("attempt already prepared or dispatched"); + if (!record.audit && !await auditClean()) return denied("worker audit not clean"); + const cleanup = record.audit ? null : await clean(record.scratch); + fs.rmSync(resultsRoot, { recursive: true, force: true }); + fs.mkdirSync(resultsRoot, { recursive: true }); + fs.writeFileSync(path.join(resultsRoot, "attempt.json"), JSON.stringify({ runId: record.runId, shardIndex: record.shardIndex, attemptId: record.attemptId, lockId: record.lockId })); + fs.writeFileSync(path.join(resultsRoot, "temporary-cleanup.json"), JSON.stringify(record.recoveryCleanup ? { ...cleanup, recovery: record.recoveryCleanup } : cleanup)); + if (record.recovery) { + // First evidence in a freshly reset tree. The bounded receipt is also + // accounted for by the surface writer's shared attempt budget. + const receipt = { schemaVersion: 1, ...record.recovery, identity: { runId: record.runId, shardIndex: record.shardIndex, attemptId: record.attemptId, surfaceId: "worker-recovery" } }; + const raw = JSON.stringify(receipt); + if (Buffer.byteLength(raw) > 128 * 1024) throw new Error("recovery receipt exceeds sidecar budget"); + fs.mkdirSync(path.join(resultsRoot, "resources")); + fs.writeFileSync(path.join(resultsRoot, "resources/recovery.json"), raw); + } + // Checkout can clone/reset the entire repository. Keep this evidence in + // attempt scratch until the shell or teardown restores it after checkout. + fs.cpSync(resultsRoot, path.join(record.scratch, "evidence"), { recursive: true }); + record.prepared = true; write(record); + } else if (operation === "dispatch") { + if (!record.prepared || record.dispatch) return denied("invalid dispatch state"); + record.dispatch = { processName: input.processName, requestedAt: now() }; write(record); + } else if (operation === "enter") { + if (!record.dispatch || record.dispatch.processName !== input.processName || record.dispatch.pid) return denied("dispatch not current"); + const pid = input.pidFromEnv ? Number(process.env.TV_SHARD_ENTRY_PID) : input.pid; + const entry = readProcess(pid); + if (!entry || entry.shardToken !== record.lockId) return denied("entry attribution missing"); + record.dispatch.pid = entry.pid; record.dispatch.startTime = entry.startTime; write(record); + } else if (operation === "teardown") { + // A request that never registered may still arrive. Keep it excluded until + // recovery, whose entry check will reject that old identity. + if (record.dispatch && !record.dispatch.pid) return denied("dispatch outcome unknown"); + const processes = await stopOwners(record.lockId); + if (!record.audit && !await auditClean()) return denied("worker audit not clean"); + if (record.prepared) { + fs.mkdirSync(resultsRoot, { recursive: true }); + const preserved = path.join(record.scratch, "evidence"); + if (fs.existsSync(preserved)) fs.cpSync(preserved, resultsRoot, { recursive: true }); + const setupFile = path.join(resultsRoot, `setup-shard-${record.shardIndex}.json`); + if (!fs.existsSync(setupFile)) { + const inputFile = path.join(record.scratch, "setup.jsonl"); + const rows = fs.existsSync(inputFile) ? fs.readFileSync(inputFile, "utf8").split("\n").filter(Boolean).map((line) => JSON.parse(line)) : []; + fs.writeFileSync(setupFile, JSON.stringify(rows)); + } + const summaryFile = path.join(resultsRoot, `shard-${record.shardIndex}.json`); + if (!fs.existsSync(summaryFile)) fs.writeFileSync(summaryFile, JSON.stringify({ ...record.summaryIdentity, collectedFiles: [], status: "infra-failed", failureKind: "worker-execution", failureStep: "interrupted-before-summary", failureMessage: "The owned process ended without a finalized shard summary.", shardIndex: record.shardIndex, tasks: [], neverProduced: ["final shard summary"] })); + } + const cleanup = await clean(record.scratch, true); + record.teardown = { confirmed: true, processes, cleanup }; write(record); + if (record.prepared) { + fs.writeFileSync(path.join(resultsRoot, "teardown.json"), JSON.stringify(record.teardown)); + const files = []; + const inventory = (dir) => { for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const file = path.join(dir, entry.name); + if (entry.isSymbolicLink()) throw new Error("symlink in retained evidence"); + if (entry.isDirectory()) inventory(file); + else if (entry.name !== "evidence.json") files.push(path.relative(resultsRoot, file)); + } }; + inventory(resultsRoot); + fs.writeFileSync(path.join(resultsRoot, "evidence.json"), JSON.stringify({ runId: record.runId, attemptId: record.attemptId, files })); + } + } else if (operation === "release") { + if (!record.teardown?.confirmed || !input.retrieved) return denied("teardown and retained evidence required"); + fs.rmSync(lock, { recursive: true }); + } else throw new Error(`unknown lease operation ${operation}`); + return { ok: true, record }; +} + +/** Direct allowed directories only; lstat traversal never follows symlinks. */ +export async function cleanWorkerTemporaryDirectories({ tmpRoot = "/tmp", current, onlyCurrent = false }, probes = {}) { + const fs = require("node:fs"); + const path = require("node:path"); + const lstat = probes.lstat ?? fs.lstatSync; + const remove = probes.remove ?? ((file) => fs.rmSync(file, { recursive: true, force: false })); + const scratchRoot = path.join(tmpRoot, "tv-testshards"); + const mountPoints = new Set(fs.readFileSync("/proc/self/mountinfo", "utf8").trim().split("\n").map((line) => line.split(" ")[4].replace(/\\([0-7]{3})/g, (_match, code) => String.fromCharCode(parseInt(code, 8))))); + const free = () => { const value = fs.statfsSync(tmpRoot); return { availableBytes: value.bavail * value.bsize, freeBytes: value.bfree * value.bsize }; }; + const report = { ok: true, before: free(), after: null, removed: {}, skips: [], errors: [] }; + for (const root of [tmpRoot, scratchRoot]) { + if (!fs.existsSync(root)) continue; + if (lstat(root).isSymbolicLink() || !lstat(root).isDirectory()) throw new Error(`unsafe cleanup root ${root}`); + if (root !== tmpRoot && (mountPoints.has(root) || lstat(root).dev !== lstat(tmpRoot).dev)) throw new Error("scratch root crosses a mount boundary"); + } + function allocated(file, device) { + const stat = lstat(file); + // Overlay files may expose their backing device while directories expose + // overlay's device. A regular file's st_dev alone is not a mount boundary. + if (mountPoints.has(file) || (stat.isDirectory() && stat.dev !== device)) throw new Error("unexpected mount boundary"); + let bytes = stat.blocks * 512; + if (stat.isDirectory()) for (const name of fs.readdirSync(file)) bytes += allocated(path.join(file, name), device); + return bytes; + } + const candidates = []; + if (onlyCurrent) { + if (!current || path.dirname(current) !== scratchRoot) throw new Error("current scratch escapes allowed root"); + candidates.push([current, "current"]); + } else { + for (const name of fs.readdirSync(tmpRoot)) { + const prefix = ["television-source-cli-e2e-", "television-web-bundle-"].find((value) => name.startsWith(value)); + if (prefix) candidates.push([path.join(tmpRoot, name), prefix]); + } + if (fs.existsSync(scratchRoot)) for (const name of fs.readdirSync(scratchRoot)) { + const file = path.join(scratchRoot, name); if (file !== current) candidates.push([file, "stale-scratch"]); + } + } + for (const [file, prefix] of candidates) { + try { + if (!fs.existsSync(file)) continue; + const stat = lstat(file); + if (!stat.isDirectory() || stat.isSymbolicLink()) { report.skips.push({ prefix, reason: "not a real directory" }); continue; } + const bytes = allocated(file, lstat(tmpRoot).dev); + const currentIdentity = lstat(file); + if (!currentIdentity.isDirectory() || currentIdentity.isSymbolicLink() || currentIdentity.dev !== stat.dev || currentIdentity.ino !== stat.ino) throw new Error("directory identity changed before removal"); + remove(file); + const counts = report.removed[prefix] ??= { directories: 0, allocatedBytes: 0 }; + counts.directories++; counts.allocatedBytes += bytes; + } catch (error) { report.ok = false; report.errors.push({ prefix, reason: error.code ?? error.message }); } + } + report.after = free(); + return report; +} diff --git a/scripts/test/cli.mjs b/scripts/test/cli.mjs index ac1aba81..85b6b9be 100644 --- a/scripts/test/cli.mjs +++ b/scripts/test/cli.mjs @@ -1,19 +1,20 @@ #!/usr/bin/env node +import { leaseRecoveryArgs } from "./blaxel-lease-acquisition.mjs"; import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import process from "node:process"; -import { loadTestConfig, owningSurfaces, selectSurfaces, validateRegistry } from "./config.mjs"; +import { loadTestConfig, selectSurfaces, validateRegistry } from "./config.mjs"; import { requiredPreflights, runPreflights, runRemotePreflight } from "./preflight.mjs"; import { normalizeSurfaceResult, safeSurfaceName } from "./reporting.mjs"; import { atomicWriteFile, createRunContext, finalizeRun, readRunDirectoryIdentity } from "./run-context.mjs"; import { BASELINE_TIMING_PROVIDER, updateTimingBaseline } from "./timing-baseline.mjs"; -import { enumerateTestInventory } from "./file-inventory.mjs"; +import { enumerateTestInventory, loadRegistrySnapshotAtCommit } from "./file-inventory.mjs"; import { recommendShardCount } from "./shard-plan.mjs"; import { normalizeBlaxelSurfaces, normalizeProviderTimingShards, resolveProviderRunGit } from "./provider-normalization.mjs"; import { providerCommandExitCode } from "./provider-outputs.mjs"; -import { groupByExecutionGroup, splitVitestWorkspaceResult, writeVitestRunnerConfig, vitestProjectsArgs } from "./execution-groups.mjs"; +import { exactPlaywrightFiles, groupByExecutionGroup, splitVitestWorkspaceResult, writeVitestRunnerConfig, vitestProjectsArgs } from "./execution-groups.mjs"; import { applyCursorAgentEnvWorkaround } from "./cursor-agent-env.mjs"; import { createSurfaceSupervisor } from "./surface-supervisor.mjs"; import { startSurfaceServices } from "./surface-services.mjs"; @@ -23,6 +24,9 @@ import { isQualifyingRun } from "./publication-eligibility.mjs"; import { appendPhaseMetricFile, appendRunSummaryPhase, phaseFromEpoch, recordPhaseMetricBestEffort } from "./phase-metrics.mjs"; import { DEFAULT_BLAXEL_POOL_SIZE, RECOMMENDED_TEST_SHARD_COUNT } from "../testshard-constants.mjs"; +import { BROAD_LOCAL, BROAD_ZERO, enforceLocalFileGuidance, resolveFileSelection, resolveRetryPolicy } from "./test-guidance.mjs"; +import { acquireLocalTestMutex, CONTENTION_BYPASS, localTestMutexPath, MUTEX_WAIT, validateMutexOptions } from "./local-test-mutex.mjs"; + applyCursorAgentEnvWorkaround(process.env); const args = process.argv.slice(2); @@ -31,8 +35,20 @@ const options = command === "pool" ? parseArgs(args.slice(3)) : command === "testpass" || command === "baseline" ? parseArgs(args.slice(2)) : parseArgs(args.slice(1), { allowPositionals: command === "verify" }); const config = command === "testpass" ? null : loadTestConfig(); +let localMutex = null; +let mutexPath; +process.on("exit", () => { + try { localMutex?.release(); } + catch (error) { console.error(`Local-test mutex release failed: ${error.message}`); } +}); try { + mutexPath = localTestMutexPath(); + validateMutexOptions(options); + if (options["allow-extreme-inefficiency"]) fail(`--allow-extreme-inefficiency has been renamed to --${BROAD_LOCAL}.`); + for (const key of Object.keys(options)) { + if (key.startsWith("against-test-guidance") && ![BROAD_LOCAL, BROAD_ZERO, CONTENTION_BYPASS].includes(key)) fail(`Unknown option --${key}. Run npm test -- help.`); + } if (Object.prototype.hasOwnProperty.call(options, "publish")) fail("--publish is not a supported option. A qualifying Blaxel verify publishes its attestation automatically; use --no-publish to opt out."); if (command === "help" || options.help) help(); else if (command === "list") await list(); @@ -44,12 +60,12 @@ try { else if (command === "baseline") await baseline(); else fail(`Unknown command ${JSON.stringify(command)}. Run npm test -- help.`); } catch (error) { - fail(error.message || error); + fail(error.message || error, error.exitCode ?? 2); } async function list() { assertRegistryValid(); - const surfaces = selectSurfaces(config, options); + const { surfaces } = resolveFileSelection({ config, options }); for (const surface of surfaces) { console.log(`${surface.id}\t${surface.kind}\t${surface.runner}\t${surface.package ?? "-"}\t${surface.config}\t${surface.tags.join(",")}`); } @@ -57,12 +73,14 @@ async function list() { async function preflight() { const provider = requireProvider(); - assertRegistryValid(); - const surfaces = selectedSurfaces({ requireSelection: false }); + if (provider === "local") assertRegistryValid(); + const selectionConfig = provider === "blaxel" ? await loadRegistrySnapshotAtCommit({ commit: options.commit ?? "HEAD" }) : config; + const selectionOptions = provider === "blaxel" ? { ...options, commit: selectionConfig.commit } : options; + const { surfaces } = resolveFileSelection({ config: selectionConfig, options: selectionOptions, commit: provider === "blaxel" ? selectionConfig.commit : null }); enforceExecutionPlacement(provider, surfaces); const results = provider === "local" ? runProviderPreflights(provider, surfaces) - : runRemotePreflightAndPrint(provider, options).results; + : runRemotePreflightAndPrint(provider, selectionOptions).results; if (results.some((result) => result.status !== "passed")) process.exit(1); } @@ -73,6 +91,7 @@ async function verify() { requireSelftestSeam("TV_TEST_RUNNER_DRY_RUN"); fail("TV_TEST_RUNNER_DRY_RUN cannot be used with verify; verify requires a finalized test run."); } + resolveRetryPolicy(options); const provider = resolveVerifyProvider(); const runDirOutput = path.join(os.tmpdir(), `tv-verify-run-dir-${process.pid}`); const phases = verifyPhases(provider, runDirOutput); @@ -81,6 +100,11 @@ async function verify() { for (const phase of phases) console.log(`[verify:${phase.name}] ${[phase.command, ...phase.args].map(shellDisplay).join(" ")}`); return; } + if (provider === "local") { + const { surfaces } = resolveFileSelection({ config, options: hasSelection(options) ? options : { suite: "all" } }); + enforceExecutionPlacement(provider, surfaces); + localMutex = await acquireLocalTestMutex({ lockPath: mutexPath, options, role: "verify", marked: fs.existsSync(path.join(os.homedir(), ".tvdev-use-blaxel")) }); + } try { fs.rmSync(runDirOutput, { force: true }); } catch (error) { @@ -95,11 +119,13 @@ async function verify() { const identity = `${process.pid}-${verifyStarted}`; const metricsFile = path.join(metricsDir, `.verify-phases-${identity}.ndjson`); let verifiedRunDir = null; + const phaseSupervisor = provider === "local" ? createSurfaceSupervisor({ runId: `verify-${identity}`, surfaceIds: ["verify"] }) : null; try { for (const phase of phases) { const phaseStarted = Date.now(); console.log(`\n[verify:${phase.name}] ${[phase.command, ...phase.args].map(shellDisplay).join(" ")}`); - const status = spawnSync(phase.command, phase.args, { cwd: process.cwd(), env: { ...process.env, TV_TEST_PHASE_METRICS_FILE: metricsFile }, stdio: "inherit" }).status ?? 1; + const phaseOptions = { cwd: process.cwd(), env: { ...process.env, TV_TEST_PHASE_METRICS_FILE: metricsFile, ...(phase.name === "tests" ? localMutex?.childEnv() : {}) }, stdio: "inherit" }; + const status = phaseSupervisor ? (await phaseSupervisor.run(phase.command, phase.args, phaseOptions)).exitCode : spawnSync(phase.command, phase.args, phaseOptions).status ?? 1; let phaseRunDir = null; let identityError = null; if (phase.name === "tests" && status === 0) { @@ -131,15 +157,21 @@ async function verify() { return; } recordPhaseMetricBestEffort("append verify:total run summary", () => appendRunSummaryPhase({ runDir: verifiedRunDir, phase: phaseFromEpoch({ name: "verify:total", category: "verification", startedMs: verifyStarted }) })); - console.log(`verify passed (${provider})`); - finishVerifyAttestation(attestationStart, verifiedRunDir, provider); } finally { + try { + const leaks = await phaseSupervisor?.finish() ?? []; + if (leaks.length) { console.error(`verify cleanup found ${leaks.length} owned process leak(s)`); process.exitCode = 1; } + } catch (error) { console.error(`verify cleanup failed: ${error.message}`); process.exitCode = 1; } recordPhaseMetricBestEffort("remove verify phase sidecar", () => fs.rmSync(metricsFile, { force: true })); recordPhaseMetricBestEffort("remove verify run identity", () => fs.rmSync(runDirOutput, { force: true })); } + if (!process.exitCode) { + console.log(`verify passed (${provider})`); + finishVerifyAttestation(attestationStart, verifiedRunDir, provider); + } } -// Tree-hash attestation handoff (specs/arch/test-runner/attestation.md#who-writes-and-when). +// Tree-hash attestation handoff (specs/arch/test-runner/attestation.md#who-may-write). // Eligibility-qualified publication: verify captures its facts around the // phases and publishes only when every clause holds; every other outcome is // a logged no-op, and no failure here can fail the verify. @@ -190,7 +222,7 @@ function finishVerifyAttestation(start, runDir, provider) { expectedSurfaceIds: selectSurfaces(config, { suite: "all" }).map((surface) => surface.id), shardSubset: Boolean(options["shard-indices"]), targetedSelection: Boolean(options.file || options.grep || options.surface || options.package || options.runner || options.tag || (options.suite && options.suite !== "all")), - canonicalRetries: canonicalRetryFacts({ retriesOption: options.retries, env: process.env }), + canonicalRetries: canonicalRetryFacts({ options, env: process.env }), attestedSkipRun: false, policyAtStart: start.policyAtStart, policyAtEnd: readAttestationPolicy(), @@ -228,12 +260,12 @@ function resolveVerifyProvider() { return "blaxel"; } - fail(`The default broad verification strategy should be blaxel, which requires a clean working tree and all code pushed to origin. Blaxel preflight failed at ${failed.name}${failed.message ? `: ${failed.message}` : ""} ${preflightRemediation(failed)}\nEscape hatch: npm run verify -- local --allow-extreme-inefficiency`); + fail(`The default broad verification strategy should be blaxel, which requires a clean working tree and all code pushed to origin. Blaxel preflight failed at ${failed.name}${failed.message ? `: ${failed.message}` : ""} ${preflightRemediation(failed)}\nEscape hatch: npm run verify -- local --${BROAD_LOCAL}`); } function enforceLocalVerifyGuardrail() { - if (!fs.existsSync(path.join(os.homedir(), ".tvdev-use-blaxel")) || options["allow-extreme-inefficiency"]) return; - fail("~/.tvdev-use-blaxel is present. Use Blaxel for full verification: npm run verify -- blaxel. With explicit human permission under testing policy, allow local verification using npm run verify -- local --allow-extreme-inefficiency."); + if (!fs.existsSync(path.join(os.homedir(), ".tvdev-use-blaxel")) || options[BROAD_LOCAL]) return; + fail("~/.tvdev-use-blaxel is present. Use Blaxel for full verification: npm run verify -- blaxel. Commit and push to origin first; do not replace broader validation with serial file loops. With explicit human permission under testing policy, allow local verification using npm run verify -- local --against-test-guidance-broad-local-run."); } function positionalProvider() { @@ -259,7 +291,7 @@ function verifyPhases(provider, runDirOutput = null) { { name: "lint", command: npmCommand(), args: ["run", "--silent", "lint"] }, { name: "type-check", command: npmCommand(), args: ["run", "--silent", "type-check"] }, { name: "package-manifests", command: npmCommand(), args: ["run", "--silent", "lint:package-manifests"] }, - { name: "tests", command: process.execPath, args: ["scripts/test/cli.mjs", "local", "--suite", "all", "--force", "--retries", options.retries ?? "2", ...(runDirOutput ? ["--run-dir-output", runDirOutput] : [])] }, + { name: "tests", command: process.execPath, args: ["scripts/test/cli.mjs", "local", "--suite", "all", "--force", ...[BROAD_LOCAL, CONTENTION_BYPASS, MUTEX_WAIT].filter(key => options[key]).map(key => `--${key}`), ...verifyRetryArgs(), ...(runDirOutput ? ["--run-dir-output", runDirOutput] : [])] }, vibeModePhase(), ]; } @@ -279,8 +311,12 @@ function vibeModePhase() { return { name: "vibe-mode", command: process.execPath, args: ["scripts/test/vibe-mode-check.mjs"] }; } +function verifyRetryArgs() { + return ["--retries", String(resolveRetryPolicy(options)), ...(options[BROAD_ZERO] ? [`--${BROAD_ZERO}`] : [])]; +} + function remoteVerifyArgs() { - const out = ["--commit", options.commit ?? "HEAD", "--retries", options.retries ?? "2"]; + const out = ["--commit", options.commit ?? "HEAD", ...verifyRetryArgs()]; for (const key of ["ref", "shards", "shard-indices", "timeout-profile", "retry-infra", "arch", "pool", "allow-shard-count-override"]) { if (options[key]) out.push(`--${key}`, options[key]); } @@ -301,7 +337,7 @@ async function pool() { process.exit(result.status ?? 1); } -// Attestation prune (specs/arch/test-runner/attestation.md#prune-and-retention). +// Attestation prune (specs/arch/test-runner/attestation.md#revoking-and-pruning). // Dry-run by default; every refusal and any partial deletion failure exits // non-zero. TV_TESTPASS_LAB=1 selects lab mode, as on every attestation path. async function testpass() { @@ -345,7 +381,7 @@ async function baseline() { } // Only Blaxel's shard count is recommended: the baseline holds Blaxel time -// (specs/arch/test-runner/sharded-execution.md#Shard-count scaling). +// (specs/arch/test-runner/sharded-execution.md#duration-baseline). function blaxelShardRecommendation({ baseline, baselineBytes }) { if (Object.keys(baseline.files).length === 0) return null; const selected = selectSurfaces(config, { suite: "all" }); @@ -388,19 +424,29 @@ function allowsUncommitted(selectionOptions) { } async function runWithSelection(provider, selectionOptions) { - assertRegistryValid(); - const surfaces = selectedSurfaces({ options: selectionOptions, requireSelection: true }); + validateRunDirOutput(selectionOptions); + if (provider === "local") assertRegistryValid(); + const selectionConfig = provider === "blaxel" ? await loadRegistrySnapshotAtCommit({ commit: selectionOptions.commit ?? "HEAD" }) : config; + if (provider === "blaxel") selectionOptions = { ...selectionOptions, commit: selectionConfig.commit }; + const resolved = resolveFileSelection({ config: selectionConfig, options: selectionOptions, commit: provider === "blaxel" ? selectionOptions.commit ?? "HEAD" : null }); + const { surfaces } = resolved; + selectionOptions = { ...selectionOptions, resolvedFiles: resolved.files }; enforceExecutionPlacement(provider, surfaces); if (process.env.TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT) requireSelftestSeam("TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT"); - const testRetries = resolveTestRetries(selectionOptions, surfaces); + if (provider === "local") enforceLocalFileGuidance(selectionOptions, resolved); + if (provider === "local") for (const surface of surfaces) for (const selector of ["file", "grep"]) { + if (selectionOptions[selector] && !surface.supports.includes(selector)) fail(`Surface ${surface.id} does not support --${selector}.`); + } + const testRetries = resolveTestRetries(selectionOptions, surfaces, resolved); if (process.env.TV_TEST_RUNNER_DRY_RUN === "1") { requireSelftestSeam("TV_TEST_RUNNER_DRY_RUN"); const selection = selectionSummary(selectionOptions, surfaces); - console.log(`[dry-run] provider=${provider} suite=${selection.suite ?? "-"} surfaces=${selection.surfaces.join(",")} files=${selection.files.join(",")}`); + console.log(`[dry-run] provider=${provider} suite=${selection.suite ?? "-"} surfaces=${selection.surfaces.join(",")} files=${selection.files.join(",")}${provider === "local" ? ` mutex=${mutexPath}` : ""}`); return; } if (provider === "blaxel") return runBlaxel(selectionOptions, surfaces, testRetries); if (provider !== "local") fail(`Provider ${provider} is not implemented in the canonical CLI yet.`); + localMutex = await acquireLocalTestMutex({ lockPath: mutexPath, options: selectionOptions, marked: fs.existsSync(path.join(os.homedir(), ".tvdev-use-blaxel")) }); const runContext = createProviderRunContext(provider); writeRunDirOutput(selectionOptions, runContext); const startedAt = new Date().toISOString(); @@ -470,6 +516,8 @@ function stopForStaleOwners({ startupOwners, surfaces, runContext, selection, se } async function runBlaxel(selectionOptions, surfaces, testRetries = 0) { + // Selection refusals precede remote preflight and the run directory, so they leave no directory. + const selectionArgs = blaxelSelectionArgs(selectionOptions, surfaces, testRetries); const remote = runRemotePreflightAndPrint("blaxel", selectionOptions); assertPreflightsPassed(remote.results); const git = remote.git; @@ -480,39 +528,14 @@ async function runBlaxel(selectionOptions, surfaces, testRetries = 0) { const args = ["scripts/run-blaxel-testshards.mjs", "--pool", selectionOptions.pool ?? "poc", "--commit", selectionOptions.commit ?? "HEAD", "--output-dir", outputDir]; if (selectionOptions["timeout-profile"]) args.push("--timeout-profile", selectionOptions["timeout-profile"]); else args.push("--timeout-profile", "cold"); + args.push(...leaseRecoveryArgs(selectionOptions)); if (selectionOptions.shards) args.push("--shards", selectionOptions.shards); if (selectionOptions["shard-indices"]) args.push("--shard-indices", selectionOptions["shard-indices"]); if (selectionOptions["allow-shard-count-override"]) args.push("--allow-shard-count-override", selectionOptions["allow-shard-count-override"]); if (selectionOptions["retry-infra"]) args.push("--retry-infra", selectionOptions["retry-infra"]); else args.push("--retry-infra", "2"); if (allowsUncommitted(selectionOptions)) args.push("--allow-dirty"); - if (testRetries > 0) args.push("--test-retries", String(testRetries)); - - if (selectionOptions.file || selectionOptions.grep || selectionOptions.surface) { - if (surfaces.length !== 1) fail("Blaxel targeted runs require a selection that resolves to exactly one surface."); - const surface = surfaces[0]; - if (!["vitest", "playwright"].includes(surface.runner)) fail(`Blaxel targeted runs do not support runner ${surface.runner}.`); - args.push( - "--suite", "target", - "--shards", "1", - "--target-surface-id", surface.id, - "--target-runner", surface.runner, - "--target-config", relativeToCwd(surface, surface.config), - "--target-cwd", surface.command ? "." : surface.cwd || ".", - "--target-retries", String(testRetries), - ); - if (surface.command) args.push("--target-command-json", JSON.stringify(surface.command)); - if (surface.preCommand) args.push("--target-pre-command-json", JSON.stringify(surface.preCommand)); - if (surface.services.length > 0) args.push("--target-services-json", JSON.stringify(surface.services)); - if (selectionOptions.file) args.push("--target-file", surface.command ? selectionOptions.file : relativeToCwd(surface, selectionOptions.file)); - if (selectionOptions.grep) args.push("--target-grep", selectionOptions.grep); - } else if (selectionOptions.package || selectionOptions.runner || selectionOptions.tag) { - args.push("--suite", "selection", "--surfaces", surfaces.map((surface) => surface.id).join(",")); - } else { - const suite = selectionOptions.suite ?? (selectionOptions.all ? "all" : null); - if (!["all", "unit", "e2e"].includes(suite)) fail("Blaxel suite runs require --suite all|unit|e2e or --all. The telemetry PostHog roundtrip suite is local-only because it requires the PostHog test read key."); - args.push("--suite", suite); - } + args.push("--test-retries", String(testRetries), ...selectionArgs); fs.writeFileSync(path.join(runContext.runDir, "request.json"), `${JSON.stringify({ provider: "blaxel", git, selection: selectionSummary(selectionOptions, surfaces), delegatedCommand: [process.execPath, ...args] }, null, 2)}\n`); console.log(`[blaxel] ${[process.execPath, ...args].map(shellDisplay).join(" ")}`); @@ -678,7 +701,7 @@ async function runLocalSurface(surface, selectionOptions, runDir, testRetries = const runnerConfigPath = surface.runner === "vitest" ? path.resolve(runDir, "native", `${name}.runner.config.mjs`) : null; if (runnerConfigPath) { fs.rmSync(attemptResultAbsPath, { force: true }); - writeVitestRunnerConfig({ configPath: runnerConfigPath, baseConfigPath: surface.config, runnerPath: "scripts/test/vitest-attempt-reporter.mjs" }); + writeVitestRunnerConfig({ configPath: runnerConfigPath, baseConfigPath: surface.config, runnerPath: "scripts/test/vitest-attempt-reporter.mjs", files: selectionOptions.resolvedFiles?.filter((file) => file.surfaceId === surface.id).map((file) => file.path) }); } const command = buildSurfaceCommand(surface, selectionOptions, nativeResultAbsPath, testRetries, runnerConfigPath); const started = Date.now(); @@ -780,24 +803,27 @@ function skippedSurfaceResult({ surface, command, durationMs, logPath, reason, p function buildSurfaceCommand(surface, selectionOptions, nativeResultPath, testRetries = 0, runnerConfigPath = null) { const file = selectionOptions.file; + const files = selectionOptions.resolvedFiles?.filter((file) => file.surfaceId === surface.id).map((file) => file.path); const grep = selectionOptions.grep; if (file && !surface.supports.includes("file")) fail(`Surface ${surface.id} does not support --file.`); if (grep && !surface.supports.includes("grep")) fail(`Surface ${surface.id} does not support --grep.`); - if (surface.command) return [...surface.command, ...selectorArgs(surface, file, grep), ...testRetryArgs(surface, testRetries), ...reporterArgs(surface, nativeResultPath)]; - if (surface.runner === "vitest") return ["npx", "vitest", "run", "--config", relativeToCwd(surface, runnerConfigPath ?? surface.config), ...selectorArgs(surface, file, grep), ...testRetryArgs(surface, testRetries), ...reporterArgs(surface, nativeResultPath)]; - if (surface.runner === "playwright") return ["npx", "playwright", "test", "--config", relativeToCwd(surface, surface.config), ...selectorArgs(surface, file, grep), ...testRetryArgs(surface, testRetries), ...reporterArgs(surface, nativeResultPath)]; + if (surface.command) return [...surface.command, ...selectorArgs(surface, file, grep, files), ...testRetryArgs(surface, testRetries), ...reporterArgs(surface, nativeResultPath)]; + if (surface.runner === "vitest") return ["npx", "vitest", "run", "--config", relativeToCwd(surface, runnerConfigPath ?? surface.config), ...selectorArgs(surface, file, grep, files), ...testRetryArgs(surface, testRetries), ...reporterArgs(surface, nativeResultPath)]; + if (surface.runner === "playwright") return ["npx", "playwright", "test", "--config", relativeToCwd(surface, surface.config), ...selectorArgs(surface, file, grep, files), ...testRetryArgs(surface, testRetries), ...reporterArgs(surface, nativeResultPath)]; fail(`Unsupported runner ${surface.runner} for ${surface.id}`); } -function selectorArgs(surface, file, grep) { +function selectorArgs(surface, file, grep, files) { const out = []; - if (file) out.push(surface.command ? file : relativeToCwd(surface, file)); + if (files) { + if (surface.runner === "playwright") out.push(...exactPlaywrightFiles(files)); + } else if (file) out.push(surface.command ? file : relativeToCwd(surface, file)); if (grep) out.push(surface.runner === "vitest" ? "-t" : "-g", grep); return out; } function testRetryArgs(surface, retries) { - if (!shouldRetrySurface(surface) || retries <= 0) return []; + if (surface.kind === "unit") retries = 0; if (surface.runner === "playwright") return [`--retries=${retries}`]; if (surface.runner === "vitest") return [`--retry=${retries}`]; return []; @@ -818,7 +844,7 @@ function reporterArgs(surface, nativeResultPath) { } function reporterEnv(surface, nativeResultPath, attemptResultPath = null, baseEnv = process.env) { - const env = { ...baseEnv, FLAKY_TEST_RETRIES: baseEnv.FLAKY_TEST_RETRIES ?? "5" }; + const env = { ...baseEnv, TV_TEST_RUNNER_CONTEXT: "1", FLAKY_TEST_RETRIES: baseEnv.FLAKY_TEST_RETRIES ?? "5" }; if (surface.runner === "vitest" && attemptResultPath) env.TV_VITEST_ATTEMPT_FILE = attemptResultPath; delete env.FORCE_COLOR; delete env.NO_COLOR; @@ -826,16 +852,11 @@ function reporterEnv(surface, nativeResultPath, attemptResultPath = null, baseEn return env; } -function resolveTestRetries(selectionOptions, surfaces) { - const explicit = selectionOptions.retries; +function resolveTestRetries(selectionOptions, surfaces, resolved) { + const value = resolveRetryPolicy(selectionOptions, resolved); const retriable = hasRetriableSurface(surfaces); - if (explicit !== undefined) { - const value = Number.parseInt(explicit, 10); - if (!Number.isInteger(value) || value < 0) fail("--retries must be a non-negative integer."); - if (!retriable && value > 0) console.error("NOTICE: --retries is ignored for unit-only selections; runner-level retries apply only to non-unit Vitest and Playwright surfaces."); - return retriable ? value : 0; - } - return retriable ? 2 : 0; + if (!retriable && selectionOptions.retries !== undefined && value > 0) console.error("NOTICE: --retries is ignored for unit-only selections; runner-level retries apply only to non-unit Vitest and Playwright surfaces."); + return retriable ? value : 0; } function hasRetriableSurface(surfaces) { @@ -846,26 +867,17 @@ function shouldRetrySurface(surface) { return surface.kind !== "unit" && ["playwright", "vitest"].includes(surface.runner); } -function selectedSurfaces({ options: localOptions = options, requireSelection }) { - if (requireSelection && !localOptions.all && !localOptions.suite && !localOptions.surface && !localOptions.package && !localOptions.file && !localOptions.runner && !localOptions.tag) { - fail("Select tests explicitly with --suite, --all, --surface, --package, --file, --runner, or --tag."); - } - if (localOptions.file) { - const owners = owningSurfaces(config.surfaces, localOptions.file); - if (owners.length === 0) fail(`No test surface owns ${localOptions.file}.`); - if (owners.length > 1 && !localOptions.surface) fail(`${localOptions.file} is owned by multiple surfaces:\n${owners.map((surface) => `- ${surface.id}`).join("\n")}\nPass --surface to disambiguate.`); - } - const surfaces = selectSurfaces(config, localOptions); - if (surfaces.length === 0) fail("Selection matched no test surfaces."); - return surfaces; -} - function assertRegistryValid() { const errors = validateRegistry(config); if (errors.length) fail(`Test registry validation failed:\n${errors.map((e) => `- ${e}`).join("\n")}`); } function enforceExecutionPlacement(provider, surfaces) { + if (surfaces.some((surface) => surface.id === "experiment:blaxel-harness")) { + if (provider !== "local" || process.env.TV_TEST_SHARD_OWNER || !process.env.TV_BLAXEL_HARNESS_COORDINATOR?.trim() || !process.env.TV_BLAXEL_HARNESS_POOL?.trim()) { + fail("The Blaxel harness driver requires its dedicated coordinator outside the worker pool. Set TV_BLAXEL_HARNESS_COORDINATOR and TV_BLAXEL_HARNESS_POOL explicitly."); + } + } const isolated = surfaces.filter((surface) => surface.tags.includes("isolated-github-only")); if (isolated.length === 0) return; const ids = isolated.map((surface) => surface.id).join(", "); @@ -933,22 +945,58 @@ function runProcess(command, args, { logPath, env = process.env, append = false, } function selectionSummary(selectionOptions, surfaces) { - return { suite: selectionOptions.suite ?? (selectionOptions.all ? "all" : null), surfaces: surfaces.map((surface) => surface.id), files: selectionOptions.file ? [selectionOptions.file] : [], grep: selectionOptions.grep ?? null }; + return { suite: selectionOptions.suite ?? (selectionOptions.all ? "all" : null), surfaces: surfaces.map((surface) => surface.id), files: selectionOptions.resolvedFiles?.map((file) => file.path) ?? (selectionOptions.file ? [selectionOptions.file] : []), grep: selectionOptions.grep ?? null }; } function readJson(file) { try { return JSON.parse(fs.readFileSync(file, "utf8")); } catch { return null; } } +function blaxelSelectionArgs(selectionOptions, surfaces, testRetries) { + const args = []; + if (selectionOptions.file || selectionOptions.grep || selectionOptions.surface) { + if (surfaces.length !== 1) fail("Blaxel targeted runs require a selection that resolves to exactly one surface."); + const surface = surfaces[0]; + if (!["vitest", "playwright"].includes(surface.runner)) fail(`Blaxel targeted runs do not support runner ${surface.runner}.`); + args.push( + "--suite", "target", + "--shards", "1", + "--target-surface-id", surface.id, + "--target-runner", surface.runner, + "--target-config", relativeToCwd(surface, surface.config), + "--target-cwd", surface.command ? "." : surface.cwd || ".", + "--target-retries", String(testRetries), + ); + if (surface.command) args.push("--target-command-json", JSON.stringify(surface.command)); + if (surface.preCommand) args.push("--target-pre-command-json", JSON.stringify(surface.preCommand)); + if (surface.services.length > 0) args.push("--target-services-json", JSON.stringify(surface.services)); + if (selectionOptions.resolvedFiles) args.push("--target-files-json", JSON.stringify(selectionOptions.resolvedFiles.map((file) => file.path))); + if (selectionOptions.file) args.push("--target-file", surface.command ? selectionOptions.file : relativeToCwd(surface, selectionOptions.file)); + if (selectionOptions.grep) args.push("--target-grep", selectionOptions.grep); + } else if (selectionOptions.package || selectionOptions.runner || selectionOptions.tag) { + args.push("--suite", "selection", "--surfaces", surfaces.map((surface) => surface.id).join(",")); + } else { + const suite = selectionOptions.suite ?? (selectionOptions.all ? "all" : null); + if (!["all", "unit", "e2e"].includes(suite)) fail("Blaxel suite runs require --suite all|unit|e2e or --all. The telemetry PostHog roundtrip suite is local-only because it requires the PostHog test read key."); + args.push("--suite", suite); + } + return args; +} + function createProviderRunContext(provider) { if (provider === "blaxel") return createRunContext("blaxel", { timingProvider: "blaxel-playwright-x64-4vcpu" }); return createRunContext("local"); } +function validateRunDirOutput(options) { + const output = options["run-dir-output"]; + if (output != null && (typeof output !== "string" || output === "1" || output.trim() === "")) fail("--run-dir-output requires a file path"); +} + function writeRunDirOutput(options, runContext) { const output = options["run-dir-output"]; if (output == null) return; - if (typeof output !== "string" || output === "1" || output.trim() === "") fail("--run-dir-output requires a file path"); + validateRunDirOutput(options); atomicWriteFile(path.resolve(output), `${runContext.absoluteRunDir}\n`); } @@ -1001,8 +1049,11 @@ function parseArgs(argv, { allowPositionals = false } = {}) { positional.push(arg); continue; } - const key = arg.slice(2); + const equals = arg.indexOf("="); + const key = arg.slice(2, equals < 0 ? undefined : equals); const next = argv[i + 1]; + if ([BROAD_LOCAL, BROAD_ZERO, CONTENTION_BYPASS, MUTEX_WAIT].includes(key) && (equals >= 0 || next && !next.startsWith("--"))) fail(`--${key} does not take a value. Pass the flag by itself to deliberately enable it, or omit it.`); + if (equals >= 0) { out[key] = arg.slice(equals + 1); continue; } if (next && !next.startsWith("--")) { out[key] = next; i += 1; } else out[key] = "1"; } @@ -1042,25 +1093,47 @@ Remote options: --ignore-uncommitted Allow remote runs to ignore local uncommitted changes. Guardrails: + - Every local run and local verify holds one mutex across checkouts for the + same operating-system user until test processes and cleanup finish. + Contention refuses immediately by default. --${MUTEX_WAIT} + waits without a time limit; SIGINT or SIGTERM cancels only the waiter. + --${CONTENTION_BYPASS} + deliberately permits concurrent local runs: major host contention and + OOM-killed processes are possible. Its use is the caller's judgment. + It does not lift the marked-host file limit and cannot be combined with wait. - Without ~/.tvdev-use-blaxel, verify defaults to local without contacting Blaxel. With the marker, verify defaults to Blaxel and refuses automatic local fallback. - Explicit local verify then requires --allow-extreme-inefficiency. + Local runs select one --file, optionally with --grep; broader work goes to Blaxel + after committing and pushing to origin. Do not use serial file loops for broader validation. + With explicit human permission under testing policy, --${BROAD_LOCAL} permits + broader local tests or local verify. It does not replace --force. - Provider shortcuts (npm test -- local|blaxel) refuse broad --suite all runs. Use 'npm run verify' instead, or pass --force when a raw broad provider run is intentional. + - Direct native Vitest/Playwright execution is refused by registered configs. + For deliberate native watch/debug/editor use, set + TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN=1 (exactly 1). + This is the caller's judgment: it skips the canonical mutex and marked-host + file limit. Native configuration and required services still apply. Retries: Non-unit Vitest and Playwright surfaces retry twice by default, including targeted runs; unit surfaces do not receive runner-level retries. - Use --retries to override the non-unit surface retry count. + Use the default budget for validation. Plain --retries 0 is allowed with + --file selecting exactly one actual test file, optionally with --grep. + Broader zero requires --${BROAD_ZERO}; + this also selects zero without --retries. It conflicts with positive counts. + Positive --retries overrides the non-unit retry count. This retry override + does not authorize broader local work; its use is the caller's judgment. Examples: npm test -- local --file packages/web/test/copy-button.test.ts --grep "renders the 'idle default' state" - npm test -- local --surface e2e:desktop + npm test -- blaxel --surface e2e:desktop npm run verify npm run verify -- blaxel - npm test -- local --suite telemetry-posthog-roundtrip - TV_DAEMON_TEST_HOST=1 npm test -- local --suite daemon-acceptance + npm test -- local --file packages/server/test/telemetry-posthog.integration.test.ts + TV_DAEMON_TEST_HOST=1 npm test -- local --file test/node/daemon-acceptance.test.ts + npm test -- local --file test/agent/acp-bridge.test.ts # requires local agent setup npm test -- blaxel --suite e2e npm test -- blaxel --shard-indices 5 --force @@ -1079,4 +1152,4 @@ function relative(file) { return path.relative(process.cwd(), file); } function shellDisplay(value) { return /\s/.test(String(value)) ? JSON.stringify(String(value)) : String(value); } function formatDuration(ms) { return !Number.isFinite(ms) ? "unknown" : ms < 1000 ? `${ms}ms` : ms < 60000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.floor(ms / 60000)}m ${Math.round((ms % 60000) / 1000)}s`; } function formatPhase(phase) { return `${phase.category}/${phase.name}=${formatDuration(phase.durationMs)} status=${phase.status}${phase.cacheStatus ? ` cache=${phase.cacheStatus}` : ""}`; } -function fail(message) { console.error(message); process.exit(2); } +function fail(message, status = 2) { console.error(message); process.exit(status); } diff --git a/scripts/test/config.d.mts b/scripts/test/config.d.mts index 93426b2b..98a9cbd5 100644 --- a/scripts/test/config.d.mts +++ b/scripts/test/config.d.mts @@ -52,6 +52,7 @@ export interface SelectionOptions { export const CANONICAL_TEST_INCLUDE_ROOTS: readonly string[]; export function loadTestConfig(options?: { root?: string }): TestConfig; +export function normalizeTestConfig(config: TestConfig, options: { root: string; packageDirs: Map }): TestConfig; export function validateRegistry(config: TestConfig, options?: { root?: string }): string[]; export function selectSurfaces(config: TestConfig, options: SelectionOptions): TestSurface[]; export function owningSurfaces(surfaces: TestSurface[], file: string): TestSurface[]; diff --git a/scripts/test/config.mjs b/scripts/test/config.mjs index 907b715b..5c951609 100644 --- a/scripts/test/config.mjs +++ b/scripts/test/config.mjs @@ -5,9 +5,12 @@ import testConfig from "../../test.config.mjs"; export const CANONICAL_TEST_INCLUDE_ROOTS = ["packages", "test"]; export function loadTestConfig({ root = process.cwd() } = {}) { - const packageDirs = discoverPackageDirs(root); - const surfaces = rawSurfaces(testConfig).map((surface) => normalizeSurface(surface, root, packageDirs)); - return { ...testConfig, surfaces }; + return normalizeTestConfig(testConfig, { root, packageDirs: discoverPackageDirs(root) }); +} + +export function normalizeTestConfig(config, { root, packageDirs }) { + const surfaces = rawSurfaces(config).map((surface) => normalizeSurface(surface, root, packageDirs)); + return { ...config, surfaces }; } export function validateRegistry(config, { root = process.cwd() } = {}) { diff --git a/scripts/test/cursor-agent-env.mjs b/scripts/test/cursor-agent-env.mjs index cf0a6f05..ffecc072 100644 --- a/scripts/test/cursor-agent-env.mjs +++ b/scripts/test/cursor-agent-env.mjs @@ -1,6 +1,6 @@ /** * When CURSOR_AGENT=1, unset env vars Cursor injects that break local tests. - * See specs/arch/test-runner/test-runner.md "Cursor-agent environment workaround". + * See specs/arch/test-runner/test-runner.md "Cursor agent environment". */ /** Set by Cursor on agent shell invocations (observed value: "1"). */ diff --git a/scripts/test/execution-groups.mjs b/scripts/test/execution-groups.mjs index 1575d486..05566c3b 100644 --- a/scripts/test/execution-groups.mjs +++ b/scripts/test/execution-groups.mjs @@ -24,15 +24,22 @@ export function vitestProjectsArgs({ configPath, configPaths }) { return ["vitest", "run", "--config", configPath]; } -export function writeVitestRunnerConfig({ configPath, baseConfigPath, runnerPath }) { +export function writeVitestRunnerConfig({ configPath, baseConfigPath, runnerPath, files = null }) { + // Vitest positional filters are substrings, even for absolute filenames. + // Replace (do not merge) include when admission has resolved actual files. + const test = { runner: path.resolve(runnerPath), ...(files ? { include: files.map((file) => path.resolve(file).replace(/[?*[\]{}()!+@]/g, "\\$&")), includeSource: [] } : {}) }; fs.writeFileSync(configPath, [ - 'import { defineConfig, mergeConfig } from "vitest/config";', + 'import { defineConfig } from "vitest/config";', `import base from ${JSON.stringify(path.resolve(baseConfigPath))};`, - `export default mergeConfig(base, defineConfig({ test: { runner: ${JSON.stringify(path.resolve(runnerPath))} } }));`, + `export default defineConfig(async (env) => { const config = await (typeof base === "function" ? base(env) : base); return { ...config, test: { ...config.test, ...${JSON.stringify(test)} } }; });`, "", ].join("\n")); } +export function exactPlaywrightFiles(files) { + return files.map((file) => `^${path.resolve(file).replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`); +} + export function splitVitestWorkspaceResult({ nativeResultPath, tasks }) { const statuses = new Map(tasks.map((task) => [task.surfaceId, 0])); if (!fs.existsSync(nativeResultPath)) return new Map(tasks.map((task) => [task.surfaceId, 1])); diff --git a/scripts/test/file-inventory.d.mts b/scripts/test/file-inventory.d.mts index f18ecafd..6b40d537 100644 --- a/scripts/test/file-inventory.d.mts +++ b/scripts/test/file-inventory.d.mts @@ -2,5 +2,5 @@ import type { TestSurface, TestConfig } from "./config.mjs"; export const TEST_FILE_PATTERN: RegExp; export interface InventoryFile { path: string; surfaceId: string; runner: "vitest" | "playwright" } export function enumerateTestInventory(options: { repoRoot?: string; surfaces: TestSurface[]; selectedSurfaceIds?: string[]; commit?: string | null }): InventoryFile[]; -export function enumerateTrackedPaths(options?: { repoRoot?: string; commit?: string | null }): string[]; -export function loadRegistrySnapshotAtCommit(options?: { repoRoot?: string; commit?: string }): Promise; +export function enumerateTrackedPaths(options?: { repoRoot?: string; commit?: string | null; includeUntracked?: boolean }): string[]; +export function loadRegistrySnapshotAtCommit(options?: { repoRoot?: string; commit?: string }): Promise; diff --git a/scripts/test/file-inventory.mjs b/scripts/test/file-inventory.mjs index d683dbf3..e72a8750 100644 --- a/scripts/test/file-inventory.mjs +++ b/scripts/test/file-inventory.mjs @@ -3,7 +3,7 @@ import crypto from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { owningSurfaces } from "./config.mjs"; +import { normalizeTestConfig, owningSurfaces } from "./config.mjs"; export const TEST_FILE_PATTERN = /(?:^|\/)[^/]+\.(?:test|spec)\.(?:[cm]?[jt]s)x?$/; @@ -27,11 +27,11 @@ export function enumerateTestInventory({ repoRoot = process.cwd(), surfaces, sel return files.sort((left, right) => left.path.localeCompare(right.path)); } -export function enumerateTrackedPaths({ repoRoot = process.cwd(), commit = null } = {}) { +export function enumerateTrackedPaths({ repoRoot = process.cwd(), commit = null, includeUntracked = false } = {}) { if (!commit) { // The index retains unstaged deletions; current-worktree consumers must not // try to inspect paths that are tracked there but absent from this tree. - return splitNull(git(["ls-files", "--cached", "-z"], { cwd: repoRoot })) + return [...new Set(splitNull(git(["ls-files", "--cached", ...(includeUntracked ? ["--others", "--exclude-standard"] : []), "-z"], { cwd: repoRoot })))] .filter((file) => worktreePathExists(path.join(repoRoot, file))); } const index = path.join(os.tmpdir(), `tv-test-index-${process.pid}-${crypto.randomBytes(8).toString("hex")}`); @@ -46,32 +46,24 @@ export function enumerateTrackedPaths({ repoRoot = process.cwd(), commit = null } export async function loadRegistrySnapshotAtCommit({ repoRoot = process.cwd(), commit = "HEAD" } = {}) { + commit = git(["rev-parse", "--verify", "--end-of-options", `${commit}^{commit}`], { cwd: repoRoot }).trim(); const source = git(["show", `${commit}:test.config.mjs`], { cwd: repoRoot }); const digest = crypto.createHash("sha256").update(source).digest("hex"); const url = `data:text/javascript;base64,${Buffer.from(source).toString("base64")}#${digest}`; let raw; try { raw = (await import(url)).default; } catch (error) { throw new Error(`could not load test registry from ${commit}: ${error.message}`); } - const groups = raw.executionGroups ?? []; - const surfaces = groups.flatMap((group) => (group.surfaces ?? []).map((surface) => normalizeSurface({ ...surface, executionGroup: { id: group.id, name: group.name, order: group.order } }, repoRoot))); - return { ...raw, surfaces, registryDigest: digest }; -} - -function normalizeSurface(surface, repoRoot) { - const config = normalizePath(surface.config); - return { - ...surface, - config, - absoluteConfig: path.join(repoRoot, config), - cwd: normalizePath(surface.cwd ?? defaultCwd(surface.config)), - roots: (surface.roots ?? []).map(normalizePath), - excludeRoots: (surface.excludeRoots ?? []).map(normalizePath), - }; -} - -function defaultCwd(config) { - const normalized = normalizePath(config); - if (normalized === "vitest.config.ts" || normalized.startsWith("test/")) return "."; - return path.posix.dirname(normalized); + // Package locations belong to the tested tree just as the registry does. + // In particular, a package's native config need not live at its package root. + const packageDirs = new Map(); + for (const file of enumerateTrackedPaths({ repoRoot, commit })) { + if (path.posix.basename(file) !== "package.json" || /(?:^|\/)(?:node_modules|dist|\.git|\.blaxel-testshards|\.testshards|\.test-runs)\//.test(file)) continue; + const source = git(["show", `${commit}:${file}`], { cwd: repoRoot }); + try { + const pkg = JSON.parse(source); + if (pkg.name) packageDirs.set(pkg.name, path.posix.dirname(file)); + } catch {} + } + return { ...normalizeTestConfig(raw, { root: repoRoot, packageDirs }), registryDigest: digest, commit }; } function contains(root, file) { diff --git a/scripts/test/local-test-mutex.mjs b/scripts/test/local-test-mutex.mjs new file mode 100644 index 00000000..f6024a3d --- /dev/null +++ b/scripts/test/local-test-mutex.mjs @@ -0,0 +1,137 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import crypto from "node:crypto"; +import { decodeOwnerToken, parseLinuxProcessStat, processIdentityMatches, readMacOSProcesses, scanLinuxProcesses } from "./process-lifecycle.mjs"; +import { reapStaleOwnerProcesses } from "./stale-owner-reaper.mjs"; + +export const CONTENTION_BYPASS = "against-test-guidance-major-host-contention-and-oom-killed-processes"; +export const MUTEX_WAIT = "wait-for-local-test-mutex"; +const VERIFY_HANDOFF = "TV_TEST_RUNNER_VERIFY_MUTEX"; + +export function localTestMutexPath() { + if (process.env.TV_TEST_RUNNER_LOCK_PATH) { + if (process.env.TV_TEST_RUNNER_SELFTEST !== "1") throw new Error("TV_TEST_RUNNER_LOCK_PATH requires TV_TEST_RUNNER_SELFTEST=1."); + console.error("WARNING: TV_TEST_RUNNER_LOCK_PATH self-test seam substitutes a private local-test mutex location."); + return path.resolve(process.env.TV_TEST_RUNNER_LOCK_PATH); + } + // userInfo reads the operating-system account, independent of HOME/TMPDIR. + return path.join(os.userInfo().homedir, ".cache/television/local-test-mutex.sqlite3"); +} + +export function validateMutexOptions(options) { + if (options[MUTEX_WAIT] && options[CONTENTION_BYPASS]) throw new Error(`--${MUTEX_WAIT} and --${CONTENTION_BYPASS} cannot be combined.`); +} + +// Stable on-disk contract: one row (slot 1), containing version 1 JSON. +// SQLite supplies atomic compare-and-swap across processes and releases its +// brief write locks after crashes. No transaction spans test execution. +export async function acquireLocalTestMutex({ lockPath, options, role = "run", marked = false }) { + const { DatabaseSync } = await import("node:sqlite").catch(error => { + throw new Error(`The local-test mutex needs node:sqlite. Select the Node runtime in .nvmrc and run npm run toolchain:check. ${error.message}`, { cause: error }); + }); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + const db = new DatabaseSync(lockPath); + db.exec("PRAGMA busy_timeout=1000; CREATE TABLE IF NOT EXISTS local_test_mutex (slot INTEGER PRIMARY KEY CHECK(slot=1), record TEXT NOT NULL)"); + const mine = { ...identity(process.pid), command: process.argv.join(" "), checkout: process.cwd(), role }; + const fresh = { version: 1, id: crypto.randomUUID(), participants: [mine] }; + const handoff = process.env[VERIFY_HANDOFF]; + delete process.env[VERIFY_HANDOFF]; + let waitingFor = null; + let interrupted = 0; + const int = () => { interrupted = 130; }; + const term = () => { interrupted = 143; }; + process.on("SIGINT", int); process.on("SIGTERM", term); + const read = () => db.prepare("SELECT record FROM local_test_mutex WHERE slot=1").get()?.record; + const replace = (before, after) => db.prepare("UPDATE local_test_mutex SET record=? WHERE slot=1 AND record=?").run(JSON.stringify(after), before).changes === 1; + try { + for (;;) { + if (interrupted) throw Object.assign(new Error("Local-test mutex wait cancelled."), { exitCode: interrupted }); + const before = read(); + if (!before) { + if (db.prepare("INSERT OR IGNORE INTO local_test_mutex(slot,record) VALUES(1,?)").run(JSON.stringify(fresh)).changes === 1) return ownership(fresh.id); + continue; + } + const holder = parseRecord(before); + // Only verify's directly spawned test child receives this handoff. It + // consumes the signal; descendants and unrelated invocations contend. + if (handoff === holder.id && holder.participants.some(p => p.role === "verify" && p.pid === process.ppid && isLive(p))) { + if (replace(before, { ...holder, participants: [...holder.participants, mine] })) return ownership(holder.id); + continue; + } + if (!holder.participants.some(isLive)) { + const recovering = { ...fresh, participants: [...fresh.participants, ...holder.participants] }; + if (!replace(before, recovering)) continue; + const held = ownership(fresh.id); + try { + const recovery = await reapStaleOwnerProcesses(); + if (recovery.staleDetected || !recovery.cleanupConfirmed) throw Object.assign(new Error(`[local-test mutex] ${recovery.message}`), { exitCode: 1 }); + if (interrupted) throw Object.assign(new Error("Local-test mutex wait cancelled."), { exitCode: interrupted }); + replace(JSON.stringify(recovering), fresh); + return held; + } catch (error) { held.release({ retain: true }); throw error; } + } + if (options[CONTENTION_BYPASS]) { + console.error(`${describe(holder)} Deliberately running concurrently: major host contention and OOM-killed processes are possible.`); + db.close(); + return { childEnv: () => ({}), release() {} }; + } + if (!options[MUTEX_WAIT]) throw new Error(`${describe(holder)} Try after this run finishes, or use --${MUTEX_WAIT}.${marked ? " For broader work, commit and push, then use Blaxel." : " Blaxel is optional if you have access."} Do not stop another agent's run to free the mutex. --${CONTENTION_BYPASS} deliberately permits major host contention and OOM-killed processes.`); + if (waitingFor !== holder.id) { + console.error(`Waiting for local-test mutex. ${describe(holder)} Interrupt this wait with Ctrl-C (SIGINT) or SIGTERM; the holder keeps running.`); + waitingFor = holder.id; + } + await new Promise(resolve => setTimeout(resolve, 500)); + } + } catch (error) { if (db.isOpen) db.close(); throw error; } + finally { process.off("SIGINT", int); process.off("SIGTERM", term); } + + function ownership(id) { + let released = false; + return { + childEnv: () => ({ [VERIFY_HANDOFF]: id }), + release({ retain = false } = {}) { + if (released) return; + released = true; + try { + if (retain) return; + // Keep the record for recovery if an unhandled exit leaves children. + if (scanProcesses().some(p => { + const owner = decodeOwnerToken(p.ownerToken); + return owner?.supervisorPid === mine.pid && owner.supervisorStartTime === mine.startTime; + })) return; + for (;;) { + const before = read(); + if (!before) return; + const holder = parseRecord(before); + if (holder.id !== id) return; + const participants = holder.participants.filter(p => !processIdentityMatches(p, mine)); + if (participants.length ? replace(before, { ...holder, participants }) : db.prepare("DELETE FROM local_test_mutex WHERE slot=1 AND record=?").run(before).changes === 1) return; + } + } finally { db.close(); } + }, + }; + } +} + +function parseRecord(text) { + const record = JSON.parse(text); + if (record.version !== 1 || typeof record.id !== "string" || !Array.isArray(record.participants) || !record.participants.length) throw new Error("Unsupported local-test mutex ownership record; keep this file for diagnosis."); + return record; +} + +function identity(pid) { + // Liveness needs only PID/start time. Reading environ here would fail if a + // dead holder's PID had been reused by a process belonging to another user. + const found = process.platform === "linux" ? parseLinuxProcessStat(fs.readFileSync(`/proc/${pid}/stat`, "utf8")) : readMacOSProcesses().find(p => p.pid === pid); + if (!found || found.state === "Z") throw Object.assign(new Error(`No live process identity for PID ${pid}`), { code: "ESRCH" }); + return { pid: found.pid, startTime: found.startTime }; +} + +function isLive(expected) { + try { return processIdentityMatches(identity(expected.pid), expected); } + catch (error) { if (["ENOENT", "ESRCH"].includes(error.code)) return false; throw error; } +} + +function scanProcesses() { return process.platform === "linux" ? scanLinuxProcesses() : readMacOSProcesses(); } +function describe(holder) { return `Local-test mutex held by ${holder.participants.map(p => `PID ${p.pid} (start ${p.startTime}), ${p.command}, checkout ${p.checkout}`).join("; ")}.`; } diff --git a/scripts/test/native-entrypoint.d.mts b/scripts/test/native-entrypoint.d.mts new file mode 100644 index 00000000..cb0ff5c3 --- /dev/null +++ b/scripts/test/native-entrypoint.d.mts @@ -0,0 +1 @@ +export {}; diff --git a/scripts/test/native-entrypoint.mjs b/scripts/test/native-entrypoint.mjs new file mode 100644 index 00000000..fe8e9af2 --- /dev/null +++ b/scripts/test/native-entrypoint.mjs @@ -0,0 +1,12 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// Imported before config setup. Context is an invocation signal, not a credential. +if (process.env.TV_TEST_RUNNER_CONTEXT !== "1" && process.env.TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN !== "1") { + const remote = fs.existsSync(path.join(os.homedir(), ".tvdev-use-blaxel")) + ? " For broader work, commit and push first, then use npm test -- blaxel --surface or npm run verify -- blaxel." + : " Broader local selections and npm run verify are available without Blaxel access."; + throw new Error("Direct native test invocation skips the canonical runner's protections. Use npm test -- local --file [--grep ]." + remote + + " For deliberate native watch/debug/editor use, set TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN=1 (exactly 1). This is the caller's judgment: it skips the local-test mutex and marked-host file limit. Native configuration and required services remain your responsibility."); +} diff --git a/scripts/test/planned-shard-worker.mjs b/scripts/test/planned-shard-worker.mjs index cf536201..e985c91d 100644 --- a/scripts/test/planned-shard-worker.mjs +++ b/scripts/test/planned-shard-worker.mjs @@ -14,6 +14,7 @@ import { runnerEnv } from "./runner-env.mjs"; import { createSurfaceSupervisor } from "./surface-supervisor.mjs"; import { startSurfaceServices } from "./surface-services.mjs"; import { reapStaleOwnerProcesses } from "./stale-owner-reaper.mjs"; +import { surfaceResourceDiagnostics } from "./resource-diagnostics.mjs"; export async function runPlannedShard(rawOptions = {}, env = process.env) { const planPath = path.resolve(rawOptions.plan ?? env.TEST_SHARD_PLAN ?? ""); @@ -179,6 +180,7 @@ function relocateTaskArtifacts(summary, resultsDir, providerOutputDir, nativeOut workspaceNativeResultPath: relocateNative(task.workspaceNativeResultPath), lifecycleResultPath: relocateProvider(task.lifecycleResultPath), processLeakResultPath: relocateProvider(task.processLeakResultPath), + resourceDiagnosticsPaths: (task.resourceDiagnosticsPaths ?? []).map(relocateProvider), generatedInputPaths: (task.generatedInputPaths ?? []).map(relocateProvider), })), }; @@ -208,10 +210,12 @@ async function runVitestWorkspace(tasks, { resultsDir, shardIndex, skipPreComman const nativeResultPath = path.join(resultsDir, `native-unit-workspaces-shard-${shardIndex}.json`); const attemptResultPath = path.join(resultsDir, `native-unit-workspaces-shard-${shardIndex}.attempts.ndjson`); const supervisor = createSurfaceSupervisor({ runId: lifecycleRunId, surfaceIds: tasks.map((task) => task.surfaceId) }); + const diagnostics = tasks.map((task) => surfaceResourceDiagnostics({ resultsDir, surfaceId: task.surfaceId, env, runId: lifecycleRunId })); let pre; let exitCode = 0; let processLeaks = []; let commandStarted = null; + let splitStatuses = new Map(); fs.rmSync(attemptResultPath, { force: true }); const commandArgs = vitestProjectsArgs({ configPath: workspacePath, configPaths: tasks.map((task) => path.resolve(task.config)) }); try { @@ -225,10 +229,16 @@ async function runVitestWorkspace(tasks, { resultsDir, shardIndex, skipPreComman exitCode = pre.exitCode; } } finally { - processLeaks = await supervisor.finish(); + let cleanupConfirmed = false; + try { processLeaks = await supervisor.finish(); cleanupConfirmed = true; } + finally { + try { splitStatuses = splitVitestWorkspaceResult({ nativeResultPath, tasks }); } + finally { + for (const [index, task] of tasks.entries()) task.resourceDiagnosticsPaths = diagnostics[index].finish({ runner: "vitest", nativeResultPath: task.nativeResultPath, attemptResultPath, exitCode, incomplete: !cleanupConfirmed || !pre || pre.failed || processLeaks.length > 0 }); + } + } } if (pre.failed) return tasks.map((task) => failedTask(task, pre.exitCode, started, { preCommandsSkipped: skipPreCommands, failureKind: pre.failureKind, failureStep: "pre-command", failureMessage: pre.failureMessage, phases: pre.phases, generatedInputPaths: [workspacePath], processLeaks })); - const splitStatuses = splitVitestWorkspaceResult({ nativeResultPath, tasks }); return tasks.map((task) => { const status = exitCode === 0 ? 0 : (splitStatuses.get(task.surfaceId) ?? exitCode); const collected = collectedVitestFiles(task.nativeResultPath, task); @@ -239,6 +249,7 @@ async function runVitestWorkspace(tasks, { resultsDir, shardIndex, skipPreComman async function runStandalone(task, { resultsDir, shardIndex, skipPreCommands, env, lifecycleRunId }) { const started = Date.now(); const supervisor = createSurfaceSupervisor({ runId: lifecycleRunId, surfaceIds: [task.surfaceId] }); + const diagnostics = surfaceResourceDiagnostics({ resultsDir, surfaceId: task.surfaceId, env, runId: lifecycleRunId }); let pre; let command; let args; @@ -300,7 +311,9 @@ async function runStandalone(task, { resultsDir, shardIndex, skipPreCommands, en pre?.phases.push(phase("surface-services-stop", serviceStopped, 1)); } } - processLeaks = await supervisor.finish(); + let cleanupConfirmed = false; + try { processLeaks = await supervisor.finish(); cleanupConfirmed = true; } + finally { task.resourceDiagnosticsPaths = diagnostics.finish({ runner: task.runner, nativeResultPath: task.nativeResultPath, attemptResultPath, exitCode, incomplete: !cleanupConfirmed || !pre || pre.failed || Boolean(serviceFailure) || processLeaks.length > 0 }); } } if (pre.failed) return failedTask(task, pre.exitCode, started, { preCommandsSkipped: skipPreCommands, failureKind: pre.failureKind, failureStep: "pre-command", failureMessage: pre.failureMessage, phases: pre.phases, processLeaks }); const collected = task.runner === "vitest" ? collectedVitestFiles(task.nativeResultPath, task) : collectedPlaywrightFiles(task.nativeResultPath, task); @@ -343,7 +356,7 @@ function taskFromSurface(surface, assignedFiles, { resultsDir, shardIndex, testR nativeResultPath, services: surface.services ?? [], preCommand: surface.preCommand ? { command: surface.preCommand[0], args: surface.preCommand.slice(1), cwd: process.cwd() } : null, - retryArgs: surface.kind === "unit" || testRetries === 0 ? [] : [surface.runner === "playwright" ? `--retries=${testRetries}` : `--retry=${testRetries}`], + retryArgs: [surface.runner === "playwright" ? `--retries=${surface.kind === "unit" ? 0 : testRetries}` : `--retry=${surface.kind === "unit" ? 0 : testRetries}`], }; if (surface.command) return { ...base, command: surface.command[0], args: surface.command.slice(1), env: {} }; if (surface.runner === "playwright") return { ...base, command: npx(), args: ["playwright", "test", "--config", path.relative(cwd, path.resolve(surface.config))], env: {} }; @@ -366,6 +379,7 @@ function resultForTask(task, exitCode, started, extras) { attemptResultPath: extras.attemptResultPath && fs.existsSync(extras.attemptResultPath) ? extras.attemptResultPath : null, workspaceNativeResultPath: extras.workspaceNativeResultPath ?? null, generatedInputPaths: extras.generatedInputPaths ?? [], + resourceDiagnosticsPaths: task.resourceDiagnosticsPaths ?? [], preCommandsSkipped: extras.preCommandsSkipped, phases: extras.phases ?? [], processLeaks, ...(mismatch diff --git a/scripts/test/prebuilt.mjs b/scripts/test/prebuilt.mjs index d815e9bb..23b0c8f0 100644 --- a/scripts/test/prebuilt.mjs +++ b/scripts/test/prebuilt.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -// Build-once artifact helper (specs/arch/test-runner/github-ci.md#build-once-fan-out). +// Build-once artifact helper (specs/arch/test-runner/github-ci.md#build-once). // CLI: check | run | pack | artifact-name import { spawnSync } from "node:child_process"; import fs from "node:fs"; diff --git a/scripts/test/provider-normalization.mjs b/scripts/test/provider-normalization.mjs index b2277b56..8b0afe50 100644 --- a/scripts/test/provider-normalization.mjs +++ b/scripts/test/provider-normalization.mjs @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { normalizeSetupTimingStatus } from "./phase-metrics.mjs"; -import { normalizeSurfaceResult } from "./reporting.mjs"; +import { normalizeSurfaceResult, retainedResourcePaths } from "./reporting.mjs"; export function normalizeProviderShardSurfaces({ provider, providerSummary, outputDir, surfaces, delegatedCommand, exitCode, coordinatorLogPath }) { const bySurface = new Map(surfaces.map((surface) => [surface.id, surface])); @@ -49,6 +49,7 @@ export function normalizeProviderShardSurfaces({ provider, providerSummary, outp durationMs: shard.durationMs ?? providerSummary.durationMs ?? 0, logPath: path.relative(process.cwd(), shard.logPath ?? coordinatorLogPath), nativeResultPath: path.relative(process.cwd(), shard.nativeResultPath), + resourceDiagnosticsPaths: shard.resourceDiagnosticsPaths, })); } } @@ -201,6 +202,7 @@ export function mergeSurfaceParts(parts) { durationSource: "task-wall", nativeResultPath: group.map((part) => part.nativeResultPath).filter(Boolean)[0] ?? null, nativeResultPaths: group.map((part) => part.nativeResultPath).filter(Boolean), + resourceDiagnosticsPaths: retainedResourcePaths(group.flatMap((part) => part.resourceDiagnosticsPaths ?? [])), failureKind: firstInfrastructureFailure?.failureKind ?? first.failureKind, failureStep: firstInfrastructureFailure?.failureStep ?? first.failureStep, failureMessage: firstInfrastructureFailure?.failureMessage ?? first.failureMessage, @@ -248,6 +250,7 @@ function normalizeTaskPart({ task, surface, shard, outputDir, coordinatorLogPath logPath: path.relative(process.cwd(), shard.logPath ?? coordinatorLogPath), nativeResultPath, attemptResultPath, + resourceDiagnosticsPaths: task.resourceDiagnosticsPaths, processLeaks: task.processLeaks ?? [], }); const infraStatus = task.infraStatus === "incomplete" || normalized.infraStatus === "incomplete" ? "incomplete" : "completed"; @@ -357,5 +360,6 @@ function providerFailureSurfaces({ provider, failures, delegatedCommand, coordin infraFailedShards: failure.status === "infra-failed" && failure.shardIndex ? [failure.shardIndex] : [], logPath: retainedLogPath(failure.logPath, coordinatorLogPath), nativeResultPath: null, + resourceDiagnosticsPaths: retainedResourcePaths(failure.resourceDiagnosticsPaths), })); } diff --git a/scripts/test/provider-outputs.d.mts b/scripts/test/provider-outputs.d.mts index c8e2809a..75625c42 100644 --- a/scripts/test/provider-outputs.d.mts +++ b/scripts/test/provider-outputs.d.mts @@ -10,6 +10,7 @@ export interface ProviderTask { exitCode?: number | null; durationMs?: number; nativeResultPath?: string | null; + resourceDiagnosticsPaths?: string[]; [key: string]: unknown; } @@ -28,6 +29,7 @@ export interface ProviderShard { leaseAcquired?: boolean; executionStarted?: boolean; nativeResultPath?: string | null; + resourceDiagnosticsPaths?: string[]; [key: string]: unknown; } diff --git a/scripts/test/provider-outputs.mjs b/scripts/test/provider-outputs.mjs index 2f8d3ff2..c2e0247b 100644 --- a/scripts/test/provider-outputs.mjs +++ b/scripts/test/provider-outputs.mjs @@ -1,6 +1,7 @@ import fs from "node:fs"; import path from "node:path"; import { normalizeSetupTimingStatus, normalizeTimingPhase } from "./phase-metrics.mjs"; +import { retainedResourcePaths } from "./reporting.mjs"; export function providerCommandExitCode({ delegatedExitCode, reportStatus }) { if (delegatedExitCode !== 0) return delegatedExitCode || 1; @@ -63,6 +64,7 @@ export function compactProviderSummary(report, { outputDir = null } = {}) { logPath: shard.logPath, summaryPath: shard.summaryPath, outputDir: shard.outputDir, + resourceDiagnosticsPaths: retainedResourcePaths(shard.resourceDiagnosticsPaths), })), failedTasks, infraFailedShards, @@ -150,6 +152,7 @@ export function summarizeInfrastructureAttempts(reports) { logPath: shard.logPath, summaryPath: shard.summaryPath, outputDir: shard.outputDir, + resourceDiagnosticsPaths: retainedResourcePaths(shard.resourceDiagnosticsPaths), })), })); } @@ -189,7 +192,13 @@ export function combineAttemptReports(attempts, expectedShardIndices, { shardTot message: `No infrastructure attempt returned a result for shard ${shardIndex}/${shardTotal}.`, })); } - const shards = expectedShardIndices.map((shardIndex) => latestByShard.get(shardIndex)).sort((a, b) => a.shardIndex - b.shardIndex); + const shards = expectedShardIndices.map((shardIndex) => ({ ...latestByShard.get(shardIndex) })).sort((a, b) => a.shardIndex - b.shardIndex); + // Retain only links from superseded attempts, never their test outcomes. + for (const shard of shards) { + const history = attempts.flatMap((attempt) => (attempt.shards ?? []).filter((part) => part.shardIndex === shard.shardIndex)); + shard.resourceDiagnosticsPaths = retainedResourcePaths(history.flatMap((part) => [...(part.resourceDiagnosticsPaths ?? []), ...(part.tasks ?? []).flatMap((task) => task.resourceDiagnosticsPaths ?? [])])); + shard.tasks = (shard.tasks ?? []).map((task) => ({ ...task, resourceDiagnosticsPaths: retainedResourcePaths(history.flatMap((part) => (part.tasks ?? []).filter((candidate) => candidate.surfaceId === task.surfaceId).flatMap((candidate) => candidate.resourceDiagnosticsPaths ?? []))) })); + } const last = attempts.at(-1); const failedTasks = failedTasksFromShards(shards, { shardTotal, outputDir }); const startedAt = attempts[0]?.startedAt ?? attempts[0]?.run?.createdAt ?? new Date().toISOString(); diff --git a/scripts/test/reporting.d.mts b/scripts/test/reporting.d.mts index eef8a7d0..c68c7bb9 100644 --- a/scripts/test/reporting.d.mts +++ b/scripts/test/reporting.d.mts @@ -20,6 +20,7 @@ export interface NormalizedSurfaceResult { logPath: string | null; nativeResultPath?: string | null; nativeResultPaths?: string[]; + resourceDiagnosticsPaths?: string[]; counts: Counts; failedTests: Array>; flakyRecoveredTests: Array>; @@ -62,4 +63,5 @@ export function buildReports(args: { startedAt: string; completedAt: string; }): { summary: ReportSummary; results: ReportResults }; -export function normalizeSurfaceResult(args: { surface: Pick & Partial & { durationSource?: NormalizedSurfaceResult["durationSource"] }; command: string[]; exitCode: number | null; durationMs: number; logPath: string; nativeResultPath: string | null; attemptResultPath?: string | null; processLeaks?: Array> }): NormalizedSurfaceResult; +export function normalizeSurfaceResult(args: { surface: Pick & Partial & { durationSource?: NormalizedSurfaceResult["durationSource"] }; command: string[]; exitCode: number | null; durationMs: number; logPath: string; nativeResultPath: string | null; attemptResultPath?: string | null; resourceDiagnosticsPaths?: string[]; processLeaks?: Array> }): NormalizedSurfaceResult; +export function retainedResourcePaths(paths?: string[]): string[]; diff --git a/scripts/test/reporting.mjs b/scripts/test/reporting.mjs index 809b9689..13242288 100644 --- a/scripts/test/reporting.mjs +++ b/scripts/test/reporting.mjs @@ -79,6 +79,7 @@ export function buildReports({ runId, provider = null, commandProvider = provide logPath: surface.logPath, nativeResultPath: Array.isArray(surface.nativeResultPath) ? (surface.nativeResultPath[0] ?? null) : surface.nativeResultPath, nativeResultPaths: surface.nativeResultPaths ?? (Array.isArray(surface.nativeResultPath) ? surface.nativeResultPath : [surface.nativeResultPath].filter(Boolean)), + resourceDiagnosticsPaths: retainedResourcePaths(surface.resourceDiagnosticsPaths).map((file) => path.relative(runDir, file).replaceAll("\\", "/")), failureKind: surface.failureKind ?? null, failureStep: surface.failureStep ?? null, failureMessage: surface.failureMessage ?? null, @@ -135,7 +136,7 @@ function compactObject(value) { return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined && item !== null)); } -export function normalizeSurfaceResult({ surface, command, exitCode, durationMs, logPath, nativeResultPath, attemptResultPath = null, processLeaks = [] }) { +export function normalizeSurfaceResult({ surface, command, exitCode, durationMs, logPath, nativeResultPath, attemptResultPath = null, resourceDiagnosticsPaths = [], processLeaks = [] }) { const nativeReport = readJsonResult(nativeResultPath); const native = nativeReport.value; const base = { @@ -148,6 +149,7 @@ export function normalizeSurfaceResult({ surface, command, exitCode, durationMs, logPath, nativeResultPath, attemptResultPath, + resourceDiagnosticsPaths: retainedResourcePaths(resourceDiagnosticsPaths), infraStatus: "completed", processLeaks: deduplicateProcessLeaks(processLeaks), }; @@ -170,6 +172,12 @@ export function normalizeSurfaceResult({ surface, command, exitCode, durationMs, return { ...base, status: exitCode === 0 ? "passed" : "failed", counts: emptyCounts(), failedTests: [], flakyRecoveredTests: [], files: [] }; } +// Exact paths only: a basename lookup could attach another infrastructure +// attempt's observation when this attempt's download is absent. +export function retainedResourcePaths(paths = []) { + return [...new Set(paths.filter((file) => typeof file === "string" && fs.existsSync(file)).map((file) => path.resolve(file)))].sort(); +} + function normalizePlaywright(native, exitCode, surface) { const tests = []; collectPlaywrightSpecs(native.suites ?? [], tests, [], surface); diff --git a/scripts/test/resource-diagnostics.d.mts b/scripts/test/resource-diagnostics.d.mts new file mode 100644 index 00000000..b69fb4d1 --- /dev/null +++ b/scripts/test/resource-diagnostics.d.mts @@ -0,0 +1,7 @@ +export function startResourceDiagnostics(options: { root: string; identity: Record; readProbe?: (name: string, interval?: Record) => unknown; now?: () => Date; write?: (file: string, contents: string) => void; warn?: (message: string) => void }): { file: string | null; finish: (outcome: { runner: string; nativeResultPath: string | null; attemptResultPath?: string | null; exitCode: number; incomplete?: boolean }) => string[] }; +export function writeResourceDiagnostic(root: string, name: string, record: Record, options?: { write?: (file: string, contents: string) => void; warn?: (message: string) => void }): string | null; +export const RESOURCE_SIDECAR_BYTES: number; +export const RESOURCE_ATTEMPT_BYTES: number; +export function surfaceResourceDiagnostics(options: { resultsDir: string; surfaceId: string; env?: NodeJS.ProcessEnv; runId: string }): ReturnType; +export function readLinuxResourceProbe(name: string, options?: Record): unknown; +export function kernelLinesInInterval(lines: string[], interval: { since: string; until: string }): { lines: string[]; omittedUnrecognizedLines: number }; diff --git a/scripts/test/resource-diagnostics.mjs b/scripts/test/resource-diagnostics.mjs new file mode 100644 index 00000000..5d1d35c7 --- /dev/null +++ b/scripts/test/resource-diagnostics.mjs @@ -0,0 +1,239 @@ +import { BASELINE_TIMING_PROVIDER } from "./timing-baseline.mjs"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { normalizeSurfaceResult } from "./reporting.mjs"; + +export const RESOURCE_SIDECAR_BYTES = 128 * 1024; +export const RESOURCE_ATTEMPT_BYTES = 8 * 1024 * 1024; +const KERNEL_BYTES = 64 * 1024; +const COMPACT_RESERVE_BYTES = 8 * 1024; +const probes = { + memoryAvailable: { unit: "bytes", source: "/proc/meminfo:MemAvailable" }, + sharedMemoryUsed: { unit: "bytes", source: "/proc/meminfo:Shmem" }, + temporaryFree: { unit: "bytes", source: "statfs" }, + oomKills: { unit: "count", source: "/proc/vmstat:oom_kill" }, +}; +const warnDefault = (message) => console.error(`[resource diagnostics] ${message}`); + +/** Shared by planned tasks and the targeted surface parent, never the native child. */ +export function surfaceResourceDiagnostics({ resultsDir, surfaceId, env = process.env, runId }) { + if (env.TV_TEST_TIMING_PROVIDER !== BASELINE_TIMING_PROVIDER) return { file: null, finish: () => [] }; + return startResourceDiagnostics({ root: resultsDir, identity: { + runId: env.TV_TEST_RUN_ID ?? runId, + shardIndex: Number(env.TV_TEST_SHARD_INDEX ?? "1"), + attemptId: env.TV_TEST_SHARD_ATTEMPT ?? "unavailable", + surfaceId, + } }); +} + +export function startResourceDiagnostics({ root, identity, readProbe = readLinuxResourceProbe, now = () => new Date(), write, warn = warnDefault }) { + const name = `surface-${randomUUID()}`; + const tmpdir = os.tmpdir(); + const start = observe(readProbe, now, tmpdir, warn); + const record = { + kind: "surface", identity: cleanIdentity(identity), start, + end: { status: "missing", reason: "parent-did-not-finalize" }, + nativeEvidence: { status: "pending", failedAttempt: null, recovered: null }, + oomDelta: { value: null, unit: "count", reason: "missing-final-observation" }, + kernel: { status: "not-requested", reason: "parent-did-not-finalize" }, + }; + let file = writeResourceDiagnostic(root, name, record, { write, warn }); + return { + get file() { return file; }, + finish(outcome) { + try { + record.end = observe(readProbe, now, tmpdir, warn); + record.oomDelta = counterDelta(start.oomKills, record.end.oomKills); + record.nativeEvidence = nativeEvidence(outcome, path.join(root, "resources"), warn); + if (outcome.exitCode !== 0 || outcome.incomplete || record.nativeEvidence.status !== "complete" || record.nativeEvidence.failedAttempt) { + const interval = { since: start.observedAt, until: record.end.observedAt }; + try { record.kernel = cleanKernel(readProbe("kernelOOM", interval)); } + catch (error) { record.kernel = { status: "unavailable", reason: errorCode(error) }; warn(`kernel evidence unavailable: ${errorCode(error)}`); } + } else record.kernel = { status: "not-requested", reason: "no-failed-native-attempt" }; + file = writeResourceDiagnostic(root, name, record, { write, warn }) ?? file; + } catch (error) { warn(`final observation unavailable: ${errorCode(error)}`); } + return file && fs.existsSync(file) ? [file] : []; + }, + }; +} + +function observe(readProbe, now, tmpdir, warn) { + const observedAt = now().toISOString(); + const sample = { status: "observed", observedAt }; + for (const [name, metadata] of Object.entries(probes)) { + const value = { ...metadata, observedAt, ...(name === "temporaryFree" ? { path: tmpdir } : {}) }; + try { + const reading = readProbe(name, { tmpdir }); + if (!Number.isSafeInteger(reading) || reading < 0) throw new Error("Invalid numeric probe"); + sample[name] = { ...value, value: reading }; + } catch (error) { + sample[name] = { ...value, value: null, reason: errorCode(error) }; + warn(`${name} unavailable: ${errorCode(error)}`); + } + } + return sample; +} + +export function readLinuxResourceProbe(name, options = {}) { + if (process.platform !== "linux") throw Object.assign(new Error("Linux required"), { code: "ENOTSUP" }); + if (name === "temporaryFree") { + const stats = fs.statfsSync(options.tmpdir ?? os.tmpdir()); + return stats.bavail * stats.bsize; + } + if (name === "kernelOOM") { + // The worker's util-linux 2.38 accepts whole-second calendar bounds. Read + // an enclosing UTC window, then filter its output to our exact interval. + const calendar = (value) => new Date(value).toISOString().slice(0, 19).replace("T", " "); + const result = spawnSync("dmesg", ["--kernel", "--time-format", "iso", "--since", calendar(Date.parse(options.since) - 1), "--until", calendar(Date.parse(options.until) + 1000)], { encoding: "utf8", env: { ...process.env, TZ: "UTC", LC_ALL: "C" }, timeout: 2000, maxBuffer: 128 * 1024 }); + const truncated = result.error?.code === "ENOBUFS"; + if (!truncated && (result.error || result.status !== 0)) { + const permissionCode = /operation not permitted/i.test(result.stderr ?? "") ? "EPERM" : /permission denied/i.test(result.stderr ?? "") ? "EACCES" : "unavailable"; + throw Object.assign(new Error("Kernel log unavailable"), { code: result.error?.code ?? permissionCode }); + } + return { ...kernelLinesInInterval((result.stdout ?? "").split("\n"), options), truncated }; + } + const file = name === "oomKills" ? "/proc/vmstat" : "/proc/meminfo"; + const key = name === "oomKills" ? "oom_kill" : name === "memoryAvailable" ? "MemAvailable:" : "Shmem:"; + const line = fs.readFileSync(file, "utf8").split("\n").find((line) => line.startsWith(`${key} `)); + if (!line) throw Object.assign(new Error("Counter unavailable"), { code: "ENOENT" }); + return Number(line.trim().split(/\s+/)[1]) * (name === "oomKills" ? 1 : 1024); +} + +function counterDelta(start, end) { + if (start.value === null || end.value === null) return { value: null, unit: "count", reason: "unavailable" }; + if (end.value < start.value) return { value: null, unit: "count", reason: "counter-reset" }; + return { value: end.value - start.value, unit: "count" }; +} + +export function kernelLinesInInterval(lines, interval) { + const since = Date.parse(interval.since); + const until = Date.parse(interval.until); + let omittedUnrecognizedLines = 0; + const retained = lines.filter((line) => { + if (!/out of memory|oom[-_ ]kill|killed process/i.test(line)) return false; + const timestamp = line.split(/\s/, 1)[0].replace(",", ".").replace(/\.(\d{3})\d+/, ".$1"); + const time = Date.parse(timestamp); + if (!Number.isFinite(time)) { omittedUnrecognizedLines++; return false; } + return time >= since && time <= until; + }); + return { lines: retained, omittedUnrecognizedLines }; +} + +function nativeEvidence(outcome, diagnosticDir, warn) { + const nativeExists = outcome.nativeResultPath && fs.existsSync(outcome.nativeResultPath); + const attemptExists = outcome.attemptResultPath && fs.existsSync(outcome.attemptResultPath); + const references = { + resultPath: nativeExists ? path.relative(diagnosticDir, outcome.nativeResultPath) : null, + attemptResultPath: attemptExists ? path.relative(diagnosticDir, outcome.attemptResultPath) : null, + }; + if (!nativeExists) return { ...references, status: "missing", failedAttempt: null, recovered: null }; + try { + const native = normalizeSurfaceResult({ surface: { id: "diagnostic", runner: outcome.runner }, command: [], logPath: null, durationMs: 0, ...outcome }); + const attempts = native.files.flatMap((file) => file.attempts ?? []); + const complete = native.infraStatus !== "incomplete" && native.files.every((file) => file.complete); + return { ...references, status: complete ? "complete" : "incomplete", failedAttempt: attempts.some((attempt) => ["failed", "timed-out", "interrupted"].includes(attempt.status)) || native.counts.testsFailed > 0, recovered: native.counts.testsFlakyRecovered > 0 }; + } catch (error) { + warn(`native attempt evidence unavailable: ${errorCode(error)}`); + return { ...references, status: "incomplete", failedAttempt: null, recovered: null }; + } +} + +// Kernel output is not a trusted text channel. Retain only timestamps, executable +// names and numeric OOM fields, never raw messages, paths, arguments or values. +function cleanKernel(input) { + const lines = []; + let bytes = 0; + let omitted = integer(input?.omittedUnrecognizedLines) ?? 0; + let truncated = Boolean(input?.truncated); + for (const raw of input?.lines ?? []) { + const line = String(raw); + if (!/out of memory|oom[-_ ]kill|killed process/i.test(line)) continue; + const killed = /Killed process (\d+) \(([A-Za-z0-9_.+-]{1,80})\)/.exec(line); + if (!killed) { omitted++; continue; } + const timestamp = /^\d{4}-\d\d-\d\dT[\d:.,+-]+/.exec(line)?.[0]; + const fields = [...line.matchAll(/\b(total-vm|anon-rss|file-rss|shmem-rss|pgtables|UID|oom_score_adj):\s*(-?\d+)(kB)?/g)].map((match) => `${match[1]}:${match[2]}${match[3] ?? ""}`); + const retained = [timestamp, `Killed process ${killed[1]} (${killed[2]})`, ...fields].filter(Boolean).join(" "); + const cost = Buffer.byteLength(retained) + (lines.length ? 1 : 0); + if (bytes + cost > KERNEL_BYTES) { truncated = true; break; } + lines.push(retained); bytes += cost; + } + return { status: "available", lines, truncated, omittedUnrecognizedLines: omitted }; +} + +/** One budget for surface records and recovery receipts in the current attempt. */ +export function writeResourceDiagnostic(root, name, input, { write = atomicWrite, warn = warnDefault } = {}) { + try { + if (!/^[A-Za-z0-9_.-]+$/.test(name)) throw new Error("Invalid diagnostic filename"); + const directory = path.resolve(root, "resources"); + fs.mkdirSync(directory, { recursive: true }); + const file = path.join(directory, `${name}.json`); + // Reserve each baseline's final compact observation before optional excerpts + // can consume the remaining budget. Parents write sequentially per attempt. + const otherUsage = fs.readdirSync(directory).filter((entry) => entry.endsWith(".json") && path.join(directory, entry) !== file) + .reduce((sum, entry) => sum + Math.max(COMPACT_RESERVE_BYTES, fs.statSync(path.join(directory, entry)).size), 0); + const budget = Math.min(RESOURCE_SIDECAR_BYTES, RESOURCE_ATTEMPT_BYTES - otherUsage); + if (budget < COMPACT_RESERVE_BYTES) { warn("attempt budget exhausted; diagnostic omitted"); return null; } + const record = cleanRecord(input); + let contents = serialize(record); + if (Buffer.byteLength(contents) > budget) { + record.truncated = true; + const field = record.kind === "recovery" ? "processes" : null; + const values = field ? record[field] : record.kernel?.lines ?? []; + if (!field && record.kernel) record.kernel.truncated = true; + let low = 0; let high = values.length; + while (low < high) { + const count = Math.ceil((low + high) / 2); + if (field) record[field] = values.slice(0, count); else if (record.kernel) record.kernel.lines = values.slice(0, count); + if (Buffer.byteLength(serialize(record)) <= budget) low = count; else high = count - 1; + } + if (field) { record[field] = values.slice(0, low); record.omittedProcesses = (integer(input.omittedProcesses) ?? 0) + values.length - low; } + else if (record.kernel) record.kernel.lines = values.slice(0, low); + contents = serialize(record); + // The omission count itself can add a few bytes at the boundary. + while (Buffer.byteLength(contents) > budget && field && record[field].length) { record[field].pop(); record.omittedProcesses++; contents = serialize(record); } + warn("diagnostic truncated to its byte budget"); + } + if (Buffer.byteLength(contents) > budget) { warn("compact diagnostic exceeds its budget; omitted"); return null; } + write(file, contents); + return fs.existsSync(file) ? file : null; + } catch (error) { warn(`diagnostic write failed: ${errorCode(error)}`); return null; } +} + +function cleanRecord(input) { + const base = { schemaVersion: 1, kind: input.kind === "recovery" ? "recovery" : "surface", identity: cleanIdentity(input.identity ?? {}) }; + if (base.kind === "recovery") return { + ...base, + previous: select(input.previous, ["runId", "lockId", "worker", "lastRenewal", "recoverAfter"]), + cleanup: token(input.cleanup), discardedReports: Boolean(input.discardedReports), + omittedProcesses: integer(input.omittedProcesses) ?? 0, + processes: (input.processes ?? []).map((entry) => ({ pid: integer(entry.pid), startTime: token(entry.startTime), command: executable(entry.command), outcome: token(entry.outcome) })), + }; + return { + ...base, start: cleanObservation(input.start), end: cleanObservation(input.end), + oomDelta: select(input.oomDelta, ["value", "unit", "reason"]), + nativeEvidence: select(input.nativeEvidence, ["status", "failedAttempt", "recovered", "resultPath", "attemptResultPath"]), + kernel: input.kernel?.status === "available" ? cleanKernel(input.kernel) : select(input.kernel, ["status", "reason"]), + }; +} + +function cleanObservation(input) { + const record = select(input, ["status", "reason", "observedAt"]); + for (const key of Object.keys(probes)) if (input?.[key]) record[key] = select(input[key], ["value", "unit", "source", "observedAt", "path", "reason"]); + return record; +} + +function cleanIdentity(input) { return { runId: token(input.runId), shardIndex: integer(input.shardIndex), attemptId: token(input.attemptId), surfaceId: token(input.surfaceId) }; } +function token(value) { return typeof value === "string" ? value.replace(/[^a-zA-Z0-9_.:/@+-]/g, "_").slice(0, 256) : null; } +function executable(value) { return typeof value === "string" ? token(path.basename(value).split(/\s/)[0]) : null; } +function integer(value) { return Number.isSafeInteger(value) && value >= 0 ? value : null; } +function select(input, fields) { return Object.fromEntries(fields.filter((key) => input?.[key] !== undefined).map((key) => [key, input[key]])); } +function serialize(value) { return `${JSON.stringify(value)}\n`; } +function errorCode(error) { return ["EACCES", "EPERM", "ENOENT", "ENOSPC", "EROFS", "EIO", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"].includes(error?.code) ? error.code : "unavailable"; } +function atomicWrite(file, contents) { + const temporary = `${file}.${process.pid}.tmp`; + try { fs.writeFileSync(temporary, contents); fs.renameSync(temporary, file); } + finally { fs.rmSync(temporary, { force: true }); } +} diff --git a/scripts/test/runner-env.mjs b/scripts/test/runner-env.mjs index aef2d689..2be7ebb9 100644 --- a/scripts/test/runner-env.mjs +++ b/scripts/test/runner-env.mjs @@ -8,6 +8,7 @@ export function runnerEnv(baseEnv = process.env, ...layers) { PATH: `${path.join(process.cwd(), "node_modules", ".bin")}${path.delimiter}${baseEnv.PATH ?? ""}`, }; for (const layer of layers) Object.assign(env, layer ?? {}); + env.TV_TEST_RUNNER_CONTEXT = "1"; delete env.FORCE_COLOR; delete env.NO_COLOR; return env; diff --git a/scripts/test/shard-plan.mjs b/scripts/test/shard-plan.mjs index 050aa53a..4bc19ff3 100644 --- a/scripts/test/shard-plan.mjs +++ b/scripts/test/shard-plan.mjs @@ -13,7 +13,7 @@ export function createShardPlan({ inventory, surfaces, baseline, baselineBytes, const surfaceById = new Map(surfaces.map((surface) => [surface.id, surface])); const knownBySurface = new Map(selectedSurfaceIds.map((id) => [id, []])); // Every plan weighs files by the baseline's Blaxel measurements, whatever - // substrate executes it (specs/arch/test-runner/sharded-execution.md#Provider weights and unknown files). + // substrate executes it (specs/arch/test-runner/sharded-execution.md#duration-baseline). const knownBaselineWide = []; for (const entry of Object.values(baseline.files)) { const timing = entry.providers[BASELINE_TIMING_PROVIDER]; diff --git a/scripts/test/supervised-command.mjs b/scripts/test/supervised-command.mjs index f2b9fa7e..9aabfdfc 100644 --- a/scripts/test/supervised-command.mjs +++ b/scripts/test/supervised-command.mjs @@ -4,15 +4,21 @@ import path from "node:path"; import process from "node:process"; import { createSurfaceSupervisor } from "./surface-supervisor.mjs"; import { startSurfaceServices } from "./surface-services.mjs"; +import { runnerEnv } from "./runner-env.mjs"; +import { surfaceResourceDiagnostics } from "./resource-diagnostics.mjs"; const { runId, surfaceIds, resultPath, preCommand, services, command, args } = parseArgs(process.argv.slice(2)); const supervisor = createSurfaceSupervisor({ runId, surfaceIds }); +const diagnostics = surfaceResourceDiagnostics({ resultsDir: path.dirname(resultPath), surfaceId: surfaceIds.join(","), runId }); +// The targeted Vitest adapter writes per-attempt rows beside its native result. +const attemptResultPath = process.env.TV_TARGET_RUNNER === "vitest" && process.env.TV_TARGET_NATIVE_RESULT ? `${process.env.TV_TARGET_NATIVE_RESULT}.attempts.ndjson` : null; let commandResult = { exitCode: 127, signal: null, error: null }; let cleanupError = null; let executionFailure = null; let processLeaks = []; let serviceScope = null; try { + if (attemptResultPath) fs.rmSync(attemptResultPath, { force: true }); if (preCommand) { const preResult = await supervisor.run(preCommand[0], preCommand.slice(1), { cwd: process.cwd(), env: process.env, stdio: "inherit" }); const preLeaks = await supervisor.checkForLeaks(); @@ -30,7 +36,7 @@ try { } } if (!executionFailure) { - commandResult = await supervisor.run(command, args, { cwd: process.cwd(), env: serviceScope?.env ?? process.env, stdio: "inherit" }); + commandResult = await supervisor.run(command, args, { cwd: process.cwd(), env: runnerEnv(serviceScope?.env ?? process.env), stdio: "inherit" }); } } finally { if (serviceScope) { @@ -50,6 +56,7 @@ try { } const failedForLifecycle = processLeaks.length > 0 || cleanupError !== null; +const resourceDiagnosticsPaths = diagnostics.finish({ runner: process.env.TV_TARGET_RUNNER, nativeResultPath: process.env.TV_TARGET_NATIVE_RESULT, attemptResultPath, exitCode: commandResult.exitCode, incomplete: failedForLifecycle || Boolean(executionFailure) }); const publishedServices = serviceScope?.services ?? []; const result = { schemaVersion: 1, @@ -62,6 +69,8 @@ const result = { exitCode: commandResult.exitCode, signal: commandResult.signal, processLeaks, + resourceDiagnosticsPaths, + attemptResultPath: attemptResultPath && fs.existsSync(attemptResultPath) ? attemptResultPath : null, failureKind: cleanupError?.name === "UnconfirmedSurfaceCleanupError" ? "unconfirmed-process-cleanup" : executionFailure?.kind ?? (failedForLifecycle ? "process-leak" : null), failureStep: executionFailure?.step ?? null, failureMessage: cleanupError?.message ?? executionFailure?.message ?? (processLeaks.length > 0 ? `${processLeaks.length} owned process leak(s) were reaped` : null), diff --git a/scripts/test/surface-supervisor.mjs b/scripts/test/surface-supervisor.mjs index 508be8aa..368962bd 100644 --- a/scripts/test/surface-supervisor.mjs +++ b/scripts/test/surface-supervisor.mjs @@ -59,7 +59,7 @@ export class SurfaceSupervisor { } childEnv(env = process.env) { - return { ...env, [OWNER_TOKEN_ENV]: this.ownerToken }; + return { ...env, TMPDIR: env.TMPDIR ?? process.env.TMPDIR, ...(process.env.TV_TEST_SHARD_OWNER ? { TV_TEST_SHARD_OWNER: process.env.TV_TEST_SHARD_OWNER } : {}), [OWNER_TOKEN_ENV]: this.ownerToken }; } run(command, args = [], { diff --git a/scripts/test/target-command.d.mts b/scripts/test/target-command.d.mts new file mode 100644 index 00000000..a0d59278 --- /dev/null +++ b/scripts/test/target-command.d.mts @@ -0,0 +1,12 @@ +export interface NativeTarget { + runner?: string | null; + cwd: string; + config?: string | null; + file?: string | null; + files?: string[] | null; + grep?: string | null; + retries: string; + command?: string[] | null; + workspace?: string | null; +} +export function targetCommand(target: NativeTarget | null): string; diff --git a/scripts/test/target-command.mjs b/scripts/test/target-command.mjs new file mode 100644 index 00000000..3c25e475 --- /dev/null +++ b/scripts/test/target-command.mjs @@ -0,0 +1,22 @@ +export function targetCommand(target) { + if (!target) return ""; + if (target.runner) return ["node", "scripts/test/target-native.mjs", JSON.stringify(target)].map(shellQuote).join(" "); + const args = [ + "npm", + "--workspace", + target.workspace, + "run", + "test:e2e", + "--if-present", + "--", + ]; + if (target.file) args.push(target.file); + if (target.grep) args.push("-g", target.grep); + args.push(`--retries=${target.retries}`, "--reporter=list"); + return args.map(shellQuote).join(" "); +} + + +function shellQuote(value) { + return `'${String(value).replaceAll("'", `'"'"'`)}'`; +} diff --git a/scripts/test/target-native.mjs b/scripts/test/target-native.mjs new file mode 100644 index 00000000..a1b485fc --- /dev/null +++ b/scripts/test/target-native.mjs @@ -0,0 +1,27 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { exactPlaywrightFiles, writeVitestRunnerConfig } from "./execution-groups.mjs"; +import { runnerEnv } from "./runner-env.mjs"; + +// Runs inside the targeted worker's existing supervised process/service scope. +const target = JSON.parse(process.argv[2]); +if (!["vitest", "playwright"].includes(target.runner)) throw new Error("target runner must be vitest or playwright"); +if (target.command && target.runner !== "playwright") throw new Error("target commands currently support Playwright surfaces only"); +const native = path.resolve(process.env.TV_TARGET_NATIVE_RESULT); +const cwd = path.resolve(target.cwd); +const env = runnerEnv(); +let command; +if (target.runner === "vitest") { + const configPath = `${native}.runner.mjs`; + env.TV_VITEST_ATTEMPT_FILE = `${native}.attempts.ndjson`; + fs.rmSync(env.TV_VITEST_ATTEMPT_FILE, { force: true }); + writeVitestRunnerConfig({ configPath, baseConfigPath: path.resolve(cwd, target.config), runnerPath: "scripts/test/vitest-attempt-reporter.mjs", files: target.files }); + command = ["npx", "vitest", "run", "--config", configPath, ...(target.files ? [] : target.file ? [target.file] : []), ...(target.grep ? ["-t", target.grep] : []), `--retry=${target.retries}`, "--reporter=json", "--outputFile", native]; +} else { + env.PLAYWRIGHT_JSON_OUTPUT_NAME = native; + command = [...(target.command ?? ["npx", "playwright", "test", "--config", target.config]), ...(target.files ? exactPlaywrightFiles(target.files) : target.file ? [target.file] : []), ...(target.grep ? ["-g", target.grep] : []), `--retries=${target.retries}`, "--reporter=json"]; +} +const result = spawnSync(command[0], command.slice(1), { cwd, env, stdio: "inherit" }); +if (result.error) console.error(result.error.message); +process.exitCode = result.status ?? 1; diff --git a/scripts/test/test-guidance.mjs b/scripts/test/test-guidance.mjs new file mode 100644 index 00000000..7ab5d359 --- /dev/null +++ b/scripts/test/test-guidance.mjs @@ -0,0 +1,84 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { owningSurfaces, selectSurfaces } from "./config.mjs"; +import { enumerateTrackedPaths, TEST_FILE_PATTERN } from "./file-inventory.mjs"; + +export const BROAD_LOCAL = "against-test-guidance-broad-local-run"; +export const BROAD_ZERO = "against-test-guidance-turn-flakes-into-failures-on-broad-runs"; + +export function hasBlaxelMarker() { + return fs.existsSync(path.join(os.homedir(), ".tvdev-use-blaxel")); +} + +// Resolve once, before admission. The native adapters consume these same files. +export function resolveFileSelection({ config, options, repoRoot = process.cwd(), commit = null }) { + const candidates = selectSurfaces(config, { ...options, file: undefined }); + if (!options.file) { + if (!candidates.length) throw new Error("Selection matched no test surfaces."); + return { surfaces: candidates, files: null, oneFile: false }; + } + const filter = path.relative(repoRoot, path.resolve(repoRoot, options.file)).split(path.sep).join("/"); + const paths = enumerateTrackedPaths({ repoRoot, commit, includeUntracked: !commit }).filter((file) => TEST_FILE_PATTERN.test(file)); + const exact = paths.includes(filter); + const matches = paths.filter((file) => exact ? file === filter : file.includes(filter)); + if (!matches.length) throw new Error(`No test files match --file ${options.file}.${commit ? "" : " Local selection includes tracked and non-ignored working-tree test files."}`); + const files = []; + for (const file of matches) { + const owners = owningSurfaces(config.surfaces, file); + if (!owners.length) throw new Error(`No test surface owns ${file}.`); + const selected = owners.filter((owner) => candidates.some((candidate) => candidate.id === owner.id)); + if (selected.length > 1) throw new Error(`${file} is owned by multiple surfaces: ${selected.map((owner) => owner.id).join(", ")}. Pass --surface to disambiguate.`); + if (selected.length === 1) files.push({ path: file, surfaceId: selected[0].id }); + } + if (!files.length) throw new Error(`${options.file} is not part of the selected surfaces. Check its owning surface and co-selectors.`); + const surfaces = candidates.filter((surface) => files.some((file) => file.surfaceId === surface.id)); + if (surfaces.length > 1) throw new Error(`${options.file} is owned by multiple surfaces: ${surfaces.map((surface) => surface.id).join(", ")}. Pass --surface to disambiguate.`); + return { surfaces, files, oneFile: files.length === 1 }; +} + +export function resolveRetryPolicy(options, { oneFile = false } = {}) { + const raw = options.retries; + if (raw !== undefined && (!/^\d+$/.test(raw) || !Number.isSafeInteger(Number(raw)))) throw new Error("--retries must be a non-negative integer."); + if (options[BROAD_ZERO] && raw !== undefined && Number(raw) !== 0) throw new Error(`--${BROAD_ZERO} conflicts with a positive --retries count.`); + if (raw !== undefined && Number(raw) === 0 && !oneFile && !options[BROAD_ZERO]) { + throw new Error(`Use the default retry budget for broader validation. Plain --retries 0 is for a focused --file check, optionally with --grep. To deliberately turn flakes into failures on a broader run, use --${BROAD_ZERO}.`); + } + return options[BROAD_ZERO] ? 0 : raw === undefined ? 2 : Number(raw); +} + +export function enforceLocalFileGuidance(options, { surfaces, oneFile }) { + if (!hasBlaxelMarker() || oneFile || options[BROAD_LOCAL]) return; + const localOnly = surfaces.filter((surface) => localOnlyRequirement(surface)); + let route; + if (localOnly.length) { + const requirements = localOnly.map(localOnlyRequirement); + const commands = localOnly.map((surface) => { + const file = surface.roots.length === 1 && TEST_FILE_PATTERN.test(surface.roots[0]) ? shellQuote(surface.roots[0]) : ""; + return `${surface.id.startsWith("daemon-acceptance:") ? "TV_DAEMON_TEST_HOST=1 " : ""}npm test -- local --file ${file}`; + }); + route = `${[...new Set(requirements)].join("; ")}. Use ${commands.join(" or ")} with those prerequisites; this selection has no equivalent supported Blaxel command.`; + } else if ((options.file || options.grep || options.surface) && surfaces.length !== 1 || options.suite && !["all", "unit", "e2e"].includes(options.suite) && !options.file && !options.surface && !options.package && !options.runner && !options.tag) { + route = "This selection has no equivalent supported Blaxel command. npm test -- blaxel supports a single targeted surface or planned all/unit/e2e, package, runner, and tag selections. Commit and push to origin before a supported Blaxel run."; + } else { + const args = ["npm", "test", "--", "blaxel"]; + for (const key of ["suite", "surface", "package", "file", "grep", "runner", "tag"]) if (options[key]) args.push(`--${key}`, options[key]); + if (options.all) args.push("--all"); + if (options.force) args.push("--force"); + route = `Commit and push to origin first, then run: ${args.map(shellQuote).join(" ")}.`; + } + throw new Error(`~/.tvdev-use-blaxel is present. Local iteration selects at most one test file with --file, optionally narrowed by --grep. A file filter can match several files. Broader runs can exhaust the shared host. ${route} Do not replace broader validation with serial file loops. With explicit human permission under testing policy, --${BROAD_LOCAL} permits a broader local run.`); +} + +// Selections with no Blaxel equivalent: refusals route these to their local one-file form. +function localOnlyRequirement(surface) { + if (surface.agent) return "agent tests require local agent setup"; + if (surface.id.startsWith("telemetry-posthog-roundtrip:")) return "PostHog tests require the local PostHog test read key"; + if (surface.id.startsWith("daemon-acceptance:")) return "daemon tests require a dedicated local host with TV_DAEMON_TEST_HOST=1"; + if (surface.id === "experiment:blaxel-harness") return "the Blaxel harness driver runs locally on its dedicated coordinator outside every worker pool"; + return null; +} + +function shellQuote(value) { + return /^[a-zA-Z0-9_./:@=-]+$/.test(value) ? value : `'${String(value).replaceAll("'", "'\\''")}'`; +} diff --git a/scripts/test/timing-baseline.mjs b/scripts/test/timing-baseline.mjs index 16510ae4..d2d8c178 100644 --- a/scripts/test/timing-baseline.mjs +++ b/scripts/test/timing-baseline.mjs @@ -38,7 +38,7 @@ export function validateTimingBaseline(baseline) { return baseline; } -// The only baseline writer (specs/arch/test-runner/sharded-execution.md#Manual update): +// The only baseline writer (specs/arch/test-runner/sharded-execution.md#duration-baseline): // complete-`all` Blaxel runs in this checkout's .test-runs/ feed a full // recomputation. Nothing is written unless every input run validates. export function updateTimingBaseline({ repoRoot = process.cwd(), config, dryRun = false } = {}) { diff --git a/specs/arch/desktop/e2e-harness.md b/specs/arch/desktop/e2e-harness.md index be8ec3cc..34fd7945 100644 --- a/specs/arch/desktop/e2e-harness.md +++ b/specs/arch/desktop/e2e-harness.md @@ -6,7 +6,7 @@ Desktop tests use a real Electron process when browser-only testing cannot exerc ## What this owns -This spec owns how `e2e:desktop` obtains and launches Electron: `packages/desktop/test/e2e/global-setup.ts`, the Electron-specific parts of `packages/desktop/test/e2e/helpers.ts`, the shared Linux environment planner in `scripts/electron-e2e-env.mjs`, the test fixture hook, and the harness's real-process seams. [runtime.md](./runtime.md) owns runtime validity. The canonical runner, registry surface, preflight timing, and provider setup remain under [test-runner](../test-runner/test-runner.md), [registry](../test-runner/test-registry.md), [preflight](../test-runner/preflight.md), [GitHub CI](../test-runner/github-ci.md), and [Blaxel](../test-runner/blaxel-testshards.md) authority. +This spec owns how `e2e:desktop` obtains and launches Electron: `packages/desktop/test/e2e/global-setup.ts`, the Electron-specific parts of `packages/desktop/test/e2e/helpers.ts`, the shared Linux environment planner in `scripts/electron-e2e-env.mjs`, the test fixture hook, and the harness's real-process seams. [runtime.md](./runtime.md) owns runtime validity. The canonical runner, registry surface, preflight timing, and provider setup remain under [test-runner](../test-runner/test-runner.md), [GitHub CI](../test-runner/github-ci.md), and [Blaxel](../test-runner/blaxel-testshards.md) authority. Electron e2e is used only when the claim depends on Electron. Browser e2e remains the less expensive boundary for ordinary web behavior. @@ -68,7 +68,7 @@ When [Blaxel's owned setup](../test-runner/blaxel-testshards.md) supplies a vali [] ``` -or the package directory plus an isolated profile for the real connect flow, followed by any extra arguments the test passes, such as the update runtime's simulation flag ([desktop updates](./updates.md#Testing)). A fixture beginning with `/` resolves against the runner-published desktop Vite origin; an absolute URL passes through unchanged. The `e2e:desktop` [registry service](../test-runner/test-registry.md#Surface services) publishes its bound fixture URL as `TV_DESKTOP_E2E_URL`. The desktop Playwright configuration requires this variable. Fixture helpers require the value to be an HTTP URL on `127.0.0.1` and use its origin. The helper's inherited environment carries `TV_TEST_MODE=true`, with per-test overrides applied explicitly. ^desktop-e2e-launch-contract +or the package directory plus an isolated profile for the real connect flow, followed by any extra arguments the test passes, such as the update runtime's simulation flag ([desktop updates](./updates.md#Testing)). A fixture beginning with `/` resolves against the runner-published desktop Vite origin; an absolute URL passes through unchanged. The `e2e:desktop` [registry service](../test-runner/test-runner.md#network-addresses-in-tests) publishes its bound fixture URL as `TV_DESKTOP_E2E_URL`. The desktop Playwright configuration requires this variable. Fixture helpers require the value to be an HTTP URL on `127.0.0.1` and use its origin. The helper's inherited environment carries `TV_TEST_MODE=true`, with per-test overrides applied explicitly. ^desktop-e2e-launch-contract Each launch phase has an **eight-second timeout**: `_electron.launch`, first `BrowserWindow`, and its `domcontentloaded` state. Runtime installation and bundle builds finish in global setup and are excluded. A later product-readiness wait can carry its own owning timeout; it does not extend a launch phase. If a phase after process creation fails, the helper closes the Electron application before rethrowing the named phase error. ^desktop-e2e-launch-timeout @@ -85,6 +85,8 @@ npm test -- local --file packages/desktop/test/e2e/smoke.test.ts npm test -- local --surface e2e:desktop ``` +The surface command follows the runner's [local guardrails](../test-runner/test-runner.md#^local-one-file): on marked hosts use `npm test -- blaxel --surface e2e:desktop` for Linux coverage, or the documented broad-local override under [testing-policy permissions](../testing-policy.md#verification-provider-and-completion) when the complete surface must run on macOS. + On macOS, `packages/desktop/test/e2e/user-data-identity.test.ts` uses Electron's real default `~/Library/Application Support/Television` application-data directory; expect the run to write Chromium profile files there. On a fresh Linux host, install Playwright's Chromium browser and shared system dependencies once with `npx playwright install --with-deps chromium`. The Electron preflight names a missing display or sandbox prerequisite; global setup owns Electron runtime preparation and desktop bundle builds, so neither is a manual prerequisite. @@ -93,7 +95,7 @@ An Electron-only scenario belongs under `packages/desktop/test/e2e/`. Add author ## Provider integration -The canonical command surface and surface selection are owned by [the test runner](../test-runner/test-runner.md) and [registry](../test-runner/test-registry.md). Whenever a provider selects `e2e:desktop`, this harness applies its shared environment plan, global setup, and launch contract. Provider setup does not change the [launch-phase timeout](#^desktop-e2e-launch-timeout). +The canonical command surface and surface selection are owned by [the test runner](../test-runner/test-runner.md) and its [registry](../test-runner/test-runner.md#the-test-registry). Whenever a provider selects `e2e:desktop`, this harness applies its shared environment plan, global setup, and launch contract. Provider setup does not change the [launch-phase timeout](#^desktop-e2e-launch-timeout). ## Testing diff --git a/specs/arch/desktop/index.md b/specs/arch/desktop/index.md index c3b9999c..022d9ffc 100644 --- a/specs/arch/desktop/index.md +++ b/specs/arch/desktop/index.md @@ -31,7 +31,7 @@ Desktop modules consume these authorities without restating them: - [Artifact isolation](../artifact-frame/isolation.md#In the desktop app) owns why artifact webviews run in partitions apart from the window's session, how the main process rewrites the artifact proxy's headers there, and the permission handlers the main process sets on the window's session and the partitions'. - [Desktop upgrade gate](../updates/desktop-upgrade-gate.md) owns cross-release compatibility gating between an installed desktop release and the server-served interface. [Desktop connection flow](./connect-flow.md) owns the surrounding main-process connection lifecycle. - [Licensing architecture](../licensing.md) owns the desktop build's third-party notices, the license gate's check of the workspace's `dependencies`, the license checks on the upload directory, and where Electron's license files sit in the built app. -- [GitHub CI](../test-runner/github-ci.md), [Blaxel shards](../test-runner/blaxel-testshards.md), and [preflight](../test-runner/preflight.md) own the provider setup and checks around the desktop test harness. +- [GitHub CI](../test-runner/github-ci.md), [Blaxel shards](../test-runner/blaxel-testshards.md), and [the test runner](../test-runner/test-runner.md) own the provider setup and checks around the desktop test harness. ToDesktop builds, signs and notarizes the app users install under [distribution.md](./distribution.md), and the app updates itself from ToDesktop releases under [updates.md](./updates.md). A different embedded-document mechanism is outside this architecture. diff --git a/specs/arch/desktop/runtime.md b/specs/arch/desktop/runtime.md index 65b71203..19d8ca1b 100644 --- a/specs/arch/desktop/runtime.md +++ b/specs/arch/desktop/runtime.md @@ -22,7 +22,7 @@ Development and test hosts run Linux or macOS, and the runtime provides the layo The Electron package root is found by resolving `electron/package.json`; validation never loads `electron`'s main entry. Loading that entry is installation-capable in Electron 43 and is reserved for [obtaining the runtime](#obtaining-the-runtime). Every existing runtime is classified from files under that resolved package root first. ^desktop-runtime-before-resolver -A successful validation produces this logical contract for [the e2e harness](./e2e-harness.md) and [preflight](../test-runner/preflight.md): +A successful validation produces this logical contract for [the e2e harness](./e2e-harness.md), including the runner preflight that shares its [environment plan](./e2e-harness.md#^desktop-e2e-environment-plan): ```ts type ValidElectronRuntime = diff --git a/specs/arch/explainer-desktop-app.md b/specs/arch/explainer-desktop-app.md index f75b6356..1e295cc3 100644 --- a/specs/arch/explainer-desktop-app.md +++ b/specs/arch/explainer-desktop-app.md @@ -68,7 +68,7 @@ The gate and the recommendation compare the shell's version with thresholds set ## How the desktop app is tested - **Electron for development and tests.** Development runs and the desktop tests use Electron from npm, at the exact version the ToDesktop build also uses ([Electron runtime](./desktop/runtime.md)). The [e2e harness](./desktop/e2e-harness.md) launches the real app for tests through Playwright, and uses Electron only when a claim depends on it. -- **Where those tests run.** The `e2e:desktop` surface is declared in the [test registry](./test-runner/test-registry.md). [GitHub CI](./test-runner/github-ci.md#desktop-runtime-preparation), [Blaxel shards](./test-runner/blaxel-testshards.md) and [preflight](./test-runner/preflight.md#^preflight-electron-runtime) own the provider setup and checks around the harness. +- **Where those tests run.** The `e2e:desktop` surface is declared in the [test registry](./test-runner/test-runner.md#the-test-registry). [GitHub CI](./test-runner/github-ci.md), [Blaxel shards](./test-runner/blaxel-testshards.md) and [the test runner](./test-runner/test-runner.md) own the provider setup and checks around the harness. - **Acceptance in both clients.** Several product specs require acceptance in a real browser and in the real Electron app, among them [artifacts](../product/artifacts.md#testing), [artifact navigation](../product/artifact-navigation.md#testing), [keyboard navigation](../product/keyboard-navigation.md#testing), [themes and appearance](../product/themes-and-appearance.md#testing) and [telemetry](../product/telemetry.md#testing). - **The installed app.** No automated test runs the ToDesktop build. Checks on an unreleased candidate build show what only the built app can show, such as signing, identity, saved connections and the update notice's restart. Before a change to the app's behavior reaches `main`, it passes the checks the change needs; agents run the ones they can, and a person on a real Mac runs the rest. Releasing a build ships it to users, so a released build is never what is tested. The product spec states these checks as an exception to the testing policy ([real-host acceptance](../product/desktop-app.md#testing)). diff --git a/specs/arch/licensing.md b/specs/arch/licensing.md index c23e1e50..08d9b527 100644 --- a/specs/arch/licensing.md +++ b/specs/arch/licensing.md @@ -279,7 +279,7 @@ The licensing tests check the desktop [upload directory](./desktop/distribution. ## Pipeline integration -Licensing enforcement runs as **ordinary tests in the standard suites** — no dedicated verify phase, no dedicated CI job, no workflow or attestation changes. The `e2e:node` surface's registry `preCommand` ([test-registry.md](./test-runner/test-registry.md)) runs **both** product builds — the full CLI composite build and the desktop build — sharing the default [inventory root](#^licensing-inventory-root), and the licensing test file runs the gate as a fast file-level check over those persisted inventories, alongside the notices, tarball and upload-directory inspections. Every `npm run verify` and every CI run executes these tests because they live in the normal suites; that is the enforcement point. In CI's build-once fan-out, where shards skip `preCommand`s, the inventory root reaches the shards inside the build artifact because it is untracked, and the artifact carries the build's untracked output ([the artifact's contents](./test-runner/github-ci.md#^gha-prebuilt-partition)). ^licensing-suite-enforcement +Licensing enforcement runs as **ordinary tests in the standard suites** — no dedicated verify phase, no dedicated CI job, no workflow or attestation changes. The `e2e:node` surface's registry `preCommand` ([test registry](./test-runner/test-runner.md#the-test-registry)) runs **both** product builds — the full CLI composite build and the desktop build — sharing the default [inventory root](#^licensing-inventory-root), and the licensing test file runs the gate as a fast file-level check over those persisted inventories, alongside the notices, tarball and upload-directory inspections. Every `npm run verify` and every CI run executes these tests because they live in the normal suites; that is the enforcement point. In CI's build-once fan-out, where shards skip `preCommand`s, the inventory root reaches the shards inside the build artifact because it is untracked, and the artifact carries the build's untracked output ([the artifact's contents](./test-runner/github-ci.md#^gha-prebuilt-partition)). ^licensing-suite-enforcement - **Publish:** no new steps. `publish.yml` runs `npm run build`, which emits the notices as part of building, and then `npm run check:publish-tarballs`, which asserts them (above). - **Desktop builds:** no step beyond `build.mjs`, which emits the desktop notices. The upload directory that the [build script](./desktop/distribution.md#The build script) generates carries the files checked [above](#^licensing-upload-assertions). diff --git a/specs/arch/node-versions.md b/specs/arch/node-versions.md index 80220f3f..66690e84 100644 --- a/specs/arch/node-versions.md +++ b/specs/arch/node-versions.md @@ -8,7 +8,7 @@ Television uses one Node/npm toolchain to develop, test, build, and publish the This spec is authoritative for the three version contracts, their current values, their declarations, the developer checks, and the procedure for changing them. -It does not own build mechanics, TypeScript configuration, test-runner mechanics, CI job mechanics, Blaxel provisioning, or release mechanics; those remain with their existing authorities. [preflight.md](./test-runner/preflight.md), [github-ci.md](./test-runner/github-ci.md), [blaxel-testshards.md](./test-runner/blaxel-testshards.md), [arch/cli/index.md](./cli/index.md), [arch/onboarding/bake.md](./onboarding/bake.md), and [the ToDesktop build](./desktop/distribution.md) consume this policy and link here for values instead of restating them. +It does not own build mechanics, TypeScript configuration, test-runner mechanics, CI job mechanics, Blaxel provisioning, or release mechanics; those remain with their existing authorities. [test-runner.md](./test-runner/test-runner.md), [github-ci.md](./test-runner/github-ci.md), [blaxel-testshards.md](./test-runner/blaxel-testshards.md), [arch/cli/index.md](./cli/index.md), [arch/onboarding/bake.md](./onboarding/bake.md), and [the ToDesktop build](./desktop/distribution.md) consume this policy and link here for values instead of restating them. ## Version contracts diff --git a/specs/arch/telemetry/sink.md b/specs/arch/telemetry/sink.md index 3ac30030..957d99f1 100644 --- a/specs/arch/telemetry/sink.md +++ b/specs/arch/telemetry/sink.md @@ -51,10 +51,10 @@ const POSTHOG_TEST_PROJECT_ID = 484904 `telemetry-posthog-roundtrip:integration` is the central-runner surface for live PostHog validation. It belongs to the `telemetry-posthog-roundtrip` suite, is excluded from `all`, and is run with: ```bash -npm test -- local --suite telemetry-posthog-roundtrip +npm test -- local --file packages/server/test/telemetry-posthog.integration.test.ts ``` -The surface declares the `posthog-test-key` preflight ([../test-runner/preflight.md](../test-runner/preflight.md)), which requires the PostHog test project read key before Vitest starts. The key is the secret `phx_` value and must be supplied as `TV_POSTHOG_TEST_READ_KEY` in the environment or in the repository-root `.env` file. `.env` is gitignored, auto-loaded by the test, and must never be committed. There is no in-test silent skip for this surface: selecting the suite without the read key is a preflight failure. +The surface declares the `posthog-test-key` preflight in the [test registry](../test-runner/test-runner.md#the-test-registry), which requires the PostHog test project read key before Vitest starts. The key is the secret `phx_` value and must be supplied as `TV_POSTHOG_TEST_READ_KEY` in the environment or in the repository-root `.env` file. `.env` is gitignored, auto-loaded by the test, and must never be committed. There is no in-test silent skip for this surface: selecting the suite without the read key is a preflight failure. The suite enables delivery through `TV_TELEMETRY_TEST` and refuses to run under enabled `CI` or `DO_NOT_TRACK`. This suite is excluded from CI and ordinary developer `verify` runs, and Blaxel workers do not receive the read key. It is run manually by an operator who knows that its persisted-daemon cases register and remove real, uniquely named user services through the host service manager. After each persisted telemetry assertion, the fixture invokes `tv stop` against that test-named service; this is suite hygiene, not CLI acceptance coverage. Direct `Daemon.uninstall()` remains failure cleanup if the normal stop path does not complete. diff --git a/specs/arch/test-runner/attestation.md b/specs/arch/test-runner/attestation.md index 9910b56e..495f5e18 100644 --- a/specs/arch/test-runner/attestation.md +++ b/specs/arch/test-runner/attestation.md @@ -1,132 +1,98 @@ -*Tree-hash test attestation: recording that an exact tree passed full validation as a bare `refs/testpass//` ref, and letting validation-bearing CI runs skip their heavy test jobs — never their cheap ones — when the identical tree is already attested under the current version.* +*Tree attestations: recording that an exact tree passed the full gate, so that GitHub CI can skip its heavy test jobs for that tree, without ever skipping on doubt.* -# Test Attestation +# Test attestation -Blaxel verification and GitHub CI can each supply full validation under the [shared-branch workflow](../../spec-workflow.md#^shared-branch-workflow). An *attestation* records the first green full validation so later runs can skip their heavy test jobs instead of re-proving it — cutting minutes from CI and letting publishes land sooner — while cheap checks still run and any doubt resolves to running the tests. Draft pull requests use the separate whole-graph skip defined by [GitHub CI](./github-ci.md). +An *attestation* records that a tree passed the full gate, so that later CI runs of the identical tree skip their heavy test jobs instead of repeating them. That saves minutes on pull requests and lets publication from `main` proceed sooner. Cheap checks always run, and any doubt resolves to running the tests. Attestation is an optimization, never a gate. -This spec is authoritative for the attestation contract: what an attestation claims, the ref shape, what may write one, the skip semantics in the CI workflow, staleness and revocation, failure modes, the activation policy file and its trusted read, and pruning. The CI workflow whose jobs skip is owned by [github-ci.md](./github-ci.md); the `verify` orchestration that writes attestations is [test-runner.md](./test-runner.md). +The CI jobs that skip are owned by [github-ci.md](./github-ci.md), and the verify command that writes attestations by [test-runner.md](./test-runner.md#verify). -## What an attestation claims +## The claim -An attestation is the claim: **the tree `` passed the repository's full validation gate.** Concretely: +An attestation is the ref `refs/testpass//` on `origin`, pointing at the commit that was validated. It claims that the tree with that hash passed lint, type-check, the package manifest checks and the complete `all` suite, desktop included, with the runner's standard retry budget. A recovered flake counts as passing, as it does in the gate. Blaxel workers have no Firefox ([test-runner.md](./test-runner.md#the-test-registry)), so an attestation from a Blaxel verify does not cover the Firefox projects, and CI skips them too for that tree. -- *Tree hash* is `git rev-parse ^{tree}` — the content identity of every tracked file, independent of commit metadata and history. -- *Full validation gate* is the CI gate's own test scope at that tree: lint, type-check, package-manifests, and the complete `all` suite (unit and e2e, desktop included) with the gate's standard retry budget ([test-runner.md](./test-runner.md)). A pass that used runner-level retries attests exactly like it gates: a recovered flake is a pass. -- **The claim is self-pinning for everything the tree controls — and only that.** The tree hash covers every tracked file: the workflow definitions, the test registry, the suite definitions, and the tests themselves. Two identical trees cannot disagree about the *repo-defined* meaning of "passed": any tracked change that alters the gate produces a different tree and never matches an existing attestation. -- **What the tree does not pin:** mutable third-party action tags (`actions/checkout@v4` is a moving pointer), the `ubuntu-latest` runner image label, repository/organization settings, Actions variables and secrets, and external services (npm registry, browser CDNs, Blaxel). Changes there can alter what "passed" means without changing the tree. Two mechanisms bound that exposure: the age limit (drift, see Staleness) and the **attestation version** (known contract changes, below). +- The tree hash, `git rev-parse ^{tree}`, covers every tracked file, including the workflows, the registry and the tests. Any tracked change that alters the gate therefore gives a different tree. +- It does not cover what lives outside the tree: the versions behind action tags, the runner image, repository settings, secrets and variables, and external services. Two things bound that exposure. An attestation is honored only while its commit's committer time is at most 7 days old; a future committer time is rejected. And the ref carries a version that the operator raises when a known outside change alters what passing means. +- The ref is bare, with no metadata. A reader fetches it by name, checks that it points at a commit whose tree matches the hash in the name, and ignores it otherwise. +- Writes are create-only and atomic: `git push --force-with-lease=:` with an empty expected value. Finding an existing valid attestation for the tree counts as success. Finding a malformed one is reported and left for pruning. +- Experiments write `refs/testpass-lab/*`, which CI never honors. -**Attestation version.** The activation policy carries a `testpassVersion`. Attestations are written under the current version, the version is part of the ref name, and skip logic honors only refs of the version it reads from the trusted `origin/main` policy blob. +## Who may write -- **Domain and encoding:** a JSON integer, `1 ≤ testpassVersion ≤ 999999999`, encoded in the ref name as its plain base-10 digits with no padding (`refs/testpass/2/`). A missing, non-integer, out-of-domain, or otherwise malformed value disables **both** writing and skipping — the same default-deny posture as a missing flag. -- **Version is bound to the validation interval, not the completion instant.** The writer reads the trusted version from `origin/main` **before the first gate phase** and again **at completion**, and publishes only when both reads succeeded and agree; a mid-run bump makes the run ineligible (the external contract changed while it was validating, so its pass may already be under the old contract). This closes the time-of-check/time-of-use hole where a run started under version N publishes under N+1. -- **Safe version-change procedure.** To make every in-flight validation ineligible under the changed contract: (1) set `testpassRefsEnabled: false` on main, which stops skipping and writing, including writes from runs that read version N at start; (2) let eligible in-flight runs drain or cancel them; (3) change the external contract (runner image, action versions, environment); (4) increment `testpassVersion` to a value that has never been used, and never revert to a used value; (5) re-enable `testpassRefsEnabled`. A version change alone cannot reach a run that already made its skip decision, but the disabled flag is checked again by that run's writer. Prune the retired version's namespace afterward. Policy history on main is the audit trail, and the version in each ref prevents retired refs from becoming eligible again. +Exactly two sources write attestations: -## Ref shape +- **A complete Blaxel verify.** Its test phase runs the whole `all` suite with no narrowing selector and no shard subset; `--shards` only changes the partition. It writes with the caller's own push credential, unless `--no-publish` is passed. +- **The GitHub CI workflow's `attest-write` job,** after the complete graph and its join have passed, for a run from the repository itself; fork runs cannot write. The writer refuses unless it runs in GitHub Actions, in workflow `CI`, job `attest-write`, with `TV_TEST_REF_PUBLICATION=required`. -``` -refs/testpass// → -``` - -- A **bare ref**: no metadata object. The ref name carries the attestation version and the attested tree hash; the ref points at the validated commit whose tree has that hash. -- **Create-only, enforced atomically.** The writer pushes with a create-only compare-and-swap — `git push --force-with-lease=:` with an empty expected value — never an existence-check-then-push, which races. Two benign outcomes are successes: creation, and the transport-level no-op when the ref already points at the pushed commit (`Everything up-to-date` — measured, not assumed: an identical re-push never reaches the lease because nothing would change). A lease **rejection** means the ref exists pointing elsewhere: the writer fetches it by name and validates (object is a commit; name↔tree invariant holds) — a valid existing attestation is idempotent success without pushing; a malformed one is **not** success — it is reported loudly and left for exact-ref prune. -- A consumer must fetch the advertised object by ref name before resolving anything from it, then verify the invariant: `rev-parse ^{tree}` must equal the `` in the name and the object must be a commit. Any mismatch is a malformed ref — ignored by skip logic (fail toward running) and deletable by prune's exact-ref path. -- Experiments use `refs/testpass-lab/*` exclusively; CI never honors lab refs and lab refs are pruned by their experiment. - -## Who writes, and when - -An attestation is written only when the **eligibility predicate** holds — every clause, no exceptions. A green run that fails any clause simply does not attest; it is never an error. - -1. **Eligible source.** A production attestation has exactly two eligible source classes: a publication-qualifying Blaxel verify, or the repository's GitHub Actions CI workflow after its complete validation graph passes. There is no third source class. In particular, a `local` provider run is ineligible regardless of cleanliness, scope, or result because a developer host is not a controlled validation environment. `--no-publish` suppresses publication on an otherwise qualifying Blaxel verify without changing its execution. -2. **One tree, everywhere.** All gate phases targeted a single captured tree: the runner captures `attestedTree = rev-parse HEAD^{tree}` before the first phase, and every phase runs against it. A verify whose test phase targets a different commit than its lint/type-check checkout (`--commit `) is ineligible — the phases would validate two different trees and the attestation would claim a gate no single tree passed. -3. **Clean and unchanged, bracketed.** The checkout is clean at start **and re-verified clean with the same HEAD at end** (`git status --short` empty, `rev-parse HEAD^{tree}` equal to the captured value). `--ignore-uncommitted` is ineligible. A recorded `workingTreeDirty: false` at start alone is insufficient. -4. **Complete scope.** Every surface in the gate scope produced a **passed** result — none skipped, none absent. `--shard-indices` subsets, partial suites, targeted selectors, and environment knobs that skip a gate surface (e.g. `SKIP_ELECTRON_E2E=1` recording desktop as skipped) are all ineligible. Environment knobs that are part of the gate's canonical configuration (e.g. `SKIP_ACP_TESTS=1`, which guards suites outside the gate scope) do not affect eligibility. -5. **Canonical retry policy.** The gate's standard retry budget ([test-runner.md](./test-runner.md)) applied; a run with a nonstandard retry override is ineligible. A pass that used the standard budget attests exactly like it gates: a recovered flake is a pass. -6. **Not itself attested-skipped.** A run whose heavy phase was skipped by an existing attestation never writes (it would refresh nothing and must not extend anything). -7. **Policy permits** (below), with the version captured before the first phase and re-read at completion — both reads valid, enabled, and equal (§Attestation version). - -Eligible writers and their credentials: - -- **`blaxel-verify`:** `verify` automatically writes after a passing publication-qualifying Blaxel run as defined by [test-runner.md#^test-publication-qualifying](./test-runner.md#^test-publication-qualifying) when clauses 2–7 hold. It tests a committed revision fetched from origin and pushes with the caller's existing origin credential. No publication flag is required. -- **`github-ci`:** the `attest-write` job writes only after a fully green GitHub Actions run (all matrix jobs, build, all shards, join). The command boundary requires GitHub Actions, workflow `CI`, job `attest-write`, and the positively classified `TV_TEST_REF_PUBLICATION=required` context before it reaches the production writer. The job uses `contents: write`. A fork carries `not-authorized` and cannot write. - -The production writer receives one of those source identities and refuses any other source before pushing. Although GitHub-hosted test jobs invoke the runner with `provider=local` inside their ephemeral VMs, the attestation source is the complete `github-ci` workflow and its strict join, never one local-provider child. A developer local verify, a narrowed Blaxel verify, and an opted-out Blaxel verify never reach the production writer. - -Trust model: identical to the team's existing repository-write trust, as ratified. Anyone who can push code can already change what the gate runs; attestation grants no authority beyond what `contents: write` and repo-write membership already carry. - -## Skip semantics in CI - -On validation-bearing events, the CI workflow gains one cheap, always-run decision job and conditions on it: +Either source writes only when all of these hold: -- **`attest-check`** (seconds, no install): computes `git rev-parse HEAD^{tree}`, reads the activation policy from the `origin/main` blob ([#Activation policy](#Activation policy)), looks up `refs/testpass//` on origin, fetches and verifies the ref (object type, name↔tree invariant, age bound), and emits `skip=true` only when **every** check affirms. Its log states the tree hash, version, the ref found or absent, and the decision — a skip must be auditable at a glance. Wiring semantics, exactly: - - **Expected negative outcomes are successes.** Lookup errors, unreachable origin, unreadable policy, malformed refs: the job catches them and completes **successfully** with the explicit output `skip=false`. An expected error never fails the job. - - **Unexpected failure is non-gating.** The job runs with `continue-on-error: true`, so even a crash cannot redden the workflow conclusion (which would block `publish.yml`'s `workflow_run` gate). A crash **before** the decide step publishes no outputs; a crash **after** it does NOT erase them — GitHub preserves outputs written by a successful step even when a later step fails the job — so absence of outputs alone is not a safe no-skip signal (next bullet). - - **Skip authorization is bound to decision-job success via the `authorized` output, not `needs.*.result`.** A decide that emits `skip=true` followed by a later step failing the job leaves the preserved output claiming a skip from a FAILED decision job — and the obvious binding does not work: under job-level `continue-on-error`, dependents read `needs.attest-check.result` as `'success'` even when the job failed (measured: run 29558875144, where result-based guards honored the preserved skip). Instead `attest-check`'s **final** step emits `authorized=true`; its implicit `success()` condition means any earlier step failure leaves it unset, and no step may follow it. Every consumer of the skip output — the heavy jobs' no-op steps and real-step guards, the join's attested branch, and `attest-write` — requires `authorized == 'true'` in addition to the literal skip `'true'`. The no-op condition is `skip == 'true' && authorized == 'true'`; the run-everything guard is its negation `(skip != 'true' || authorized != 'true')`. Known residual, accepted and documented: a failure in a post-action (e.g. checkout cleanup) runs after the final step and cannot retract an already-emitted authorization — the decision itself was validly computed by then, so the skip remains sound. - - **Heavy-job conditions are skip-only-on-explicit-signal:** an absent, empty, or false output runs everything and only the literal `'true'` no-ops. The join's third condition conversely requires the literal `'true'`: it affirms only on the explicit signal, never on absence. - - **Never-skip-on-doubt applies to the job graph itself.** A heavy job's `needs` on `attest-check` must not let GitHub's implicit `success()` gate suppress it when the decision job **crashes**: each heavy job's condition carries a status-check function (`!cancelled()`) so a failed `attest-check` still runs full validation — without it, a crashed decision job skips the entire heavy tier and the join can green a run with zero heavy validation. The e2e shards additionally require `needs.build.result == 'success'` explicitly, preserving the failed-build ⇒ shards-skipped ⇒ join-red edge that the implicit gate used to provide. - - **The skip is enforced at step level, never at job level, on matrix jobs.** Branch protection requires the concrete matrix check contexts (`unit`, `e2e:desktop`, each `e2e shard n/16`), and GitHub evaluates a job-level `if` before matrix expansion — a job-level skip therefore suppresses the concrete contexts entirely and a required-context ruleset blocks the PR forever despite the green join (measured: run 29554767152 emitted only the unresolved templated names). Instead every heavy job always expands; under an attested skip each of its steps is guarded by `needs.attest-check.outputs.skip != 'true'` (composed with any existing step condition, including `always()` post-steps, which would otherwise run anyway) and the job fast-succeeds as an explicit logged no-op, emitting its concrete required context green. -- **What always runs on validation-bearing events:** lint, type-check, package-manifests, and `attest-check` itself. Only the heavy jobs (`unit`, `e2e:desktop`, `build`, the e2e shards) honor an attestation `skip`. Draft pull requests skip the complete graph under their separate job guards. -- **The join** (`e2e-required`, [github-ci.md](./github-ci.md)) is neutral/skipped on a draft pull request because its job-level condition composes `always()` with the same draft guard as the validation jobs. On every non-draft event it runs despite dependency outcomes and is green only when build and e2e both report success. An authorized attestation reaches that state through successful step-level no-ops; each artifact-download, unpack, and plan-agreement step is skipped by its explicit step-level guard on the authorized attestation outputs. Missing, skipped, cancelled, upstream-failed, and test-failed dependencies are red on this run path. -- **Main-push:** the same mechanism with no special casing — a squash-merge that reproduces an attested tree (branch up to date at merge) skips identically, and the CI conclusion stays green, so `publish.yml`'s `workflow_run` gate proceeds. A merge that produces a novel tree (conflict resolution, stale branch) misses and runs everything. +- Every phase checked one tree. The checkout was clean when the run started and is still clean, with the same tree, when it ends. A verify whose tests target a commit with a different tree from its checkout cannot write, and neither can a run with `--ignore-uncommitted`. +- Every surface of the gate passed, and none was skipped, for example by `SKIP_ELECTRON_E2E=1`. +- The standard retry budget applied. +- The run did not itself skip heavy jobs because of an attestation. +- The policy permitted writing, under the same version, both when the run first read it, before any of its tests, and when the run finished. -## Failure modes — never skip on doubt +A local run never writes, whatever it ran, because a development host is not a controlled environment. A run that misses a condition simply does not write; that is not an error. A failed write is logged and leaves the run's result unchanged. -Every failure resolves to running the tests: +Anyone who can push to the repository can already change what the gate runs, so attestation grants no authority beyond that write access. -- origin unreachable, `ls-remote` timeout or error → run everything. -- policy blob unreadable, malformed, or missing the flag → treat as disabled: no skip (and no write). -- ref present but name↔tree invariant fails, or the age bound cannot be computed → ignore the ref, run everything. -- `attest-check` job itself fails → heavy jobs run (its conditions are written so job failure means no-skip, not workflow failure). -- Write failures are non-fatal to the run that tried to write: a green run whose attestation push fails stays green and logs the failure; attestation is an optimization, never a gate. +## Policy -## Staleness, age, and revocation - -- **Definition changes:** self-invalidating by tree identity (above). No revocation machinery is needed for "the workflow changed" — the hash changes. -- **Environment drift:** an attestation is honored only when `0 ≤ now − committerTime ≤ 7 days`. Age is computed from the validated commit's committer timestamp — a conservative proxy, since bare refs carry no timestamp of their own: a commit validated long after it was authored *understates* its remaining validity and expires early, failing toward running tests. The lower bound matters too: git permits future committer timestamps, and a future-dated commit would otherwise stay eligible past 7 days after its validation — so a future timestamp, like an unparseable one, is rejected outright (no-skip). The bound and the proxy are named in `attest-check`'s log line. -- **Revocation:** deleting the ref revokes the attestation. The prune command's exact-ref path is the tool; the policy flag is the global kill switch — flipping it off stops every **new** skip decision immediately (each `attest-check` reads the trusted policy fresh), while a run whose decision already emitted `skip=true` finishes as an attested-skip; an emergency revocation that must also stop in-flight skips drains or cancels running workflows, as in step 2 of the version-change procedure. Refs are untouched either way — they are inert without the flag. - -## Activation policy - -Both writing production attestations and honoring them in CI are **default-deny**. The committed policy has this shape: +Writing and honoring attestations are both off unless the policy on `main` turns them on: ```ts +// test/testpass-policy.json interface AttestationPolicy { schemaVersion: 1; testpassRefsEnabled: boolean; - testpassVersion: number; + testpassVersion: number; // integer from 1 to 999999999, written in ref names as plain digits } ``` -It lives at `test/testpass-policy.json`. `testpassRefsEnabled` gates production writes and skips; `testpassVersion` is the attestation version defined above. Every reader resolves the current `refs/heads/main` object ID from `origin`, fetches that exact object when necessary, and reads the policy blob from that object. It never trusts the tested checkout's copy, so a work branch cannot activate itself; only the operator's policy change on main turns production writes and skips on. An unreachable origin, failed fetch, absent default branch, missing policy blob, malformed JSON, unsupported schema, a key set other than the three above, or a non-boolean flag makes the policy unavailable: no write, no skip, and no prune deletion. Lab-namespace experiments (`refs/testpass-lab/*`) do not consult the policy but are never honored by CI. +Every reader, whether the writer, CI's decision job or prune, reads this file from the current `main` on `origin`, never from the checkout under test, so a branch cannot enable itself. Anything unexpected means disabled, with no write, no skip and no prune deletion. That includes an unreachable origin, a missing file, malformed JSON, keys other than these three, and a version out of range. Only refs under the current version are honored. -## Prune and retention +## Skipping in CI -``` -npm test -- testpass prune --older-than 30d # dry run, current version -npm test -- testpass prune --older-than 30d --apply -npm test -- testpass prune --dead-version --apply # sweep an entire retired version -npm test -- testpass prune --delete-ref refs/testpass// --apply -``` +On every run that is not for a draft, CI's `attest-check` job decides `skip=true` only when every check affirms: the policy is enabled, a ref exists for the tree under the current version, its commit matches the name, and its age is in range. Its log names the tree, version, ref and decision. GitHub Actions behaves in ways that make the obvious wiring unsafe: + +- Expected problems, such as an unreachable origin, an unreadable policy or a malformed ref, end the job successfully with `skip=false`. +- The job is marked `continue-on-error`, so a crash cannot turn the workflow red, which would block publication. Under `continue-on-error`, though, dependent jobs see the job's result as success even when it failed, and outputs written before a failing step survive. Consumers therefore trust a skip only together with `authorized=true`, an output written by the job's last step, which any earlier failure prevents. A post-action that fails after that step cannot retract the authorization. That limitation is accepted, because the decision was already made correctly. +- Heavy jobs carry `!cancelled()`, so they still run when the decision job crashes. End-to-end shards also require that the build succeeded. +- On matrix jobs, the skip is applied to each step, never as a job-level condition. GitHub evaluates a job-level condition before expanding the matrix, so a matrix job skipped that way never reports its concrete check names, such as `unit` and `e2e:desktop`, and the ruleset waits for them forever. A skipped heavy job instead runs as a logged no-op that reports success. + +Only the heavy jobs skip: unit, desktop end-to-end, the build and the end-to-end shards. The join passes an authorized skip through. A push to `main` that reproduces an attested tree, as an up-to-date squash merge does, skips the same way and stays green, so publication proceeds. + +## Revoking and pruning + +Deleting a ref revokes that attestation. Setting `testpassRefsEnabled` to `false` on `main` stops new skips and writes at once. A run that has already decided to skip still finishes skipped; cancel it if the revocation is urgent. + +`npm test -- testpass prune` is a dry run unless `--apply` is passed: + +- `--older-than d` deletes current-version refs strictly older than the cutoff, by committer time; +- `--dead-version ` deletes every ref of a retired version and refuses the current version or any later one; +- `--delete-ref ` deletes one exact, version-qualified ref under `refs/testpass/`. -The command fetches the namespace explicitly (custom namespaces are not fetched by branch refspecs), is a dry run by default, deletes on origin and locally only with `--apply`, and uses exact-ref deletion for revocation and malformed refs. Before choosing refs to delete, the namespace fetch updates the local refs from origin and removes local refs that no longer exist there. Safety semantics, all modes: an unavailable or malformed trusted policy read means **zero deletions**; the age cutoff is exact and pinned (a ref at exactly the cutoff age is retained — deletion requires strictly older); `--dead-version` refuses the current version and any version ≥ current; `--delete-ref` rejects globs, non-version-qualified names, and any ref outside `refs/testpass/`. If deletion fails for a ref, the command reports that ref and exits non-zero. It stops at the first failure; later candidates remain untouched and are reported as not deleted. Three modes: +Prune refreshes the namespace from `origin` first, deletes nothing when the policy cannot be read, and stops at the first failed deletion. Pruning is always safe: the cost of removing too much is one redundant test run. -- **Age-based** (default): operates within the current trusted version, deleting refs older than the cutoff by the same committer-timestamp proxy as skip logic. -- **Dead-version sweep:** `--dead-version ` deletes every ref under `refs/testpass//*`. Per the safety semantics above it refuses when `` equals or exceeds the current trusted `testpassVersion` read from `origin/main` — sweeping the live version is mass revocation and must go through the policy disable or exact-ref deletion instead. This completes the retired-namespace cleanup in the version-change procedure. -- **Exact-ref:** revocation and malformed-ref removal, version-qualified name required. +## Operations -Attestations are pure optimization state and are always safe to delete (the cost of over-pruning is one redundant test run). Growth is naturally bounded at one ref per unique validated tree per version; age-based pruning bounds the live namespace and dead-version sweeps empty retired ones. +When a known outside change, such as a new runner image or new action versions, alters what passing means, change the version so that no run validated under the old conditions can write or be honored afterwards: -## Testing +1. Set `testpassRefsEnabled` to `false` on `main`. +2. Let in-flight runs finish, or cancel them. +3. Make the outside change. +4. Raise `testpassVersion` to a value never used before. Never reuse a version. +5. Set `testpassRefsEnabled` back to `true`, then prune the old version with `--dead-version`. -Under the testing policy's [seam-test rule](../testing-policy.md#^shape-seam), the repository's tests of the attestation writer and prune command use real Git. They create objects, fetch and push refs, collide with an existing ref, and delete refs. They do not replace any part of Git. The repository's tests inspect the workflow file. They provide evidence about its contents, not about a live GitHub Actions run. +A run that read different versions at its start and its end cannot write. -Before a change to this mechanism reaches main, an isolated copy of the complete CI graph must run only on the branch that carries the change. That copy must demonstrate: +## Testing guidance -- an eligible lab write; -- reuse of the identical tree with every concrete required check context green; -- all heavy jobs running when an attestation is absent or unreachable, or when the decision job fails before or after emitting `skip=true`; and -- the join turning red after a build failure. +The workflow's behavior can be shown only by a live run. Before a change to this mechanism reaches `main`, an isolated copy of the CI graph on the branch carrying the change must show these behaviors against lab refs: -In explicit lab mode, `npm run verify -- blaxel` must publish the attestation automatically as part of the same command. The lab refs are pruned afterward. +- a write; +- a skip with every required check green; +- the heavy jobs running when the decision job fails before or after deciding to skip. -No test can exercise the trusted `origin/main` policy read or the credentials for writing to the production namespace. No test can show that `publish.yml` proceeds after an attested-skip run on main. After enabling the policy, the operator checks these behaviors during the first production Blaxel verify, the first production CI run, and the first attested-skip run on main. Disabling the policy flag provides the immediate rollback. +In lab mode, a Blaxel verify publishes as part of the command. Lab refs are pruned afterwards. +No test can exercise the trusted read of `main`'s policy, the production write credential, or publication proceeding after an attested skip on `main`. After enabling the policy, the operator checks the first production Blaxel verify, the first production CI run and the first attested skip on `main`; turning the flag off is the rollback. diff --git a/specs/arch/test-runner/blaxel-testshards.md b/specs/arch/test-runner/blaxel-testshards.md index 94f05a97..e3ee6981 100644 --- a/specs/arch/test-runner/blaxel-testshards.md +++ b/specs/arch/test-runner/blaxel-testshards.md @@ -1,131 +1,132 @@ -*The Blaxel shard coordinator: leasing a persistent sandbox pool, selecting the repository workers fetch, sourcing a GitHub token, preparing and locking workers, running shards with dependency-hash caching, and classifying Blaxel shard outcomes.* +*Blaxel: the pool of persistent remote workers that runs broad validation, how runs from many branches and repositories share it safely, and how to set it up.* -# Blaxel Test Shards +# Blaxel test shards -Blaxel is the recommended way for agents and humans to run broad validation during development. It cuts the time a full run takes substantially — usually under three minutes, sometimes under two — which is the whole point. It works by keeping a pool of machines on hot standby, preconfigured so a run pays none of the usual setup cost: copying the container image, installing Playwright browsers, and the like. Routing runs to infrastructure like this, for speed and reliability, is also the principal reason Television has a single centralized test runner. +Blaxel is the recommended way for people and agents to run broad validation during development, including `npm run verify -- blaxel`. A pool of persistent sandboxes is kept ready with browsers, the Node runtime and dependencies installed, so a run pays almost no setup and a full verification usually takes about three minutes. Even a run on one worker takes its load off a shared development host. `npm test -- blaxel` drives the coordinator, `scripts/run-blaxel-testshards.mjs`, and `npm test -- pool blaxel` manages the pool. -This spec is authoritative for the Blaxel coordinator: how it leases a persistent sandbox pool, selects the repository workers fetch and sources the GitHub token they fetch it with, prepares and locks each worker, runs the shard, caches dependencies by hash, classifies a Blaxel shard's outcome, and its options and output layout. The selected Node/npm values and `.nvmrc` authority are owned by [arch/node-versions.md](../node-versions.md). It extends the shared model in [sharded-execution.md](./sharded-execution.md); the git-safety and auth checks before dispatch are [preflight.md](./preflight.md); the canonical command surface that invokes it is [test-runner.md](./test-runner.md). +Every branch and checkout that uses the pool runs its own revision of the runner, and several repositories use it. Much of this spec is therefore the protocol that lets those runs share workers safely, and changes to it must stay compatible with revisions still in use. -Blaxel is the only remote provider. `npm test -- blaxel ` runs the coordinator `scripts/run-blaxel-testshards.mjs`; `npm test -- pool blaxel ` manages the pool through `scripts/manage-blaxel-testshards.mjs`. +How a suite run is divided into shards and how shard outcomes combine are owned by [sharded-execution.md](./sharded-execution.md). Resource evidence and recovery receipts are owned by [reporting.md](./reporting.md#resource-diagnostics). The Node version workers use comes from [node-versions.md](../node-versions.md). -## Repository - -Workers fetch the commit under test from a GitHub repository. By default it is the repository the invoking checkout calls `origin`: the remote whose branches the [remote preflight](./preflight.md#Blaxel remote preflight) requires to contain that commit. `--repo-url` names a different repository instead; it does not change the coordinator's checks against `origin`. The coordinator converts the selected URL, whether a GitHub HTTPS, `git@github.com:`, or `ssh://git@github.com/` URL, to `https://github.com//.git` and drops any credentials embedded in it. A missing `origin`, or a selected URL that does not name a GitHub repository, fails the run before dispatch. A checkout of public Television therefore runs against the public repository, and a checkout of a private development repository runs against that repository, with no repository option. +## Pool -One pool serves every repository, in any order. Each worker keeps one persistent checkout at `/workspace/television`, and the `checkout` step points that checkout's `origin` at the run's repository before fetching the commit into it. A worker that last ran another repository keeps its clone and its installed dependencies; Git transfers only the objects the checkout lacks. Dependency reuse depends only on the dependency declarations ([#Dependency-hash caching](#Dependency-hash caching)), so changing repositories reinstalls dependencies only when those declarations differ. +Workers are sandboxes labelled `project=television`, `purpose=testshards`, `pool=` and `arch=x64`. The coordinator uses only `DEPLOYED` workers of the requested pool, checks that each is x64 Linux before use, and fails if there are fewer workers than shards. The runner uses the pool `poc`. -Repositories that share a pool share a trust boundary. A run's code can read the Git objects and ignored files that earlier runs of other repositories left in the worker's checkout, as well as the token in its own environment. +Blaxel's account tier allows at most 75 workers to run at once. The pool has 72 workers and a standard run uses 36 shards, so two verifications fit side by side within that limit. Suite runs refuse other shard counts unless `--allow-shard-count-override 1` is passed. -## GitHub token +Provisioning is fresh-only: `ensure` creates and provisions every worker of an absent pool and refuses a pool that already has workers. Blaxel's create-if-absent call applies a sandbox specification only when the sandbox does not exist, so it cannot update an existing worker, and the pool manager does not try to reconcile unknown runtime state. A new worker installs nvm and the Node version selected by the repository's `.nvmrc`. It becomes eligible for test runs only after provisioning and its architecture check succeed. To change worker provisioning, delete the pool and ensure it again. -Workers fetch the repository with a GitHub token, sourced from the `.blaxel-gh-token` file in the repository root first (trimmed), then the `BLAXEL_TV_GH_TOKEN` environment variable (trimmed), else none. A missing token fails the run. The token is passed to each worker as `GH_TOKEN` and consumed through `GIT_ASKPASS`; it is never written into repository files or persistent git config. +## Repository and credentials -## Pool +Workers fetch the commit under test from GitHub, from the repository the invoking checkout calls `origin`. The URL is normalized to `https://github.com//.git` without embedded credentials, and a URL that is not a GitHub repository fails before dispatch. A checkout of the public repository therefore tests against it, and a checkout of a private development repository tests against that, with no option. -Workers are persistent sandboxes labelled `project=television`, `purpose=testshards`, `pool=`, `arch=x64`. The coordinator leases only sandboxes in the `DEPLOYED` state for the requested pool and architecture, and fails if fewer workers exist than requested shards. The shared pool size is 72 sandboxes (`DEFAULT_BLAXEL_POOL_SIZE`) and the standard run is 36 shards (`RECOMMENDED_TEST_SHARD_COUNT`), allowing two standard runs to pack exactly within the Blaxel tier's hard limit of 75 concurrent shards. Non-target runs reject any other shard count unless `--allow-shard-count-override 1` is passed. Only verified x64/Linux sandboxes are supported; a worker is verified with `uname -m` and `process.arch` before use, and any non-`x64` `--arch` is rejected. +One pool serves several repositories. Each worker keeps a single checkout at `/workspace/television`, points its `origin` at the run's repository and fetches what it lacks; dependencies are reinstalled only when the dependency declarations differ. Repositories that share a pool share a trust boundary: a run can read the Git objects and ignored files that earlier runs of other repositories left on the worker. -The 36-shard default reduces the file-atomic plan's mean baseline work from 42.957 seconds at 18 shards to 21.479 seconds. Its 30-second modeled maximum is the heaviest atomic file weight rather than insufficient shard capacity. The unchanged 72-worker pool provides two independent verify lanes (`2 × 36 = 72 ≤ 75`) without oversubscribing the tier. +Workers authenticate to GitHub with a token from `.blaxel-gh-token` in the repository root, or else `BLAXEL_TV_GH_TOKEN`; a run without one fails. The token must be able to read the run's repository. Workers receive it through the environment and use it through `GIT_ASKPASS`; it is never written to repository files or Git configuration. -The pool manager `ensure` creates sandboxes named `tv-testshard---NN` (default pool `default`, size 72, image `blaxel/playwright-chromium:latest`, memory 8192 MB). The base image supplies the browser and system environment; its ambient Node is not part of the toolchain contract. Each fresh sandbox installs nvm from pinned release tag `v0.40.3`, installs the Node selected by the manager's repository `.nvmrc`, makes that runtime the nvm default, and verifies x64/Linux under it. +## Leases -Provisioning requires an absent pool: delete the target pool before `ensure`, then let `ensure` create and provision every worker. Blaxel's `createIfNotExists` applies a creation specification only when a sandbox is absent; it does not reconcile provisioning commands or runtime state inside an existing sandbox. `ensure` therefore refuses a populated target pool, keeping provisioning fresh-only instead of carrying convergence logic for unknown nvm/Node state. +A shard attempt leases one worker for its whole life: from before checkout, through setup, tests and teardown, until its reports have been downloaded and checked. Releasing earlier would let the next run's checkout overwrite results that are still being read. -## Locking +- The lease is `/tmp/tv-testshard.lock/owner.json` on the worker. Every change to it, whether acquiring, renewing, registering a dispatch, recovering or releasing, happens under a short `flock` on `/tmp/tv-testshard.guard` and rechecks the current owner first, so a delayed renewal or release can never overwrite a newer owner. The guard is never held during downloads. +- Only the coordinator renews. Each renewal sets `recoverAfter` to the worker's clock plus the recovery interval: 20 minutes by default, or `--lock-ttl-seconds`. An expired lease cannot be renewed, and a shard process cannot renew on behalf of a coordinator that has gone. +- Every process the attempt starts inherits `TV_TEST_SHARD_OWNER`, set to the lease's `lockId`, alongside each surface's own owner token ([test-runner.md](./test-runner.md#processes-started-by-tests)). The coordinator records a dispatch before requesting it. The remote entry point registers its PID and start time in the lease before touching the checkout, and only while the lease is still the dispatching coordinator's, so a delayed request cannot start work after recovery. +- Release requires that the attempt's processes are confirmed gone and its reports retrieved. A shell exit or a Blaxel process stop does not prove that descendants have exited. The coordinator audits for processes carrying the attempt's tokens and stops survivors, validating PID and start time before each signal. If the audit cannot be read, the dispatch outcome is unknown, or an owner survives, the worker stays leased and is not reused for the run's retries. +- On `SIGINT` or `SIGTERM`, the coordinator writes an interrupted report that keeps whatever it has already retrieved, cleans up, and exits `130`; a second signal exits at once. A lease it cannot release safely is left for recovery. -The coordinator leases one worker per shard through an in-sandbox lock directory `/tmp/tv-testshard.lock` with an `owner.json` recording `lockId`, `runId`, shard, owner, `expiresAt`, and `acquiredAt`. Normal runs use a bounded worker pool with at most 32 concurrent lease acquisitions (`MAX_PARALLEL_LEASE_ACQUISITIONS`); the pool starts another acquisition as each active attempt settles. Each attempt retains its unique shard lock ID, validates `uname -m` and `process.arch`, and records a lease as soon as it succeeds. The coordinator waits for every requested lease before starting any shard process. If pool discovery fails, the deployed pool is undersized, or acquisition is incomplete, no shard process starts. The attempt records one infrastructure-failed result per requested shard, including whether a lease was acquired, `executionStarted: false`, the assignment, and the exact discovery, acquisition, or barrier cause. Every successful partial lease is released, and the incomplete shards remain eligible for infrastructure retry. A lock whose `expiresAt` has passed may be reclaimed. +Other revisions read the lease, so the fields that decide ownership are shared across revisions: -Ordinary attempt cleanup stops a dispatched remote process when completion was not confirmed, then runs the existing owner-lifecycle reaper while retaining the lease. Its remote shell activates nvm before enabling nounset because nvm activation is not nounset-safe; strict error, pipe, and unset-variable handling remains active for the reaper body. Named Blaxel process stop is not a process-tree guarantee: the reaper uses owner tokens and revalidated process identity/group signalling to remove detached descendants. Cleanup releases the lease only when the reaper confirms no live or stale owner-token processes remain; an unreadable, inconsistent, or contaminated report fails closed and leaves the exact lock to its TTL rather than exposing a dirty worker. The reaper report is appended to the shard log as lifecycle evidence. A process whose wait completed successfully is not stopped or reaped again, preserving the normal completed-process path; a lease whose execution was never requested is only released. This ordering prevents a timed-out or transport-failed process or descendant from continuing in an unlocked persistent sandbox while keeping completed and pre-dispatch cleanup unchanged. +```ts +// /tmp/tv-testshard.lock/owner.json; other fields belong to the writer +type WorkerLeaseRecord = WorkerLease | DeletionReservation; -On `SIGINT` or `SIGTERM` while an attempt exists, the attempt becomes interrupted even if its shard promises have all just completed. Completed shard results remain intact. An unfinished shard retains downloaded summaries, native results, task artifacts, parsed task evidence, assignments, collections, and logs, and records `interrupted/coordinator-interruption` plus its known lease and execution state. A pending shard has no synthetic task result. A dispatched process whose start is unconfirmed omits `executionStarted`; absence means unknown, while `false` means execution was not requested. The coordinator makes this interrupted snapshot authoritative before cleanup, stops running shard processes, applies the same owner-confirmation boundary before releasing each known lease, and ensures cleanup cannot replace it with an ordinary completion report. A report-write failure does not suppress lease and process cleanup. A cleanup that cannot confirm zero owners retains its lock until TTL. The first signal exits 130 after cleanup; a second signal exits immediately, with lock expiry as the final bound. A signal handled before the first attempt has no shard result to preserve; the canonical runner treats the absent provider evidence as incomplete when it receives the coordinator exit. Console summaries name only retained log files. +interface WorkerLease { + lockId: string; // also the value of TV_TEST_SHARD_OWNER + runId: string; + attemptId: string; + lastRenewal: number; // worker clock, milliseconds since the epoch + recoverAfter: number; // worker clock, milliseconds since the epoch + expiresAt: 0; // see Older runners +} -## Shard lifecycle +interface DeletionReservation { + lockId: string; + deleting: true; // see Deleting workers +} +``` -After acquiring the complete batch, the coordinator uploads each shard's plan and worker script before starting its remote process. A rejected upload returns an infrastructure-failed shard result with the sandbox, assignment, and `transport/worker-dispatch` cause; it cannot reject the whole attempt without a report. +### Abandoned runs -Each shard runs a script on its worker whose steps are recorded as setup timings: `checkout` (clone if the worker has no checkout, point `origin` at the run's repository, then fetch and hard-reset to the commit), `runtime-activation`, `runtime-versions`, `deps`, `system-deps` (`xauth`, `libgtk-3-0`, and `iproute2` for `ss`), `owner-lifecycle`, `playwright-cache`, `electron-repair`, then `test-run`. The `electron-repair` phase invokes the installed Electron package's own `install.js` with `/cache/electron` as its archive cache. Electron's installer reuses a complete runtime or prepares an absent one; the phase carries no separate download/extraction implementation and does not delete runtime files. ^blaxel-electron-repair After checkout, `runtime-activation` loads the provisioned nvm and runs `nvm use` from the checked-out `.nvmrc`; it does not run a second version checker or install/repair a runtime. All dependency, repair, and test commands inherit the activated Node/npm path. The `system-deps` step asserts these tools on every lease and installs the package set when any prerequisite is absent. `owner-lifecycle` leaves live-supervisor tokens untouched, reaps stale token-bearing groups with revalidated TERM/KILL, and fails before `test-run` whenever it found contamination. That sandbox is excluded from every later attempt by the coordinator. A configured infrastructure retry may run the shard on a different sandbox, whose own lease and owner audit must pass before `test-run`; without an exact contaminated-sandbox identity or enough distinct clean capacity, the shard remains infrastructure-failed. The test run is `scripts/run-test-shard.mjs` ([sharded-execution.md](./sharded-execution.md)) for suite and selection runs, or a single targeted command for a `target` run. A target executes its registry-declared `preCommand` and then its registry `command` when one is declared, otherwise its native runner, inside one surface process scope carrying the selected surface ID and owner token. Selectors, retry arguments, and the native JSON reporter are forwarded through a declared command, so wrappers such as the headless-Linux Electron launcher remain active remotely. It always writes a target shard summary after `test-run`, including native results and any normalized process leaks, so a native test failure is classified as `failed` rather than a setup failure. The worker exports `CI=1` and `TEST_RETRIES`, so child runners see the `FLAKY_TEST_RETRIES` default ([flaky-tests.md](./flaky-tests.md)). +A lease whose coordinator has stopped renewing for the recovery interval is abandoned, whether the coordinator died, hung or lost its connection. The next ordinary acquisition then, under the guard: -If a setup step fails before the test run and no shard summary was produced, the worker synthesizes an `infra-failed` shard summary naming the failed step (`checkout` failures are reported distinctly). Native result paths rooted at the worker checkout (`/workspace/television`) are normalized to repository-relative paths after download. +1. rechecks the record and its deadline; +2. stops only the abandoned attempt's processes, found by its tokens and validated by PID and start time; +3. confirms that no live or stale test owner remains on the worker; +4. removes the abandoned scratch and the known leftovers listed under [Temporary files](#temporary-files); +5. discards the abandoned attempt's undownloaded reports; +6. takes the lease. -### Dependency-hash caching +It writes a recovery receipt describing the previous run, its deadline, the processes stopped and the reports discarded into the new attempt's evidence. Discarded reports can never count as a pass for the abandoned run. Expiry authorizes cleanup, never running beside live processes: if the owners cannot be identified or stopped, the worker is held and reported for an operator. An unreadable or incomplete lease record is reclaimed only after a clean audit of the worker. A coordinator that returns after losing its lease cannot renew, release, dispatch or touch the attempt; it reports an infrastructure failure. -The `deps` step hashes the tracked root `.nvmrc`, `package-lock.json`, root `package.json`, and workspace `*/package.json` files into a marker `/cache/blaxel-testshards/deps-.ok`. `.nvmrc` carries the Node selector, while the root manifest carries the declared Node/npm ranges; changing either toolchain declaration therefore invalidates dependency state. With the marker present and `node_modules` in place the step is a cache hit; otherwise it runs `npm ci --prefer-offline --no-audit --fund=false` under the activated runtime and rewrites the marker. +### Older runners -## Classification +**Transitional, while branches still run a runner that predates this lease protocol.** Those runners keep a single `expiresAt` deadline in seconds and reclaim a lock once that time has passed. New records set `expiresAt: 0`, which older runners never treat as passed, so they leave current leases alone. A current runner respects a legacy record until its `expiresAt` passes, then acquires only after a clean audit of the worker. Older runners release before downloading their reports, so the lease cannot protect their results; a branch gets that protection when it takes up the current runner. -A Blaxel shard is classified `passed`, `failed`, or `infra-failed` from its worker exit, readable shard summary, and recorded setup timings: +### Deleting workers -- A known nonzero exit whose setup timings show a failed step *before* `test-run` is `infra-failed`. -- No readable shard summary is `infra-failed`, even when setup timings show that `test-run` was reached. Reaching the command does not establish a test result. -- A summary with `status: "passed"` and worker exit `0` is `passed`. -- A summary with `status: "passed"` and a known nonzero worker exit is a self-contradictory `failed` result. It is not retried. Its `result-report/summary-validation` provider result is infrastructure-incomplete after normalization, making the run incomplete and its test status unknown without inventing a failed test. -- A summary with `status: "passed"` but no known worker exit is `infra-failed` because completion is unconfirmed. -- A summary declaring `status: "failed"` is `failed`; a summary declaring `status: "infra-failed"` or an unsupported status is `infra-failed`. +`npm test -- pool blaxel delete` follows the lease rules. It refuses workers with unexpired leases and reserves an idle worker only after a clean audit, by replacing the lease with a `DeletionReservation`. Every acquisition refuses a reserved worker, whatever its audit finds. It may destroy a worker whose lease has expired even if process cleanup failed, because deleting the sandbox is the cleanup. Before deleting, it removes the worker's eligibility for test runs, so a slow or failed deletion cannot admit a new run. A worker that cannot answer the ownership check is held for operator repair. A new worker that was never made eligible for test runs may be deleted without a lease. -This realizes the infrastructure-vs-test distinction from [sharded-execution.md](./sharded-execution.md) for Blaxel. After download, the coordinator also validates the summary's plan ID, timing provider, tested commit/tree, index/total, and assigned/collected files against the provider plan. Any identity disagreement overrides the classifier result to `infra-failed` with `failureKind` and `failureStep` set to `plan-validation`. +## Temporary files -The coordinator attributes failures through remote wait and final log capture to `worker-execution`; a matching worker deadline is `timeout` and other coordinator errors are `transport`. After report retrieval begins, failures while retaining the summary and artifacts are `transport/report-download`. Either path preserves any result progress already downloaded before the failure. +Worker `/tmp` is memory-backed, and Chromium keeps its shared memory there, so temporary files cost memory, not just disk. Each shard attempt gets a fresh private directory under `/tmp/tv-testshards/`, named for its run, shard, attempt and lease, and `TMPDIR` points inside it for everything the attempt runs. The path stays short enough for browser Unix sockets. Persistent caches for npm, Electron, browsers and dependencies stay where they are. ^blaxel-scratch -`stale-owner` fails closed at the sandbox boundary. The contaminated sandbox runs no selected test, is excluded from every later attempt by that coordinator, and retains its failed-attempt evidence. A configured infrastructure retry may use a different sandbox only after that sandbox acquires its own lease and passes its owner audit. If no distinct clean sandbox is available, the shard remains infrastructure-failed. Attempt files and final provider-report construction preserve the contaminated attempt and follow [sharded-execution.md](./sharded-execution.md). +Once it holds the lease and no earlier execution remains, a run removes only: -## Options +- directories directly under `/tmp` named `television-source-cli-e2e-*` or `television-web-bundle-*`: generated runtimes that test helpers on some branches leave behind; +- earlier attempts' directories under `/tmp/tv-testshards/`. -The coordinator's option set, with code-true defaults. The canonical runner ([test-runner.md](./test-runner.md)) overrides some of these, noted inline. +Removal rejects unexpected mount boundaries and does not follow links out of these locations. The current attempt, the lease and guard, unrelated temporary files, the checkout and persistent caches are kept. The run reports what it removed and the free space before and after. -| Option | Default | Notes | -|---|---|---| -| `--suite` | `all` | `all` \| `unit` \| `e2e` \| `target` \| `selection`. | -| `--surfaces` | — | required for `--suite selection`. | -| `--shards` | `36` | other counts rejected unless `--allow-shard-count-override 1` (non-target). | -| `--shard-indices` | all | run a subset while preserving the total. | -| `--pool` | `default` | the canonical runner passes `poc`. | -| `--arch` | `x64` | other values rejected. | -| `--commit` | `HEAD` | the committed revision to test. | -| `--timeout-profile` | `normal` | the canonical runner passes `cold`. See drift note below. | -| `--worker-timeout-seconds`, `--lock-ttl-seconds` | per profile | override the profile values. | -| `--retry-infra` | `0` | the canonical runner passes `2`. | -| `--allow-dirty` | off | dispatch from a dirty tree, ignoring local edits. The canonical runner's `--ignore-uncommitted` is translated to this. | -| `--test-retries` | `0` | non-unit retry count passed to the worker. | -| `--repo-url` | the checkout's `origin` as a GitHub HTTPS URL | repository fetched in the sandbox ([#Repository](#Repository)). | -| `--playwright-browsers-path` | `/home/playwright/.cache/ms-playwright` | browser cache path. | -| `--output-dir` | `.blaxel-testshards/latest` | run output location. | -| `--audit-stale-owners` | off | lease one worker, install/check system prerequisites, run the read-only stale-owner diagnostic, print its JSON result, and release the lease without dispatching tests. It may be combined only with `--pool` and `--arch`. | +After the attempt, whether it succeeded, failed, timed out or was interrupted, the worker stops the attempt's processes, moves its evidence out of scratch and removes the attempt's directory, still holding the lease until download. Deleting files does not free Chromium's open, unlinked shared memory; only stopping the processes does. If teardown is uncertain, scratch is kept, and if cleanup fails the worker takes no new tests until a later cleanup succeeds. Workers keep only the current attempt's results and diagnostics under `/workspace/television/.testshards/results`; no archive accumulates on them. -Timeout profiles (code-true): `normal` = worker 240 s, lock TTL 900 s; `cold` = worker 480 s, lock TTL 1200 s; `repair` = worker 480 s, lock TTL 1200 s. +## Runs -**Default profile.** The coordinator's own default timeout profile is `normal`, but the canonical runner invokes Blaxel with `--timeout-profile cold`, so a `npm test -- blaxel` run uses `cold` unless overridden. +A selection made with `--surface`, `--file` or `--grep` must resolve to one surface, and runs as a single targeted shard. The suites `all`, `unit` and `e2e`, and selections by package, runner or tag, are divided across shards ([sharded-execution.md](./sharded-execution.md)). Other suites are refused. -## Output +Either way, a Blaxel run uses the same registry declarations as a local run, including build steps, services, wrapper commands and the runner's retry budget. That way a remote result stands for the local one. Each worker activates the Node version in the checked-out `.nvmrc` before installing or running anything. -The run output directory (default `.blaxel-testshards/latest`) contains `request.json`, combined `report.json`, compact `summary.json`, and any retained per-shard `shard-.log` and `shard-.json` files ([sharded-execution.md](./sharded-execution.md)). A path is exposed only after its local file is retained; shards that never produced a log or result do not advertise an inferred filename. Planned suite runs also contain `shard-plan.json`. Completed shard downloads add `setup-shard-.json` and, when the worker produced it, top-level `native-shard-.json`. Per-surface native reports, Vitest attempt sidecars, lifecycle results, process-leak results, and generated runner inputs are downloaded under `artifacts/shard-/`, preserving their worker-relative paths. Infrastructure retries write `attempt-/` subdirectories with the same applicable layout. Each shard summary exposes normalized restore/wake, checkout, dependency cache hit/miss, system setup, readiness, test-compute, and report-download phases; the coordinator prints those rows and passes them into the canonical run events and summary. Ordinary completion exits `0` only for a passed combined report and `1` for a failed report. A handled `SIGINT` or `SIGTERM` exits `130`; when an attempt exists it writes the interrupted provider report first. When the canonical runner receives that coordinator exit, it finalizes the normalized incomplete run and propagates `130`. +The coordinator is the invoking checkout's revision of the runner, and each worker runs the tested commit's. When they differ, as they can with `--commit`, everything that passes between them crosses revisions: the registry and duration baseline the coordinator reads from the tested commit, the plan and worker command it sends, and the shard summary it reads back. A change to any of these should keep working with revisions still in use. Where it cannot, the mismatch must fail the shard or the run as an infrastructure problem and never be read as test results. As an exception to the [spec policy's rule](../../spec-policy.md#typescript-in-arch-specs) that data another revision reads has a pinned type, the shapes of these exchanges are left to the code, under the rule above. Both ends validate what they receive, and the worker checks the plan against a digest of its whole content. -## Operations +Before tests, each worker looks for stale owner tokens ([test-runner.md](./test-runner.md#processes-started-by-tests)). A worker that finds them runs no tests for that attempt, and the coordinator excludes it from the run's later attempts. An infrastructure retry may use a different worker once that worker has its own lease and a clean audit. Without a clean worker to use, the shard stays infrastructure-failed. -### Local setup +## Setup -Running Blaxel shards requires a Blaxel account with the CLI logged in: +Running Blaxel shards needs a Blaxel account with the CLI logged in: ```bash brew install blaxel bl login ``` -The GitHub token the workers use is obtained from the corporate 1Password item **“Television Blaxel sharded test runner github PAT”**. The token supplied for a run must be able to read that run's repository, so a token shared by checkouts of several repositories needs access to each of them. Make it available either by exporting `BLAXEL_TV_GH_TOKEN` (which works across all worktrees) or by writing it to a `.blaxel-gh-token` file in the repository root; the runner's read precedence is in [#GitHub token](#GitHub token). Creating `~/.tvdev-use-blaxel` opts into Blaxel-preferred broad verification ([test-runner.md](./test-runner.md)). +The GitHub token is in the 1Password item **“Television Blaxel sharded test runner github PAT”**. Export it as `BLAXEL_TV_GH_TOKEN`, which works across worktrees, or write it to `.blaxel-gh-token` in the repository root. Creating `~/.tvdev-use-blaxel` makes the host a [marked host](./test-runner.md#^blaxel-host-marker), so that verify uses Blaxel by default. -### Pool provisioning +To provision the pool, delete it and recreate it, which needs the pool to be idle: -The canonical runner uses pool `poc`. Provision it by deleting the complete pool with `npm test -- pool blaxel delete --pool poc`, then recreating all 72 fresh workers with `npm test -- pool blaxel ensure --pool poc` ([#Pool](#Pool)). Use `list --pool poc` only for inspection. `ensure` refuses a populated pool and intentionally does not reconcile existing runtime state. A bare command targets the manager's `default` pool, which the canonical runner does not use. - -### Opt-in local suite exclusions - -The `telemetry-posthog-roundtrip` and `daemon-acceptance` suites are excluded from `all`, so Blaxel broad verification (`npm run verify -- blaxel`, which delegates `--suite all`) runs neither. Blaxel workers do not receive the PostHog test read key; run the local telemetry PostHog roundtrip suite when validating telemetry delivery changes, using [its authoritative setup](../telemetry/sink.md#real-posthog-integration-test-surface). The daemon acceptance suite deliberately replaces a host's active global `tv` package and literal production user service, so it runs only on an explicitly acknowledged designated developer host under [its local-suite contract](./test-runner.md#production-daemon-acceptance-suite), never as remote broad verification. +```bash +npm test -- pool blaxel delete --pool poc +npm test -- pool blaxel ensure --pool poc +``` -## Testing +Deletion reports any worker it refused and exits nonzero if any requested worker remains. `npm test -- pool blaxel list --pool poc` shows the pool. Without `--pool`, these commands act on a pool named `default`, which the runner does not use. -Under the testing policy's [rule for integration against paid services or complex host dependencies](../testing-policy.md#integration-against-paid-services-or-complex-host-dependencies-is-important), a change that could affect the Blaxel coordinator's remote operations requires evidence from a live Blaxel run before it reaches main. One targeted shard run (`--suite target`) is sufficient. The run must call the real Blaxel API and communicate with the remote process through the real transport. It must exercise worker leasing, the lock protocol, and architecture validation. It must also exercise checkout, runtime and dependency setup, streaming logs, and result download. Recorded responses cannot replace this live evidence. +## Testing guidance -`--suite target` is not sufficient for a change that affects plan transfer; [sharded-execution.md's Testing section](./sharded-execution.md#Testing) defines the required evidence. +A change that could affect the coordinator's remote operation needs a live Blaxel run before it reaches `main`; recorded responses cannot stand in for one. One targeted run is enough to exercise the API, the transport, leasing, setup, log streaming and result download. Runs that check other changes: -A change to pool provisioning or runtime selection requires the live run to use freshly recreated `poc` workers, since `ensure` refuses a populated pool ([#Pool](#Pool)). +- a planned suite run checks a change to how plans reach workers, since a targeted run carries no plan; +- freshly provisioned workers on a dedicated pool check a change to provisioning or runtime selection, leaving the shared `poc` workers alone; +- runs from checkouts of two repositories on one pool, including a worker whose checkout last served the other repository, check a change to repository selection or switching. -A change to repository selection, or to how a worker's checkout moves between repositories, requires live runs from checkouts of two repositories on the same pool, including a run on workers whose checkout last served the other repository. +Recovery and cleanup experiments use disposable processes on a dedicated pool and never contaminate shared workers. Temporary-file placement is shown only by observing, on a live worker, the files Node creates and Chromium's shared-memory descriptors under the attempt's `TMPDIR`. +The automated live qualification is the `experiment:blaxel-harness` surface, with the fixtures it dispatches in `experiment:blaxel-harness-fixtures`. Both stay out of `all` and CI. The driver runs only on a dedicated, credentialed Blaxel coordinator sandbox outside every worker pool, never on a shared development host or a leased worker. It runs from a committed and pushed checkout, with `TV_BLAXEL_HARNESS_COORDINATOR` naming that sandbox and `TV_BLAXEL_HARNESS_POOL` naming a disposable pool it owns, deletes and recreates. The runner refuses the driver on Blaxel, inside a leased worker, or without both names; choosing the right sandbox is the operator's job. Its usage is in `test/runner-fixtures/blaxel-harness/README.md`. ^blaxel-harness-placement diff --git a/specs/arch/test-runner/flaky-tests.md b/specs/arch/test-runner/flaky-tests.md deleted file mode 100644 index d4ee0314..00000000 --- a/specs/arch/test-runner/flaky-tests.md +++ /dev/null @@ -1,74 +0,0 @@ -*The flaky-test policy: the runner's retry budget, the per-test `FLAKY_TEST_RETRIES` annotation, and how recovered flakes are surfaced.* - -# Flaky Tests - -Flaky tests are an unfortunate fact of end-to-end testing, so Television handles them with a two-layer mechanism. The first layer is a default retry budget the runner applies to the non-unit Vitest and Playwright e2e surfaces — unit surfaces get no runner-level retries; on those surfaces it surfaces which tests are flaky and provides a thin line of defense. The second is an annotation in code that marks a single known-flaky test as warranting more retries to keep it passing — a per-test retry option in Vitest, or a retry-configured group wrapping the one test in Playwright. - -This spec is authoritative for the flaky-test policy: the retry budget the runner applies, the `FLAKY_TEST_RETRIES` annotation that opts a single test into retries, and the rule that recovered flakes are reported rather than hidden. Budget application is owned by [test-runner.md](./test-runner.md); recovered-flake reporting is owned by [reporting.md](./reporting.md); the per-worker retry pass for remote runs is in [sharded-execution.md](./sharded-execution.md). - -A *flaky test* is a test that passes and fails non-deterministically for reasons other than a product regression — a test-harness or environment problem, not a real defect. - -## Retry budget - -Non-unit Vitest and Playwright surfaces retry twice at the runner level by default, including targeted runs; unit surfaces receive no runner-level retries ([test-runner.md](./test-runner.md), [sharded-execution.md](./sharded-execution.md)). A test that fails then passes within its runner retries is a recovered flake: the surface passes and the recovery is reported, not counted as a failure ([reporting.md](./reporting.md)). Package-level Playwright `retries` and Vitest `retry` stay at `0`, so the runner is the single place runner retries are configured. - -A provider's infrastructure retry is a separate boundary governed by [test-runner.md#Retries](./test-runner.md#Retries). It may repeat unit work because observations from an incomplete shard do not establish a completed shard result. Task observations retained from the incomplete attempt remain diagnostic attempt evidence and do not enter the final failure list after a completed retry. This provider-level retry does not increment `testsFlakyRecovered`, because that counter records attempts exposed by the native runner within a completed task result. - -## FLAKY_TEST_RETRIES - -```ts -// test/flaky-retries.ts -export const FLAKY_TEST_RETRIES: number; -``` - -`FLAKY_TEST_RETRIES` resolves at import time from `test/flaky-retries.ts`: - -- if the `FLAKY_TEST_RETRIES` environment variable is set, its value parsed with `Number.parseInt(value, 10)` — which reads a leading integer and ignores any trailing non-numeric characters, so `"5abc"` resolves to `5`; a value with no leading integer, or a negative one, throws; -- otherwise `5` when `CI` is set; -- otherwise `0`. - -The canonical runner and the shared shard worker inject `FLAKY_TEST_RETRIES=5` into the child test environment unless it is already set, so a runner-driven or CI run defaults to `5` annotated-retries, while an ad hoc direct `vitest`/`playwright` invocation with neither the variable nor `CI` resolves to `0`. - -## Annotation patterns - -A flaky annotation is applied to exactly one test case, never a whole file or feature suite. - -**Playwright** — wrap the single case in a nested `describe` whose title begins with `flaky:` and configure its retries: - -```ts -import { FLAKY_TEST_RETRIES } from "../../../../test/flaky-retries.js"; - -test.describe("flaky: short reason", () => { - test.describe.configure({ retries: FLAKY_TEST_RETRIES }); - test("actual test name", async ({ page }) => { /* ... */ }); -}); -``` - -**Vitest** — set the per-test retry option and keep an explicit title: - -```ts -import { FLAKY_TEST_RETRIES } from "../../test/flaky-retries.js"; - -test("flaky: actual test name or short reason", { retry: FLAKY_TEST_RETRIES }, () => { /* ... */ }); -``` - -A dedicated flaky-quarantine file is allowed only when the filename makes that purpose explicit. - -## Policy - -On a first failure, investigate. A real failure is fixed. Only persistent, investigated infrastructure or harness flakiness earns an annotation, and only on the one offending case. The `flaky:` title prefix is what marks a case as annotated; [reporting.md](./reporting.md) sets `flakyAnnotated` from it when surfacing recovered flakes. - -## Recovered-flake reporting - -[reporting.md](./reporting.md) classifies a passed-after-runner-retry case as a recovered flake. For Vitest, the attempt-recorder sidecar is authoritative: a test is recovered when its recorded attempts include a failure and its final recorded attempt passed. For Playwright, the native `flaky` status identifies the recovery. The report lists each recovery under `testsFlakyRecovered` with its attempt count, distinct from failures. - -## Lifecycle failures are not recoverable flakes - -The outer surface owner evaluates process cleanup after the native runner has completed all test attempts ([test-runner.md](./test-runner.md)). A leaked process fails the surface even when a test retry passed, and the result preserves both the recovered-flake record and lifecycle failure ([reporting.md](./reporting.md)). Runner retries and `FLAKY_TEST_RETRIES` never retry or suppress this lifecycle failure. A completed shard carrying the lifecycle failure is likewise not eligible for an infrastructure retry. - -A network reset or unhandled socket error emitted by test infrastructure is a harness defect. It is investigated and contained under [testing-policy.md](../testing-policy.md); it does not qualify a case for a flaky annotation while the infrastructure error remains unhandled. - -## Testing - -Tests do not check whether a flaky annotation follows the patterns and policy above. That question is judged in review under the testing policy's rule for coverage and test honesty across the spec system ([testing-policy.md#Tests are the validation mechanism](../testing-policy.md#Tests are the validation mechanism)). - diff --git a/specs/arch/test-runner/github-ci.md b/specs/arch/test-runner/github-ci.md index 3b6c13c7..9e06d1a8 100644 --- a/specs/arch/test-runner/github-ci.md +++ b/specs/arch/test-runner/github-ci.md @@ -1,137 +1,55 @@ -*The GitHub Actions CI execution contract: PR concurrency, per-job dependency and browser environment, build-once fan-out with an artifact manifest, cleanup tool prerequisites, worker counts, and the retirement record for the Blacksmith-dispatched shard provider.* +*The GitHub Actions workflow that validates pull requests and shared branches: when it runs, the checks branch protection requires, how its jobs share setup, and how failures stay visible.* -# GitHub Actions CI +# GitHub CI -GitHub Actions validates every ready pull request targeting `main` or `integration/**`, and every push to those branches. Draft pull requests skip the validation graph, including the `e2e-required` join; marking one ready starts validation. Shared-branch policy comes from the [contribution workflow](../../spec-workflow.md#^shared-branch-workflow). The CI workflow fans test suites out across parallel jobs and gates mergeable changes on the result. Ordinary test jobs restore an exact-keyed dependency cache that is equivalent to a clean install, e2e shards restore one image-bound archive containing those dependencies and the Playwright Chromium and Firefox browsers, and workspace builds happen once before their outputs fan out. This contract prevents each job from repeating dependency, browser, system-library, and workspace-build setup. +`.github/workflows/ci.yml` validates every ready pull request into `main` or a branch under `integration/`, and every push to those branches. The [contribution workflow](../../spec-workflow.md#^shared-branch-workflow) decides when that validation is needed. CI runs the same registry and runner as local and Blaxel runs ([test-runner.md](./test-runner.md)), divides its end-to-end tests with the shared shard plan ([sharded-execution.md](./sharded-execution.md)), and skips heavy jobs for an already attested tree ([attestation.md](./attestation.md)). -This spec is authoritative for how the GitHub Actions CI workflow (`.github/workflows/ci.yml`) cancels superseded PR runs and sets up and executes test jobs: the dependency-install contract, the browser-environment contract, the build-once fan-out, cleanup/leak-detection tool prerequisites, and CI worker counts. The repository's Node/npm values and `.nvmrc` selector are owned by [arch/node-versions.md](../node-versions.md). The canonical command surface is [test-runner.md](./test-runner.md) (which also owns the CI retry contract); the per-worker shard runner CI invokes is [sharded-execution.md](./sharded-execution.md); surface builds are declared in [test-registry.md](./test-registry.md). +## When it runs -What CI runs — the suites, duration-aware shard fan-out, run artifacts, and the `e2e-required` join for branch protection — is owned by the workflow under [test-runner.md](./test-runner.md)'s retry contract. This spec owns both that GitHub topology and the setup each job pays before tests execute; [sharded-execution.md](./sharded-execution.md) owns the shared plan and worker semantics. Whether the heavy test jobs run at all on an already-attested tree is owned by [attestation.md](./attestation.md). +Draft pull requests run nothing, including the join described below; marking a pull request ready starts validation. A new run for a pull request cancels the run it supersedes. Pushes are not cancelled. -The developer-to-CI reuse path starts with a normal `npm run verify -- blaxel`. When that complete, unfiltered verify passes and the trusted policy permits writing, it automatically publishes the tree attestation; no publication opt-in is part of the command. A ready pull request for the identical tree then reaches `attest-check` and uses the step-level no-op path below. `--no-publish` is the developer's explicit escape hatch when this reuse is unwanted. +Every run that is not for a draft runs lint, type-check, the package manifest checks and the [vibe-mode check](./test-runner.md#^verify-vibe-mode). The heavy jobs, which are unit tests, desktop end-to-end tests, the build and the end-to-end shards, run unless an attestation lets them skip. -GitHub CI is the other eligible attestation source. Its `attest-write` job sets `TV_TEST_REF_PUBLICATION` from the workflow's same-repository/fork classification and invokes the CI writer only after the complete graph and strict join pass. The writer command additionally requires `GITHUB_ACTIONS=true`, workflow `CI`, job `attest-write`, and `TV_TEST_REF_PUBLICATION=required`; a developer shell, an individual `provider=local` test job, and a fork context cannot use this path. +## Required checks -## Superseded pull request runs +The repository's ruleset for `main` and `integration/**` requires the checks `lint`, `type-check`, `package-manifests`, `unit`, `e2e:desktop` and `e2e-required`. People maintain that ruleset in GitHub's settings, so these job names are a contract with it. Renaming a job, or making one stop reporting, blocks merges or silently drops a requirement unless the ruleset changes with it. -A new CI run for the same pull request cancels its superseded run. Runs for different pull requests are independent; push and manual runs are not cancelled or serialized by this PR concurrency policy. The complete required-check graph remains the validation requirement for the head commit of a ready pull request. ^gha-pr-concurrency +`e2e-required` stands for the whole end-to-end matrix. On every run that is not for a draft it runs whatever happened upstream, and it passes only when all of these hold: -## Dependency install: exact-key caching of the install's full effect +- the build and every shard succeeded, counting the successful no-ops of an attested skip; +- the vibe-mode job succeeded, so an attested tree is not exempt from that check; +- when the shards ran tests, they agree: each index from 1 to the shard total appears once, all ran the same plan on the same commit, tree and total, and each collected exactly its assigned files. -Every job that executes repository code selects Node from `.nvmrc` exactly once. The `ci` and `test` jobs select it through `setup-workspace`; the build and e2e shard jobs select it through `setup-shard-readiness`; `attest-check`, `e2e-required`, and `attest-write` call `actions/setup-node` directly before their repository command. Continue-on-error phase-metric recorders may run on the runner default before a setup composite because their failure is non-gating and they neither install dependencies nor validate the repository. +An authorized attested skip needs the successful build and shard no-ops and a successful vibe-mode check; it produces no shard reports, and none are required. Missing, skipped, cancelled or failed dependencies fail the join, so a job skipped by an upstream failure is never read as success. -Every CI job that needs the workspace installed follows this contract: +## Jobs -- **What is cached:** the install's *complete* effect on the dependency tree — every `node_modules` directory the install produces. -- **The key covers the cached dependency tree.** Key components are the OS, CPU architecture, Node major version, the execution environment's identity (the container image tag for container jobs; the runner image label for VM jobs), and a content hash over the committed `package-lock.json`. The hash also retains `packages/skills/package.json` as a conservative input. The concrete `hashFiles` list is pinned in the workflow and mirrored by the repo assertion below; when in doubt, over-include an input rather than risk a stale hit. -- **No `restore-keys` fallback.** A partial or stale tree restored from a near-miss key is worse than a cold install. -- **On a hit, `npm ci` is skipped entirely.** The guarantee: a hit is behaviorally equivalent to a pristine `npm ci` at the same inputs. The key construction above is what makes this hold — any drift in an input produces a miss, never a stale hit. -- **On a miss, `npm ci` runs and the paths are saved under the exact key** in the same job, so the next run at the same inputs hits. -- Version-bump commits rewrite the lockfile and therefore miss; that cost is accepted (a version-normalized key is a possible later refinement, out of scope here). +The `unit` and `e2e:desktop` jobs run `npm test -- local` with `--suite unit` and `--surface e2e:desktop`. The end-to-end shards cover the `e2e-ci` suite: every end-to-end surface except Electron, which the desktop job covers. Each shard job computes the plan for itself and runs its index with the shared shard worker directly. Because that bypasses `npm test`, the shard jobs pass the runner's retry budget explicitly. The matrix has 16 shards; a measured run with 24 shards finished under 20 seconds sooner for about a quarter more runner time. The shard total appears in the job names, the plan and the join's expectation, which change together. -The `setup-node` npm-store cache (`~/.npm`) is kept as the miss-path accelerator; it never substitutes for the contract above, because a store hit still runs the full install and lifecycle. +### Build once -The `setup-shard-readiness` production namespace is a manually versioned payload-schema boundary: it must increment whenever the required browser set or another cached-path requirement changes without changing the source-input hash. The current default is `shard-readiness-v2`, which identifies a payload containing both locked Playwright browsers. +One build job runs the deduplicated build recipes declared in `test.prebuilt.mjs`. Every build step declared by a surface in the `all` suite is covered by exactly one recipe, and the build job fails otherwise. The job then uploads every untracked file and symlink in its workspace, except `node_modules`, `.git` and `.test-runs`, as a tar archive that keeps file modes; the artifact service drops modes, and tests run the built CLI directly. Each end-to-end shard takes tracked files from its own checkout, dependencies from its cache and build output from the archive, and skips its surfaces' build steps. A missing or broken archive fails the shard, with no fallback build. No list of build outputs is kept anywhere, so a new output ships because it exists. ^gha-prebuilt-partition -The e2e shard path specializes this contract through `setup-shard-readiness`: one exact cache archive contains `node_modules` and `~/.cache/ms-playwright`. Its key includes the same source-input hash plus OS, architecture, the **major version derived from the Node runtime actually selected by `setup-node`**, and GitHub's concrete hosted-image `ImageOS` and `ImageVersion`. `ubuntu-latest` is not a sufficient image identity: an image rollover must miss rather than restore readiness produced on an older image. The build job is the sole combined-cache producer. On a miss it runs `npm ci`, prepares Chromium and Firefox, proves real Chromium and Firefox launches, and saves the combined paths under the resolved key. +That arrangement places limits on builds and on what shards may assume: -A consuming shard normally restores the build's exact key. On a miss it emits an Actions warning, runs its own `npm ci` and Chromium/Firefox system-dependency installation, and records a cache-miss `shard-readiness-fallback` phase before continuing to the mandatory launch probe. GitHub rolls hosted images out gradually, so a build and its shards can legitimately receive different `ImageVersion` values during one workflow. Failing on that miss would turn image-fleet skew into a red matrix before tests execute. The fallback preserves availability without weakening cache identity: it does not restore a near-match or publish a combined cache entry, and the build remains the sole producer with a strict pre-save launch proof. +- A build must not leave a tracked file with content different from the commit, because shards take tracked files from their checkout and would never see the change. The build job checks this after the recipes. +- The archive only adds files. It carries no deletions and no empty directories, and nothing a build writes inside `node_modules`. +- The desktop job runs its own build steps and does not depend on the build job, so a build failure cannot hide desktop results. -## Browser environment +### Dependency and browser caches -The dedicated `e2e:desktop` job gets Chromium and Firefox with their system libraries through the `setup-playwright` composite action: the browser binaries restore from a cache keyed on the exact Playwright version, and system dependencies install per job. `setup-playwright` deliberately carries no Node selector or default; every call follows `setup-workspace` or `setup-shard-readiness` and inherits that job's already-selected `.nvmrc` runtime for its `node` and `npx` commands. The e2e shard jobs normally receive Chromium and Firefox through the image-bound combined readiness archive above; a shard reuses `setup-playwright` only on its explicit self-seed fallback. During initial promotion, every shard performs minimal real Chromium and Firefox launch/close probes after cache restore or fallback on its own VM. This verifies the resulting payload against that shard's actual host libraries and fails before tests on corruption or incompatibility. The build's pre-save launches prevent publishing a known-bad archive; the per-shard launches are independently load-bearing because producer launches on another VM cannot prove the consumer host. The operator may remove the per-shard probes only after observing the promoted path across a hosted-image rollover; removing them is a deliberate contract update, not incidental cleanup. +Every job that needs dependencies restores an exact-keyed cache of every `node_modules` directory a clean install produces, and skips `npm ci` on a hit. The key covers OS, CPU architecture, Node major version, the runner image and a hash of `package-lock.json` (and `packages/skills/package.json`). There is no fallback to a near match, because a hit must be equivalent to a fresh `npm ci`. -**Deferred: the pinned Playwright container.** Running shard jobs inside `mcr.microsoft.com/playwright:v-` was implemented and validated with an in-container Node pin, an in-container cache, and a mixed 1/8 shard selection (runs 29525458743 and 29526644799). Its wall time was neutral compared with the apt path (~32 s image pull vs 15–31 s apt; the goal was determinism, not seconds). The first full-suite validation run (29531734369, shard 1/3) surfaced an in-container behavior difference that the narrower selection did not reach: font-metric-sensitive assertions in `packages/view-markdown/test/e2e/markers.spec.ts` failed on all three attempts. (That run's second failure, `test/node/missing-artifacts.test.ts` hook timeouts, initially looked container-specific but reproduced on the VM and was root-caused to the artifact transport stripping the built CLI's executable bit — fixed by the tar packing in the fan-out contract, run 29533230103.) Diagnosing and resolving the font difference (font pinning, environment-specific setup) belongs to the surface that owns the test, and changing its assertions is outside this contract's scope, so containerization is **deferred**: the composite path above is the contract. Revisit trigger: the named incompatibility is fixed or explicitly waived by its owning surface. +End-to-end shards restore one archive holding dependencies and Playwright's Chromium and Firefox. Its key also includes the hosted image's concrete `ImageOS` and `ImageVersion`, because `ubuntu-latest` names a moving image. Only the build job writes that archive, after proving both browsers launch. GitHub rolls new images out gradually, so a shard can land on a newer image than its build. On a miss, the shard installs dependencies and browsers for itself and records the fallback rather than failing. Each shard also launches both browsers before testing, because a launch in the build job on another machine does not prove the shard's host. The cache namespace, currently `shard-readiness-v2`, must be raised when the cached browsers or paths change without changing the key's inputs. -Jobs that run no browser surface (lint, type-check, package-manifests, vibe-mode, unit) use neither browser setup path. `e2e:desktop` additionally needs the Electron runtime environment and follows the fan-out scope rule below (own builds, no build-job dependency). +### Run artifacts -### Desktop runtime preparation +Every test job uploads its complete run directory as `test-run-`, including hidden files, even when tests fail, and keeps it for 30 days. A job finds its run directory from the identity its runner reports, never by listing `.test-runs/`: jobs that run `npm test` pass `--run-dir-output` ([reporting.md](./reporting.md#run-directory)), and shard jobs read it from the shard worker's summary. -The dedicated `e2e:desktop` job obtains its Electron binary through [the test surface's own setup](../desktop/e2e-harness.md#^desktop-e2e-global-setup), not through `npm ci` or a workflow step. The workflow adds no Electron install step and no Electron cache. The harness's setup and handoff carry the behavioral proof; this negative workflow-structure decision is maintained by review and owes no separate assertion under [the testing policy's guidance for contextual statements](../testing-policy.md#^spec-purposes). ^gha-electron-runtime-preparation +### Browser worker counts +`e2e:browser-app-real-stack` runs one Playwright worker. With two, its product-server tests failed when their servers competed on the 2-vCPU hosted machine, although each passed alone. -## Duration-aware matrix execution +## Testing guidance -The e2e matrix plans the registry-owned `e2e-ci` suite, which includes `kind:e2e` and excludes `tag:electron`. A registry assertion requires that selection to equal all e2e surfaces except Electron surfaces. `e2e:desktop` remains selected and exercised by its dedicated matrix job, so the exclusion cannot silently drop desktop coverage. - -Every e2e shard job independently computes the complete plan from the checked-out commit, the committed baseline, the `github-ubuntu-24.04-x64-2vcpu-vm` timing provider, and the configured shard total. It passes that file through the shared worker's single `--plan ` input and executes one index. Every GitHub test job sets the VM timing provider explicitly. The plan weighs files by the baseline's Blaxel measurements ([sharded-execution.md](./sharded-execution.md#Provider weights and unknown files)). - -Each test job records schedule, checkout, dependency/cache, browser-readiness, plan, test-compute, and report phases under the normalized reporting contract, then uploads its complete `.test-runs//` child path with hidden files, error-on-missing behavior, and 30-day retention. Direct local test jobs pass `--run-dir-output "$RUNNER_TEMP/test-run-dir"`; their artifact step reads that exact identity and verifies it names a direct child of `$GITHUB_WORKSPACE/.test-runs`. It never enumerates `.test-runs`, where repository tests may create legitimate sibling directories. The e2e join downloads those run artifacts and invokes the shared agreement validator. In addition to the build/matrix status rules below, it requires indices `1..N` exactly once, equal valid plans, timing provider, tested commit/tree, and total, plus exact plan/assigned/collected file equality. A passing worker cannot hide a duplicated, omitted, or broadened file assignment. - -## Build-once fan-out - -Surfaces declare `preCommand` builds in the registry ([test-registry.md](./test-registry.md)), and the underlying builds overlap: the CLI package build (`packages/cli/build.mjs`) itself builds the web, server, view, and skills workspaces before bundling. Running each surface's `preCommand` in every parallel job therefore repeats the same work. One job builds; the e2e shards receive its output. - -**The artifact is the build's output, shipped whole.** A shard assembles its workspace from three sources: ^gha-prebuilt-partition - -- **its checkout** of the same commit, for every tracked file; -- **the readiness cache** (or, on a miss, its own `npm ci`, equivalent at the same inputs by the caching contract above), for dependency state; -- **the artifact**, for the build's output. - -The artifact is **every untracked file and symlink in the build job's workspace, excluding `node_modules`, `.git`, and `.test-runs`** — packed without output selection. The three exclusions are the paths a shard must not take from the build job: dependency state it restores from its own cache, repository metadata from its own checkout, and test scratch that belongs to whichever run produced it (on a developer machine `.test-runs` is the overwhelming majority of untracked files, and it is never build output). That set is what git itself reports as untracked, so nothing chooses which build outputs matter: nothing can choose wrongly, nothing needs maintaining, and a new build output ships because it exists rather than because someone listed it. **No hand-maintained output list appears in this contract, the workflow, or the manifest.** - -The promise is bounded to exactly that: **every file the build wrote is on the shard.** It is not a claim that the two workspaces are identical, and this contract deliberately buys no machinery to make one. Four things are out of scope, and tests must not depend on them: repository metadata under `.git`, which differs between independent checkouts; anything a build writes *inside* `node_modules`, which the shard takes from the cache; *deletions*, since extraction adds and overwrites but never removes — a build that expresses itself by removing a pre-existing untracked file does not transport that removal; and *empty directories*, which git does not report as untracked content and which therefore do not ship (a directory holding output travels with its files). ^gha-prebuilt-bounds - -**The one rule this places on a build:** it must not leave a tracked file whose content differs from the commit, because a shard takes tracked files from its checkout and would never see the difference. The build job asserts this after the recipes — no tracked file differs from the commit — so a violation fails the producer loudly instead of silently diverging a consumer. Rewriting a tracked file with identical bytes is fine and invisible; the published packages' `LICENSE` copies do exactly that. ^gha-prebuilt-tracked-rule - -The rest of the contract: - -- **The manifest declares recipes, not paths** — which deduplicated builds run (each **once**, not one invocation per surface `preCommand`) and which surfaces they cover. It is a committed module at the repository root beside the registry, `test.prebuilt.mjs`: - - ```ts - interface PrebuiltManifest { - artifactName: "prebuilt-dists"; - builds: PrebuiltBuild[]; - } - interface PrebuiltBuild { - id: string; // deduplicated build recipe, e.g. "products" - command: string[]; // the one command the build job runs for this recipe - coversPreCommands: string[]; // surface ids whose registry preCommand this recipe subsumes - } - ``` - - The registry stays the declaration of what each *surface* needs. The consistency check: every selected surface with a non-null `preCommand` appears in exactly one recipe's `coversPreCommands`. A violation fails the build job, not the consumer. -- The build job uploads one run artifact under the manifest-derived name with `retention-days: 1` (it is consumed only within its own workflow run, which also guarantees same-run validity; one day keeps it inspectable for post-run debugging without accumulating storage). It is packed as a **tar archive extracted with stored permissions**, because the artifact transport strips modes and tests spawn the built CLI directly. -- **Consumers fail loudly, not silently.** Shards download and extract; a missing artifact or a failed extraction fails the job, and there is no fallback to rebuilding. No path is inspected, because none was selected: the artifact arrives whole or extraction fails. ^gha-prebuilt-consumer -- Test jobs invoke the shard runner with pre-commands disabled: the worker input `--skip-pre-commands` (env `TEST_SHARD_SKIP_PRECOMMANDS=1`) skips every surface `preCommand`; the caller takes responsibility for having supplied the built outputs. The worker's sole plan input is `--plan ` (or `TEST_SHARD_PLAN`), which validates and executes an explicit shard plan. Both inputs are defined by the shared worker contract in [sharded-execution.md](./sharded-execution.md). -- The build job produces the image-bound combined shard-readiness cache before running the recipes, so its cost is bounded by one readiness restore (or one miss-path install/browser preparation) plus one deduplicated round of builds. -- **Fan-out scope:** the parallel e2e shard jobs consume the artifact. The `e2e:desktop` job does not — it keeps running its own registry `preCommand` builds and takes no dependency on the build job, so a build failure cannot mask desktop results and desktop stays independent. The cost (one duplicated build round on one job) is accepted. -- **Join integrity:** introducing the build job makes test jobs skippable by an upstream failure, and the branch-protection join must never read that skip as success. On a draft pull request, the join has the same draft guard as the validation jobs and reports neutral/skipped: it is not applicable while GitHub prohibits merging the draft. On every non-draft event, `always()` makes the join run despite dependency outcomes; it passes only when the build and e2e matrix both report success, including the successful step-level no-ops produced by an authorized attestation. Missing, skipped, cancelled, and failed dependencies fail the join. The join also depends on the `vibe-mode` job, which runs the [vibe-mode check](./test-runner.md#^verify-vibe-mode); any vibe-mode result other than success fails the join, so a branch in vibe mode fails the required check while every other job reports its own result. The `vibe-mode` job takes no dependency on `attest-check` and has no skip step, and the join applies this requirement before its attested-skip path, so an attested tree is not exempt. The join **checks out the repository like every other job** and runs the plan-agreement validator from that checkout: shipping the validator inside the artifact would mean deciding which of its files to ship, which is the maintenance this contract exists to abolish. An isolated live run with an induced build failure proves this failure path in addition to the YAML assertion. - -## Tool prerequisites on CI workers - -The shard worker's owner-lifecycle supervisor and leak detector (contracts owned by their runner/process-lifecycle specs) need specific host tools, and CI images do not reliably provide them. The rule this spec sets for every CI job environment (runner VM and container alike): - -- **Never assume a tool.** Every binary the owner-lifecycle path invokes is either asserted present by a check that fails the job loudly, or installed explicitly as a setup step in the same workflow. -- The current required set: `ss` (iproute2) and a readable `/proc` (procfs) for socket-to-process attribution. The owning spec for the leak detector defines what it does with them; this spec guarantees the environment provides them. -- Changes to the required tool set land here and in the workflow's assert/install step in the same change. - -## CI worker counts - -**GitHub e2e matrix:** **16 shards** is the measured cost/latency knee. Run `29561697838` completed the build-through-join critical path in 3m31s at 29.42 e2e runner-minutes. The 24-way run `29565323284` saved only 17 seconds while consuming 37.10 runner-minutes (26% more), so regular CI does not buy that last increment. The job name denominator, matrix indices, planner total, worker total, and agreement-join expectation change together. - -**`e2e:browser-app`** (`packages/web/playwright.config.ts`) keeps its CI Playwright worker pin at **2** — an explicitly conservative default, not a measured conclusion. The 2-vs-4 measurement is **inconclusive due to runner variance**: run 29523328500 measured 4 workers ~9% slower with per-test p95 roughly doubled, while run 29525458743 measured 4 workers ~23% faster (162 s → 124 s, p95 4.6 s → 6.1 s, zero retries in both). Two one-shot samples with opposite conclusions demonstrate that per-run runner variance dominates the effect size, so further one-shot sampling is not spent on it. **Revisit trigger:** the test jobs' run artifacts provide enough GitHub-hosted timing samples to compare worker counts across many runs. Every surface keeps its currently configured worker behavior; any per-surface change cites its own evidence here. In CI, `TV_PW_WORKERS` can override the two-worker pin for a controlled measurement run. It has no effect outside CI. The production workflow leaves it unset. - -**`e2e:browser-app-real-stack`** runs browser acceptances that boot product servers with one Playwright worker. `path-artifact-real-stack-coverage.test.ts` raised the app's disconnected modal on every attempt in run 32435629379 when a second Playwright worker competed with its per-test servers, while it passed alone on the same 2-vCPU hosted VM. `onboarding-browser.test.ts` likewise passed alone locally and on Blaxel, but its application root never rendered when full TV-492 shard plans paired it with the setup-heavy onboarding skill-assets file: three full verifies failed only that assignment, including after extending the initial readiness deadline from five to twenty seconds. A full verify of bare `archive/feat-ui-redesign-2026-08` passed all 36 shards under its different plan. The surface's explicit `workers: 1` serializes both product-server files and prevents unrelated browser setup from competing inside their Playwright invocation; the cost remains one extra invocation on whichever shard each file lands on. - -## Testing - -No repository test uploads to or downloads from the GitHub Actions cache and artifact services. Run the isolated live validation below before merging changes that could plausibly disrupt CI workflow mechanics, such as cache reuse, build-output transfer between jobs, browser preparation, or failure propagation. An implementation change that preserves those mechanics does not require this exercise. When needed, run a complete isolated copy of the CI graph only on the branch carrying the change, using both services. The isolated copy passes scratch `cache-namespace` values to both setup composites ([#Test hooks](../../../proofs/arch/test-runner/github-ci.md#Test hooks)). After validation, its cache entries are deleted to protect the shared repository cache quota. - -The live evidence must show all of the following: - -- a forced miss in the combined shard-readiness cache completes the fallback described above and creates an entry under the exact key in its scratch namespace; -- a following cache hit consumes that entry and skips `npm ci` and browser preparation; -- on both paths, the Chromium and Firefox probes pass, surfaces consume the build artifact with their `preCommand`s disabled, and the strict plan-agreement join is green; and -- an induced build failure leaves the join red. - -An isolated workflow cannot establish required-check contexts under the production job names. The first production run after the change reaches main must show that those contexts are emitted under the names branch protection requires. - -## Retirement record — Blacksmith-dispatched shard provider (2026-07) - -Historical record; the living contract above does not depend on it. Television previously carried a third remote execution path: a `workflow_dispatch` shard workflow on third-party Blacksmith runners, coordinated by `scripts/run-gha-testshards.mjs` and selectable as provider `gha`. It existed to compare sharded runners against Blaxel; it failed frequently at container provisioning, and the operator decided to retire it. The retirement removed: - -- the `gha` provider from the canonical command surface and from `verify` resolution ([test-runner.md](./test-runner.md)); `verify` auto-resolution is Blaxel-or-local, -- the `github-cli` preflight ([preflight.md](./preflight.md)), -- the coordinator `scripts/run-gha-testshards.mjs`, its infrastructure-failure classifier, and its normalization coverage, -- the workflows `.github/workflows/gha-testshards.yml` and `.github/workflows/gha-testshards-image.yml`, and the `.github/gha-testshards/` image definition. +No repository test can use GitHub's cache and artifact services. A change that could disturb CI mechanics, such as caching, the build transfer, browser preparation or failure propagation, therefore needs a live run of a complete isolated copy of the CI graph, only on the branch that carries the change. The copy passes scratch `cache-namespace` values to both setup actions, and its cache entries are deleted afterwards to protect the repository's shared cache quota. An isolated copy cannot report checks under the production job names, so the first production run after the change merges must show the required checks under the names the ruleset expects. diff --git a/specs/arch/test-runner/preflight.md b/specs/arch/test-runner/preflight.md deleted file mode 100644 index ab17bce7..00000000 --- a/specs/arch/test-runner/preflight.md +++ /dev/null @@ -1,92 +0,0 @@ -*The checks the test runner runs before tests: local capability checks for the selected surfaces, and the git-safety and provider-auth checks that gate a committed-revision remote run.* - -# Preflight - -This spec is authoritative for *preflight*: the checks the runner runs before executing tests, how the required set is chosen per provider, what each check verifies, and the process-inspection capabilities required on an execution host. The runner that invokes preflight is [test-runner.md](./test-runner.md); the per-surface list of required local checks is declared in the registry ([test-registry.md](./test-registry.md)); the repository Node/npm values come from [arch/node-versions.md](../node-versions.md); the remote provider that depends on the git-safety and auth checks is [blaxel-testshards.md](./blaxel-testshards.md). Plain GitHub CI executes the checked-out revision directly and performs execution-host preflight before tests. - -A *preflight* is a single named check that must pass before tests run. Each check produces a `PreflightResult`. Checks run in order and the runner stops at the first failure. - -## Required checks per provider - -`node` is always required. Canonical local execution and the planned-shard worker run `process-lifecycle` before their first selected task. Local execution additionally runs the union of the `preflight` names declared by the selected surfaces ([test-registry.md](./test-registry.md)), e.g. `playwright-chromium`, `playwright-firefox`, `electron`. Unit-only local selections are not exempt: runner and harness unit tests can spawn real subprocesses, and basing ownership safety on today's surface contents would let a future subprocess bypass the check. Blaxel target dispatch uses `supervised-command.mjs`, and the legacy non-plan branch of `run-test-shard.mjs` does not invoke `process-lifecycle`; these diagnostic paths are exceptions to the canonical planned-worker preflight. - -A Blaxel coordinator runs `blaxel-github-token` and `blaxel-auth`; each planned suite worker independently runs the execution-host check. Target dispatch follows the exception above. Selected-surface capability checks apply only to local execution because Blaxel workers prepare browser and system dependencies through provider setup. The coordinator result covers dispatch capabilities and git safety, not worker capability. - -Plain GitHub CI invokes the local or planned-shard entrypoint on its checked-out revision, so its test jobs receive the execution-host check without introducing a `ci` provider or dispatch-auth preflight. - -## Execution-host capability checks - -- `node` — delegates toolchain validity to the checker owned by [arch/node-versions.md](../node-versions.md). The machine-readable name remains `node` because preflight names are stable identities in provider results and normalized reports; the result reports both selected versions. -- `process-lifecycle` — the host is POSIX and provides a working inspection backend for the ownership and leak-detection contract in [test-runner.md](./test-runner.md): - - on Linux, `/proc` exposes process identity and the environment of a disposable child owned by the current user, and `ss -H -ltnp` executes successfully; - - on macOS, `ps` can read a disposable child's process group and inherited owner token, and `lsof` can inspect its TCP descriptors; - - another platform, an unreadable process table, or a missing command fails with an installation message. The check never silently degrades to process-name matching or a known-port list. -- `playwright-chromium` — the Playwright Chromium dependency is present. -- `playwright-firefox` — the Playwright Firefox executable exists at Playwright's configured path. The preflight and Playwright config share that predicate. When it is unavailable outside GitHub Actions the check passes as skipped with the missing path as its reason and the Firefox project is omitted; GitHub Actions fails configuration because its setup step is required to install Firefox. -- `electron` — unless `SKIP_ELECTRON_E2E=1` makes the check pass as skipped, the installed Electron npm package must resolve. The check consumes the pre-setup result of the [shared environment planner](../desktop/e2e-harness.md#shared-environment-plan): it accepts the planner's absent or valid runtime result when the plan has no failures and rejects its invalid result. Preflight never invokes Electron's installer, downloads an archive, or changes runtime files. ^preflight-electron-runtime -- `daemon-test-host` — the operator has set `TV_DAEMON_TEST_HOST=1` to acknowledge that the selected surface installs or replaces the active global `tv` package and literal `com.television.server` user service. Any other value fails before the surface build pre-command or Vitest process starts and explains that the suite belongs only on a designated developer host not intended to run a normal Television server. The test file repeats the exact-value check as a direct-run backstop. This acknowledgement prevents accidental invocation; it is not a security or authorization boundary. -- `posthog-test-key` — the PostHog test project read key is available as `TV_POSTHOG_TEST_READ_KEY` in the environment or in the repository-root `.env` file and begins with `phx_`. This local-provider check gates the `telemetry-posthog-roundtrip:integration` surface before Vitest starts; missing keys fail with a message that names both accepted sources, while a key with another prefix fails as the wrong key format. The telemetry PostHog roundtrip suite is local-only through the central runner, and the test file retains a direct-run backstop that throws if someone bypasses the runner. The secret key must never be committed. -- `blaxel-github-token` — a Blaxel GitHub token is available (see [blaxel-testshards.md](./blaxel-testshards.md) for sourcing). -- `blaxel-auth` — `@blaxel/core` can list sandboxes with the current credentials. - -`fuser` and its `psmisc` package are not dependencies of `process-lifecycle`. The runner establishes ownership when it spawns a process and discovers descendants by owner token, then uses `/proc`/`ss` or `ps`/`lsof` for attribution. Provider images must include the matching backend explicitly: Linux GitHub Actions and Blaxel workers must provide `iproute2` (`ss`) and readable procfs; a Linux local host must install `iproute2`; macOS local execution uses the operating system's `ps` and `lsof`. Any missing requirement is a failed preflight, not a skipped cleanup step. - -## Blaxel remote preflight - -A Blaxel run additionally requires that the commit under test is safe to dispatch. Its remote preflight runs the provider capability checks first; if they all pass it runs the git-safety sequence, short-circuiting at the first failure. - -The git-safety checks, in order: - -1. `working-tree` — `git status --short`. A dirty tree fails unless `--ignore-uncommitted` is passed; remote runs test a committed revision from origin and do not include local edits. With the flag, the check passes and records `workingTreeDirty: true`. -2. `commit-resolves` — `git rev-parse --verify ^{commit}` resolves the target (default `HEAD`). -3. `origin-fetch` — `git fetch origin --prune` succeeds. -4. `origin-reachable` — `git branch -r --contains ` includes at least one `origin/*` ref. - -Only when every git-safety check passes does the result carry a non-null `git` summary. - -After Blaxel suite dispatch, each planned shard worker runs its execution-host `process-lifecycle` check before the first surface. It then records `stale-owner-cleanup`, which leaves live supervised owners alone and blocks execution after stale-owner contamination. Plain GitHub CI also uses the planned worker. A failed planned-worker check is represented as a pre-test infrastructure failure and no assigned test command starts on that shard. Blaxel target dispatch and the non-plan shard path remain the exceptions described above. Provider setup must supply the capability consistently. - -## Contributor branches and remote revisions - -Contributors work on their own development branches. Before a Blaxel run, commit the intended changes and push the branch to `origin` so workers can fetch that revision. Use a development branch for unfinished checkpoints; see the [shared-branch workflow](../../spec-workflow.md#^shared-branch-workflow). - -The checks above enforce remote reachability, not branch naming or ownership: any containing `origin/*` branch satisfies the check. `--ignore-uncommitted` tests only the selected committed revision and gives no evidence for local edits. The same rules apply to macOS and Linux contributors; execution-host differences are stated above. - -## Contract surface - -```ts -interface PreflightResult { - name: string; - provider: string; - status: "passed" | "failed"; - message?: string; - startedAt: string; - completedAt: string; - durationMs: number; - [key: string]: unknown; // per-check detail, e.g. version, commit, containingRemotes -} - -interface RemoteGitPreflight { - commit: string; - reachableFromOrigin: true; - workingTreeDirty: boolean; - containingRemotes: string[]; -} - -interface RemotePreflightResult { - provider: string; - results: PreflightResult[]; - git: RemoteGitPreflight | null; -} -``` - -## Testing - -Under the testing policy's [rule for platform-dependent behavior](../testing-policy.md#^platform-breadth), cases where a required host capability is missing or the platform is unsupported use contract coverage. A seam on the CI host must prove that `process-lifecycle` uses the host's real process-inspection backend. On Linux, that seam must use `/proc` and `ss` to attribute a real, disposable child process and its listener, then clean up both. This Linux evidence does not prove the macOS backend, which uses `ps` and `lsof`. Proving the macOS backend requires the same seam on a macOS host. - -Injected `ElectronE2EPlan` values may cover how the Electron check handles each planner state. The suite must also exercise the real handoff from that check to the shared environment planner. It must use the repository's installed Electron package and that package's real files on disk. It must prove that preflight leaves the Electron runtime tree unchanged. - -A seam must invoke the real Git process to run the production git-safety sequence in an actual repository with an `origin` remote. Once that seam exists, the failing case for each git-safety check may use contract coverage under the testing policy's [compositional coverage rule](../testing-policy.md#Compositional coverage across clean seams). - -A change that could affect the Blaxel provider capability checks requires live evidence. That evidence is governed by the testing policy's [rule for integration against paid services or complex host dependencies](../testing-policy.md#Integration against paid services or complex host dependencies is important) and the [Blaxel test-shard testing directive](./blaxel-testshards.md#Testing). - diff --git a/specs/arch/test-runner/reporting.md b/specs/arch/test-runner/reporting.md index 0cc4fab1..cb9bfddd 100644 --- a/specs/arch/test-runner/reporting.md +++ b/specs/arch/test-runner/reporting.md @@ -1,411 +1,129 @@ -*The run directory, normalized reports, timing events, and CI run artifacts every test run produces.* +*What every test run leaves behind: the run directory, a compact summary for people and agents, honest status, timing records, and resource diagnostics from Blaxel workers.* # Reporting -Every test run leaves a compact result for people and agents, detailed files for debugging, and machine-readable timing records. The timing records that Blaxel runs leave in their run directories are the input for balancing future test runs across parallel machines. - -This spec is authoritative for the *run directory*, the *normalized surface result* model, the per-run timing event stream, compact-summary discipline, status accounting, and the run artifacts GitHub Actions jobs upload. Local runs, regular GitHub CI jobs, and Blaxel runs all produce this model; Blaxel provider results are translated into it by [sharded-execution.md](./sharded-execution.md). The runner that drives reporting is [test-runner.md](./test-runner.md); the testing discipline behind the honesty of these reports is [testing-policy.md](../testing-policy.md). Duration rollups and shard planning are owned by [sharded-execution.md](./sharded-execution.md). +Every test run, whether local, on Blaxel, or in a GitHub CI job, leaves results in one shape. A compact summary tells people and agents what failed. Detailed files support debugging, and timing records feed the planning of later runs ([sharded-execution.md](./sharded-execution.md#duration-baseline)). ## Run directory -Each run gets a *run directory* at `.test-runs//`. Its exact run ID is `-p-r`, where `safe-start` is `new Date().toISOString().replace(/[:.]/g, "-")`, `pid` is the unsigned decimal process ID, and `random` is 8 random bytes encoded as 16 lowercase hexadecimal characters. Example: `2026-07-16T04-22-32-979Z-p4095823-r0123456789abcdef`. The random suffix prevents concurrent processes and separate hosts with the same clock and process ID from colliding. +Each run writes a *run directory*, `.test-runs//`, in the checkout. Run IDs are unique even across concurrent processes and hosts. The directory holds: -``` -.test-runs// - summary.json compact run summary - results.json per-surface normalized results - events.ndjson normalized timing event stream - logs/ raw runner logs, per surface - native/ native Vitest/Playwright reports - provider/ provider-specific outputs for remote runs, including the shard plan -``` +- `summary.json`, the compact summary, whose path the command prints at the end; +- `results.json`, with one normalized result per surface that has evidence; +- `events.ndjson`, the timing records; +- `logs/`, `native/` (the Vitest and Playwright reports) and, for Blaxel, `provider/`. -`summary.json` carries an `outputs` pointer block naming `results.json`, `events.ndjson`, `logs/`, `native/`, and `provider/` when present. Files are written atomically through a sibling temporary file followed by rename. A run that exits before native test output exists still writes all three top-level report files with the infrastructure failure it reached. +Files are written atomically. Once a run directory exists, a run that fails still writes its summary with the failure it reached, even when no test ran. The local runner owns the directory of a local run and creates it before local preflight. For a Blaxel run, the canonical runner creates the directory once its remote preflight and selection checks have passed, and writes one combined report after downloading each shard's parts. Each GitHub CI test job writes its own. A guardrail refusal, or a Blaxel run refused before dispatch, leaves no run directory. -The process that owns the test invocation owns its run directory: +`--run-dir-output ` makes a run write the absolute path of its own run directory into the caller's file as soon as the directory exists. A caller of `npm test`, such as a CI test job, reads that file instead of searching `.test-runs/`, where tests themselves create other run directories. -- `scripts/test/cli.mjs` produces it for direct local runs, including unit and desktop commands invoked inside GitHub CI. -- `scripts/run-test-shard.mjs` produces it for each regular GitHub CI matrix shard that invokes the worker directly; its existing `.testshards/results/shard-.json` is an input retained under `provider/`, not the final report. -- The canonical runner's Blaxel coordinator finalizer produces one combined run directory after downloading shard fragments. Blaxel workers produce fragments only. +## Honest status -Regular GitHub matrix jobs therefore have distinct run IDs and one-shard event streams. Their required join validates plan agreement under [sharded-execution.md](./sharded-execution.md); it does not rewrite those job runs into a synthetic run. +The summary's `run` block answers three questions separately: -A caller that must retain this invocation's directory passes `--run-dir-output `. The owning runner atomically writes its absolute run-directory path to that caller-owned file immediately after creation. Orchestration uses this identity instead of enumerating `.test-runs/`: repository tests deliberately create sibling run directories while exercising retry capture and real CLI boundaries, so directory count and modification time do not identify the outer run. +- `infraStatus`: did the infrastructure deliver complete evidence for the selection? `completed` or `incomplete`. +- `testStatus`: did the tests pass? `passed`, `failed`, or `unknown` whenever the evidence is incomplete. +- `status`: the run as a whole. It is `incomplete` whenever the evidence is incomplete, and otherwise equals `testStatus`. -## Normalized surface result +Reports state only what was observed. Known failures, passes, recovered flakes and skipped work stay visible in an incomplete run. Missing work is never counted as run, and an unreadable result never becomes a failed test. Nothing claims that a selection passed or failed as a whole without evidence for all of it. -A *normalized surface result* is one surface's outcome, shaped the same regardless of runner or provider. +A process leak fails each surface it belongs to and marks it incomplete, because cleanup was not clean; the leak appears beside that surface's test results. A recovered flake leaves its test passed and is listed with its attempts. Vitest's own JSON report does not record individual retry attempts, so Vitest runs use the repository's attempt recorder, and the recorded attempts are the only evidence of a Vitest recovery. Failure messages are never read as attempts. -```ts -interface Counts { - testsTotal: number; - testsPassed: number; - testsFailed: number; - testsSkipped: number; - testsFlakyRecovered: number; -} +## The compact summary -interface ProcessLeakSocket { - protocol: "tcp"; - family: "ipv4" | "ipv6" | "unknown"; - host: string; - port: number; -} +`summary.json` is written so that an agent can read it without flooding its context. It lists failed tests, recovered flakes and process leaks, and does not list passing tests. Each listed test has the fields below and a short error excerpt when one is available. Each leak keeps its PID, command, owning surfaces and listening sockets, since those identify the harness that leaked. Detailed evidence stays in the other files. -interface NormalizedProcessLeak { - leakId: string; - ownerToken: string; - owningSurfaceIds: string[]; - pid: number; - parentPid: number | null; - processGroupId: number | null; - processStartedAt: string; - detectedAt: string; - command: string; - listeningSockets: ProcessLeakSocket[]; - cleanup: { - termSent: boolean; - killSent: boolean; - outcome: "terminated" | "killed" | "already-exited" | "survived" | "unknown"; - }; -} - -interface NormalizedSurfaceResult { - id: string; - runner: string; - status: "passed" | "failed" | "skipped"; - infraStatus?: "completed" | "incomplete"; - command: string[]; - exitCode?: number | null; - durationMs: number; - durationSource: "surface-wall" | "owned-file-sum" | "task-wall" | "unknown"; - logPath: string | null; - nativeResultPath?: string | null; - nativeResultPaths?: string[]; - counts: Counts; - failedTests: Array>; - flakyRecoveredTests: Array>; - processLeaks: NormalizedProcessLeak[]; - skipReason?: string | null; - assignedFiles?: string[]; - collectedFiles?: string[]; - // lifecycle and provider infrastructure failures also carry: - failureKind?: string | null; - failureStep?: string | null; - failureMessage?: string | null; - infraFailedShards?: number[]; -} -``` - -`normalizeSurfaceResult` reads the surface's native report and classifies tests. For Vitest, a failed assertion is a failure, while recovered flakes come from the attempt-recorder sidecar: a test is recovered when its recorded attempts include a failure and the final recorded attempt passed. Aggregate Vitest failure messages never establish a recovery. For Playwright, a `flaky` test status is a recovered flake. A missing or malformed native report is infrastructure-incomplete result evidence; normalization preserves the read or parse cause and does not invent a failed test. - -A standalone surface uses its process wall for `durationMs`. A surface that ran inside the combined Vitest run uses the sum of its owned file walls and `durationSource: "owned-file-sum"`; the combined process wall is recorded once as the shard phase `execution-group:unit:workspaces`. The group wall is never copied onto every member surface. Provider-normalized standalone task parts use `task-wall`. - -## Reports - -`TimingProvider` is the execution-substrate identity defined by [sharded-execution.md](./sharded-execution.md); it is independent of the command provider. `buildReports` writes `summary.json` and `results.json`. +People, agents and scripts read these fields, including from earlier runs: ```ts -interface ReportSummary { +// summary.json; other fields belong to the writer +interface RunSummary { schemaVersion: 1; run: { id: string; - commandProvider: "local" | "blaxel" | null; - timingProvider: TimingProvider; status: "passed" | "failed" | "incomplete"; infraStatus: "completed" | "incomplete"; testStatus: "passed" | "failed" | "unknown"; - startedAt: string; - completedAt: string; - durationMs: number; - }; - selection: Record; - git: Record | null; - preflights: Array>; - counts: { - surfacesSelected: number; - surfacesRun: number; - surfaces: number; - surfacesPassed: number; - surfacesFailed: number; - surfacesSkipped: number; - testsFailed: number; - testsFlakyRecovered: number; - processLeaks: number; - infraFailures: number; - }; - failedSurfaces: Array>; - skippedSurfaces: Array>; - failedTests: Array>; - flakyRecoveredTests: Array>; - processLeaks: NormalizedProcessLeak[]; - phaseMetrics: { - schemaVersion: 1; - run: { - durationMs: number; - observedStartedAt?: string; - observedCompletedAt?: string; - observedWallDurationMs?: number; - phases: TimingPhase[]; - }; - shards: Array<{ - index: number; - total: number; - status: EventStatus; - durationMs?: number; - phases: TimingPhase[]; - surfaces: Array<{ surfaceId: string; status: EventStatus; durationMs?: number; durationSource: string; phases: TimingPhase[] }>; - }>; - }; - outputs: { - results: string; - events: string; - logsDir: string; - nativeDir: string; - providerDir?: string; }; + failedTests: SummaryTest[]; + flakyRecoveredTests: SummaryTest[]; + processLeaks: Array<{ + pid: number; + command: string; // the executable, without arguments + owningSurfaceIds: string[]; + listeningSockets: Array<{ host: string; port: number }>; + }>; } -interface ReportResults { - schemaVersion: 1; - run: ReportSummary["run"]; - selection: Record; - git: Record | null; - preflights: Array>; - counts: { - surfacesSelected: number; - surfacesRun: number; - surfacesSkipped: number; - }; - surfaces: NormalizedSurfaceResult[]; - infraFailures: Array>; +interface SummaryTest { + surfaceId: string; + file: string; + line: number | null; + title: string; + attempts: number; + flakyAnnotated: boolean; } ``` -`ReportSummary` declares `failedSurfaces` once. Any generated type declaration must match that shape. `surfacesRun` counts selected surfaces with execution evidence: a wholly skipped surface is excluded, while an aggregate that retains passed tests, normalized files, or collected files from one shard remains counted when another shard was skipped by fail-fast. `surfacesSkipped` counts only wholly skipped selected surfaces. Infrastructure-only `provider:` records remain visible as failures but are not selected test work and do not contribute to `surfacesRun`. - -`selection` is the complete requested scope. `results.surfaces` contains normalized selected-surface outcomes only where task or native-result evidence exists, plus explicit provider/lifecycle failure records. When provider evidence is absent, the report does not synthesize one failed or passed result per selected surface; the selection remains visible, `surfacesRun` stays zero for work not observed, and an infrastructure-incomplete provider result carries the reporting cause. A shard containing only `not-assigned` tasks is known idle work, not missing provider evidence. Such a shard may pass as one participant in a valid plan because other shards own the files; [sharded-execution.md](./sharded-execution.md) requires at least one file across the complete plan. - -### Status and evidence claims - -Three run fields answer separate questions: - -- `infraStatus` is `incomplete` when a preflight failed or any result needed to account for the selection is infrastructure-incomplete; otherwise it is `completed`. -- `testStatus` is `unknown` whenever `infraStatus` is `incomplete`. With complete infrastructure evidence, it is `failed` when any surface failed and `passed` otherwise. -- `status` is `incomplete` whenever `infraStatus` is `incomplete`; otherwise it equals `testStatus`. - -An incomplete run may report outcomes that have direct evidence: completed surface results, native failed tests, recovered flakes, skipped assignments, and partial task observations from an incomplete shard. It may not infer an outcome for missing work, count an unstarted selected surface as run, turn an unreadable result into a failed test, or claim that the complete selection passed or failed. Known failures remain visible even though the run-level `testStatus` is `unknown`; unknown describes the unresolved complete selection, not the absence of all observations. - -`failedSurfaces` contains failed or infrastructure-incomplete results. `skippedSurfaces` contains only wholly skipped selected surfaces and carries each reason. A partially executed aggregate retains `status: "skipped"` because its complete assignment did not finish. Its assigned and collected files remain explicit in `results.json`, and it counts as run rather than wholly skipped. Aggregate `status` describes completion of the whole assigned surface; `surfacesRun` records whether any selected work executed. - -An interrupted provider run is infrastructure-incomplete even when every shard happened to complete before the signal was handled. Results completed before interruption remain unchanged. A running shard retains any summary, native result, task artifact, assignment, collection, and log evidence already downloaded, and its unfinished shard outcome records the interruption. Pending work has no synthetic task outcome. The report therefore preserves known failures and passes without inventing outcomes for unobserved work. - -A completed shard whose passed summary conflicts with a known nonzero worker exit produces a `result-report/summary-validation` provider failure. The shard is non-retryable, but the provider failure is infrastructure-incomplete in the normalized report, so the run has `status: "incomplete"` and `testStatus: "unknown"`. No failed test is synthesized from the contradictory exit. - -A process leak makes every surface named by `owningSurfaceIds` failed with `infraStatus: "incomplete"`, `failureKind: "process-leak"`, and `failureStep: "surface-cleanup"`. Native failures and recovered flakes remain present. A stale owner found before test execution retains the original `owningSurfaceIds` and is represented by incomplete synthetic lifecycle surfaces when those owners are outside the current selection; its failure fields are `stale-owner` / `startup-owner-cleanup`, no selected test surface starts, and a selected owning surface does not contribute to `surfacesRun`. The report counts unique `leakId` values, because one leak from a combined Vitest run can be attached to several surface results. Lifecycle failures are not runner retries; [flaky-tests.md](./flaky-tests.md) owns that retry boundary, and [test-runner.md](./test-runner.md) owns detection and cleanup. - -### Compact discipline - -`summary.json` enumerates failed tests, recovered flakes, and process leaks but never enumerates passed test cases; counts represent observed passes. Each enumerated test is compacted to file, line, title, attempts, `flakyAnnotated`, status, and an `errorSummary` truncated to 800 characters with ANSI codes stripped. A leak is already compact and is preserved in full because PID, command, owning surfaces, and sockets are the evidence needed to identify the leaking harness. `results.json` enumerates each selected surface for which an outcome was retained and its leak records, while `selection` remains the authority for the complete requested set. Work with no evidence receives no per-surface pass or failure. +## Timing records -`events.ndjson` is an explicit exception to compact-summary discipline. It enumerates every test attempt because its purpose is machine analysis; it is not printed into agent context as a normal command result. +`events.ndjson` records the run, its shards, surfaces and files, and every test attempt, with durations and setup phases. The same input gives the same bytes. Phases cover every step that can cost tens of seconds, such as worker wake-up, checkout, dependencies, browser readiness, planning, tests and report download, so that slow setup is visible. The summary repeats the phases. -## Timing event stream +Later invocations read earlier runs' event streams. `npm test -- baseline update` ([sharded-execution.md](./sharded-execution.md#duration-baseline)) validates the stream of every Blaxel run of the full suite it finds in `.test-runs/` and stops on one it cannot read, and those runs may come from earlier revisions of the runner. A revision that changes the stream must therefore keep accepting streams with `schemaVersion: 1`, or introduce a new version and decide how the update treats older runs. -`events.ndjson` is UTF-8 newline-delimited JSON. Each event is serialized with RFC 8785 JSON canonicalization followed by one LF byte; the file also ends in LF. Each line is one `TimingEvent`. The stream is complete only after the run has finished; it is produced from native reports and provider output, not treated as a live progress protocol. - -Test execution events form the hierarchy **run → shard → surface → file → test attempt** through parent IDs. Surface-end process-leak events are additional shard descendants that reference every owning surface event, because a leak from a combined Vitest run can belong to several surfaces. An unsharded local process has one synthetic shard with index and total `1`. A provider failure before task assignment may stop at the run or shard level, with descendants absent. Once a plan assigns a task, planned fail-fast preserves an assigned-but-not-started surface event as `skipped` with no file or leak descendants. A selected surface with no files assigned by a shard plan gets a surface event with status `not-assigned` and no file or leak descendants. +The update checks every event before using any. Each event carries `schemaVersion: 1`, the run's `runId` and a zero-based `sequence` equal to its line position. Every event except a `process-leak` event carries one of the statuses below. Blaxel runs record the timing provider `blaxel-playwright-x64-4vcpu`, and the update reads only runs with that identity. Beyond that envelope, the update uses these fields: ```ts -type EventStatus = - | "passed" - | "failed" - | "skipped" - | "timed-out" - | "interrupted" - | "not-assigned" - | "incomplete" - | "unknown"; - -interface TimingPhase { - name: string; - category: - | "schedule" - | "checkout" - | "dependencies" - | "readiness" - | "plan" - | "join" - | "test" - | "report" - | "publication" - | "verification" - | "other"; - status: "passed" | "failed" | "skipped" | "unknown"; - startedAt?: string; - completedAt?: string; - durationMs?: number; - cacheStatus?: "hit" | "miss" | "not-applicable" | "unknown"; -} +// events.ndjson: one JSON event per line, the run event first; other fields belong to the writer +type EventStatus = "passed" | "failed" | "skipped" | "timed-out" | "interrupted" | "not-assigned" | "incomplete" | "unknown"; -interface TimingEventBase { +interface EventEnvelope { schemaVersion: 1; - sequence: number; - kind: "run" | "shard" | "surface" | "file" | "test-attempt" | "process-leak"; - runId: string; + runId: string; // the same on every event of a run + sequence: number; // zero-based line position } -interface RunTimingEvent extends TimingEventBase { - kind: "run"; - commandProvider: "local" | "blaxel" | null; - timingProvider: TimingProvider; - status: "passed" | "failed" | "incomplete"; - startedAt: string; - completedAt: string; - durationMs: number; - selection: Record; - git: { - commit: string | null; - tree: string | null; - dirty: boolean; - }; - environment: { - ci: boolean; - os: string; - arch: string; - node: string; - vitest?: string; - playwright?: string; - logicalCpuCount?: number; - runnerImage?: string; - }; - shardPlanId?: string; - timingBaselineDigest?: string; - phases: TimingPhase[]; -} - -interface ShardTimingEvent extends TimingEventBase { - kind: "shard"; - shardId: string; - index: number; - total: number; - status: EventStatus; - startedAt?: string; - completedAt?: string; - durationMs?: number; - predictedDurationMs?: number; - phases: TimingPhase[]; -} - -interface SurfaceTimingEvent extends TimingEventBase { - kind: "surface"; - shardId: string; - surfaceEventId: string; - surfaceId: string; - runner: "vitest" | "playwright"; - status: EventStatus; - startedAt?: string; - completedAt?: string; - durationMs?: number; - durationSource: "surface-wall" | "owned-file-sum" | "task-wall" | "unknown"; - workerCount?: number; - retryBudget: number; - phases: TimingPhase[]; -} - -interface FileTimingEvent extends TimingEventBase { - kind: "file"; - shardId: string; - surfaceEventId: string; - fileEventId: string; - path: string; - status: EventStatus; - complete: boolean; - startedAt?: string; - completedAt?: string; - durationMs: number; - planningDurationMs: number; - durationSource: "file-wall" | "attempt-sum" | "runner-total" | "unknown"; - testsTotal: number; - retryCount: number; - recoveredFlakeCount: number; -} - -interface TestAttemptTimingEvent extends TimingEventBase { - kind: "test-attempt"; - shardId: string; - surfaceEventId: string; - fileEventId: string; - testId: string; - attemptIndex: number; // zero-based - path: string; - line?: number; - column?: number; - project?: string; - titlePath: string[]; - status: EventStatus; - startedAt?: string; - completedAt?: string; - durationMs: number; - retry: boolean; - recoveredFlake: boolean; - flakyAnnotated: boolean; -} - -interface ProcessLeakTimingEvent extends TimingEventBase { - kind: "process-leak"; - shardId: string; - surfaceEventIds: string[]; - leak: NormalizedProcessLeak; -} - -type TimingEvent = - | RunTimingEvent - | ShardTimingEvent - | SurfaceTimingEvent - | FileTimingEvent - | TestAttemptTimingEvent - | ProcessLeakTimingEvent; -``` - -### Phase metrics - -Phase metrics are standing run output, not an ad-hoc performance probe. Every canonical run writes normalized phases into `events.ndjson` and mirrors them under `summary.json.phaseMetrics`, where the run, every shard, and every surface remain individually addressable. The CLI prints run and shard phases plus the slowest surface-compute rows. `observedWallDurationMs` spans the earliest recorded setup phase through the latest recorded run phase; `run.durationMs` remains the test invocation's own wall. - -The stable `category` values let dashboards compare different execution paths without depending on provider-specific phase names. Every step capable of costing tens of seconds is represented where it applies: schedule/restore-wake, checkout/fetch, dependencies, browser or application readiness, planning, test compute, report collection/finalization, and verification phases. Dependency and browser phases record `cacheStatus` as `hit` or `miss` when the execution substrate exposes it. Provider-specific detail remains in `name`: for example, Blaxel keeps `setup:deps`, `setup:playwright-cache`, and `report-download`, while regular GitHub shards keep `dependencies`, `browser-readiness`, and `shard-plan`. - -Regular GitHub jobs query their own Actions run after checkout to measure schedule delay from workflow creation—or from the completed build prerequisite for shard jobs—to VM job start. Workflow setup phases are appended to a caller-owned NDJSON file and ingested by the canonical runner, so they survive in the same run event and summary as test compute. The required agreement job runs after shard artifacts are immutable; it therefore uploads its join phase as a distinct `test-phase-metrics--join` artifact and writes the same value to the Actions step summary rather than rewriting shard evidence from another job. - -Blaxel records sandbox acquisition/restore, checkout, dependency cache outcome, system prerequisites, owner cleanup, browser/Electron readiness, test execution, and result download per shard. Surface phases include service/application readiness, runner wall, and time from runner start to the first native test attempt when the runner reports that timestamp. A developer-machine `verify` passes its lint, type-check, package-manifest, and test-phase metrics into the resulting canonical run summary. - -Report finalization is measured while constructing the artifact. The enclosing `verify:tests` and `verify:total` phases are recorded after the child run exits and `events.ndjson` is sealed, so they are appended to `summary.json.phaseMetrics` only; the sealed event bytes are never rewritten. - -Attaching `phaseMetrics` while finalizing `summary.json` is part of producing a valid run artifact and remains load-bearing. Every later metrics append and every workflow-only sidecar is best-effort observability: a write, GitHub API lookup, parse, or upload failure emits a warning and omits that phase, but never changes the command, test, or agreement result. A wrapper that measures a real command always re-raises that command's original status after attempting the metric, so observability can neither turn green red nor mask red as green. - -All durations are non-negative integer milliseconds; fractional native durations are converted with `Math.round`. Paths are normalized repo-relative POSIX paths. A native result downloaded from a remote worker may carry an absolute path rooted at `/workspace/television/`; event construction removes that remote workspace root before validating and hashing the path, rather than interpreting it relative to the coordinator's checkout. IDs are deterministic within a run. `testId` is the lowercase hexadecimal SHA-256 digest of the UTF-8 RFC 8785 serialization of this exact array: - -```ts -["test-id-v1", surfaceId, path, project ?? null, titlePath, line ?? null, column ?? null] +type TimingEvent = EventEnvelope & ( + | { + kind: "run"; + status: "passed" | "failed" | "incomplete"; + commandProvider: "local" | "blaxel" | null; + timingProvider: string; // "blaxel-playwright-x64-4vcpu" for Blaxel runs + completedAt: string; + selection: { suite?: string | null; surfaces?: string[]; files?: string[]; grep?: string | null }; + } + | { kind: "shard"; status: EventStatus; index: number; total: number } + | { kind: "surface"; status: EventStatus; surfaceEventId: string; surfaceId: string } + | { + kind: "file"; + status: EventStatus; + surfaceEventId: string; + path: string; // repository-relative + complete: boolean; // false when the native report cannot account for every attempt + planningDurationMs: number; // the file's duration without its retry attempts + retryCount: number; + recoveredFlakeCount: number; + } + | { kind: "test-attempt"; status: EventStatus } + | { kind: "process-leak" } +); ``` -`titlePath` remains a nested JSON array, so its entry count and boundaries are part of the input. `line` and `column` are non-negative JSON integers or `null`; `project` is a string or `null`. This makes absent values distinct from strings and title entries. The ID is an event identity, not the duration-baseline key; duration planning is file-based. - -A leak record represents one surviving process captured at surface end that required a cleanup signal or remained after cleanup. A candidate that revalidation finds already exited before either cleanup signal is omitted rather than reported as a leak; `cleanup.outcome: "already-exited"` remains valid when an earlier signal was sent and the process exits before a later cleanup step. `leakId` uses the same digest and canonical UTF-8 rules over `["process-leak-id-v1", ownerToken, pid, processStartedAt]`, with `pid` encoded as a JSON integer. `owningSurfaceIds` and `surfaceEventIds` are non-empty, deduplicated, and lexicographically sorted; they contain several entries only for a shared execution scope such as the combined Vitest run. `processStartedAt` and `detectedAt` are UTC ISO-8601 timestamps. `command` is the executable path or executable name only, never argv. Socket records are captured before cleanup, use ports 1–65535, and sort by protocol, family, host, and port. `cleanup.outcome: "survived"` keeps the run incomplete after the final scan. - -`durationMs` on a file is its observed elapsed wall when the native runner exposes one and otherwise the sum of its attempt durations. `planningDurationMs` subtracts the durations of retry attempts from that observed duration; when no file wall is available it is the sum of first attempts. This keeps setup and hook time where the runner exposes it while keeping retry time out of ordinary shard weights. A file is `complete: false` when the native data is insufficient to enumerate every attempt or the runner stopped during that file; incomplete files never feed the duration rollup. - -Every native attempt becomes exactly one `test-attempt` event. For a test that failed and then passed, every attempt carries `recoveredFlake: true`; `retry` is true when `attemptIndex > 0`. A final failure after retries carries `recoveredFlake: false`. The normalizer must preserve an unknown value as `status: "unknown"` or mark the file incomplete; it must not invent a passing attempt from aggregate counts. +The event stream and the initial summary are required parts of a run: failing to write them fails the command. Optional diagnostics are best-effort whenever they are collected. They include resource probes, metrics that GitHub workflows record, and phases appended later; verify, for example, adds its own phases to its test run's summary after the test command finishes. A failure to record or upload a diagnostic is a warning and never changes a command's or a test's result. -Playwright 1.58 JSON contains ordered result records with retry indices and durations and is sufficient input. Vitest's built-in JSON supplies final assertion outcomes, not the complete ordered attempt record needed here. Reporter callbacks likewise lack each retry attempt's duration. Vitest execution attaches a custom runner-lifecycle recorder through each generated project or standalone run config's `test.runner`; it records each attempt's identity, index, status, and duration alongside the built-in file report while delegating ordinary behavior to `VitestTestRunner`. The normalizer joins those records by test identity. It never derives attempt count, order, or duration from `failureMessages`. ^vitest-attempt-recording +## Nothing secret -Events are ordered deterministically: run first; then shards by index; within a shard, surfaces in registry execution order; files by path; tests by declared location and title path; attempts by index; then surface-end process leaks by owning-surface list, PID, and process start time. `sequence` starts at zero and increases by one with no gaps. This ordering makes a stream byte-stable for the same normalized input. +GitHub CI uploads run directories as workflow artifacts, which anyone who can read the repository can download. Run directories therefore never contain credentials. The timing records contain no environment values, command arguments, output or error text, and process records give the executable without its arguments. -The event stream contains no environment-variable names or values, stdout, stderr, command arguments, error messages, attachments, or source contents. `RunTimingEvent.environment` is an allowlist of non-secret runtime facts produced directly by the runner, not a copy of `process.env`. A process-leak `command` is limited to the executable identity defined above and excludes arguments. Raw process-table and socket-tool output remains elsewhere in the run directory. This boundary keeps the event stream a compact, secret-free record for machine analysis. +## Resource diagnostics -## GitHub Actions artifacts +Blaxel workers have memory-backed temporary storage, and tests there can die of memory exhaustion. Each Blaxel surface execution therefore records, at its start and end: -Every GitHub Actions test job uploads its full `.test-runs//` directory as `test-run-` with `if: always()`, `if-no-files-found: error`, and 30-day retention. Matrix jobs use distinct run IDs and artifact names. Artifacts contain the native reports and logs needed for diagnosis. Credentials are never copied into a run directory or event. +- available memory; +- shared memory in use; +- free space on its temporary filesystem; +- the kernel's out-of-memory kill counter. -## Testing +The start is saved before the surface prepares, so a crash during the surface still leaves it. When a native attempt failed or the evidence is incomplete, the record adds the kernel's out-of-memory messages from that interval. Readings that could not be taken are recorded as unavailable, never as zero. These records give context and a change in the kill counter. They do not measure peak memory, and a missing kernel message does not prove that no kill happened. ^resource-diagnostics -A seam must exercise the handoff from the installed Vitest runner's lifecycle to the production attempt recorder. It must preserve separate failed and passed attempts and their durations. Under the testing policy's [compositional coverage rule](../testing-policy.md#Compositional coverage across clean seams), fixtures made from recorded native reports may cover the other normalization cases. +Each record identifies its run, shard attempt and surface, and the surface's result links to it. Records survive retries and download, and an infrastructure retry does not overwrite the records of the attempt it superseded. They are bounded per surface and per shard attempt so they cannot exhaust a worker, and they never change a test outcome. A [recovery](./blaxel-testshards.md#abandoned-runs) leaves a receipt among the new attempt's records under the same bounds. The receipt says which run was abandoned, which processes were stopped and whether undownloaded reports were discarded. diff --git a/specs/arch/test-runner/sharded-execution.md b/specs/arch/test-runner/sharded-execution.md index 07f702b8..c5d36d8d 100644 --- a/specs/arch/test-runner/sharded-execution.md +++ b/specs/arch/test-runner/sharded-execution.md @@ -1,341 +1,37 @@ -*The shared remote execution model: deterministic duration-aware plans, per-worker execution, provider reports, retries, and normalization.* +*How suite runs divide test files among parallel machines, the committed durations that balance them, and how shard outcomes combine into one honest result.* -# Sharded Execution +# Sharded execution -Remote test runs divide the repository's test files among parallel workers and combine their outcomes into one report. A committed baseline of file durations measured on Blaxel keeps those workers balanced while an explicit plan makes every assignment reproducible and skips runner startup where a shard has no work. +Blaxel suite runs and GitHub CI's end-to-end matrix divide the selected test files among parallel machines, called *shards*, and combine what each shard reports. Both use the shared shard worker, `scripts/run-test-shard.mjs`. The rules here make that division complete and reproducible, so that a passing result accounts for every selected file. Blaxel's own coordination is owned by [blaxel-testshards.md](./blaxel-testshards.md), and the CI workflow by [github-ci.md](./github-ci.md). -This spec is authoritative for shared sharded execution: the committed file-duration baseline, manual baseline update, shard-plan contract and algorithm, shard-count recommendation, the shard worker used by Blaxel and regular GitHub CI, Blaxel provider report shape, retry-attempt combination, the *infrastructure-failure*-versus-*test-failure* distinction, and normalization into [reporting.md](./reporting.md). The Blaxel coordinator extends this model in [blaxel-testshards.md](./blaxel-testshards.md); provider git safety and authentication are owned by [preflight.md](./preflight.md). Regular GitHub CI invokes the worker directly as matrix jobs and has no dispatch coordinator. Test surfaces, file ownership, and execution-group order come from [test-registry.md](./test-registry.md). +## Plans -## Duration baseline - -The repository contains exactly one committed duration rollup: - -``` -test/timing-baseline.json -``` - -It is a file-level planning input, not a pass attestation and not a substitute for the per-run event stream. Its measurements come only from Blaxel runs, where broad verification normally runs on uniform persistent sandboxes. Every shard plan weighs files by these measurements, whichever substrate executes it ([#Provider weights and unknown files](#Provider weights and unknown files)). - -```ts -type TimingProvider = - | `local-${"linux" | "darwin"}-${"x64" | "arm64"}-${number}cpu` - | "blaxel-playwright-x64-4vcpu" - | "github-ubuntu-24.04-x64-2vcpu-vm" - | "github-playwright-noble-x64-2vcpu"; - -interface ProviderFileTiming { - medianDurationMs: number; - sampleCount: number; - retryCount: number; - recoveredFlakeCount: number; - lastUpdated: string; -} - -interface BaselineFileTiming { - surfaceId: string; - providers: { "blaxel-playwright-x64-4vcpu": ProviderFileTiming }; -} - -interface TimingBaseline { - schemaVersion: 1; - sourceThrough: string | null; - files: Record; // repo-relative POSIX path -} -``` - -The `providers` map names the timing provider that produced each measurement. It holds exactly one entry, for the Blaxel timing provider; an entry with any other key makes the baseline invalid. - -Before any run history is available, the committed file is the canonical empty baseline: - -```json -{ - "schemaVersion": 1, - "sourceThrough": null, - "files": {} -} -``` - -`sourceThrough: null` means no run has been examined. The planner requires the committed file and rejects a missing or malformed baseline; the empty baseline is valid input, has a stable raw-byte digest, and gives every planned file the 60,000 ms no-history weight. - -A *timing provider* identifies the execution substrate that determines duration, not the command used to reach it. Every run event records one ([reporting.md](./reporting.md)), and every shard plan names the substrate that executes it. A local ID records operating system, architecture, and positive decimal logical-CPU count, for example `local-linux-x64-16cpu`, so materially different local hosts do not share an identity. The Blaxel ID pins its current architecture and CPU tier. Regular GitHub CI records `github-ubuntu-24.04-x64-2vcpu-vm` for jobs on the plain hosted VM, including the direct 16-worker e2e matrix. Unit and desktop CI jobs that invoke `npm test -- local` therefore record command provider `local` and the GitHub VM timing provider, never a `local-*` timing provider. Direct e2e matrix workers have no command provider and record the same GitHub VM timing provider. `github-playwright-noble-x64-2vcpu` is reserved for the deferred pinned Playwright-container substrate and receives no active workflow measurements. A future substrate gets a distinct reviewed ID when provider, image family, architecture, or CPU tier changes materially. An ID outside the listed literals or local pattern is invalid; local CPU counts must be positive base-10 integers. - -The execution host receives its ID as `TV_TEST_TIMING_PROVIDER`. Blaxel coordinator requests and regular GitHub workflows set it explicitly. A non-CI direct local run defaults it to `local-${process.platform}-${process.arch}-${availableParallelism()}cpu`. A process with `GITHUB_ACTIONS=true` fails reporting initialization when its timing provider is absent or begins with `local-`, including when a transferred plan supplies the provider directly to the run context. It never guesses from the `local` command path, so a hosted run never records a developer-host substrate. - -`medianDurationMs` is the median of the file events' `planningDurationMs` values from [reporting.md](./reporting.md). Eligible duration samples are complete files whose final status is `passed`, including recovered flakes because retry time has already been removed. `sampleCount` is the number of those samples. `retryCount` is the number of attempts after attempt zero, and `recoveredFlakeCount` is the number of tests that failed and then passed, summed across all complete observations for that file in the input runs. `lastUpdated` is the latest completion time among those observations. `sourceThrough` is the latest completion time among the input runs, including runs that contributed no eligible duration sample. - -For an odd sample count, the median is the middle value after numeric sort. For an even count, it is the arithmetic mean of the two middle values rounded to the nearest integer, with `.5` rounded up. Durations are non-negative integer milliseconds, sample counts are positive integers, timestamps are UTC ISO-8601, paths are normalized repo-relative POSIX paths, and each path's `surfaceId` must match current registry ownership. Deleted and currently unowned paths are omitted. - -Keys are written in lexicographic path order. The complete value is serialized as `JSON.stringify(value, null, 2)` plus one LF byte. Given the same input runs and checkout, baseline generation produces identical bytes. - -### Manual update - -The only writer is: - -```bash -npm test -- baseline update -``` - -The command reads the checkout's `.test-runs/` directory. Each direct child directory containing `events.ndjson` is a candidate. A candidate is an *input run* when it is the complete run a Blaxel `verify` produces: - -- its run event records command provider `blaxel`, the `blaxel-playwright-x64-4vcpu` timing provider, and a selection of the `all` suite with no file or grep filter; -- its selected surfaces include every surface the current registry places in the `all` suite; -- its shard events cover every index of one shard total; -- its run status is not `incomplete`, and every surface event ended `passed`, `failed`, or `not-assigned`. - -The `all` label alone is not evidence of a complete run: surface, package, runner, and tag filters and shard subsets keep it, and so does a run that stopped early. Because the update carries nothing forward, such a run would otherwise replace the measurements of every file it did not execute. Local, GitHub, narrowed, and unfinished runs are ignored. The command validates, under [reporting.md](./reporting.md), the complete event stream of each candidate whose run event meets the first condition, recomputes the complete rollup from the input runs, and atomically replaces `test/timing-baseline.json`. It prints added, removed, and changed file entries plus the shard-count recommendation described below. `--dry-run` performs the same validation and calculation without changing the file. - -A candidate whose run event cannot be read, or a Blaxel `all` candidate whose event stream fails validation, aborts the update without changing the baseline and names the directory. With no input runs, the command fails without changing the baseline, so running it in a checkout without Blaxel history cannot erase the committed measurements. The command does not commit or push its result. Test runs and CI never edit the baseline automatically; a person runs the command in a checkout that holds recent Blaxel run directories, reviews the diff, and commits the one rollup file. - -A baseline entry is derived only from runs the command actually read. Existing values are not carried forward, so a file that no input run measured receives the unknown-file weight in later plans. - -## Shard plan - -A *shard plan* assigns every selected test file to exactly one of `N` shards. Files are atomic: a plan never divides tests within one file. The plan contains all selected surfaces, including an empty file list when a shard has no assignment for a surface. - -```ts -type PlanWeightSource = "baseline" | "unknown-surface" | "unknown-timing-provider"; - -interface PlannedFile { - path: string; - weightMs: number; - weightSource: PlanWeightSource; - baselineSampleCount?: number; -} - -interface PlannedSurface { - surfaceId: string; - runner: "vitest" | "playwright"; - files: PlannedFile[]; -} - -interface PlannedShard { - index: number; // one-based - total: number; - predictedDurationMs: number; - surfaces: PlannedSurface[]; -} - -interface ShardPlan { - schemaVersion: 1; - planId: string; - timingProvider: TimingProvider; - testedCommit: string; - testedTree: string; - shardTotal: number; - selectedSurfaceIds: string[]; - timingBaselinePath: "test/timing-baseline.json"; - timingBaselineDigest: string; - shards: PlannedShard[]; -} -``` - -A valid plan assigns at least one test file across the complete run. Individual surfaces and individual machines may still receive empty file lists; those per-shard idle assignments are represented as `not-assigned` and are normal outcomes of splitting a smaller category across more machines. - -`selectedSurfaceIds` and every per-shard surface array follow registry execution order; file arrays are lexicographic. Canonical JSON means the RFC 8785 JSON Canonicalization Scheme encoded as UTF-8. `planId` is the lowercase hexadecimal SHA-256 digest of the canonical plan with `planId` omitted. `timingBaselineDigest` is the lowercase hexadecimal SHA-256 digest of the committed baseline file's raw UTF-8 bytes, including its final newline. Plans contain no generation time, machine-local path, random value, or mutable remote input. The plan is retained under the run directory's `provider/` output and its ID is copied into provider reports and timing events. - -### File inventory - -The planner derives its inventory from tracked paths and registry ownership in the tested tree. It enumerates the target index with `git ls-files --cached -z`; a coordinator planning a commit other than its checkout first loads that commit into a temporary `GIT_INDEX_FILE` with `git read-tree`, uses the same enumeration command, and removes the temporary index. Untracked caller files and the coordinator's checked-out revision cannot affect the result. - -A tracked path inside at least one registered `root` is a candidate test file when it matches `/(?:^|\/)[^/]+\.(?:test|spec)\.(?:[cm]?[jt]s)x?$/`. Each candidate must have exactly one owner across the registry's normalized `roots` and `excludeRoots`; canonical test files outside every registered root are outside the test runner's inventory. The selected inventory contains candidates whose owner is a selected surface; paths and owners are normalized and sorted before weighting. A missing config, duplicate path, zero or multiple owners, path outside the tested tree, or disagreement between assignment and worker collection is a configuration failure, never an empty passing shard. The registry and filename convention are the planning authority; planner processes do not import runner configs or test modules. - -Workers load the owning Vitest or Playwright config only when executing their assigned files. The assigned-versus-collected comparison is the runner-aware guard: it catches a config that rejects an assigned path, broadens a substring filter, or reports a different normalized identity. This split keeps independent GitHub jobs deterministic without weakening the execution boundary. - -### Provider weights and unknown files - -A file uses its baseline `medianDurationMs` when the baseline entry's `surfaceId` matches the file's registry owner, whatever timing provider the plan names. A GitHub CI plan therefore balances its matrix with Blaxel measurements. Balancing depends on the files' relative costs, which Blaxel measurements approximate for other substrates; the plan's predicted durations are in Blaxel time and do not predict wall time on another substrate. - -A file is *unknown* when the baseline has no entry for its path or its registry owner differs from the entry's `surfaceId`. Its weight is intentionally generous: - -1. Take the nearest-rank p90 of known file medians for the same surface. If the surface has no known medians, take the nearest-rank p90 across the baseline. -2. When a p90 exists, use `max(30_000, ceil(1.5 × p90))` milliseconds. -3. When the baseline has no known medians, use `60_000` milliseconds. - -Nearest-rank p90 selects sorted element `ceil(0.9 × count)`, using one-based indexing. A known baseline duration uses `max(1, medianDurationMs)`. A known file records `weightSource: "baseline"`. An unknown file records `"unknown-surface"` when either the same-surface p90 or baseline-wide p90 supplies its weight, and `"unknown-timing-provider"` when the baseline has no medians and the 60-second fallback applies. Unknown files are therefore spread as expensive work rather than collected on a shard that appears cheap. - -### LPT assignment +Before any shard runs, a *shard plan* assigns every selected test file to exactly one of the run's shards. Files are never split between shards. -The planner applies longest-processing-time-first scheduling globally across all selected surfaces: +- The file inventory comes from the files tracked in the tested commit and the registry's ownership rules. Planning never imports runner configs or test modules, and never looks at the coordinator's own working tree. +- Each shard runs exactly its assigned files. After the native runner has collected tests, the shard checks that it collected exactly those files and no others; any difference fails that surface as an infrastructure problem. A shard with no files for a surface skips that surface entirely: no build step, services or runner start. +- Plans are deterministic. The same commit, selection, shard count, timing provider and baseline give the same plan, identified by a digest of its content. This matters because each GitHub CI matrix job computes the plan for itself, and CI's required join then checks that every shard ran the same plan and collected exactly its files ([github-ci.md](./github-ci.md#required-checks)). The Blaxel coordinator computes the plan once and sends it with each shard; the worker checks it before running anything. The coordinator and the workers can run different revisions of the runner, so the plan, the baseline and the shard summary cross revisions. Their shapes are left to the code, under the rule and the stated exception in [blaxel-testshards.md](./blaxel-testshards.md#runs). +- Shards are balanced by expected duration from the baseline below. A file with no recorded duration is weighted as expensive, so that new files spread across shards instead of collecting on one that looks cheap. Predictions add up per-file durations and leave out per-surface startup and runner parallelism, so they guide the balance but do not predict wall time. +- The plan is fixed for the whole run. Shards do not pull work from a queue, and an infrastructure retry reruns the same assignment. -1. Sort files by descending `weightMs`; break ties by `surfaceId`, then path, both ascending. -2. Start `N` bins with predicted duration zero. -3. Assign each file to the bin with the lowest predicted duration; break bin ties by lower one-based shard index. -4. Add the file weight to that bin and continue until every file is assigned. -5. Materialize every selected surface in every shard, sorting its assigned files by path. - -This algorithm prevents independently heavy surface slices from accumulating on the same worker. `predictedDurationMs` is the sum of assigned file weights. It is a comparable planning estimate, not a promise of wall time: runner worker counts, setup, and host contention can make observed wall time differ. - -The plan is static for the run. Workers do not pull from a queue, and infrastructure retries reuse the same plan and shard assignment. This preserves reproducibility and avoids repeating process and web-server startup for small dynamic batches. - -### Stage-one cost-model boundary - -The first planner deliberately uses additive file medians only. It does not divide a surface's assigned weight by its runner worker count and does not charge a fixed cost when the first file activates a surface on a shard. This keeps the committed rollup to the ratified per-file format and avoids encoding guessed setup constants before normalized phase events exist. - -The tradeoff is explicit: a two-worker browser surface can have its shard wall overestimated by about 2×, and global LPT may spread a small surface across more shards and forfeit part of empty-surface skipping. Existing empty-surface startup observations reach 24 seconds on one shard, so predicted durations are advisory and shard count must not be raised from this model alone. - -The model must be revised before any shard-count increase, or earlier when three consecutive successful broad runs on one timing provider show either (a) fixed surface phases above 10 seconds or 10% of the critical shard, whichever threshold is lower, or (b) an observed surface wall differing from its additive assigned-file weight by more than 25%. That revision adds provider-and-surface fixed-cost and effective-worker terms from recorded run events, bumps the plan schema, and replays the same captured inventories before adoption. This spec owns that back pressure; it is not an untracked implementation improvement. - -### Determinism and GitHub plan agreement - -Every regular GitHub CI matrix job invokes the shard worker directly and computes the complete plan in-job from the checked-out commit, selected surfaces, timing-provider ID, shard count, and committed baseline. Before execution, a job recomputes the digest from the canonical plan body and rejects an internally inconsistent `planId`. No job consults a mutable timing source while planning. - -Internal digest verification does not establish cross-job agreement. Every shard summary carries `planId`, `timingProvider`, tested tree, shard index/total, assigned files, and collected files. The required GitHub CI `e2e-required` join downloads every shard summary and fails unless: indices `1..N` appear exactly once; all plan IDs, timing providers, tested trees, and totals are equal; and each summary's assigned and collected files agree. [github-ci.md](./github-ci.md) is authoritative for that join and its branch-protection result. This consensus check detects environment-dependent inventory or planning that could otherwise duplicate or omit files while individual jobs pass. - -The Blaxel coordinator computes the complete plan once from the target commit before leasing workers, places the plan in each worker request, and tells each worker which shard index to execute. A worker validates the plan digest, tested commit/tree, timing provider, index, and total before starting a surface. - -## Explicit file execution - -A planned worker invokes runners with the assigned files and does not pass native `--shard` flags. - -- Vitest receives absolute file paths resolved from the normalized repo-relative assignments. Vitest treats positional filters as substrings, so the worker first uses `String.includes` against the complete planned inventory and requires every repo-relative filter to match exactly one file: itself. An ambiguous filter fails plan validation before Vitest starts. The post-collection assigned-versus-collected check remains mandatory as a second boundary. The `unit:workspaces` execution group remains one combined Vitest process for all unit files assigned to that shard. -- Playwright receives a generated `--test-list` file containing one config-root-relative file path per line. A line with only the file path selects every test in that file without treating path characters as a regular expression. Custom Playwright commands, including the Electron wrapper, forward this option unchanged. -- Generated Playwright list files and combined Vitest config files live under the shard output directory and are retained with provider output. - -The worker verifies after collection that every assigned file was collected by its owning surface and that no unassigned file ran. A mismatch fails the surface as infrastructure-incomplete with the plan and observed lists preserved. - -When a planned surface has no files on a shard, the worker records it as `not-assigned` and skips its pre-command, runner process, web server, global setup, and cleanup hooks. When an entire execution group has no assigned files, the worker skips the group process. At aggregation, a surface is selected and run when at least one shard owns one of its files; per-shard `not-assigned` records do not turn that aggregate surface into a skipped surface. - -A caller that has supplied all prebuilt artifacts may pass `--skip-pre-commands` or set `TEST_SHARD_SKIP_PRECOMMANDS=1`. The worker then skips every assigned surface's registry `preCommand`, records that phase as skipped with reason `prebuilt-artifact`, and still runs the configured test command, web server, global setup, and cleanup. The flag never makes the worker infer whether artifacts are sufficient and never falls back to a build: the caller owns artifact completeness, and a missing artifact must fail through the real test command. Without the flag, assigned surfaces retain their registry pre-commands. - -This planned execution contract is an explicit exception to native per-runner sharding. Native `--shard=/` remains available to targeted diagnostics outside a provider plan, but shared remote suite execution uses only the explicit plan. - -## Shard-count scaling - -Planning accepts an explicit shard count from the Blaxel invocation or GitHub workflow. The committed count remains a reviewed coordinator/workflow setting; a run never changes its own matrix or pool demand. - -A manual baseline update reports a recommendation for the Blaxel shard count. For candidate counts from 1 through the Blaxel pool size ([blaxel-testshards.md](./blaxel-testshards.md)), it plans the `all` suite with the exact LPT algorithm and chooses the smallest count whose maximum predicted shard duration is at most **120,000 ms**. This target keeps planned file work near the program's two-minute test budget while avoiding unbounded worker growth; provider setup time and internal runner parallelism remain visible beside the estimate. The command also reports: - -- the current configured count and its predicted maximum; -- the provider's configured recommendation capacity; -- total predicted file work; -- the heaviest atomic file; -- the arithmetic lower bound `max(heaviest file, ceil(total weight / shard count))`; -- an explicit “split or optimize this file” warning when the heaviest file alone exceeds 120,000 ms. - -Blaxel capacity is owned by [blaxel-testshards.md](./blaxel-testshards.md). A recommendation never overrides Blaxel pool constraints or concurrency limits. A person changes the committed count after reviewing the recommendation and available capacity. This gives suite growth a measurable scaling path without automatic infrastructure expansion. The GitHub matrix count is a measured setting owned by [github-ci.md](./github-ci.md#CI worker counts); because the baseline holds Blaxel time, the update makes no GitHub recommendation. - -## Shared shard worker - -Blaxel workers and regular GitHub CI matrix jobs run `scripts/run-test-shard.mjs`. - -- Selection comes from the plan file passed as `--plan ` or `TEST_SHARD_PLAN=`; `--plan` always requires a file and has no diagnostic or dry-run meaning. Surfaces are grouped by execution group ([test-registry.md](./test-registry.md)) and run in order, with the `unit:workspaces` group run as one Vitest process when it has assigned files, using the combined configuration described in [test-runner.md](./test-runner.md#Surface process lifecycle). -- `--test-retries ` applies `--retries` or `--retry` to non-unit surfaces only; unit surfaces never retry. The worker injects `FLAKY_TEST_RETRIES` for child runners (see [flaky-tests.md](./flaky-tests.md)). -- The runner-child environment starts with the host environment, the retry default, and the repository's `node_modules/.bin` prepended to the host `PATH`. Surface task values and declared-service values apply in that order. Immediately before launch, the worker removes both `FORCE_COLOR` and `NO_COLOR` from the completed environment. -- `--skip-pre-commands` or `TEST_SHARD_SKIP_PRECOMMANDS=1` applies the prebuilt-artifact contract from [#Explicit file execution](#Explicit file execution). -- `--fail-fast` stops the shard after the first failed group or task. Every later task that had files assigned by the plan remains in the shard summary with `status: "skipped"`, `skipReason: "fail-fast"`, its assigned files, and no collected files or runner artifacts. -- The worker writes `shard-.json` into its results directory, default `.testshards/results`, and exits non-zero if any task failed or plan validation failed. - -## Blaxel provider report shape - -The Blaxel coordinator collects the plan and per-shard summaries into a provider report. Regular GitHub CI retains each direct worker's shard summary and timing stream as job artifacts; the required agreement join is owned by [github-ci.md](./github-ci.md) under the invariants above. - -```ts -interface ProviderTask { - surfaceId?: string; - runner?: string; - command?: string; - args?: string[]; - status: "passed" | "failed" | "skipped" | "not-assigned"; - skipReason?: string | null; - suite?: string; - name?: string; - exitCode?: number | null; - durationMs?: number; - infraStatus?: "completed" | "incomplete"; - predictedDurationMs?: number; - assignedFiles?: string[]; - collectedFiles?: string[]; - preCommandsSkipped?: boolean; - failureKind?: string | null; - failureStep?: string | null; - processLeaks?: NormalizedProcessLeak[]; - nativeResultPath?: string | null; - [key: string]: unknown; -} - -interface ProviderShard { - shardIndex: number; - shardTotal?: number | null; - planId?: string; - timingProvider?: TimingProvider; - testedTree?: string; - status: "passed" | "failed" | "infra-failed"; - durationMs?: number; - predictedDurationMs?: number; - tasks?: ProviderTask[]; - logPath?: string | null; - leaseAcquired?: boolean; - executionStarted?: boolean; - outputDir?: string; - nativeResultPath?: string | null; - [key: string]: unknown; // sandbox, failureKind, failureStep, failureMessage, setupTimings, ... -} - -interface AttemptReport { - startedAt?: string; - completedAt?: string; - durationMs?: number; - interruptedBy?: "SIGINT" | "SIGTERM"; - plan?: ShardPlan; - run?: { createdAt?: string; updatedAt?: string; [key: string]: unknown }; - shardIndices?: number[]; - shards?: ProviderShard[]; - [key: string]: unknown; -} - -interface CombinedReport extends AttemptReport { - status: "passed" | "failed"; - startedAt: string; - completedAt: string; - durationMs: number; - plan?: ShardPlan; - shards: ProviderShard[]; - failedTasks: Array>; - infraFailedShards: number[]; -} -``` - -`NormalizedProcessLeak` is defined by [reporting.md](./reporting.md). Generated TypeScript declarations must preserve these status unions. - -## Infrastructure failure vs test failure - -A shard's status is `passed`, `failed`, or `infra-failed`. The status describes whether the shard completed before it describes any individual test observation. - -- `infra-failed` means the shard's execution or result evidence is incomplete. Setup, plan or collection validation, transport, interruption, and an unavailable completion result are infrastructure failures. Reaching the test command or retaining some task observations does not establish that the shard completed. -- A *test failure* is a completed shard with usable native or task evidence that reports a failed test. It is `failed` and is not eligible for an infrastructure retry. -- A passed summary and a known nonzero worker exit are contradictory completed evidence. The shard is `failed` and non-retryable, but the contradiction is reported as `result-report/summary-validation` with no invented failed test. A passed summary with no known worker exit is `infra-failed` because completion is unconfirmed. -- A process leak is a completed lifecycle failure. Its task and shard are `failed` and are not retried as infrastructure; its normalized surfaces remain infrastructure-incomplete because cleanup was not confirmed. Native test failures and recovered flakes remain visible beside the lifecycle cause. -- An unreadable or unparseable summary is `infra-failed`. The read or parse cause is retained without creating a test or failed-task record. - -An infrastructure-failed shard may retain known passed, failed, skipped, or not-assigned task observations. Those observations describe only the evidence recovered from that attempt. They do not change the overall run from unknown to passed or failed while the shard remains incomplete. - -The Blaxel coordinator classifies provider shards as owned by [blaxel-testshards.md](./blaxel-testshards.md). Regular GitHub CI receives the worker's exit and shard summary directly. This spec owns the shared vocabulary and report fields. - -## Report files, retries, and final results - -Each provider attempt writes a full `report.json` and compact `summary.json`. Planned attempts also retain `shard-plan.json`. An unreadable shard summary becomes an infrastructure-failed shard with its read or parse cause and no synthetic test task. - -The shard retry condition and supersession rule are owned by [test-runner.md#Retries](./test-runner.md#Retries). `stale-owner` is the provider-specific exception defined by [blaxel-testshards.md](./blaxel-testshards.md). Regular GitHub CI has no provider retry coordinator; an infrastructure-failed matrix shard fails its job and the required join. - -Attempt history and retained per-attempt files remain available for diagnosis. The combined report takes the result returned by the latest attempt that requested each shard and retains the existing result for shards outside that retry subset. A retry that returns no result contributes a `missing-result/attempt-aggregation` infrastructure result. A requested shard missing from every attempt receives the same infrastructure classification. Every planned attempt must carry the same complete plan and plan ID; disagreement invalidates aggregation. - -The combined report keeps the original requested shard set, while attempt history records the subset requested in each attempt. Its failed-task list is rebuilt from the retained final shard results. Only tasks whose status is exactly `failed` appear there or in the coordinator's `Failed task` output. `skipped` and `not-assigned` tasks remain visible in shard and surface results but are not reported as failures. Failed-task observations from a superseded infrastructure attempt remain in that attempt's artifacts and do not enter the combined failure list. If the final retained shard is still infrastructure-failed, any task observations recovered from it remain diagnostic evidence while the normalized run's test status is unknown. The combined provider report passes only when every retained shard passed and its rebuilt failed-task list is empty; every other provider outcome is failed. - -Provider tasks are normalized by selected surface. Counts, failed tests, recovered flakes, assignments, and collected files are combined across retained shard parts; process leaks are deduplicated by `leakId` and attached to every owning surface. A `not-assigned` part contributes no execution duration, test count, or leak. A fail-fast `skipped` part contributes its assignment but no test count. When another part of that surface ran, the aggregate is counted as run and retains both assigned and collected files, while its skipped status records that the complete assignment did not finish. +## Duration baseline -Each retained infrastructure-failed shard produces its own provider failure record so its cause remains attributable. Contradictory completed evidence also produces an infrastructure-incomplete provider record: the shard is non-retryable, while the normalized run remains `incomplete` with `testStatus: "unknown"`. When no task or native-result evidence exists, one infrastructure-incomplete `provider:` result reports the missing evidence with zero test failures; no selected surface is substituted. An individual passed shard containing only `not-assigned` tasks is a known idle assignment within a nonempty plan, not missing evidence. +`test/timing-baseline.json` holds a duration for each test file, measured on Blaxel. It is the median planning duration, which leaves out time spent on retry attempts ([reporting.md](./reporting.md#timing-records)). Only complete observations of a file that finally passed supply samples; that includes recovered flakes. Failed and incomplete observations supply none, so failures and flake overhead do not inflate the weights. Every plan, including GitHub CI's, balances with these Blaxel measurements, because balance depends on the files' relative cost rather than their absolute time. -Provider and native results also produce the timing hierarchy from [reporting.md](./reporting.md), including retained plan identity, predicted durations, setup phases, file durations, native retry attempts, and one process-leak event per unique `leakId`. Output links name only files retained in the directory that owns the report; an attempt directory does not claim the root request file. +The baseline changes only when a person runs `npm test -- baseline update` in a checkout that holds recent Blaxel run directories, reviews the diff and commits it. Test runs and CI never edit it. The update recomputes every entry from the runs it reads and carries nothing forward, so it reads only complete Blaxel runs of the full suite: a partial run would erase the measurements of every file it did not run. With no such run available, the command fails without touching the file. `--dry-run` shows the change without writing it. -## Testing +The update also prints a recommended Blaxel shard count: the smallest count, up to the pool's size, whose busiest shard has at most two minutes of predicted test work, or the pool's size when no count meets that target. A person changes the configured shard counts after reviewing the recommendation; no run changes its own shard count or pool demand. -Under the testing policy's [rule for platform-dependent behavior](../testing-policy.md#^platform-breadth), coverage for `resolveTimingProvider` must include its platform-dependent outcomes. A seam must also show that the production sources named in [#Test hooks](../../../proofs/arch/test-runner/sharded-execution.md#Test hooks) supply the values used to choose the timing provider. +## Shard outcomes -Under the testing policy's [compositional coverage rule](../testing-policy.md#Compositional coverage across clean seams), seams for explicit file assignment must launch the installed Vitest and Playwright processes. Each seam must show that its runner collects exactly the files assigned to it. The Vitest seam must launch the installed runner with the generated combined config for the `unit:workspaces` execution group. The Playwright seam must launch the default Playwright command with the generated `--test-list` file. +Each shard ends `passed`, `failed` or `infra-failed`. What decides it is whether the shard completed, before any individual test result. -A seam must also pass the generated `--test-list` file through the real Electron wrapper process. The default Playwright command cannot prove that the custom command declared in the registry forwards the list unchanged. +- `infra-failed`: the shard's execution or evidence is incomplete. Examples are a setup failure, a plan or collection mismatch, a transport failure, an interruption, a missing or unreadable summary, or a passed summary with no known exit status. Reaching the tests, or recovering some results, does not make a shard complete. On Blaxel an infrastructure failure is retried up to twice with the same assignment. A completed retry supersedes the failed attempt, whose evidence stays in the run's attempt history. +- `failed`: the shard completed and reported a failed test or a process leak. A completed shard whose summary says it passed but whose exit status was nonzero is also `failed`; that contradiction invents no failed test, but leaves the run incomplete. Failed shards are never retried, because rerunning would hide a real result. +- `passed`: the shard completed with a passed summary and a zero exit status. -Under the testing policy's [rule for integration against paid services or complex host dependencies](../testing-policy.md#integration-against-paid-services-or-complex-host-dependencies-is-important), a live planned Blaxel suite run must cross the real Blaxel API and the real transport to the remote process. The run must show that the worker validates and executes the `ShardPlan` it receives. A `--suite target` run carries no provider plan, so it cannot prove this handoff. +The run's result takes each shard from its latest attempt. Its list of failed tests comes only from those final shard results, so failures seen in a superseded attempt do not reappear. If a shard is still infrastructure-failed at the end, the run is incomplete: its known passes and failures are reported, but its overall test status is unknown ([reporting.md](./reporting.md#honest-status)). Regular GitHub CI has no retry coordinator; an infrastructure-failed matrix shard fails its job and the required join. -Live GitHub evidence follows the [isolated CI requirements in github-ci.md](./github-ci.md#Testing). The evidence must show that independently started matrix jobs derive the same valid plan from their shared inputs. The strict join must validate the jobs' summaries and exact file assignments. +## Testing guidance +Only a live planned Blaxel suite run shows that workers receive and execute the plan the coordinator sent; a targeted run carries no plan. Only a live CI run shows that independently started matrix jobs derive the same plan; the isolated CI run in [github-ci.md](./github-ci.md#testing-guidance) supplies that evidence. diff --git a/specs/arch/test-runner/test-registry.md b/specs/arch/test-runner/test-registry.md deleted file mode 100644 index 3a6737d6..00000000 --- a/specs/arch/test-runner/test-registry.md +++ /dev/null @@ -1,181 +0,0 @@ -*The test registry: how `test.config.mjs` declares the surfaces, suites, and execution groups the test runner selects, owns files against, and validates.* - -# Test Registry - -This spec is authoritative for the test registry: `test.config.mjs` as the single source of truth for which test areas exist, the services they require, how they group into suites and execution groups, how a selection resolves to a set of them, how a file maps to the one that owns it, and how the registry is validated against the repository. The command surface that consumes the registry is owned by [test-runner.md](./test-runner.md). - -A *test surface* is one independently runnable test area — usually one runner config and the tests it covers. A workspace package may be associated with zero, one, or many surfaces; for example a package containing both `vitest.config.ts` and `vitest.e2e.config.ts` is associated with a unit surface and an e2e surface. "Surface" is the registry's word for a runnable area; it has no relation to the Television *artifact* domain concept. - -## Source of truth - -`test.config.mjs` is the registry. Nothing infers test classification — unit vs. e2e, agent-dependent vs. not, electron-specific, browser — from file names or package names; every such fact is declared in the registry. The runner loads and normalizes the registry before any selection or validation. - -A registry uses either grouped declarations or top-level surfaces. When `executionGroups` is an array, the loader flattens those groups into `surfaces`, attaches each group as metadata, and ignores any separately declared top-level `surfaces`; without `executionGroups`, it loads the top-level array. Loading also normalizes paths and fills derived fields. An explicit declared `cwd` wins. Otherwise, a surface that names a package runs from that package directory; a non-package surface using root `vitest.config.ts` or any config under `test/` runs from the repository root; other non-package surfaces run from their config directory. `roots` and `excludeRoots` are normalized to forward-slash repo-relative paths, and `absoluteConfig` is the config path resolved against the repository root. - -## Surface - -```ts -interface TestSurface { - id: string; - runner: "vitest" | "playwright"; - config: string; - kind: string; - package: string | null; - roots: string[]; - excludeRoots: string[]; - supports: ("file" | "grep" | "shard")[]; - preflight: string[]; - tags: string[]; - agent: boolean; - command: string[] | null; - preCommand: string[] | null; - services: TestService[]; - executionGroup: { id: string; name?: string; order?: number } | null; - cwd: string; - absoluteConfig: string; -} -``` - -`TestSurface` is the normalized surface that selection and ownership operate on. A surface is *declared* in `test.config.mjs` as a `DeclaredSurface`: it omits the derived `executionGroup` and `absoluteConfig` fields and makes `cwd` an optional override; loading resolves that override or derives the required normalized `cwd`. - -- `id` — stable surface identity, e.g. `unit:root`, `e2e:desktop`. -- `runner` — `vitest` or `playwright`. The runner builds a Vitest or Playwright invocation from this; any other value is unsupported. -- `config` — repo-relative path to the runner config file for this surface. -- `kind` — classification label. The kinds in use are `unit`, `e2e`, `agent`, `experiment`, `telemetry-posthog-roundtrip`, and `daemon-acceptance`; suites select on it. -- `package` — associated workspace package name, or `null` for root-level surfaces. -- `roots` — repo-relative directories (or files) this surface owns for file-ownership resolution. -- `excludeRoots` — sub-paths carved out of `roots`, so a sibling surface can own them (e.g. a package's `test/e2e` tree excluded from its unit surface). -- `supports` — which targeted selectors apply to this surface, drawn from `file`, `grep`, `shard`. Default `["file", "grep", "shard"]`. The local command-building path rejects a `--file` or `--grep` selection against a surface that does not list it; the targeted Blaxel path does not currently apply this check. -- `preflight` — names of the capability checks this surface requires before it can run locally. The checks themselves are owned by [preflight.md](./preflight.md). Default `["node"]`. -- `tags` — selection tags such as `browser`, `electron`, `node`, `skill`, and `agent`. The reserved `isolated-github-only` tag marks destructive lifecycle fault fixtures. Such a surface has `kind: "experiment"`, is excluded from `all`, and is refused before preflight or dispatch on developer-host local and every Blaxel path. It runs only from an operator-authorized, branch-only GitHub workflow named `Process Lifecycle Fault Injection` with the explicit `TV_TEST_ISOLATED_GITHUB=1` opt-in. That workflow is not committed to the repository. The environment-based identity check prevents accidental execution in trusted automation; it is not a security boundary because callers can set those environment values. -- `agent` — whether the surface depends on a live agent/inference backend. -- `command` — an explicit invocation that overrides the default runner command (e.g. an Electron e2e wrapper script). `null` means use the default runner command. -- `preCommand` — a command run before the surface (typically a workspace build). `null` means none. -- `services` — Vite services the canonical runner starts for the surface. Default `[]`. Their declaration and URL-publication contract are below; allocation and lifecycle are owned by [test-runner.md](./test-runner.md). -- `executionGroup` — the ordered group this surface belongs to, or `null`. -- `cwd`, `absoluteConfig` — derived during loading, as described above. - -## Surface services - -```ts -interface TestService { - id: string; - kind: "vite"; - config: string; - publishUrlEnv: string; -} -``` - -A surface service is test infrastructure whose lifetime belongs to the surface runner, outside the Vitest or Playwright worker process. `id` is unique within the surface. `config` is a repository-relative Vite config path. `publishUrlEnv` matches `^[A-Z_][A-Z0-9_]*$` and names the environment variable through which the runner gives the bound service URL to later services and the test command. - -Services start in declaration order and stop in reverse order. A later service may read the URLs published by earlier services, which supports a host harness that must embed another service's independently allocated URL. Each service publishes a full URL, including scheme, host, and kernel-selected port; consumers do not derive one service's address from another. Published variables are scoped to service startup and the test child environment; teardown restores any parent value so one surface cannot supply another surface's URL. The declarations are preserved unchanged across local execution, planned shard tasks, and targeted Blaxel execution. The allocation and teardown rules are in [test-runner.md](./test-runner.md). - -## Suites - -A *suite* is a named selection of surfaces, defined in the registry: - -```ts -// TestConfig.suites — the full TestConfig is defined under "Execution groups". -suites: Record; -``` - -`include` and `exclude` are lists of `key:value` rules where `key` is one of `kind`, `runner`, `tag`, or `agent`. A surface matches a rule when its corresponding field equals the value (for `tag`, when the tag is present; for `agent`, when `String(Boolean(surface.agent))` equals the value). A surface is in the suite when it matches at least one `include` rule (an empty `include` matches all surfaces) and no `exclude` rule (an empty `exclude` excludes nothing). - -The registered suites: - -- `unit` → `kind:unit` -- `e2e` → `kind:e2e` -- `e2e-ci` → `kind:e2e`, excluding `tag:electron`; regular GitHub CI plans this suite while the dedicated `e2e:desktop` job retains Electron coverage ([github-ci.md#duration-aware-matrix-execution](./github-ci.md#duration-aware-matrix-execution)) -- `agent` → `kind:agent` -- `experiment` → `kind:experiment` -- `telemetry-posthog-roundtrip` → `kind:telemetry-posthog-roundtrip` -- `daemon-acceptance` → `kind:daemon-acceptance` -- `all` → `kind:unit`, `kind:e2e` - -`all` is the broad suite and is deliberately unit + non-agent e2e: agent, experiment, live PostHog roundtrip, and production-identity daemon acceptance surfaces are not part of `all`. Agent, telemetry, and daemon acceptance surfaces must be selected explicitly with their named suites. The experiment suite contains two placement classes. Lifecycle-fault experiments carry `isolated-github-only`, so selection alone cannot bypass their execution-placement guard. `experiment:dynamic-services` is a normal runnable service fixture without that tag; its protection is exclusion from `all`, so it runs only when selected explicitly. `telemetry-posthog-roundtrip` owns the live telemetry ↔ PostHog integration surface that requires secrets. `daemon-acceptance` owns the host-mutating packed-CLI and production-service lifecycle surface. Both are excluded from CI and broad verification. Build-config-integrity coverage is a normal unit surface, `unit:build-config`, so it runs in `all` and CI without secrets. - -## Execution groups - -An *execution group* is an ordered band of surfaces declared in `test.config.mjs`: - -```ts -interface DeclaredExecutionGroup { - id: string; - name: string; - order: number; - surfaces: DeclaredSurface[]; // raw declarations, not yet normalized -} -``` - -`loadTestConfig` flattens every group's declared surfaces into the top-level `surfaces` array, normalizing each and attaching its group as `executionGroup` metadata (`{ id; name?; order? }`). It does not re-normalize the groups themselves: the loaded config's `executionGroups` field is left as the raw declarations, so it does not hold normalized `TestSurface` objects. A surface's group is read through the `executionGroup` metadata on each normalized surface, not through `TestConfig.executionGroups`. Surfaces sort by group `order` ascending, then group `name`; the runner's use of this ordering for local execution is owned by [test-runner.md](./test-runner.md). `unit:build-config` is a separate unit surface that owns `test/repo/build-config-integrity.test.ts`; it runs the real CLI build script and inspects the bundled output, so it is kept out of the general `unit:root` surface while still belonging to `all`. - -The loaded registry returned by `loadTestConfig`: - -```ts -interface TestConfig { - suites: Record; - executionGroups?: DeclaredExecutionGroup[]; // raw declarations, as authored - surfaces: TestSurface[]; // declared surfaces, flattened and normalized -} -``` - -The `.d.mts` sidecar types both the raw group surfaces and the loaded surfaces as `TestSurface` (and declares `executionGroups?: TestExecutionGroup[]` whose `surfaces` are `TestSurface[]`), conflating the authored declaration with the normalized result. The split above — `DeclaredSurface` / `DeclaredExecutionGroup` as authored, `TestSurface` / `TestConfig` as loaded — is the intended contract. - -## Selection - -```ts -interface SelectionOptions { - all?: unknown; - suite?: string; - surface?: string; - package?: string; - file?: string; - grep?: string; - runner?: string; - tag?: string; - [key: string]: unknown; -} - -function selectSurfaces(config: TestConfig, options: SelectionOptions): TestSurface[]; -``` - -`SelectionOptions` is the selection contract. `all` is treated as a boolean flag (alias for the `all` suite); the index signature carries the other parsed command options through without affecting selection. `selectSurfaces` applies the present filters in turn: - -- `all` / `suite` — restrict to the named suite. -- `surface`, `package`, `tag` — comma-separated lists; `surface` matches by id, `package` by associated package, `tag` requires every listed tag to be present (AND-composed). -- `runner` — restrict to one runner. -- `file` — intersect with the surfaces that own the file (see below). When the file has owners but the intersection with the other selectors is empty, `selectSurfaces` throws, naming the owning surface(s); this catches a `--file` that contradicts a co-selector. - -The mapping from command flags to these fields, and the runner-side rules for an unselectable or ambiguous file, are owned by [test-runner.md](./test-runner.md). - -## File ownership - -```ts -function owningSurfaces(surfaces: TestSurface[], file: string): TestSurface[]; -``` - -A surface owns a file when the file is inside one of the surface's `roots` and not inside any of its `excludeRoots`. Ownership is path containment, not glob matching. - -The registry is maintained so that **every committed test file under the canonical include roots has exactly one owning surface** — no file is unowned, and no file is owned by two surfaces. `excludeRoots` is the mechanism that keeps overlapping root trees disjoint. - -## Discovery and validation - -```ts -const CANONICAL_TEST_INCLUDE_ROOTS: readonly string[]; // ["packages", "test"] - -function loadTestConfig(options?: { root?: string }): TestConfig; -function validateRegistry(config: TestConfig, options?: { root?: string }): string[]; -``` - -`loadTestConfig` uses `root` as the repository for package discovery and path resolution. `validateRegistry` uses `root` as the repository for checking declared paths and discovering runner configs. Each function defaults `root` to the current working directory. - -Registry discovery and validation scan only the *canonical include roots* `packages/` and `test/`. The repository root itself is not scanned, so an ad hoc top-level folder such as `prototypes/` or `experiments/` is out of scope by being outside the include roots — not by an ignore list. A test config under such a folder is not part of the registry's concern until it is moved under a canonical include root. - -`validateRegistry` returns a list of human-readable error strings; an empty list means the registry is valid. It reports: - -- a duplicate surface `id`, -- a surface whose `config` file does not exist, -- a surface whose `roots` or `excludeRoots` entry does not exist, -- a service with a duplicate `id`, a missing Vite config, an unsupported `kind`, an invalid `publishUrlEnv` name, or a `publishUrlEnv` duplicated within its surface, -- an unregistered runner config file found under the include roots — any file matching `vitest.config.ts`, `vitest.e2e.config.ts`, or `playwright.config.ts` that no surface declares. - diff --git a/specs/arch/test-runner/test-runner.md b/specs/arch/test-runner/test-runner.md index 3a376055..2ceba99b 100644 --- a/specs/arch/test-runner/test-runner.md +++ b/specs/arch/test-runner/test-runner.md @@ -1,261 +1,196 @@ -*The canonical `npm test` and `npm run verify` command surface: how a caller selects tests, picks where they run, and runs them, with the guardrails that keep broad runs honest.* +*Television's one test command: how callers select and run tests, the guardrails that protect shared development hosts, retries, verification, the test registry, and the rules for processes and network addresses that tests create.* -# Test Runner +# Test runner -This spec is authoritative for Television's one canonical test command surface and its behavior: the commands, the selectors, provider selection, the broad-run guardrail, retry conditions, `verify` orchestration, exit codes, local execution, and the Cursor-agent environment workaround. It is the boundary spec for the testing area and delegates to its companions: the surface registry it selects from is [test-registry.md](./test-registry.md); the preflight checks it runs are [preflight.md](./preflight.md); the normalized results and run directory it writes are [reporting.md](./reporting.md); remote execution is [sharded-execution.md](./sharded-execution.md), realized by [blaxel-testshards.md](./blaxel-testshards.md); the GitHub Actions CI workflow's setup and execution contract is [github-ci.md](./github-ci.md). The discipline every test is held to is [testing-policy.md](../testing-policy.md). +Television has one way to run its tests: `npm test -- ` and `npm run verify`, implemented by `scripts/test/cli.mjs`. People and coding agents use it for anything from one test case to the full gate, locally or on Blaxel, and get results in one shape. Agents readily write plausible test commands whose arguments are quietly ignored; a single documented command that honors its options is what makes a test result trustworthy. Package-level `test` scripts and direct `vitest` or `playwright` invocations are not supported ways to run the suite. -This spec is also authoritative for test-service address allocation, the responses of registry-declared services, and process lifecycle. Service declarations are owned by [test-registry.md](./test-registry.md). +This spec owns the command, its guardrails, retries, `verify`, the test registry, and the rules for processes and addresses that tests create. Blaxel is owned by [blaxel-testshards.md](./blaxel-testshards.md), dividing runs across machines by [sharded-execution.md](./sharded-execution.md), run output by [reporting.md](./reporting.md), the CI workflow by [github-ci.md](./github-ci.md), and tree attestations by [attestation.md](./attestation.md). How tests are written, when to run what, and what agents may do without asking belong to the [testing policy](../testing-policy.md). -## Why this exists +## Guidance, not enforcement -Television has one test runner so that anyone — and especially a coding agent — has a single, reliable way to run tests in whatever form a task needs: one case, one file, one suite, one package, a whole surface, locally or on a remote provider. The command grammar below is the contract: it is the one documented way to select and run tests, so callers stop improvising per-package invocations. +The runner steers callers towards fast feedback and a usable shared host. Its checks recognize departures at the ordinary entrypoints, and every refusal says what to run instead. Where a departure from the shared-host or retry guardrails below is sometimes the right call, a deliberately named override permits it. Each of those overrides has `against-test-guidance` in its name, so using one reads as a considered choice. None of this is a security boundary. Anyone can run other commands, and the runner does not try to police the host. -Without a single runner, package areas accumulate separate invocations, the permutations drift apart, and a caller cannot trust that a given flag is even respected. That unreliability falls hardest on agents, which readily emit plausible-looking test commands whose arguments are silently ignored or mean something other than intended. Consolidating invocation into one canonical surface is what makes running tests trustworthy. - -The consolidation pays off three ways: - -- **Operationally, for agents and people** — one predictable, documented grammar to learn and use, rather than a different ad-hoc invocation per package. -- **Operationally, across runners** — a normalized way to route the same run to different underlying runners and providers and collect results in one shape. -- **Architecturally** — a single place that knows where all the tests are and holds the knobs for managing them: suites, surfaces, sharding, and selection. +## Providers A *provider* is where tests run: -- `local` — the current checkout, including uncommitted edits. -- `blaxel` — a persistent remote sandbox pool, running a committed revision. +- `local` runs the current checkout, including uncommitted and untracked files. +- `blaxel` runs a committed revision on a pool of remote workers that fetch it from GitHub ([blaxel-testshards.md](./blaxel-testshards.md)). -GitHub Actions runs tests as the CI workflow ([github-ci.md](./github-ci.md)), not as a runner-selectable provider. +GitHub CI runs the same tests through its own workflow; it is not a provider a caller selects. `npm test -- local` and `npm test -- blaxel` name their provider; only `verify` chooses one automatically. -There is no default provider for direct test execution: a provider shortcut names it explicitly, and remote providers run a committed revision fetched from `origin` rather than local edits. `verify` is the one command that may resolve a provider automatically. +Because workers fetch the revision from GitHub, a Blaxel run first checks that the working tree is clean and that the commit under test is on a branch on `origin`. Contributors commit and push before using Blaxel. `--ignore-uncommitted` lets a run start from a dirty tree; it still tests only the committed revision. `--commit ` tests a revision other than `HEAD`. ^remote-revision ## Commands -``` -npm test -- [options] -npm run verify # broad verification; auto-resolves a provider -npm run verify -- local # explicit provider (also: blaxel) -npm run verify -- --no-publish # run the gate without publishing its attestation -npm run verify -- --plan # print the resolved provider and phase commands, run nothing -``` - -`npm run verify -- ` forwards to the `verify` command; `npm test -- verify ` is equivalent. - | Command | Purpose | |---|---| -| `local` \| `blaxel` | Provider shortcut: run a selection on that provider. With no selector it defaults to the `all` suite, which is a *broad run* and is refused without `--force`. | -| `verify` | Run lint, type-check, package-manifests, and the `all` suite (preflight too, locally), resolving a provider. The broad-verification entrypoint. | -| `list` | Print the surfaces a selection resolves to. | -| `preflight --provider

` | Run only the preflight checks for a provider and selection. See [preflight.md](./preflight.md). | -| `pool blaxel ` | Manage the Blaxel sandbox pool. See [blaxel-testshards.md](./blaxel-testshards.md). | -| `baseline update [--dry-run]` | Recompute the committed duration baseline from the checkout's Blaxel run directories. See [sharded-execution.md](./sharded-execution.md). | -| `testpass prune <--older-than d\|--dead-version \|--delete-ref > [--apply]` | Inspect or apply attestation-ref retention, version sweeps, and exact-ref revocation. See [attestation.md](./attestation.md). | -| `help` | Print usage. | - -Package-level `test` / `test:e2e` scripts are not supported entrypoints; the root runner invokes each surface's config directly. This command surface is the only supported way to run tests. +| `npm test -- local [selectors]`, `npm test -- blaxel [selectors]` | Run a selection on that provider. | +| `npm run verify [-- local\|blaxel]` | The full gate, described under [Verify](#verify). | +| `npm test -- list [selectors]` | Print the surfaces a selection resolves to. | +| `npm test -- preflight --provider [selectors]` | Run only the checks that precede tests for a selection, such as `--suite all`. | +| `npm test -- pool blaxel ` | Manage the Blaxel pool ([blaxel-testshards.md](./blaxel-testshards.md)). | +| `npm test -- baseline update` | Refresh the durations used to plan shards ([sharded-execution.md](./sharded-execution.md)). | +| `npm test -- testpass prune` | Remove attestations ([attestation.md](./attestation.md)). | +| `npm test -- help` | Usage, including the guardrails and overrides below. | -## Selectors +Selectors choose what runs: -A *selector* chooses which tests run; the runner resolves selectors against [test-registry.md](./test-registry.md) and translates them to the underlying runner's flags. +- `--suite `, or `--all` for `--suite all`; +- `--surface [,…]`, `--package `, `--runner `, `--tag `; +- `--file `: an exact path selects that test file; any other value selects every test file whose path contains it. The matched files must belong to one selected surface; `--surface` or another selector narrows a match that spans surfaces or a file that several surfaces own; +- `--grep `: a test-title filter within the rest of the selection. On its own it narrows nothing. -| Selector | Meaning | -|---|---| -| `--suite ` | A named suite. `e2e-ci` is the registry-owned non-Electron CI shard selection; desktop runs in its dedicated job. | -| `--all` | Alias for `--suite all`. | -| `--surface ` | One or more registered surfaces (comma-separated). | -| `--package ` | Surfaces associated with a workspace package. | -| `--file ` | The surface that owns a test file, running that file. | -| `--grep ` | Test-title filter, translated to `-t` for Vitest and `-g` for Playwright. | -| `--runner ` | Surfaces using a runner. | -| `--tag ` | Surfaces carrying a registry tag, such as `browser` or `electron`. | +With no selector, `npm test -- local` and `npm test -- blaxel` select the `all` suite, while `list` and `preflight` start from every registered surface, including those outside `all`. -The local provider accepts every registered suite. A Blaxel suite selection accepts only `all`, `unit`, or `e2e`; narrower remote work uses surface, file, or grep selection through the target path. +The runner resolves `--file` to actual files itself and hands the native tool exactly those files. Vitest treats a file argument as a substring and Playwright as a regular expression, so passing the user's string through would also run sibling files. -`--file` must resolve to exactly one owning surface: if no surface owns the file the runner fails; if several do, it fails and asks for `--surface` to disambiguate. On the local execution path, a `--file` or `--grep` selection against a surface that does not declare support for it is rejected while building that surface's command. The targeted Blaxel path does not currently apply this `supports` check. The runner parses these selectors itself and rejects unknown positional arguments; the sole positional accepted is the `verify` provider. +Exit codes: -## Publication qualification +- `0`: the run passed, or an informational command completed. `verify --plan` exits `0` without verifying anything. +- `1`: tests failed, or the run failed while executing. +- `2`: a usage error or a guardrail refusal. +- `130`: `SIGINT` cancelled a wait for the local-test mutex, or a Blaxel run was interrupted by `SIGINT` or `SIGTERM` and finished its cleanup. +- `143`: `SIGTERM` cancelled a wait for the local-test mutex. -A *publication-qualifying test execution* uses the `blaxel` provider for the complete `all` suite with no narrowing selector and no shard subset. Precisely, its effective suite is `all`; `--surface`, `--package`, `--file`, `--grep`, `--runner`, and `--tag` are all absent; and `--shard-indices` is absent. An explicit `--suite all` or `--all` remains complete. `--shards ` changes how the complete suite is partitioned, so it does not narrow the run when every index executes. A local-provider execution never qualifies, even when it runs `all`, because a developer host does not provide the controlled execution environment required for durable pass evidence. ^test-publication-qualifying +A Blaxel command exits `0` only when its coordinator succeeded and the normalized report passed. Missing or contradictory evidence cannot turn a zero exit into a pass, and a nonzero coordinator exit is never reported as success. -A *publication-qualifying verify* is a `verify` whose test phase is a publication-qualifying test execution. The normal `npm run verify -- blaxel` command has this shape. Selection options supplied to `verify` are part of the qualification decision even when verify's phase construction would otherwise force `all`; a caller cannot attach a narrowing option and receive publication credit for the resulting run. +## The test registry -Publication requires no opt-in. A passing publication-qualifying verify is eligible for its `refs/testpass/*` attestation under [attestation.md](./attestation.md). `--no-publish` is an opt-out that suppresses that write. It does not alter which phases or tests execute. There is no publication opt-in flag: narrowed and local runs never publish regardless of caller intent, and passing `--publish` is a usage error. +`test.config.mjs` declares every *test surface*: an independently runnable test area, usually one Vitest or Playwright config and the files it covers. A surface declares its id, runner, config, the paths it owns, its kind and tags, and anything it needs: a build step, Vite services, a wrapper command, or checks that must pass before it runs. Facts such as unit or end-to-end, browser or Electron, are declared there and never inferred from file or package names. -Production attestations have exactly two source classes: a publication-qualifying Blaxel verify and the complete GitHub Actions CI workflow. The workflow's `attest-write` job is a CI-only writer after the strict join; individual GitHub-hosted commands that use `provider=local` are not attestation writers. A developer-host local verify cannot publish an attestation under any scope or result. [attestation.md](./attestation.md) owns the complete source predicate. +Every test file under `packages/` and `test/` belongs to exactly one surface, and a `vitest.config.ts`, `vitest.e2e.config.ts` or `playwright.config.ts` there that no surface declares is a registry error. A test therefore cannot silently drop out of every run. -## Run-directory identity output +A *suite* is a named selection over kinds, tags and runners. `all` is the unit and end-to-end surfaces; it is what verification, CI and attestation mean by the full suite. Some surfaces stay out of `all` and run only when selected: -`--run-dir-output ` asks a local or Blaxel provider invocation to atomically write the absolute path of the run directory it owns. The write happens immediately after run-directory creation, before local preflight and test execution, so the identity remains available when either of those phases fails. The option is per invocation rather than inherited through the environment: a test that spawns another canonical runner cannot replace its parent's identity unless it explicitly passes the same option. Workflow orchestration reads this file and validates that the path is a direct child of the expected `.test-runs/` root; it never discovers ownership by enumerating that root. +- agent surfaces, which need a local agent setup and a live inference service; +- the PostHog roundtrip suite, which needs a secret read key and runs only locally; its setup is owned by the [telemetry sink](../telemetry/sink.md#real-posthog-integration-test-surface); +- daemon acceptance, which installs a freshly packed `tv` into the global npm prefix and replaces the real `com.television.server` user service. It runs only on a designated host that is not meant to run a normal Television server, and it refuses to start unless `TV_DAEMON_TEST_HOST=1` is set; +- experiment surfaces: fixtures for testing the runner itself, and live qualifications. -## Broad-run guardrail +Some browser surfaces have Firefox projects. Outside GitHub Actions, their configs omit those projects with a warning when Playwright's Firefox is not installed. GitHub CI installs Firefox, and its configs fail when it is missing. Blaxel workers have no Firefox, so a Blaxel run, including a Blaxel verify, covers Chromium only. -A *broad run* is a provider shortcut that runs the full `all` set with no narrowing selector. A broad run skips lint and type-check, so the provider shortcuts refuse it with a message directing the caller to `npm run verify`. +Surfaces tagged `isolated-github-only` deliberately leak, orphan or kill processes to test cleanup. The runner refuses them before any work, on developer hosts and on every Blaxel path. They run only inside an operator-run GitHub workflow named `Process Lifecycle Fault Injection`, with `GITHUB_ACTIONS=true` and `TV_TEST_ISOLATED_GITHUB=1`. That workflow is not committed. It must run only on branches, on an ephemeral runner, with its own timeout, cleanup by owner token, and upload of the complete run directory. The workflow name and variable prevent accidents in trusted automation; they do not stop someone who sets them on purpose. -The `telemetry-posthog-roundtrip` and `daemon-acceptance` suites are not broad verification. They are specialized, opt-in local suites and are excluded from `all`. The first crosses the live telemetry ↔ PostHog boundary; the second replaces the active global `tv` package and the literal `com.television.server` user service on a designated developer host. Build-config-integrity tests are not part of either suite: `unit:build-config` is a normal unit surface in `all` and uses no secrets, network, or host service mutation. +## Shared development hosts -The guardrail trips only on the effective `all` suite with nothing narrowing it: a bare provider (which defaults to `--suite all`), an explicit `--suite all` or `--all`, or a bare `--grep` with no other selector. Everything else passes through: +Several agents often share one development host. Concurrent test runs compete for memory and CPU, push the host into swap and get processes killed by the kernel's out-of-memory killer, which disrupts every agent on it. Keeping the host usable comes first. The guardrails in this section keep local runs small and one at a time, and send broader work to Blaxel. The [testing policy](../testing-policy.md#verification-provider-and-completion) says when agents may use the overrides. -- any narrowing selector — `--file`, `--surface`, `--package`, `--runner`, or `--tag`; -- a non-`all` suite — for example `--suite unit`, `--suite e2e`, `--suite telemetry-posthog-roundtrip`, or `--suite daemon-acceptance`. +### Marked hosts -`--grep` does not narrow on its own: a bare `--grep` defaults to the `all` suite and is refused, but `--grep` rides along with any of the selections above, including `--suite unit --grep ...`. To run a genuinely broad provider shortcut anyway, pass `--force`; `verify` passes `--force` internally to its own broad test phase. +A *marked host* has the file `~/.tvdev-use-blaxel`, in the home directory that `HOME` names. Team development hosts carry it; public contributors' and fork hosts normally do not, and keep full local use without any Blaxel access. The file's contents are ignored. On a marked host, verify uses Blaxel by default and local runs are limited to one file. ^blaxel-host-marker -## Verify orchestration +### One local file on marked hosts -`verify` runs the broad verification that the guardrail steers callers toward. It resolves a provider, then runs phases in order, stopping at the first failure. +On a marked host, a local test run, including local verify, must select exactly one test file with `--file`, optionally narrowed by `--grep`. Other selectors may accompany `--file` to choose its surface. Anything broader is refused with exit `2` before any test work. Uncommitted and untracked files count; the limit applies to the files that will actually run, not to the filter string. `--force` does not lift it. ^local-one-file -The presence of `.tvdev-use-blaxel` in the invoking user’s home directory (`os.homedir()`) makes Blaxel the verification default and enables the local-verification guardrail across checkouts and sessions; its contents are ignored. The marker affects only `verify` provider selection, not targeted local test commands. The runner consults the home-directory marker, with no dedicated environment-variable switch. ^blaxel-host-marker +The refusal explains the limit. When Blaxel can run the same selection, the refusal prints the equivalent `npm test -- blaxel` command and the reminder to commit and push first; otherwise it names the selections Blaxel supports. Agent, PostHog and daemon surfaces and the Blaxel harness driver cannot run on Blaxel, so their refusal gives the one-file local command instead. The runner never commits, pushes or dispatches on the caller's behalf. -**Provider resolution.** The provider comes from `--provider`, or a single positional (`local` | `blaxel`), or defaults to `auto`. Passing both a positional and `--provider` is an error, as is an invalid positional. +`--against-test-guidance-broad-local-run` lifts the limit. -- An explicit provider (`local` or `blaxel`) is used as given; resolution itself runs no preflight. An explicit `local` when `~/.tvdev-use-blaxel` exists is refused unless `--allow-extreme-inefficiency` is passed, to keep broad verification on the chosen remote path. Explicit Blaxel commands remain available regardless of the marker. -- Without `~/.tvdev-use-blaxel`, `auto` (the default) selects `local` without remote preflight, Blaxel access attempts, or Blaxel warnings. Local verification requires no bypass flag. -- With the marker, `auto` runs the Blaxel remote preflight ([preflight.md](./preflight.md)): if it passes, the provider is `blaxel`; if it fails, verification is refused with exit `2`, remediation and the explicit `local --allow-extreme-inefficiency` escape hatch. There is no automatic local fallback on a marked host. +### One local run at a time -**Phases.** Local verify runs six phases: local preflight (`all`), lint, type-check, package-manifests, the `all` suite locally with `--force`, then the vibe-mode check. Remote verify runs five: lint, type-check, package-manifests, the `all` suite on the chosen provider against the committed revision (`--commit HEAD` by default) with `--force`, then the vibe-mode check. There is no separate remote-preflight phase in the list; the remote preflight runs inside the provider's own test phase, before it dispatches. So an explicit remote provider runs the remote preflight once, in the test phase, while `auto` that resolved to `blaxel` runs the Blaxel preflight twice — once during resolution, once inside the test phase. Non-unit surfaces retry twice by default in the test phase; `--retries ` overrides that count. On success verify prints `verify passed ()`. A passing publication-qualifying verify automatically publishes a tree-hash attestation when the attestation eligibility predicate and activation policy permit; `--no-publish` suppresses that write ([attestation.md](./attestation.md)). Local and narrowed verifies do not publish attestations. +Every local test run and local verify holds a mutex shared by all checkouts of the same operating-system user on the host. It is taken before preflight and held until the run's test processes have exited and cleanup has finished, including after failures and handled interruptions. Verify takes it before its first phase, and the test phase it starts shares that hold. Blaxel runs, `list`, `help`, `preflight` and `verify --plan` do not take it. Different operating-system users are not coordinated. ^local-test-mutex -The *vibe-mode check*, `scripts/test/vibe-mode-check.mjs`, is the last phase for both providers. It fails when `specs/vibe-waiver.md` exists in the checkout, because a branch in vibe mode is never merged into `main`; verify then reports failure and publishes no attestation. It runs after every other phase has run and reported normally, so the lint, type-check, and test results on a vibe branch are real, and its message names vibe mode as the reason. ^verify-vibe-mode +While the mutex is held, another local run is refused at once with exit `2`. The refusal identifies the holder's process, command and checkout, explains the contention risk, advises against stopping another agent's run to free the lock, and offers these choices: -`--plan` prints the resolved provider and the phase commands without running them. A `--plan` exit of 0 is not a verification — it reports what would run, nothing more. Verify has no path that reports a pass without running its phases. +- try again after that run finishes; +- pass `--wait-for-local-test-mutex` to wait. The wait has no time limit, and `SIGINT` or `SIGTERM` cancels it without affecting the holder; +- use Blaxel; +- pass `--against-test-guidance-major-host-contention-and-oom-killed-processes` to run alongside the holder. -## Build-config integrity surface +The wait option and the contention bypass cannot be combined. The bypass leaves the one-file limit in place, and the broad-local override leaves the mutex in place; `--force` lifts neither. -`unit:build-config` runs in `all` and CI. It runs the real CLI build script and inspects the default output for the ordinary configuration and an isolated `--outfile` bundle for the release configuration to verify the [ordinary and npm-release telemetry build configurations](../telemetry/sink.md#Behavior and operations). It is separate from `telemetry-posthog-roundtrip:integration`: it does not contact PostHog, does not require `TV_POSTHOG_TEST_READ_KEY`, and is safe for Blaxel and GitHub Actions. +A holder is stale only when none of its recorded processes is still alive, judged by PID and start time. The next run then takes over the mutex. If the stale run left test processes behind, the new run reaps them and fails, as any run does that finds stale processes ([Processes started by tests](#processes-started-by-tests)); otherwise it proceeds. Elapsed time never makes a holder stale, because a long run is not a dead one. -## Telemetry PostHog roundtrip suite +Checkouts on one host can run different revisions of the runner, so the mutex's location and record are a compatibility contract between revisions: a change must keep coordinating with checkouts still using the current form. The mutex is a SQLite database at `.cache/television/local-test-mutex.sqlite3` under the operating-system account's home directory. The path is read from the account, not from `$HOME`, so test environments that override `HOME` still share it. The database holds a single row. Acquisition is atomic, every change to the row checks the recorded identity first, and no transaction stays open while tests run. -`--suite telemetry-posthog-roundtrip` selects the live telemetry ↔ PostHog integration surface, currently `telemetry-posthog-roundtrip:integration`. The canonical invocation is: - -```bash -npm test -- local --suite telemetry-posthog-roundtrip +```ts +// CREATE TABLE local_test_mutex (slot INTEGER PRIMARY KEY CHECK(slot = 1), record TEXT NOT NULL) +// `record` holds this value as JSON. +interface LocalTestMutexRecord { + version: 1; + id: string; // unique per acquisition + participants: Array<{ + pid: number; + startTime: string; // process start time as the host's process table reports it + command: string; + checkout: string; + role: "run" | "verify"; + }>; +} ``` -The suite is excluded from `all`, so `npm run verify`, CI, and Blaxel `--suite all` do not run it. It is intentionally local-only because it requires the secret PostHog test read key; remote providers do not receive that key. The selected surface declares the `posthog-test-key` preflight, so a missing key fails before Vitest starts. The PostHog key setup and the test project safety guard are specified in [../telemetry/sink.md#real-posthog-integration-test-surface](../telemetry/sink.md#real-posthog-integration-test-surface). +### Native test entrypoints -## Production daemon acceptance suite +Registered Vitest and Playwright configs refuse to load when started directly rather than by the runner, on every host. The refusal gives the canonical command and, on marked hosts, the Blaxel route for broader work. For deliberate native use, such as watch mode, a debugger or an editor's test explorer, `TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN=1` (exactly `1`) lets the config load. Such a run is outside the mutex and the one-file limit, and native setup such as required services remains the caller's job. Every way the runner itself starts a native tool, locally, on Blaxel and in CI, marks its environment so that its own runs load. -`--suite daemon-acceptance` selects `daemon-acceptance:cli`, the packed-and-installed CLI acceptance surface for persisted service installation and removal. The canonical invocation is: +## The full-suite shortcut -```bash -TV_DAEMON_TEST_HOST=1 npm test -- local --suite daemon-acceptance -``` - -The suite is excluded from `all`, ordinary `verify`, CI, and Blaxel broad verification. It is manually run only on a designated developer host that is not intended to run a normal Television server. The `daemon-test-host` preflight requires the exact acknowledgement `TV_DAEMON_TEST_HOST=1` before the surface's build pre-command or test process starts; the test file repeats that check if invoked outside the canonical runner. The acknowledgement is an accidental-execution guard, not an authorization boundary. - -The surface builds the publishable CLI, packs it, and force-installs that tarball into the active npm global prefix. Its bulldozer first removes any existing `com.television.server` user service through the real `Daemon`, logs the service and PATH-visible `tv` state it found, and always replaces the global package with the freshly packed build. The asserted path then invokes the installed `tv` symlink, production `createDaemon`, literal service identity and arguments, and the host service manager. It proves daemon boot through `/health`, removes the service through `tv stop`, installs a service whose server cannot bind its listener and observes the command fail with the service left installed, repeats installation over that service for `tv serve --persist-uninstall`, and verifies no service definition or active registration remains. The final bulldozer observes that successful CLI removal left no service, invokes idempotent real cleanup, and removes residue if an earlier assertion failed. The freshly packed global `tv` package remains installed after the suite. - -Persisted serve rejects port `0`, so the surface cannot keep a kernel-owned port reservation while handing a stable number to the service's config file. For each installation it creates a temporary [Television home](../../product/cli.md#^cli-home), binds loopback port `0`, reads the assigned nonzero port, closes the reservation, writes that port with `tv --home config set port `, and immediately installs with `tv --home serve --persist` and probes the daemon on that number. This is a narrow declared exception to [the normal continuous-reservation rule](#^test-dynamic-ports). `TV_DAEMON_TEST_DIRTY_START=1` asks the surface to create a production-identity installation before its foundation bulldozer; this is a test-harness mode for proving convergence from prior service residue, and it remains protected by `TV_DAEMON_TEST_HOST=1`. +`npm test -- local` or `npm test -- blaxel` with the whole `all` suite is refused with exit `2` and directed to `npm run verify`. That covers no selector, `--suite all`, `--all`, and `--grep` alone. The full suite without lint and type-check is not a verification. `--force` permits it deliberately, and verify passes `--force` to its own test phase. On a marked host, a full local shortcut also needs the broad-local override. ## Retries -Non-unit Vitest and Playwright surfaces retry twice at the runner level by default, including targeted file/grep/surface runs; unit surfaces receive no runner-level retries. `--retries ` overrides the non-unit count and is ignored (with a notice) for a unit-only selection. This is the runner's blanket retry budget; the per-test flaky annotation that opts a single case into retries is owned by [flaky-tests.md](./flaky-tests.md). - -The canonical runner does not impose an automatic cap on observed failure counts through Playwright `maxFailures` or an equivalent mechanism. Tests are typically sharded across independent processes, so such a cap does not meaningfully bound a complete run; it only leaves assigned tests unexecuted and reduces the diagnostic evidence available from that run. - -Blaxel infrastructure retries are separate from this per-test budget. Infrastructure failure is the only condition that causes a shard to run again. A completed rerun supersedes the prior infrastructure-failed result. The canonical Blaxel command permits two such retries and reuses the shard's unchanged assignment, including for unit work and when the incomplete attempt retained failed task observations. Retry eligibility is independent per shard, so a completed test failure elsewhere does not suppress it. `stale-owner` is the sandbox-safety exception defined by [blaxel-testshards.md](./blaxel-testshards.md): the contaminated sandbox runs no selected test and is excluded from later attempts, while a configured retry may run the unchanged shard assignment on a distinct sandbox only after that sandbox acquires its own lease and passes its owner audit. Without an exact contaminated-sandbox identity or distinct clean capacity, the shard remains infrastructure-failed. If the retry budget ends without a complete result, the normalized run remains incomplete and the command fails. - -The GitHub Actions CI workflow at .github/workflows/ci.yml is part of the runner-retry contract. Its browser and Node end-to-end shard job calls scripts/run-test-shard.mjs directly instead of entering through npm test, so that job must pass --test-retries 2. The desktop end-to-end job enters through npm test -- local --surface e2e:desktop, so it receives the same default retry count from the runner. If the default retry count for non-unit surfaces changes, this workflow must change in the same update. - -## Local execution - -A local run writes a run directory and normalized reports ([reporting.md](./reporting.md)) and exits 0 when the run passed, 1 otherwise. - -- A suite or broad local selection with no `--file` / `--grep` runs surfaces grouped by execution group, in group order ([test-registry.md](./test-registry.md)). The group whose id is `unit:workspaces` runs as one Vitest process when all of its members use Vitest and declare no services. The combined native result is split back to per-surface results by file ownership. Other groups run surface by surface, even when every member uses Vitest. -- A `--file` or `--grep` selection runs the selected surfaces one at a time. -- Each independently spawned surface task runs inside the owned process scope defined below. The runner starts its declared services, publishes their URLs, and completes teardown and leak detection before the next task starts. -- A surface may declare a build to run first, services to run alongside it, or a command that overrides the default runner invocation; these declarations are registry-owned ([test-registry.md](./test-registry.md)). -- When `SKIP_ELECTRON_E2E=1`, the desktop command still starts through the normal surface path. If it exits cleanly without leaks or cleanup failure, the runner reclassifies that result as `skipped` with `SKIP_ELECTRON_E2E=1` as its reason. - -## Test infrastructure addresses - -A *test infrastructure listener* is a network listener created to support a canonical test run: an application server, development server, proxy, fixture server, or other harness service. Every such listener asks the operating system for an available port by binding port `0` (or the runtime's equivalent) and reads back the assigned address after the bind succeeds. Test infrastructure has no fixed port, planned port range, adjacent-port convention, or reserve-then-release allocator. This is required because only the successful bind reserves a port; selecting a number before binding has a time-of-check/time-of-use race with every other process on the host. ^test-dynamic-ports - -A consumer receives a full URL from the component that owns the successful bind. This applies to runner configs, helpers, test cases, browser navigation, and URLs embedded in served fixtures; none may use a numeric test-infrastructure URL literal or a fallback to one. - -- For a registry-declared service, the runner creates an HTTP listener, mounts Vite's Node-API middleware on it, connects Vite HMR to that same listener, disables speculative request pretransforms, binds the listener to loopback port `0`, and reads its address after `listen()`. Vite treats `server.port: 0` as a falsy value and substitutes its default port, so Vite does not own this bind. Its speculative pretransforms can also retain pending transform promises after the public idle barrier; on-demand transforms remain enabled while the runner disables that optimization so bounded teardown can complete. The runner writes the full bound URL to the service's `publishUrlEnv` variable ([test-registry.md](./test-registry.md)). Services start in declaration order, and each published URL is present while later services load their config and while the test command loads its config. A missing required variable is a configuration failure before tests start. -- A test helper that starts its own listener returns the bound URL directly. A subprocess that implements the listener may publish it through a machine-readable readiness message; the helper waits for that message before returning. -- A test that spawns the product's `tv serve` gives it a temporary [Television home](../../product/cli.md#^cli-home) whose config file sets port `0`, passes that home with `--home` to the server and to every client command it spawns, and passes the server [the hidden `--print-links` flag](../cli/index.md#Server process boundary), so that it prints its connect links even though its output is a pipe. The test reads the acquired URL from those links. Spawned client commands also pass that URL's port with `--port`, as the [client-port rule](../../product/cli.md#^cli-client-port) requires for config port `0`. -- A Playwright config reads its base URL from the published environment. It does not start a second copy through Playwright's `webServer` option and does not fall back to a numeric localhost URL. A test whose framework `baseURL` type is optional asserts that it is present before use. -- A committed static fixture uses relative `src` and `href` values for dependencies served by its own origin. If it needs another independently published service, the serving Vite config uses `define` injection to supply that service's named URL to the consuming module. If a fixture needs a second origin on the same service, it derives the alternate hostname from `location` at runtime while preserving the published port. The committed fixture never contains a numeric live-service origin. - -Each URL is independent. A harness with two origins receives two published URLs and never derives one by adding to another port. The artifact host and artifact view are two independently bound services under this rule; their addresses are not allocated from a precomputed range. - -A test may contain a numeric port as inert input or expected output only when the value is never dereferenced as the address of a live test-infrastructure listener. This permits parser/serialization fixtures and a deliberate unreachable target to which no test process binds. It does not permit a URL used by navigation, `fetch`, a socket client, `src`, or `href` to identify a running test service. - -When a browser origin must remain stable while its backend restarts, one front listener binds port `0` for the whole test and updates its forwarding target after each independently bound backend starts. The test never closes a reservation and asks a replacement server to rebind its number. - -## Vite service responses - -A registry-declared service answers every request in full and never with `304 Not Modified`: its listener removes the `If-None-Match` and `If-Modified-Since` headers from each request before Vite handles it. ^test-service-full-responses - -The reason is a Chromium behavior, observed on Linux in the Chromium 145 build that Playwright 1.58.2 installs, both its headless shell and full Chromium. There, each script, module, stylesheet, or image response with status 304 left the page's renderer holding a 2 MB shared-memory buffer until the page's browser context closed, even after the page navigated elsewhere. Vite serves the application's development build as hundreds of separate modules that the browser revalidates on every load, so in that build each further load of the application in one page held about 465 MB more. Other Chromium versions may differ. Playwright launches Chromium with `--disable-dev-shm-usage`, which places those buffers in the temporary directory. Where that directory is kept in memory, as on [Blaxel workers](./blaxel-testshards.md), a test that loads the application several times in one page can run out of memory and crash the page. - -## Destructive fixture placement - -A surface tagged `isolated-github-only` is refused with usage exit `2` before local preflight, Blaxel preflight, dispatch, or lease acquisition. It is never eligible for developer-host local execution or any Blaxel path. The only accepted placement is `provider=local` inside an operator-authorized, branch-only GitHub workflow named `Process Lifecycle Fault Injection`, with `GITHUB_ACTIONS=true` and `TV_TEST_ISOLATED_GITHUB=1`. The repository does not commit this fault-injection workflow. Its exact identity and explicit opt-in prevent accidental execution in trusted automation, but they are not a security boundary because callers can set the same environment values. Any authorized workflow must use an isolated ephemeral runner, an independent timeout, owner-token cleanup, and complete hidden run-artifact upload. Repository workflow controls, registry exclusion, provider refusal, host isolation, and independent cleanup provide the protection. This execution restriction is permanent. - -## Surface process lifecycle - -```ts -const TEST_SURFACE_OWNER_ENV: "TV_TEST_SURFACE_OWNER"; -const TEST_PROCESS_TERM_GRACE_MS: 2_000; -``` - -A *surface process scope* is the complete set of operating-system processes started for one independently spawned surface task: its `preCommand`, declared services, test command, runner workers, browsers, Electron applications, and subprocesses started by tests. The local runner owns local scopes; the shared shard worker owns remote scopes ([sharded-execution.md](./sharded-execution.md)). A worker process and a test callback may perform prompt cleanup, but the outer scope owner is the final reaper because it remains alive when a worker crashes or a test callback is interrupted. ^test-surface-owner - -Before starting a scope, its owner creates a unique owner token whose encoding contains the run ID, surface identity, supervisor PID, and supervisor process start time. The token is opaque to tests and child consumers; the scope owner and preflight scanner decode it. The owner exports it to every child and launches each root command in a POSIX process group. Repository test helpers preserve the token in child environments. Declared services and helpers that create detached children give those children their own process groups so the owner can signal each tree. Descendants that create another process group remain discoverable through the inherited token. +Flaky tests are a fact of end-to-end testing, so failed tests are retried, and every recovery is reported rather than hidden ([reporting.md](./reporting.md)). -A surface scope proceeds in this order: +- **Runner budget.** On surfaces other than unit surfaces, a failed test is retried up to twice by default, whatever the selection. Unit surfaces get no runner retries. A positive `--retries ` sets the count for the non-unit surfaces. A test that fails and then passes counts as passed and is reported as a *recovered flake*. +- **Zero retries.** Plain `--retries 0` is for running a new or changed test once. It is accepted when `--file` selects exactly one test file, optionally narrowed by `--grep`, with either provider. For any broader selection, including verify, it is refused with exit `2` unless `--against-test-guidance-turn-flakes-into-failures-on-broad-runs` is passed, which sets the runner budget to zero on purpose. Neither zero form changes unit surfaces, per-test annotations or Blaxel's infrastructure retries. +- **Per-test annotation.** One known-flaky case may get more retries from `FLAKY_TEST_RETRIES` in `test/flaky-retries.ts`, with a title that starts with `flaky:`. Runner and CI runs give it 5 unless the `FLAKY_TEST_RETRIES` environment variable is set. Reports use that prefix to mark the recovery as annotated. An annotation is for investigated flakiness in the harness or environment, on the one offending case and never on a whole file or suite. A real failure is fixed instead. +- **Infrastructure retries.** Blaxel reruns a shard whose infrastructure failed, up to twice, with the same assignment. A completed shard with a failed test is never rerun ([sharded-execution.md](./sharded-execution.md)). -1. Run `preCommand`, if declared. It must exit without a live descendant before service startup. -2. Start declared services in declaration order and wait for each bound URL. -3. Run the test command and wait for its exit. -4. Stop declared services in reverse order and allow a bounded graceful-exit interval. -5. Run the leak check below, reap any survivors, and only then complete the surface result or start another task. +No retry clears a process leak: a surface that leaves a process running fails even when every test passed or recovered ([Processes started by tests](#processes-started-by-tests)). -The same teardown runs after a non-zero child exit, thrown runner error, timeout, `SIGINT`, or `SIGTERM`. It sends `SIGTERM` to registered process groups and token-matched processes, waits up to `TEST_PROCESS_TERM_GRACE_MS`, then sends `SIGKILL` to survivors. Two seconds lets a test server close sockets and flush output without allowing a leaked process to consume a material share of the run. Cleanup functions are idempotent. No test-owned process may transfer out of the scope or survive it. +Ordinary runs impose no automatic cap on failures, so independent work continues after a test fails. Tests run in many independent processes, so a cap would only leave assigned tests unrun and the evidence incomplete. -`SIGKILL` of the scope owner itself cannot execute teardown. On an ephemeral GitHub host, job disposal removes its processes. A persistent or local execution host finds the abandoned token during its next preflight and reaps it before running a surface. The lifecycle guarantee is that no subsequent surface starts alongside an abandoned process; it does not claim that a killed supervisor can execute code. +## Verify -A combined Vitest run uses a generated config with one `test.projects` entry per member surface. Each entry extends the surface's Vitest config and uses that config file's directory as its root. Attempt recording follows [reporting.md](./reporting.md#^vitest-attempt-recording). This structure applies to local and sharded execution. The command is one process scope whose token names every member surface. It may contain only surfaces with no declared services and no test that intentionally leaves a long-running subprocess alive while another surface executes. A leak fails every member because the shared process boundary cannot attribute the spawn more narrowly. A surface that needs services or long-running subprocesses runs as an independent task. +`npm run verify` is the full gate. It stops at the first failing phase: -Persistent workers inspect Television owner tokens before accepting a new scope. A token is stale only when its recorded supervisor PID and process start time no longer identify a live supervisor; a concurrent run with a live supervisor is left untouched. A stale process is captured and reaped, and the worker reports an infrastructure preflight failure naming the token and process; it does not run a new surface in contaminated state. An ephemeral provider still performs the check so its correctness does not depend on host disposal. +- locally: preflight for the `all` suite, lint, type-check, package manifest checks, the `all` suite, then the vibe-mode check; +- on Blaxel: lint, type-check and package manifest checks in the checkout, the `all` suite on Blaxel at the selected committed revision (`HEAD` by default), then the vibe-mode check. -## End-of-surface leak detection +On success it prints `verify passed ()`. A passing complete Blaxel verify publishes an attestation for its tree unless `--no-publish` is passed ([attestation.md](./attestation.md)). A local verify never publishes. `--plan` prints the chosen provider and the phase commands without running the phases; choosing the provider on a marked host still runs its Blaxel checks. -After registered teardown and its graceful interval, the scope owner enumerates every live process carrying the owner token. Each candidate is revalidated immediately before cleanup. A candidate that exits naturally before any cleanup signal is sent is not a *process leak* and is omitted; this closes the normal child-exit race where a runner exits just before one of its helper processes. A process that requires `SIGTERM` or `SIGKILL` remains a leak even if a later revalidation finds that it has exited. For every actual leak, the owner captures its identity and listening sockets before sending `SIGTERM`, escalates to `SIGKILL` after the bounded interval, and scans once more to confirm that no survivor remains. Before each signal it confirms that the PID still has the same process start time and owner token, preventing PID reuse from targeting another process. The diagnostic identifies at least the surface owner, PID, parent PID, process group, process start time, executable command, and every attributable listening TCP socket. [reporting.md](./reporting.md) owns the normalized record that carries this attribution. ^test-process-leak +`npm run verify -- local` or `npm run verify -- blaxel` chooses the provider explicitly. Otherwise an unmarked host verifies locally without contacting Blaxel. A marked host runs the Blaxel checks first: credentials, a clean tree and a pushed commit. When they pass it uses Blaxel; when they fail it refuses with exit `2` and the remediation, and never silently falls back to local. An explicit local verify on a marked host also needs `--against-test-guidance-broad-local-run`. -Linux discovery reads process identity and inherited owner tokens from `/proc` and attributes listening sockets with `ss`. macOS discovery reads process identity and inherited owner tokens with `ps` and attributes sockets with `lsof`. The required backend is checked before execution ([preflight.md](./preflight.md)). Port-based process killing and executable-name matching are not lifecycle mechanisms: they cannot establish which surface owns a process and can kill an unrelated concurrent run. +The *vibe-mode check* fails when `specs/vibe-waiver.md` exists. That file marks a branch in vibe mode, an experiment branch with the spec-driven process switched off, which must never merge into `main`. The check runs last, so the lint, type-check and test results on such a branch are still real, and its failure names vibe mode as the reason. CI applies the same check ([github-ci.md](./github-ci.md)). ^verify-vibe-mode -A leak makes its owning surface fail even when all native test results passed or a retry recovered. Native failures and recovered-flake records remain visible alongside the lifecycle failure. [reporting.md](./reporting.md) owns the normalized field names and run-status classification for this concept. In a combined Vitest run scope, each member surface receives the lifecycle failure and the report names the shared owner token. +## Processes started by tests -## Exit codes +Developer machines and Blaxel workers run many test runs over time, often concurrently. A process a test leaves behind can hold ports, memory and files that break later runs, so the runner owns every process a surface starts. -- `0` — the run passed, or an informational command completed (`list`, `help`, `verify --plan`). -- `1` — a test run failed, or a preflight/run-time failure during execution. -- `2` — a usage error or a guardrail refusal (unknown command, invalid or conflicting provider argument, a refused broad run, a refused local verify). -- `130` — a delegated Blaxel coordinator handled `SIGINT` or `SIGTERM`, finalized an interrupted provider report, and completed cleanup. When the canonical provider command receives this coordinator exit, it finalizes the normalized incomplete run and preserves `130`; `verify` likewise propagates that status from its test phase. +A *surface process scope* is everything started for one surface run: its build step, services, the test runner and its workers, browsers, Electron, and anything tests spawn. Its owner is the local runner, or the shard worker on Blaxel and in CI. The owner stays alive when a test worker crashes, so it is the final reaper. ^test-surface-owner -A remote provider command exits `0` only when both the delegated coordinator exits successfully and the normalized run report is `passed`; missing, contradictory, or infrastructure-incomplete report evidence cannot produce a successful command through a zero delegated exit. A nonzero delegated exit remains nonzero even if retained report evidence appears passed. +- The owner puts a token in `TV_TEST_SURFACE_OWNER`, and every child inherits it. Test helpers that build child environments must preserve it. +- After the tests exit, whether normally, by failure, timeout or interruption, the owner stops the scope's services and any of its commands still running, then finds every live process still carrying the token. It sends each `SIGTERM`, then `SIGKILL` after a short interval, and confirms none survived. Before each signal it rechecks the process's start time and token, so a reused PID is never hit. A process that needed a signal is a *process leak*. It fails the surface even when every test passed, and the report identifies it by PID, command and listening sockets ([reporting.md](./reporting.md)). ^test-process-leak +- Processes are found only through the inherited token, using `/proc` and `ss` on Linux, or `ps` and `lsof` on macOS. Killing by port or process name is not used: on a shared host it can kill another run's processes. Canonical local runs and planned shard workers check that the host's process backend works before tests, and fail rather than run without cleanup. Other paths, such as targeted Blaxel runs, skip that check; on Blaxel, worker setup installs the tools. +- When an owner is itself killed, nothing can clean up at that moment. Before running tests, the runner and each shard worker look for tokens whose supervisor is gone. They reap those processes and fail that run as an infrastructure problem instead of running beside them. Tokens whose supervisor is alive belong to a concurrent run and are left alone. -## Cursor-agent environment workaround +Hosts inspect tokens left by runs of other runner revisions, so the token format is shared across revisions. A token is `tv1.` followed by the base64url encoding of the JSON object below. It is stale when no live process has that PID and start time. ```ts -const CURSOR_AGENT_ENV_FLAG: string; // "CURSOR_AGENT" -const CURSOR_AGENT_ENV_VARS: readonly string[]; // "ELECTRON_RUN_AS_NODE", "PLAYWRIGHT_BROWSERS_PATH" +interface SurfaceOwner { + runId: string; + surfaceId: string; // comma-separated when one scope runs several surfaces + supervisorPid: number; + supervisorStartTime: string; // as the host's process table reports it +} ``` -When `CURSOR_AGENT=1`, the Cursor environment injects variables that break local Electron and Playwright tests. The runner strips those variables for its own process and child test processes. Without the flag it is a no-op. The flag is set only inside the Cursor agent environment; terminals, the remote providers, and CI never set it. +## Network addresses in tests -## Testing +Test infrastructure, such as application and development servers, proxies and fixture servers, binds port `0` and reads back the port the kernel assigned. There are no fixed ports, port ranges or reserve-then-release allocators. Choosing a number before binding races every other process on the host, and concurrent runs on shared hosts make that race real. The component that bound a listener hands consumers its full URL: as a return value, through a readiness message, or, for a registry-declared service, in the environment variable the registry names. Nothing derives one service's address from another's port, and no test uses a numeric URL literal for a live test service. A port number may still appear as inert test data that nothing connects to. ^test-dynamic-ports -Proof of guardrail decisions, provider resolution, and `verify` orchestration must spawn the canonical CLI. The proof must exercise the CLI's real argument parsing and observe its exit status, standard output, and standard error. Under the testing policy's [mocking policy](../testing-policy.md#Mocking policy), self-test seams may replace only downstream work that is not part of the decision being proven. +Persisted `tv serve` rejects port `0`, so the daemon acceptance suite takes a kernel-assigned port, releases it and immediately configures the installed service with it. This is the one exception. -Under the testing policy's [acceptance-test rule](../testing-policy.md#^shape-acceptance), coverage of local execution must include a real suite in which a nonzero exit from the child test process makes the canonical CLI exit with status `1`. +Registry-declared services are Vite middleware on a listener the runner binds itself, because Vite treats `server.port: 0` as unset and falls back to its default port. The runner also turns off Vite's speculative pre-transforms, which can leave transform work pending after the server reports idle and stall teardown. -Coverage for surface services must begin with a `TestService` declaration. It must use the service manager and process environment to pass the service's URL to a real Playwright configuration. It must use real Vite listeners and real HTTP requests. Coverage must also show that, after successful execution and after a service fails to start, services stop in reverse order and the parent environment is restored. +Registry-declared Vite services always answer in full and never with `304 Not Modified`: the listener strips `If-None-Match` and `If-Modified-Since` from every request. In the Chromium 145 build that Playwright 1.58.2 installs, each 304 response to a script, stylesheet or image left the renderer holding a 2 MB shared-memory buffer until the browser context closed. The development build is hundreds of modules, so every reload of the application in one page held about 465 MB more. On Blaxel workers, where Chromium's shared memory lives in memory-backed temporary storage, tests that reloaded the application several times crashed. ^test-service-full-responses -Under the testing policy's [mocking policy](../testing-policy.md#Mocking policy), claims about surface cleanup and process leaks must be proven through real subprocesses, process groups, inherited owner tokens, signals, process inspection, and listening sockets. +## Cursor agent environment -Under the testing policy's [compositional coverage rule](../testing-policy.md#Compositional coverage across clean seams), this spec owns the decision that selects a provider automatically from the home marker and, on marked hosts, a remote preflight result. [Preflight's Testing section](./preflight.md#Testing) requires a real Git process to run the production git-safety sequence in a repository with an `origin` remote. It also requires live evidence for the Blaxel capability checks. The live planned run required by [sharded execution's Testing section](./sharded-execution.md#Testing) supplies the real provider report for the remote run. +When `CURSOR_AGENT=1`, the runner removes `ELECTRON_RUN_AS_NODE` and `PLAYWRIGHT_BROWSERS_PATH` from its own environment and its children's. The Cursor agent environment sets both, and they break Electron and Playwright tests. -Reading `.github/workflows/ci.yml` proves only which command the file contains. The isolated live run required by [GitHub CI's Testing section](./github-ci.md#Testing) establishes that GitHub starts the job and that the job invokes the test runner. +## Testing guidance +Tests that deliberately leak, orphan or kill processes, or kill a supervisor, belong in `isolated-github-only` surfaces. On a shared host they could disturb other runs or leave residue for them. diff --git a/specs/arch/testing-policy.md b/specs/arch/testing-policy.md index 9d4f5086..8ecf29ff 100644 --- a/specs/arch/testing-policy.md +++ b/specs/arch/testing-policy.md @@ -1,8 +1,8 @@ -*How Television is tested: coverage requirements, the fixture/mock distinction, the three test shapes and their declaration schema, and mock vs integration discipline.* +*How Television is tested: test authoring and coverage, focused iteration, verification, and agent permissions.* # Testing policy -This spec is authoritative for the project's testing authoring discipline. How tests are *executed* at runtime is governed by the test-runner specs under [test-runner.md](./test-runner/test-runner.md) and its companions. +This spec is authoritative for the project's test authoring, iteration, and verification discipline, including agent permissions. How tests are *executed* at runtime is governed by the test-runner specs under [test-runner.md](./test-runner/test-runner.md) and its companions. ## Scope during the spec migration @@ -39,7 +39,13 @@ If full test coverage would require extremely complex or numerous tests, that is Test the narrowest thing that proves the change, then widen — never the other way around. -While implementing, run only the tests directly exercising the behavior under change (a file, a `--grep`, a surface). They are fast, focused, and tell you whether the thing you are working on is working. As that signal turns green and the behavior matures, widen the surface: the owning suite, then the full broad verification (`npm run verify`). +Use the canonical runner for local feedback on the behavior under change: start with one file, optionally narrowed with `--grep`. Repeated focused runs while investigating or editing that behavior are normal iteration. As the focused check passes and the behavior matures, widen validation to the owning surface or suite and then full verification (`npm run verify`), using the provider discipline below. + +Keeping the shared host usable comes first. Concurrent test runs compete for CPU and memory, drive the host into swap, and can get processes OOM-killed, affecting every agent on the host. The runner's [local mutex](test-runner/test-runner.md#^local-test-mutex) coordinates runs by the same operating-system user across checkouts. Pass `--wait-for-local-test-mutex` to wait for another local run to finish; the default is immediate refusal. Use the canonical entrypoint and follow its refusals; the [native-entrypoint contract](test-runner/test-runner.md#native-test-entrypoints) covers direct Vitest and Playwright invocations. + +On hosts with `~/.tvdev-use-blaxel`, local iteration selects one file with optional grep; anything broader goes to Blaxel after the intended revision is committed and pushed. For a known multi-file check, use that remote selection instead of splitting it into serial local commands to evade the file limit. Even a Blaxel selection running on one worker removes its test load from the shared host. Distribution across files improves speed where available; lack of fan-out for a targeted surface is never a reason for broader local work. Unmarked public-contributor and fork hosts retain local selections and need no Blaxel access. + +Running a new or changed test once is normal focused iteration: plain `--retries 0` is within the guidance for one explicitly selected file, optionally narrowed with `--grep`. Broader validation uses the default retry budget; disabling it turns recoverable flakes into failures across unrelated tests. Broader zero-retry runs require `--against-test-guidance-turn-flakes-into-failures-on-broad-runs`. Using that override is the caller's judgment, including for agents, without human permission. Recovered flakes remain reported, and flaky new tests still require investigation and repair under [Timing, observable status, cleanup, and flakiness](#timing-observable-status-cleanup-and-flakiness). The runner owns [selection and budget mechanics](test-runner/test-runner.md#retries), including the separate per-test and infrastructure layers. Broad runs are a **discovery tool, not your inner loop.** Use them to find what is failing somewhere else as a result of the change. The moment a broad run surfaces a failure, scope back down to a narrow test that reproduces it, get it green there, and only then re-run the broad verification. Don't iterate against the broad gate to chase a single failing test — each cycle is many minutes long, you stop seeing cause from effect, and you waste time the narrow test would have saved. @@ -47,17 +53,21 @@ The work is not done until **broad verification passes fully green** on its own. ### Verification provider and completion -Run full verification with `npm run verify`. Without `~/.tvdev-use-blaxel`, verification runs locally without Blaxel checks or warnings, a bypass flag, or a separate permission requirement. The [runner’s provider selection](test-runner/test-runner.md#Verify orchestration) owns provider resolution. +Run full verification with `npm run verify`. Without `~/.tvdev-use-blaxel`, verification defaults to local without Blaxel checks, Blaxel warnings, or a separate permission requirement; the local mutex still applies. The [runner’s provider selection](test-runner/test-runner.md#verify) owns provider resolution. + +On hosts with `~/.tvdev-use-blaxel`, Blaxel is the verification default. Agents must obtain explicit human permission before using `--against-test-guidance-broad-local-run` to lift the [one-file local limit](test-runner/test-runner.md#^local-one-file), including for local verify; the flag is a mechanism, not permission. Reluctance to commit or convenience does not justify broader local execution. Fix a reported remote preflight problem unless the broader local run is authorized. Local-only suites follow their runner-owned placement and invocation guidance. GitHub CI retains its specified execution path. -On hosts with `~/.tvdev-use-blaxel`, Blaxel is the default and local full verification requires `--allow-extreme-inefficiency`. Agents must obtain explicit human permission before using that bypass on a marked host; the flag is a mechanism, not permission. Reluctance to commit or convenience does not justify a local run. Fix a reported remote preflight problem unless local verification is authorized. GitHub CI retains its specified execution path. +The independent `--against-test-guidance-major-host-contention-and-oom-killed-processes` bypass is the caller's judgment, including for agents, and requires no human permission. It lifts only the mutex, as defined by the [runner](test-runner/test-runner.md#^local-test-mutex). -The `tvdev-setup`, `tvdev-contribute`, and `tvdev-review` skills apply the repository’s development workflow for all contributors, including forks. Normal setup and contribution work must not create the Blaxel marker or require Blaxel access. [Blaxel access setup](test-runner/blaxel-testshards.md#Local setup) is needed only for contributors choosing Blaxel, whether through an existing host marker or an explicit command. +Deliberate lower-level tool use through the [native-entrypoint override](test-runner/test-runner.md#native-test-entrypoints) is also the caller's judgment, without human permission. It enables native capabilities needed for the task outside the runner's protections; the default workflow remains canonical local iteration and broader remote validation on marked hosts. + +The `tvdev-setup`, `tvdev-contribute`, and `tvdev-review` skills apply the repository’s development workflow for all contributors, including forks. Normal setup and contribution work must not create the Blaxel marker or require Blaxel access. [Blaxel access setup](test-runner/blaxel-testshards.md#setup) is needed only for contributors choosing Blaxel, whether through an existing host marker or an explicit command. The [contribution workflow](../spec-workflow.md#^shared-branch-workflow) owns full-verification timing for pull requests and merges. Report the revision actually tested, the result, recovered flakes, and relevant limits; do not describe targeted checks as a full gate or claim unperformed validation. Run verification synchronously and wait for its result. Required specialized checks still apply where the full gate excludes them. -The [canonical runner](test-runner/test-runner.md) owns commands and selectors. Its [remote preflight](test-runner/preflight.md#Contributor branches and remote revisions) owns the committed, pushed revision requirement. Local targeted checks remain available for iteration. +The [canonical runner](test-runner/test-runner.md) owns commands and selectors. It also owns the [committed, pushed revision requirement](test-runner/test-runner.md#^remote-revision). Local targeted checks remain available for iteration. ## Mocking policy @@ -207,6 +217,10 @@ In order to liberally cover boundary seams across the codebase, complex harness Prefer real, committed fixtures over invented data. Better yet, generate fixtures dynamically from current code paths, driven by simpler inputs with fewer assumptions, whenever possible. +A reusable build created by a test helper belongs to its worker: it is shared within that worker, published only after preparation succeeds, and removed at worker teardown rather than per test case. The [Blaxel shard attempt lifecycle](test-runner/blaxel-testshards.md#Temporary files) supplies the backstop when a killed worker cannot run its teardown. ^worker-build-lifetime + +For new or substantially changed browser acceptances that use the `ProductServer` helper, the default is its built application unless the assertion needs development-module behavior. This exercises shipped assets and avoids loading hundreds of development modules for a product assertion. This default does not require retroactive conversion of other tests or apply to in-process `Server` walks using `appURLForServer()`, component fixtures, or Electron module/dispatcher seams. ^product-acceptance-built-app + ## Timing, observable status, cleanup, and flakiness Prefer real readiness and state observation over sleeps. Take the time to make complex tasks reliably observable for testing purposes. ^flake-avoidance @@ -229,6 +243,6 @@ Treat cleanup behaviors of long-running modules (with servers, listeners, networ The test development proxy installs `error` handlers on both inbound streams and their outbound peers before piping begins. For an HTTP proxy this includes the inbound request; for an upgraded connection it includes the accepted socket. A peer reset closes both sides idempotently and is never allowed to escape as an uncaught process error. Its seam tests drive abrupt HTTP and upgraded-socket resets through the real bridge and assert that neither side remains tracked. The separate stable-front proxy has no equivalent reset seam test. -Flakiness is sometimes inevitable in browser and Electron e2e tests, so the test runner provides a two-layer flaky-test retry mechanism — see [flaky-tests.md](./test-runner/flaky-tests.md) for the mechanism and annotation patterns. +Flakiness is sometimes inevitable in browser and Electron e2e tests, so the test runner provides a two-layer flaky-test retry mechanism — see [the runner's retries](./test-runner/test-runner.md#retries) for the mechanism and annotation patterns. However, the existence of a mechanism for handling flakes does not make it acceptable to write them. When writing a test, if it turns out to be flaky, stop and apply aggressive, disciplined refactoring to the code or test or both. It is worth it: a one-time effort up front beats permanent pain over the long run. diff --git a/specs/index.md b/specs/index.md index 2e5833ec..d1dbe471 100644 --- a/specs/index.md +++ b/specs/index.md @@ -10,9 +10,9 @@ Generated from the first-line description of every spec by `scripts/specs-index. - `spec-policy.md` (250 lines) — *How and why specs are written, what makes something a spec, and the rules for specs.* - `spec-proofs.md` (77 lines) — *Proofs — the agent-owned document under `proofs/` that says how one spec's promises are proven: why it exists, where it lives, its shape, when it changes, how it cites, and who owns and reviews it.* - `spec-ui.md` (82 lines) — *How visual design is specified and held authoritative: a spec per surface whose reference frames are rendered truth, staged for judgment in the workshop ([staging.md](staging.md)), and enforced by automated conformance.* -- `spec-workflow.md` (264 lines) — *The authority chain and the spec-first workflow that keeps specs the source of truth.* +- `spec-workflow.md` (268 lines) — *The authority chain and the spec-first workflow that keeps specs the source of truth.* - `staging.md` (42 lines) — *The staging workshop: where spec renders are staged and judged — the frameset viewer, the frames tree, and its conventions.* -- `terms.md` (200 lines) — *Glossary of Television spec terms; each term links to the spec that owns it. Maintained by review, not generated.* +- `terms.md` (199 lines) — *Glossary of Television spec terms; each term links to the spec that owns it. Maintained by review, not generated.* - **product/** - `artifact-navigation.md` (54 lines) — *The user's mental model for navigating inside an artifact that has links or multiple pages: browser-like back/forward history, discarding the forward trail on a new move, and remembering where the user was.* - `artifacts.md` (135 lines) — *What the user can do with and rely on from an artifact: its document's independence, sandbox and interactivity, its name, its unguessable ID, its store and share link, how its record is saved, deleting it, moving it to another channel, a shared one outliving its producer, and what survives moving around the app.* @@ -40,7 +40,7 @@ Generated from the first-line description of every spec by `scripts/specs-index. - `licensing.md` (294 lines) — *The licensing implementation: bundle-derived third-party notices generation for the esbuild, Vite, and skill outputs, the config file for elections, notice texts, and ignored packages, the vendored-asset provenance manifest, the committed source-surface notices, the suite-run allowlist gate, and the assertions on the CLI tarball and the desktop upload directory — wired into the existing build, verify, and publish pipeline.* - `making-skills.md` (30 lines) — *Arch spec: how a bundled skill gets made — authority, bundle derivation, and final consumer delivery.* - `node-versions.md` (68 lines) — *The Node and npm versions used for repository work, automation, publishing, and published-package support.* - - `testing-policy.md` (234 lines) — *How Television is tested: coverage requirements, the fixture/mock distinction, the three test shapes and their declaration schema, and mock vs integration discipline.* + - `testing-policy.md` (248 lines) — *How Television is tested: test authoring and coverage, focused iteration, verification, and agent permissions.* - **artifact-frame/** - `artifact-bridge.md` (354 lines) — *The artifact bridge: how an embedded artifact document and the app cooperate across the iframe/webview boundary — lifecycle and readiness, navigation reporting, input observation, keyboard forwarding, and live updates for shared artifacts.* - `index.md` (51 lines) — *The artifact-frame architecture root: the domain's module map, the frame lifecycle rules, and the deliberate boundary between what is specified here and what remains governed by code.* @@ -59,7 +59,7 @@ Generated from the first-line description of every spec by `scripts/specs-index. - `artifact-partitions.md` (75 lines) — *Desktop artifact partitions: which Electron session each artifact webview uses, never the window's, how the served interface and the main process agree on it, the record of each server's partitions, and the reaper that deletes the partitions of deleted artifacts.* - `connect-flow.md` (50 lines) — *The desktop main-process connection flow: the local page, connection entry from a link, saved connections and their reconnection, Disconnect from Server, server-page loading and recovery, and the local-only IPC bridge.* - `distribution.md` (100 lines) — *The ToDesktop build of the desktop app: the private workspace, the ToDesktop configuration and build target, the upload directory, the build script, candidate and test builds, desktop releases, and the download link.* - - `e2e-harness.md` (101 lines) — *The Electron end-to-end harness: preparing the exact runtime, planning its Linux environment, handing a validated executable to Playwright, launching the package, and proving real-Electron seams.* + - `e2e-harness.md` (103 lines) — *The Electron end-to-end harness: preparing the exact runtime, planning its Linux environment, handing a validated executable to Playwright, launching the package, and proving real-Electron seams.* - `index.md` (65 lines) — *The desktop architecture root: server connection, appearance, the ToDesktop build, the app's updates, the Electron runtime for development and tests, the test harness, and main-process identity.* - `runtime.md` (86 lines) — *The Electron runtime for development runs and tests: its exact version and declarations, installed-file validity on development and test hosts, how those hosts obtain it, and the recurring major-upgrade procedure.* - `updates.md` (47 lines) — *Desktop updates: how the main process starts ToDesktop's update runtime, records a downloaded update, tells the served interface about it, and restarts the app to install it.* @@ -89,15 +89,12 @@ Generated from the first-line description of every spec by `scripts/specs-index. - `sessions.md` (51 lines) — *The telemetry session module: server-side per-client session state, the content-free client activity signal, server-managed UUIDv7 `$session_id` minting and lazy rotation, and how every event is attributed to a session so PostHog derives sessions natively.* - `sink.md` (63 lines) — *The telemetry sink: the PostHog transport, its bounded in-memory buffer, fire-and-forget delivery that never blocks or crashes the server, and the platform operations (region, key, agent access).* - **test-runner/** - - `attestation.md` (132 lines) — *Tree-hash test attestation: recording that an exact tree passed full validation as a bare `refs/testpass//` ref, and letting validation-bearing CI runs skip their heavy test jobs — never their cheap ones — when the identical tree is already attested under the current version.* - - `blaxel-testshards.md` (131 lines) — *The Blaxel shard coordinator: leasing a persistent sandbox pool, selecting the repository workers fetch, sourcing a GitHub token, preparing and locking workers, running shards with dependency-hash caching, and classifying Blaxel shard outcomes.* - - `flaky-tests.md` (74 lines) — *The flaky-test policy: the runner's retry budget, the per-test `FLAKY_TEST_RETRIES` annotation, and how recovered flakes are surfaced.* - - `github-ci.md` (137 lines) — *The GitHub Actions CI execution contract: PR concurrency, per-job dependency and browser environment, build-once fan-out with an artifact manifest, cleanup tool prerequisites, worker counts, and the retirement record for the Blacksmith-dispatched shard provider.* - - `preflight.md` (92 lines) — *The checks the test runner runs before tests: local capability checks for the selected surfaces, and the git-safety and provider-auth checks that gate a committed-revision remote run.* - - `reporting.md` (411 lines) — *The run directory, normalized reports, timing events, and CI run artifacts every test run produces.* - - `sharded-execution.md` (341 lines) — *The shared remote execution model: deterministic duration-aware plans, per-worker execution, provider reports, retries, and normalization.* - - `test-registry.md` (181 lines) — *The test registry: how `test.config.mjs` declares the surfaces, suites, and execution groups the test runner selects, owns files against, and validates.* - - `test-runner.md` (261 lines) — *The canonical `npm test` and `npm run verify` command surface: how a caller selects tests, picks where they run, and runs them, with the guardrails that keep broad runs honest.* + - `attestation.md` (98 lines) — *Tree attestations: recording that an exact tree passed the full gate, so that GitHub CI can skip its heavy test jobs for that tree, without ever skipping on doubt.* + - `blaxel-testshards.md` (132 lines) — *Blaxel: the pool of persistent remote workers that runs broad validation, how runs from many branches and repositories share it safely, and how to set it up.* + - `github-ci.md` (55 lines) — *The GitHub Actions workflow that validates pull requests and shared branches: when it runs, the checks branch protection requires, how its jobs share setup, and how failures stay visible.* + - `reporting.md` (129 lines) — *What every test run leaves behind: the run directory, a compact summary for people and agents, honest status, timing records, and resource diagnostics from Blaxel workers.* + - `sharded-execution.md` (37 lines) — *How suite runs divide test files among parallel machines, the committed durations that balance them, and how shard outcomes combine into one honest result.* + - `test-runner.md` (196 lines) — *Television's one test command: how callers select and run tests, the guardrails that protect shared development hosts, retries, verification, the test registry, and the rules for processes and network addresses that tests create.* - **themes/** - `authoring.md` (204 lines) — *Theme-authoring guidance: the self-contained theming document bundled inside the `television` skill, the authoritative UI material it carries, and the workflow it gives an agent.* - `bundled-installation.md` (77 lines) — *Bundled theme installation: validated package assets, CLI path resolution, minimum-version installation and replacement with backup copies, and the one-time default-theme selection.* diff --git a/specs/product/desktop-app.md b/specs/product/desktop-app.md index 04b6256a..877a5480 100644 --- a/specs/product/desktop-app.md +++ b/specs/product/desktop-app.md @@ -57,4 +57,4 @@ The pull request that takes a change to `main` runs the checks its changes need. - **Updates** (manual). Needed for changes to the update runtime or where the app starts it, the Electron or electron-builder version, the [desktop self-update notice](./update-notifications.md#^desktop-self-update) and its restart, or the upgrade gate, including its screen. The installed build is opened with ToDesktop's simulated update, `--runtime-simulate-updates=update-available`, and connected to a server run from the same code. It must present the desktop self-update notice with the simulated version, and pressing **Restart to update** must quit the app and open it again. For a change to the gate, the app connected to a server that gates it must show the downloaded update and **Restart to update** on the gate screen. - **Bridge and IPC** (manual). Needed for changes to the native bridge, or to the messages between the app and the page or its artifact views. Connected to a server run from the same code, the installed build must do what each changed operation provides, used through the part of the interface that calls it. -The pull request that moves the app to ToDesktop must also record a run of the complete local verification gate, `npm run verify -- local`, on a supported Mac. This run provides the evidence that [preflight testing](../arch/test-runner/preflight.md#Testing) requires from a real macOS host. +The pull request that moves the app to ToDesktop must also record a run of the complete local verification gate, `npm run verify -- local`, on a supported Mac. This run exercises the test runner on a real macOS host, including its macOS process backend ([processes started by tests](../arch/test-runner/test-runner.md#processes-started-by-tests)). diff --git a/specs/spec-migration.md b/specs/spec-migration.md index f29044a3..62633c76 100644 --- a/specs/spec-migration.md +++ b/specs/spec-migration.md @@ -22,7 +22,7 @@ A redesign contract may impose one requirement on an otherwise unmigrated module The migrated areas, authoritative here: -- **Testing** → the cross-cutting [testing-policy.md](arch/testing-policy.md) (how tests are written and judged), and the test-runner subsystem under `specs/arch/test-runner/`: [test-runner.md](arch/test-runner/test-runner.md) and its companions — the test registry, preflight, reporting, sharded execution, the Blaxel provider, GitHub CI, attestation, and flaky tests. +- **Testing** → the cross-cutting [testing-policy.md](arch/testing-policy.md) (how tests are written and judged), and the test-runner subsystem under `specs/arch/test-runner/`: [test-runner.md](arch/test-runner/test-runner.md), which includes the test registry, preflight checks and retries, and its companions — reporting, sharded execution, the Blaxel provider, GitHub CI, and attestation. - **Node and npm versions** → [arch/node-versions.md](arch/node-versions.md) for the development/build toolchain, the CI/publish toolchain, and the published consumer floor. - **Product versioning** → [product/versioning.md](product/versioning.md) for Television release naming, exact Semantic Versions, the version each desktop release carries, and the Television 1.3 theme-release baseline; [arch/updates/index.md](arch/updates/index.md) for version validation, comparison rules, workspace synchronization, npm publishing eligibility, serialization and action pinning, and publication order. - **Desktop application and connection flow** → [product/desktop-app.md](product/desktop-app.md) for installation, platform support, connection to a server, identity, saved data, and the npm package's status; [arch/desktop/connect-flow.md](arch/desktop/connect-flow.md) for the local page, connection entry from a link, saved connections and their reconnection, Disconnect from Server, server-page loading and recovery, and the local-only IPC bridge; [arch/desktop/appearance.md](arch/desktop/appearance.md) for confirmed appearance state across the renderer, preload, main, and Electron native theme; [arch/desktop/artifact-partitions.md](arch/desktop/artifact-partitions.md) for the partitions artifact webviews use, their record, and their reaper; the remaining architecture modules under `specs/arch/desktop/` own the ToDesktop build, the app's updates, the Electron runtime for development and tests, and the Electron e2e harness. The connect screen's interaction, markup, and styling are owned by [ui/setup/index.md](ui/setup/index.md); how a failed desktop connect check is classified, and the wording of its message, remain code-authoritative under the connection spec's stated boundary. diff --git a/specs/spec-workflow.md b/specs/spec-workflow.md index c6dd6617..3ad362ec 100644 --- a/specs/spec-workflow.md +++ b/specs/spec-workflow.md @@ -223,9 +223,13 @@ Television does not currently accept external pull requests. Forking, building a ## Pull requests and human revisions -This section owns pull-request timing and the shared-branch workflow. Shared branches are `main` and every branch under `integration/`, at any depth; `integration/desktop/refresh` is inside the namespace, while `feature/integration/desktop` is outside it. ^shared-branch-workflow +This section owns development-branch commits and pushes, their coordination, pull-request timing, and the shared-branch workflow. Shared branches are `main` and every branch under `integration/`, at any depth; `integration/desktop/refresh` is inside the namespace, while `feature/integration/desktop` is outside it. ^shared-branch-workflow -Developers work on their own development branches. These branches may be unfinished or failing, and commits, pushes, and draft pull requests may be used as checkpoints. Draft pull request CI starts only when the pull request is marked ready for review. +Contributors and agents work on their own development branches and commit and push regularly in safe, coherent commits. A commit captures a meaningful unit of work that is safe to publish; unfinished work or failing tests may be pushed as a development checkpoint while review and implementation proceed. Draft pull requests may also serve as checkpoints. Publishing a checkpoint does not establish readiness to merge. Draft pull request CI starts only when the pull request is marked ready for review. + +Pushing makes a revision available for [Blaxel validation](arch/test-runner/test-runner.md#^remote-revision). Withholding pushes prevents remote workers from testing current work and pressures agents into broad local runs that can drive a shared host into swap and make it unusable for everyone. Prepare and push coherent development checkpoints before broader remote checks, following the [testing policy](arch/testing-policy.md#test-iteration-discipline). + +A supervisor or coordinator must not casually instruct a worker not to commit or push on its development branch. Routine review coordination is not a reason to suspend this practice. A concrete safety or conflicting-edit issue can justify a temporary hold; withheld or blocked pushes do not lift local-run restrictions. Regular development pushes need no fixed interval or separate approval step. When a developer wants to contribute that work to a shared branch: diff --git a/specs/terms.md b/specs/terms.md index baa2218e..3b729c22 100644 --- a/specs/terms.md +++ b/specs/terms.md @@ -111,20 +111,19 @@ These are the terms used to talk about the spec system itself. Domain terms are ## Testing - *provider* — where tests run: `local` or `blaxel`. Owned by `arch/test-runner/test-runner.md`. +- *marked host* / *unmarked host* — a host with / without the invoking user's `.tvdev-use-blaxel` marker. Owned by [the runner's host-marker contract](arch/test-runner/test-runner.md#^blaxel-host-marker). - *attestation* — a bare `refs/testpass//` ref recording that an exact tree passed full validation under an attestation version. Owned by `arch/test-runner/attestation.md`. - *selector* — a flag that narrows which tests run (suite, surface, package, file, grep, runner, tag). Owned by `arch/test-runner/test-runner.md`. -- *broad run* — an unnarrowed run of everything, gated by the broad-run guardrail. Owned by `arch/test-runner/test-runner.md`. -- *test surface* — an independently runnable test area declared in `test.config.mjs`. Owned by `arch/test-runner/test-registry.md`. -- *suite* — a named selection of surfaces (`unit`, `e2e`, `agent`, `all`). Owned by `arch/test-runner/test-registry.md`. -- *execution group* — an ordered band of surfaces declared in the registry. Owned by `arch/test-runner/test-registry.md`. -- *preflight* — checks run before a test run: local capability, and remote git-safety and auth. Owned by `arch/test-runner/preflight.md`. +- *broad run* — a run that selects more than one test file. Local admission on marked hosts follows [the one-file limit](arch/test-runner/test-runner.md#^local-one-file). +- *test surface* — an independently runnable test area declared in `test.config.mjs`. Owned by [the test registry](arch/test-runner/test-runner.md#the-test-registry). +- *suite* — a named selection of surfaces (`unit`, `e2e`, `agent`, `all`). Owned by [the test registry](arch/test-runner/test-runner.md#the-test-registry). +- *preflight* — the checks that run before a selection's tests, such as a surface's declared checks and Blaxel's credential and revision checks. Owned by `arch/test-runner/test-runner.md`. - *run directory* — the `.test-runs//` tree holding a run's normalized output. Owned by `arch/test-runner/reporting.md`. -- *normalized surface result* — the per-surface result shape every runner and provider is normalized into. Owned by `arch/test-runner/reporting.md`. -- *shard* — one slice of a remote provider run. Owned by `arch/test-runner/sharded-execution.md`. -- *attempt* — one execution of a shard set, combined across retries. Owned by `arch/test-runner/sharded-execution.md`. -- *infrastructure failure* — a shard failure in setup or around the run, distinct from a test failure. Owned by `arch/test-runner/sharded-execution.md`. -- *test failure* — a shard failure caused by a failing test, distinct from an infrastructure failure. Owned by `arch/test-runner/sharded-execution.md`. -- *flaky test* — a test that passes on retry; governed by the retry budget. Owned by `arch/test-runner/flaky-tests.md`. +- *shard* — one of the parallel machines among which a Blaxel suite run or GitHub CI's end-to-end matrix divides the selected test files. Owned by `arch/test-runner/sharded-execution.md`. +- *attempt* — one execution of a shard's assignment; an infrastructure retry makes another attempt with the same assignment. Owned by [shard outcomes](arch/test-runner/sharded-execution.md#shard-outcomes). +- *infrastructure failure* — a shard whose execution or evidence is incomplete (`infra-failed`), distinct from a test failure. Owned by [shard outcomes](arch/test-runner/sharded-execution.md#shard-outcomes). +- *failed shard* — a completed shard classified `failed` under [shard outcomes](arch/test-runner/sharded-execution.md#shard-outcomes). +- *recovered flake* — a test that fails and then passes on retry; it counts as passed and is reported. Owned by [the runner's retries](arch/test-runner/test-runner.md#retries). ## Product versioning diff --git a/test.config.mjs b/test.config.mjs index 1b279a3d..92d30b21 100644 --- a/test.config.mjs +++ b/test.config.mjs @@ -92,6 +92,14 @@ export default { executionGroup("agent:root", "agent: root", 14, [ surface("agent:root", "vitest", "test/agent/vitest.config.ts", { kind: "agent", roots: ["test/agent"], agent: true, tags: ["agent"] }), ]), + executionGroup("experiment:blaxel-harness", "experiment: Blaxel harness", 88, [ + surface("experiment:blaxel-harness", "vitest", "test/runner-fixtures/blaxel-harness/driver/vitest.config.ts", { kind: "experiment", roots: ["test/runner-fixtures/blaxel-harness/driver/harness.test.ts"], tags: ["blaxel-harness-driver"] }), + surface("experiment:blaxel-harness-fixtures", "vitest", "test/runner-fixtures/blaxel-harness/fixtures/vitest.config.ts", { kind: "experiment", roots: ["test/runner-fixtures/blaxel-harness/fixtures"], tags: ["blaxel-harness-fixture"] }), + ]), + executionGroup("experiment:guidance", "experiment: guidance fixtures", 88, [ + surface("experiment:guidance-vitest", "vitest", "test/runner-fixtures/guidance/vitest/vitest.config.ts", { kind: "experiment", roots: ["test/runner-fixtures/guidance/vitest"], tags: ["runner-guidance"] }), + surface("experiment:guidance-playwright", "playwright", "test/runner-fixtures/guidance/playwright/playwright.config.ts", { kind: "experiment", roots: ["test/runner-fixtures/guidance/playwright"], tags: ["runner-guidance"] }), + ]), executionGroup("experiment:dynamic-services", "experiment: dynamic services", 89, [ surface("experiment:dynamic-services", "playwright", "test/runner-fixtures/dynamic-services/playwright.config.ts", { kind: "experiment", diff --git a/test/agent/vitest.config.ts b/test/agent/vitest.config.ts index d0fdcb87..88687d92 100644 --- a/test/agent/vitest.config.ts +++ b/test/agent/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/test/fixtures/resource-diagnostics/native.fixture.ts b/test/fixtures/resource-diagnostics/native.fixture.ts new file mode 100644 index 00000000..9254bdc3 --- /dev/null +++ b/test/fixtures/resource-diagnostics/native.fixture.ts @@ -0,0 +1,8 @@ +import { afterAll, expect, test } from "vitest"; +import { writeFileSync } from "node:fs"; +import path from "node:path"; + +afterAll(() => writeFileSync(path.join(process.env.RESOURCE_FIXTURE_ROOT!, "teardown.json"), JSON.stringify({ observedAt: new Date().toISOString() }))); +test("retains a failed attempt before recovery", (context) => { + expect(context.task.result?.retryCount ?? 0).toBeGreaterThan(0); +}); diff --git a/test/fixtures/resource-diagnostics/prepare.mjs b/test/fixtures/resource-diagnostics/prepare.mjs new file mode 100644 index 00000000..b1f8b222 --- /dev/null +++ b/test/fixtures/resource-diagnostics/prepare.mjs @@ -0,0 +1,12 @@ +import fs from "node:fs"; +import path from "node:path"; + +const root = process.env.RESOURCE_FIXTURE_ROOT; +const resources = path.join(root, "results", "resources"); +const records = fs.existsSync(resources) ? fs.readdirSync(resources).filter((name) => name.endsWith(".json")).map((name) => JSON.parse(fs.readFileSync(path.join(resources, name), "utf8"))) : []; +fs.writeFileSync(path.join(root, "preparation.json"), JSON.stringify({ observedAt: new Date().toISOString(), records })); +if (process.env.RESOURCE_FIXTURE_MODE === "interrupt") { + // Abruptly end our own fixture parent, then exit this only descendant too. + // TERM permits the real supervisor to finalize, which is a different path. + process.kill(process.ppid, "SIGKILL"); +} diff --git a/test/fixtures/resource-diagnostics/setup.mjs b/test/fixtures/resource-diagnostics/setup.mjs new file mode 100644 index 00000000..e1930cac --- /dev/null +++ b/test/fixtures/resource-diagnostics/setup.mjs @@ -0,0 +1,3 @@ +export default function setup() { + if (process.env.RESOURCE_FIXTURE_MODE === "exit") process.exit(17); +} diff --git a/test/fixtures/worker-builds/exercise.ts b/test/fixtures/worker-builds/exercise.ts new file mode 100644 index 00000000..9804d025 --- /dev/null +++ b/test/fixtures/worker-builds/exercise.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import { appendFileSync, existsSync, readFileSync, readdirSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { disposeAllProductServers, launchProductServer } from "../../helpers/product-server.ts"; +import { buildVersionedWebBundle } from "../../helpers/versioned-web-bundle.ts"; + +export async function exerciseWorkerBuilds(caseNumber: number): Promise { + const temporaryRoot = os.tmpdir(); + const recordPath = process.env.TV_WORKER_BUILD_RECORD; + assert.ok(recordPath, "the parent supplies a private observation file"); + + if (caseNumber === 1) { + await assert.rejects(launchProductServer({ + onboardingBundle: path.join(temporaryRoot, "absent-bundle"), + env: { VITEST: "" }, + }), /ENOENT/); + assert.deepEqual(readdirSync(temporaryRoot).filter((entry) => entry.startsWith("television-product-e2e-")), [], + "the failed launch removes its home and private runtime before another launch"); + } + + const server = await launchProductServer({ env: { VITEST: "" } }); + let webBundle: string; + try { + const response = await fetch(server.serverURL); + assert.equal(response.status, 200); + const document = await response.text(); + const script = document.match(/src="([^"]+\.js)"/); + assert.ok(script, "the source CLI serves the built application"); + const asset = await fetch(new URL(script[1]!, server.serverURL)); + assert.equal(asset.status, 200); + assert.ok((await asset.text()).length > 0); + + webBundle = await buildVersionedWebBundle("worker-lifetime-fixture"); + const builtDocument = readFileSync(path.join(webBundle, "index.html"), "utf8"); + const builtScript = builtDocument.match(/src="([^"]+\.js)"/); + assert.ok(builtScript, "the real versioned builder emits an application entry"); + assert.ok(readFileSync(path.join(webBundle, builtScript[1]!)).length > 0); + } finally { + await server.dispose(); + await disposeAllProductServers(); + } + + const sourceDirectories = readdirSync(temporaryRoot) + .filter((entry) => entry.startsWith("television-source-cli-e2e-")) + .map((entry) => path.join(temporaryRoot, entry)); + assert.equal(sourceDirectories.length, 1, "one source build is shared across cases"); + assert.ok(existsSync(path.join(sourceDirectories[0]!, "tv.cjs"))); + assert.ok(existsSync(path.join(webBundle!, "index.html")), "case cleanup preserves the shared bundle"); + assert.equal(existsSync(server.home), false, "case cleanup removes its private home"); + appendFileSync(recordPath, `${JSON.stringify({ caseNumber, pid: process.pid, source: sourceDirectories[0], webBundle })}\n`); +} diff --git a/test/fixtures/worker-builds/playwright.fixture.ts b/test/fixtures/worker-builds/playwright.fixture.ts new file mode 100644 index 00000000..44be09a9 --- /dev/null +++ b/test/fixtures/worker-builds/playwright.fixture.ts @@ -0,0 +1,5 @@ +import { test } from "../../helpers/playwright.ts"; +import { exerciseWorkerBuilds } from "./exercise.ts"; + +test("first case prepares usable builds", async () => exerciseWorkerBuilds(1)); +test("second case reuses the worker builds", async () => exerciseWorkerBuilds(2)); diff --git a/test/fixtures/worker-builds/vitest.fixture.ts b/test/fixtures/worker-builds/vitest.fixture.ts new file mode 100644 index 00000000..8bc4c7a4 --- /dev/null +++ b/test/fixtures/worker-builds/vitest.fixture.ts @@ -0,0 +1,5 @@ +import { test } from "../../helpers/vitest.ts"; +import { exerciseWorkerBuilds } from "./exercise.ts"; + +test("first case prepares usable builds", async () => exerciseWorkerBuilds(1)); +test("second case reuses the worker builds", async () => exerciseWorkerBuilds(2)); diff --git a/test/helpers/application-requests.ts b/test/helpers/application-requests.ts new file mode 100644 index 00000000..08fad8d5 --- /dev/null +++ b/test/helpers/application-requests.ts @@ -0,0 +1,35 @@ +import { expect, type Page, type TestInfo } from "@playwright/test"; + +/** Retain paths only: authentication fragments and query values are never recorded. */ +export function recordApplicationRequests(pages: readonly Page[]) { + const inventories = pages.map((page) => { + const entries = new Set(); + page.on("request", (request) => { + entries.add(JSON.stringify({ + path: new URL(request.url()).pathname, + type: request.resourceType(), + application: request.frame() === page.mainFrame(), + })); + }); + return entries; + }); + const snapshot = () => inventories.map((entries) => [...entries].sort().map((entry) => + JSON.parse(entry) as { path: string; type: string; application: boolean } + )); + return { + expectBuilt(): void { + for (const requests of snapshot()) { + const application = requests.filter((request) => request.application); + expect(application.some((request) => /^\/(packages|@vite|@id|@fs|node_modules)\//.test(request.path)), + "the application loads shipped assets without development modules").toBe(false); + expect(application.some((request) => request.type === "document" && request.path === "/")).toBe(true); + expect(application.some((request) => request.type === "script" && /^\/assets\/.+\.js$/.test(request.path))).toBe(true); + } + }, + async attach(info: TestInfo): Promise { + await info.attach("application-request-inventory", { + body: Buffer.from(JSON.stringify(snapshot(), null, 2)), contentType: "application/json", + }); + }, + }; +} diff --git a/test/helpers/owned-process.ts b/test/helpers/owned-process.ts index 4410672f..ac0ed595 100644 --- a/test/helpers/owned-process.ts +++ b/test/helpers/owned-process.ts @@ -35,8 +35,10 @@ export function spawnOwnedProcess(command: string, args: readonly string[] = [], const { termGraceMs = TEST_PROCESS_TERM_GRACE_MS, env: requestedEnv, ...spawnOptions } = options; if (!Number.isFinite(termGraceMs) || termGraceMs < 0) throw new Error("termGraceMs must be a non-negative finite number"); const env: NodeJS.ProcessEnv = { ...(requestedEnv ?? process.env) }; - const ownerToken = process.env[OWNER_TOKEN_ENV]; - if (ownerToken !== undefined) env[OWNER_TOKEN_ENV] = ownerToken; + for (const key of [OWNER_TOKEN_ENV, "TV_TEST_SHARD_OWNER"]) { + if (process.env[key] !== undefined) env[key] = process.env[key]; + } + env.TMPDIR ??= process.env.TMPDIR; const detached = process.platform !== "win32"; const child = spawn(command, [...args], { ...spawnOptions, env, detached }); if (!child.pid) { diff --git a/test/helpers/playwright.ts b/test/helpers/playwright.ts index d2e51024..39bef2cb 100644 --- a/test/helpers/playwright.ts +++ b/test/helpers/playwright.ts @@ -1,10 +1,20 @@ import { expect, test as base } from "@playwright/test"; -import { disposeAllProductServers } from "./product-server.ts"; +import { disposeAllProductServers, disposeSourceCLIBuilds } from "./product-server.ts"; +import { disposeVersionedWebBuilds } from "./versioned-web-bundle.ts"; export { expect }; /** Playwright test registration with per-test cleanup for product servers, including interrupted launches, and their app proxies. */ -export const test = base.extend<{ productServerCleanup: void }>({ +export const test = base.extend<{ productServerCleanup: void }, { workerBuildCleanup: void }>({ + workerBuildCleanup: [async ({}, use) => { + try { + await use(); + } finally { + // Keep runtime files if a live server could not be stopped. + await disposeAllProductServers(); + await Promise.all([disposeSourceCLIBuilds(), disposeVersionedWebBuilds()]); + } + }, { scope: "worker", auto: true }], productServerCleanup: [async ({}, use) => { try { await use(); diff --git a/test/helpers/product-server.ts b/test/helpers/product-server.ts index 15a3f3cb..a8fc6eb3 100644 --- a/test/helpers/product-server.ts +++ b/test/helpers/product-server.ts @@ -9,11 +9,13 @@ import { fileURLToPath } from "node:url"; import { parseConnectURL } from "@telepath-computer/television-shared"; import { spawnOwnedProcess, type OwnedProcess } from "./owned-process.ts"; import { writeHomeConfig } from "./television-home.ts"; +import { WorkerBuilds } from "./worker-builds.ts"; const HERE = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(HERE, "../.."); const CLI_BUILD_SCRIPT = path.join(REPO_ROOT, "packages/cli/build.mjs"); -let sourceCLIEntry: string | null = null; +const sourceCLIBuilds = new WorkerBuilds("television-source-cli-e2e-"); +export const disposeSourceCLIBuilds = (): Promise => sourceCLIBuilds.dispose(); const serverStartupTimeoutMs = 30_000; export interface ProductServer { @@ -64,16 +66,15 @@ export interface LaunchProductServerOptions { env?: Record; } -function getSourceCLIEntry(): string { - if (sourceCLIEntry) return sourceCLIEntry; - const buildDir = mkdtempSync(path.join(os.tmpdir(), "television-source-cli-e2e-")); - sourceCLIEntry = path.join(buildDir, "tv.cjs"); - execFileSync(process.execPath, [CLI_BUILD_SCRIPT, "--outfile", sourceCLIEntry], { - cwd: REPO_ROOT, - stdio: "pipe", +async function getSourceCLIEntry(): Promise { + const buildDir = await sourceCLIBuilds.get("source", (directory) => { + execFileSync(process.execPath, [CLI_BUILD_SCRIPT, "--outfile", path.join(directory, "tv.cjs")], { + cwd: REPO_ROOT, + stdio: "pipe", + }); + installProductRuntimeAssets(directory); }); - installProductRuntimeAssets(buildDir); - return sourceCLIEntry; + return path.join(buildDir, "tv.cjs"); } /** @@ -81,28 +82,33 @@ function getSourceCLIEntry(): string { * The shared cached layout stays bundle-free so ordinary launches keep the * missing-bundle behavior. */ -function getCLIEntryWithBundles(options: { +function getCLIEntryWithBundles(sharedEntry: string, options: { onboardingBundle?: string; bundledThemesBundle?: string; }): { cliEntry: string; dispose: () => void } { - const sharedEntry = getSourceCLIEntry(); const runDir = mkdtempSync(path.join(os.tmpdir(), "television-product-e2e-bundle-")); - cpSync(path.dirname(sharedEntry), runDir, { recursive: true }); - if (options.onboardingBundle !== undefined) { - cpSync(options.onboardingBundle, path.join(runDir, "onboarding"), { recursive: true }); - } - if (options.bundledThemesBundle !== undefined) { - cpSync(options.bundledThemesBundle, path.join(runDir, "themes"), { recursive: true }); + try { + cpSync(path.dirname(sharedEntry), runDir, { recursive: true }); + if (options.onboardingBundle !== undefined) { + cpSync(options.onboardingBundle, path.join(runDir, "onboarding"), { recursive: true }); + } + if (options.bundledThemesBundle !== undefined) { + cpSync(options.bundledThemesBundle, path.join(runDir, "themes"), { recursive: true }); + } + return { + cliEntry: path.join(runDir, path.basename(sharedEntry)), + dispose: () => rmSync(runDir, { recursive: true, force: true }), + }; + } catch (error) { + rmSync(runDir, { recursive: true, force: true }); + throw error; } - return { - cliEntry: path.join(runDir, path.basename(sharedEntry)), - dispose: () => rmSync(runDir, { recursive: true, force: true }), - }; } function installProductRuntimeAssets(buildDir: string): void { const webDist = path.join(REPO_ROOT, "packages/web/dist"); const artifactMissingView = path.join(webDist, "views/artifact-missing"); + const urlUnsupportedView = path.join(webDist, "views/url-unsupported"); const markdownView = path.join(REPO_ROOT, "packages/view-markdown/dist"); const canonical = path.join(REPO_ROOT, "packages/canonical/dist/canonical"); const sdk = path.join(REPO_ROOT, "packages/server/dist/sdk"); @@ -111,6 +117,9 @@ function installProductRuntimeAssets(buildDir: string): void { if (!existsSync(path.join(artifactMissingView, "index.html"))) { throw new Error(`Web build produced no artifact-missing view at ${artifactMissingView}`); } + if (!existsSync(path.join(urlUnsupportedView, "index.html"))) { + throw new Error(`Web build produced no url-unsupported view at ${urlUnsupportedView}`); + } if (!existsSync(path.join(markdownView, "index.html"))) { throw new Error(`View build produced no markdown view at ${markdownView}`); } @@ -133,6 +142,7 @@ function installProductRuntimeAssets(buildDir: string): void { cpSync(sdk, path.join(buildDir, "sdk"), { recursive: true }); const staticDir = path.join(buildDir, "web"); mkdirSync(staticDir, { recursive: true }); + cpSync(urlUnsupportedView, path.join(staticDir, "views/url-unsupported"), { recursive: true }); cpSync(webIndex, path.join(staticDir, "index.html")); cpSync(webAssets, path.join(staticDir, "assets"), { recursive: true }); } @@ -157,22 +167,12 @@ export async function disposeAllProductServers(): Promise { export async function launchProductServer(options: LaunchProductServerOptions = {}): Promise { const createdHome = options.home === undefined; const home = options.home ?? mkdtempSync(path.join(os.tmpdir(), "television-product-e2e-")); - // The test-runner recipe for spawned `tv serve`: config port 0, with the - // acquired port read from startup output and passed to client commands. - writeHomeConfig(home, options.auth ? { port: 0 } : { port: 0, auth: false }); const { TELEVISION_ACP_AGENT: _ignoredAgent, ...inheritedEnv } = process.env; const env = { ...inheritedEnv, ...options.env }; - let bundledRuntime = options.onboardingBundle !== undefined || options.bundledThemesBundle !== undefined - ? getCLIEntryWithBundles({ - ...(options.onboardingBundle !== undefined - ? { onboardingBundle: options.onboardingBundle } - : {}), - ...(options.bundledThemesBundle !== undefined - ? { bundledThemesBundle: options.bundledThemesBundle } - : {}), - }) - : undefined; - const cliEntry = (): string => bundledRuntime?.cliEntry ?? getSourceCLIEntry(); + let sharedEntry: string; + let preparation: Promise | undefined; + let bundledRuntime: ReturnType | undefined; + const cliEntry = (): string => bundledRuntime?.cliEntry ?? sharedEntry; const proxies = new Map(); let current: { process: ProductServerProcess; serverURL: string } | null = null; let pendingProcess: ProductServerProcess | null = null; @@ -201,23 +201,32 @@ export async function launchProductServer(options: LaunchProductServerOptions = const dispose: ProductServerDisposer = () => { disposalRequested = true; disposalPromise ??= (async () => { - try { - await Promise.all([...proxies.values()].map((proxy) => proxy.dispose())); - const processes = new Set([current?.process, pendingProcess].filter((process): process is ProductServerProcess => process != null)); - await Promise.all([...processes].map(stopProcess)); - bundledRuntime?.dispose(); - if (options.cleanupHome ?? createdHome) { - rmSync(home, { recursive: true, force: true }); - } - } finally { - activeProductServerDisposers.delete(dispose); + // A disposal requested during preparation must not race removal of its home. + await preparation?.catch(() => undefined); + await Promise.all([...proxies.values()].map((proxy) => proxy.dispose())); + const processes = new Set([current?.process, pendingProcess].filter((process): process is ProductServerProcess => process != null)); + await Promise.all([...processes].map(stopProcess)); + bundledRuntime?.dispose(); + if (options.cleanupHome ?? createdHome) { + rmSync(home, { recursive: true, force: true }); } + // Failed cleanup remains visible to worker teardown, which preserves builds. + activeProductServerDisposers.delete(dispose); })(); return disposalPromise; }; activeProductServerDisposers.add(dispose); try { + // Register cleanup before configuration, shared preparation or copying can fail. + // Port 0 remains the recipe for every real backend, including restarts. + writeHomeConfig(home, options.auth ? { port: 0 } : { port: 0, auth: false }); + preparation = getSourceCLIEntry(); + sharedEntry = await preparation; + if (disposalRequested) throw new Error("Product server was disposed during preparation"); + if (options.onboardingBundle !== undefined || options.bundledThemesBundle !== undefined) { + bundledRuntime = getCLIEntryWithBundles(sharedEntry, options); + } current = await startServe(); pendingProcess = null; if (disposalRequested) { @@ -246,7 +255,7 @@ export async function launchProductServer(options: LaunchProductServerOptions = rmSync(sibling, { recursive: true, force: true }); cpSync(restartOptions.onboardingBundle, sibling, { recursive: true }); } else { - bundledRuntime = getCLIEntryWithBundles({ + bundledRuntime = getCLIEntryWithBundles(sharedEntry, { onboardingBundle: restartOptions.onboardingBundle, ...(options.bundledThemesBundle !== undefined ? { bundledThemesBundle: options.bundledThemesBundle } diff --git a/test/helpers/television-home.ts b/test/helpers/television-home.ts index 45b46e4b..9fe60254 100644 --- a/test/helpers/television-home.ts +++ b/test/helpers/television-home.ts @@ -5,7 +5,7 @@ import path from "node:path"; * Writes `/config.json`, creating the home when it is missing. A test * that spawns `tv serve` gives the server a home whose config file sets port * 0, then passes the acquired port to client commands with --port - * (specs/arch/test-runner/test-runner.md, "Test infrastructure addresses"). + * (specs/arch/test-runner/test-runner.md, "Network addresses in tests"). */ export function writeHomeConfig(home: string, config: Record): string { mkdirSync(home, { recursive: true }); diff --git a/test/helpers/versioned-web-bundle.ts b/test/helpers/versioned-web-bundle.ts index 0a926926..083038c1 100644 --- a/test/helpers/versioned-web-bundle.ts +++ b/test/helpers/versioned-web-bundle.ts @@ -1,8 +1,7 @@ -import { mkdtempSync } from "node:fs"; import { createRequire } from "node:module"; -import os from "node:os"; import path from "node:path"; import { pathToFileURL, fileURLToPath } from "node:url"; +import { WorkerBuilds } from "./worker-builds.ts"; // Versioned web-bundle builder for the update-notifications harnesses: a real // `vite build` of packages/web stamped at a chosen version via the sanctioned @@ -15,7 +14,8 @@ import { pathToFileURL, fileURLToPath } from "node:url"; const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); const WEB_PACKAGE_DIR = path.join(REPO_ROOT, "packages", "web"); -const distByVersion = new Map>(); +const builds = new WorkerBuilds("television-web-bundle-"); +export const disposeVersionedWebBuilds = (): Promise => builds.dispose(); // Builds are serialized: the version reaches the vite config through // process.env, which concurrent in-process builds would race on. let buildChain: Promise = Promise.resolve(); @@ -36,9 +36,8 @@ async function loadVite(): Promise { return typeof loaded.build === "function" ? loaded : loaded.default!; } -async function runBuild(version: string | undefined): Promise { +async function runBuild(version: string | undefined, outDir: string): Promise { const vite = await loadVite(); - const outDir = mkdtempSync(path.join(os.tmpdir(), "television-web-bundle-")); const previous = process.env.TV_TEST_WEB_VERSION; if (version === undefined) delete process.env.TV_TEST_WEB_VERSION; else process.env.TV_TEST_WEB_VERSION = version; @@ -52,7 +51,6 @@ async function runBuild(version: string | undefined): Promise { if (previous === undefined) delete process.env.TV_TEST_WEB_VERSION; else process.env.TV_TEST_WEB_VERSION = previous; } - return outDir; } /** @@ -61,11 +59,9 @@ async function runBuild(version: string | undefined): Promise { */ export function buildVersionedWebBundle(version?: string): Promise { const key = version ?? ""; - let dist = distByVersion.get(key); - if (dist === undefined) { - dist = buildChain.then(() => runBuild(version), () => runBuild(version)); - buildChain = dist.catch(() => undefined); - distByVersion.set(key, dist); - } - return dist; + return builds.get(key, (directory) => { + const build = buildChain.then(() => runBuild(version, directory)); + buildChain = build.catch(() => undefined); + return build; + }); } diff --git a/test/helpers/vitest.ts b/test/helpers/vitest.ts new file mode 100644 index 00000000..8adfb808 --- /dev/null +++ b/test/helpers/vitest.ts @@ -0,0 +1,16 @@ +import { test as base } from "vitest"; +import { disposeAllProductServers, disposeSourceCLIBuilds } from "./product-server.ts"; +import { disposeVersionedWebBuilds } from "./versioned-web-bundle.ts"; + +/** Test registration for Vitest callers of the reusable product builders. */ +export const test = base.extend<{ workerBuildCleanup: void }>({ + workerBuildCleanup: [async ({}, use) => { + try { + await use(); + } finally { + await disposeAllProductServers(); + await Promise.all([disposeSourceCLIBuilds(), disposeVersionedWebBuilds()]); + } + }, { scope: "worker", auto: true }], +}); +export const it = test; diff --git a/test/helpers/worker-builds.ts b/test/helpers/worker-builds.ts new file mode 100644 index 00000000..d6c7a6e7 --- /dev/null +++ b/test/helpers/worker-builds.ts @@ -0,0 +1,50 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +/** Owns reusable preparation for one runner worker. */ +export class WorkerBuilds { + private readonly builds = new Map>(); + private readonly directories = new Set(); + private disposal: Promise | undefined; + private readonly prefix: string; + private readonly parent: string; + + constructor(prefix: string, parent = os.tmpdir()) { + this.prefix = prefix; + this.parent = parent; + } + + get(key: string, prepare: (directory: string) => void | Promise): Promise { + if (this.disposal) return Promise.reject(new Error("Worker build lifetime has ended")); + let build = this.builds.get(key); + if (!build) { + const directory = mkdtempSync(path.join(this.parent, this.prefix)); + this.directories.add(directory); + build = Promise.resolve().then(() => prepare(directory)).then(() => directory, (error: unknown) => { + this.builds.delete(key); + this.remove(directory); + throw error; + }); + this.builds.set(key, build); + } + return build; + } + + dispose(): Promise { + this.disposal ??= (async () => { + await Promise.allSettled(this.builds.values()); + const failures: unknown[] = []; + for (const directory of this.directories) { + try { this.remove(directory); } catch (error) { failures.push(error); } + } + if (failures.length) throw new AggregateError(failures, "Failed to remove worker builds"); + })(); + return this.disposal; + } + + private remove(directory: string): void { + rmSync(directory, { recursive: true, force: true }); + this.directories.delete(directory); + } +} diff --git a/test/node/acp-home-context.test.ts b/test/node/acp-home-context.test.ts index a6d18c89..5f7319fb 100644 --- a/test/node/acp-home-context.test.ts +++ b/test/node/acp-home-context.test.ts @@ -2,7 +2,8 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vitest"; +import { describe, expect } from "vitest"; +import { it } from "../helpers/vitest.ts"; import { WebSocket } from "ws"; import { ACPClient } from "../../packages/web/src/services/acp-client.ts"; import { launchProductServer } from "../helpers/product-server.ts"; diff --git a/test/node/cli-assets.test.ts b/test/node/cli-assets.test.ts index c690cff5..2e92af35 100644 --- a/test/node/cli-assets.test.ts +++ b/test/node/cli-assets.test.ts @@ -18,7 +18,7 @@ import { // run the e2e:node production-build preCommand. GitHub shards deliberately // skip repeated preCommands and instead restore every packaged and // producer-side tree below from the same CLI build recipe through -// [[arch/test-runner/github-ci.md#^gha-prebuilt-cli-outputs]]. No provider +// proofs/arch/test-runner/github-ci.md#^gha-prebuilt-cli-outputs. No provider // skips a completeness assertion when one of those trees is absent. const REPO_ROOT = path.resolve(import.meta.dirname, "../.."); diff --git a/test/node/daemon-acceptance.test.ts b/test/node/daemon-acceptance.test.ts index 0d1f0672..f54e222e 100644 --- a/test/node/daemon-acceptance.test.ts +++ b/test/node/daemon-acceptance.test.ts @@ -6,7 +6,7 @@ * package in the active npm global prefix. It must run only on a designated * developer host through: * - * TV_DAEMON_TEST_HOST=1 npm test -- local --suite daemon-acceptance + * TV_DAEMON_TEST_HOST=1 npm test -- local --file test/node/daemon-acceptance.test.ts * * Persisted serve cannot accept port 0, so each service port is obtained by * binding loopback port 0 and releasing it immediately before installation. diff --git a/test/node/product-server-proxy.e2e.test.ts b/test/node/product-server-proxy.e2e.test.ts index 7370f107..50583715 100644 --- a/test/node/product-server-proxy.e2e.test.ts +++ b/test/node/product-server-proxy.e2e.test.ts @@ -1,6 +1,7 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import net, { type Socket } from "node:net"; -import { afterEach, describe, expect, test } from "vitest"; +import { afterEach, describe, expect } from "vitest"; +import { test } from "../helpers/vitest.ts"; import { disposeAllProductServers, launchProductServer, startDevelopmentProxy, type DevelopmentProxy } from "../helpers/product-server.ts"; import { ownedProcessCount } from "../helpers/owned-process.ts"; @@ -14,11 +15,13 @@ afterEach(async () => { describe("product server lifecycle", () => { test("disposes a child process while product-server startup is still in flight", async () => { - const startup = launchProductServer(); + const startup = launchProductServer({ env: { VITEST: "" } }); + const rejected = expect(startup).rejects.toThrow("tv serve exited before startup"); + await expect.poll(() => ownedProcessCount(), { interval: 5 }).toBe(1); await disposeAllProductServers(); - await expect(startup).rejects.toThrow("tv serve exited before startup"); + await rejected; expect(ownedProcessCount()).toBe(0); }); }); diff --git a/test/node/update-reload-guard.e2e.test.ts b/test/node/update-reload-guard.e2e.test.ts index 1ff46e11..567a3183 100644 --- a/test/node/update-reload-guard.e2e.test.ts +++ b/test/node/update-reload-guard.e2e.test.ts @@ -1,4 +1,5 @@ -import { describe, it } from "vitest"; +import { describe } from "vitest"; +import { it } from "../helpers/vitest.ts"; import { expect as pwExpect } from "@playwright/test"; import { buildVersionedWebBundle, diff --git a/test/node/update-reload.e2e.test.ts b/test/node/update-reload.e2e.test.ts index 1f359bbe..b8c68916 100644 --- a/test/node/update-reload.e2e.test.ts +++ b/test/node/update-reload.e2e.test.ts @@ -1,4 +1,5 @@ -import { describe, it } from "vitest"; +import { describe } from "vitest"; +import { it } from "../helpers/vitest.ts"; import { expect as pwExpect } from "@playwright/test"; import { buildVersionedWebBundle, diff --git a/test/node/vitest.config.ts b/test/node/vitest.config.ts index 8c0bdfc4..4b2bc48f 100644 --- a/test/node/vitest.config.ts +++ b/test/node/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/test/node/vitest.daemon-acceptance.config.ts b/test/node/vitest.daemon-acceptance.config.ts index ca19ce0a..6190930c 100644 --- a/test/node/vitest.daemon-acceptance.config.ts +++ b/test/node/vitest.daemon-acceptance.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/test/node/worker-build-lifetime.e2e.test.ts b/test/node/worker-build-lifetime.e2e.test.ts new file mode 100644 index 00000000..21020fd7 --- /dev/null +++ b/test/node/worker-build-lifetime.e2e.test.ts @@ -0,0 +1,72 @@ +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { expect, test } from "vitest"; +import { spawnOwnedProcess } from "../helpers/owned-process.ts"; + +const require = createRequire(import.meta.url); +const root = path.resolve(import.meta.dirname, "../.."); +type Observation = { caseNumber: number; pid: number; source: string; webBundle: string }; + +// proofs/arch/testing-policy.md#^testing-t-build-worker-seam +test.each(["playwright", "vitest"] as const)("%s workers reuse builds and remove them at teardown", async (runner) => { + let previous: Observation | undefined; + for (let worker = 0; worker < 2; worker += 1) { + const temporaryRoot = mkdtempSync(path.join(os.tmpdir(), "television-worker-build-seam-")); + const record = path.join(temporaryRoot, "observations.jsonl"); + // Nested fixtures belong to this acceptance, not independently runnable surfaces. + // Their native configs are temporary, as in the runner-reporting acceptance. + const config = path.join(temporaryRoot, `${runner}.config.mjs`); + writeFileSync(config, `export default ${JSON.stringify(runner === "playwright" ? { + testDir: path.join(root, "test/fixtures/worker-builds"), + testMatch: "playwright.fixture.ts", workers: 1, fullyParallel: false, + retries: 0, timeout: 90_000, reporter: "line", outputDir: path.join(temporaryRoot, "results"), + } : { test: { + include: ["test/fixtures/worker-builds/vitest.fixture.ts"], maxWorkers: 1, + fileParallelism: false, retry: 0, testTimeout: 90_000, + } })};\n`); + const cli = runner === "playwright" + ? require.resolve("@playwright/test/cli") + : path.join(path.dirname(require.resolve("vitest/package.json")), "vitest.mjs"); + const owned = spawnOwnedProcess(process.execPath, [ + cli, runner === "playwright" ? "test" : "run", + "--config", config, + ], { + cwd: root, + env: { ...process.env, TMPDIR: temporaryRoot, TV_WORKER_BUILD_RECORD: record }, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + owned.child.stdout!.on("data", (chunk: Buffer) => { output += chunk.toString(); }); + owned.child.stderr!.on("data", (chunk: Buffer) => { output += chunk.toString(); }); + try { + const exitCode = await new Promise((resolve, reject) => { + owned.child.once("error", reject); + owned.child.once("close", resolve); + }); + expect(exitCode, output).toBe(0); + const observations = readFileSync(record, "utf8").trim().split("\n").map((line) => JSON.parse(line) as Observation); + expect(observations).toHaveLength(2); + const [first, second] = observations as [Observation, Observation]; + expect(first.caseNumber).toBe(1); + expect(second).toEqual({ ...first, caseNumber: 2 }); + if (previous) { + expect(first.source).not.toBe(previous.source); + expect(first.webBundle).not.toBe(previous.webBundle); + } + previous = first; + + const remaining = readdirSync(temporaryRoot).filter((name) => + name.startsWith("television-source-cli-e2e-") || + name.startsWith("television-web-bundle-") || + name.startsWith("television-product-e2e-") + ); + expect(remaining, `${runner} left worker builds or failed-launch copies after teardown`).toEqual([]); + } finally { + const cleanup = await owned.dispose(); + expect(cleanup.outcome, output).not.toBe("survived"); + rmSync(temporaryRoot, { recursive: true, force: true }); + } + } +}); diff --git a/test/repo/blaxel-lease-contracts.test.ts b/test/repo/blaxel-lease-contracts.test.ts new file mode 100644 index 00000000..e305eb82 --- /dev/null +++ b/test/repo/blaxel-lease-contracts.test.ts @@ -0,0 +1,192 @@ +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, existsSync, symlinkSync, lstatSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, test as platformTest, vi } from "vitest"; +import { workerLeaseOperation, cleanWorkerTemporaryDirectories } from "../../scripts/test/blaxel-worker-lease.mjs"; +import { finishLease, renewLease, retainAttemptEvidence } from "../../scripts/test/blaxel-lease-lifecycle.mjs"; +import { publishWorker, deleteWorker } from "../../scripts/test/blaxel-pool-lifecycle.mjs"; + +// Contract clocks, owner audits/signals and SDK operations are substitutions. +// Filesystem assertions cross the real cleaner; the live driver proves transport. +const test = platformTest.skipIf(process.platform !== "linux"); +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +function fixture(record?: unknown) { + const tmpRoot = mkdtempSync(path.join(os.tmpdir(), "lease-contract-")); roots.push(tmpRoot); + const ownerFile = path.join(tmpRoot, "tv-testshard.lock/owner.json"); + if (record !== undefined) { mkdirSync(path.dirname(ownerFile)); writeFileSync(ownerFile, typeof record === "string" ? record : JSON.stringify(record)); } + const input = { lockId: "incoming", runId: "incoming-run", shardIndex: 1, attemptId: "incoming-attempt", tmpRoot, scratch: path.join(tmpRoot, "tv-testshards/current"), resultsRoot: path.join(tmpRoot, "results") }; + return { tmpRoot, input, ownerFile }; +} +const expired = { lockId: "old", runId: "old-run", attemptId: "old-attempt", lastRenewal: 1, recoverAfter: 2, expiresAt: 0 }; + +test("expired active and quiet leases use the same cleanup; uncertain owners hold", async () => { + for (const active of [false, true]) { + const f = fixture(expired); const calls: string[] = []; + let owners = active ? [{ shardToken: "old" }] : []; + const result = await workerLeaseOperation(f.input, { now: () => 3000, scanOwners: async () => owners, stopOwners: async (token: string) => { calls.push(token); owners = []; return []; } }); + expect(result.ok).toBe(true); expect(calls).toEqual(["old"]); + expect(result.record.recovery?.discardedReports).toBe(true); + } + const f = fixture(expired); + const result = await workerLeaseOperation(f.input, { now: () => 3000, stopOwners: async () => [], scanOwners: async () => [{ shardToken: "unrelated" }] }); + expect(result).toMatchObject({ ok: false }); + expect(JSON.parse(readFileSync(f.ownerFile, "utf8")).lockId).toBe("old"); +}); + +test("legacy, incomplete, unreadable and audit leases fail closed without clean ownership", async () => { + for (const record of [{ lockId: "legacy", expiresAt: 1 }, {}, "unreadable"]) { + for (const operation of ["acquire", "delete"]) { + const f = fixture(record); + expect(await workerLeaseOperation({ ...f.input, operation }, { now: () => 3000, scanOwners: async () => [{}] })).toMatchObject({ ok: false }); + expect((await workerLeaseOperation({ ...f.input, operation }, { now: () => 3000, scanOwners: async () => [] })).ok).toBe(true); + } + } + for (const record of [expired, { lockId: "legacy", expiresAt: 100 }]) { + const f = fixture(record); + expect(await workerLeaseOperation({ ...f.input, audit: true }, { now: () => 3000, scanOwners: async () => { throw Error("must not inspect held worker"); } })).toMatchObject({ ok: false }); + } + const f = fixture({ lockId: "legacy", expiresAt: 100 }); + expect(await workerLeaseOperation(f.input, { now: () => 3000, scanOwners: async () => [] })).toMatchObject({ ok: false }); + const audit = fixture(); + const residue = path.join(audit.tmpRoot, "television-web-bundle-preserve"); mkdirSync(residue); + const staleScratch = path.join(audit.tmpRoot, "tv-testshards/stale"); mkdirSync(staleScratch, { recursive: true }); + const auditInput = { ...audit.input, audit: true }; + const auditHooks = { scanOwners: async () => { throw Error("audit must observe existing owners before cleanup"); }, stopOwners: async () => [] }; + expect((await workerLeaseOperation(auditInput, auditHooks)).ok).toBe(true); + expect((await workerLeaseOperation({ ...auditInput, operation: "prepare" }, auditHooks)).ok).toBe(true); + expect((await workerLeaseOperation({ ...auditInput, operation: "teardown" }, auditHooks)).ok).toBe(true); + expect(existsSync(residue)).toBe(true); expect(existsSync(staleScratch)).toBe(true); +}); + +test("cleanup failures keep the old identity, while explicit deletion reserves an expired worker", async () => { + const f = fixture(expired); + await expect(workerLeaseOperation(f.input, { now: () => 3000, scanOwners: async () => [], stopOwners: async () => { throw Error("cannot signal"); } })).rejects.toThrow("cannot signal"); + expect(JSON.parse(readFileSync(f.ownerFile, "utf8")).lockId).toBe("old"); + expect(await workerLeaseOperation({ ...f.input, operation: "delete" }, { now: () => 3000, scanOwners: async () => { throw Error("unavailable"); } })).toMatchObject({ ok: true, deleting: true }); + expect(await workerLeaseOperation(f.input)).toMatchObject({ ok: false }); +}); + +test("only known real directories are removed, with allocated-byte evidence and outside links preserved", async () => { + const f = fixture(); const outside = path.join(f.tmpRoot, "unrelated"); mkdirSync(outside); writeFileSync(path.join(outside, "keep"), "keep"); + for (const name of ["television-source-cli-e2e-old", "television-web-bundle-old", "tv-testshards/stale", "tv-testshards/no-metadata", "tv-testshards/current"]) { mkdirSync(path.join(f.tmpRoot, name), { recursive: true }); writeFileSync(path.join(f.tmpRoot, name, "data"), "payload"); } + symlinkSync(outside, path.join(f.tmpRoot, "television-web-bundle-link")); + symlinkSync(outside, path.join(f.tmpRoot, "television-source-cli-e2e-old/link")); + const result = await cleanWorkerTemporaryDirectories({ tmpRoot: f.tmpRoot, current: f.input.scratch }); + expect(result.ok).toBe(true); + expect(result.removed["stale-scratch"].directories).toBe(2); + expect(result.removed["television-source-cli-e2e-"].allocatedBytes).toBeGreaterThanOrEqual(0); + expect(result.before.availableBytes).toBeGreaterThan(0); expect(result.after.availableBytes).toBeGreaterThan(0); + expect(existsSync(path.join(outside, "keep"))).toBe(true); expect(existsSync(f.input.scratch)).toBe(true); + expect(lstatSync(path.join(f.tmpRoot, "television-web-bundle-link")).isSymbolicLink()).toBe(true); + await cleanWorkerTemporaryDirectories({ tmpRoot: f.tmpRoot, current: f.input.scratch, onlyCurrent: true }); + expect(existsSync(f.input.scratch)).toBe(false); + const linked = path.join(f.tmpRoot, "link"); symlinkSync(outside, linked); + await expect(cleanWorkerTemporaryDirectories({ tmpRoot: linked, current: null })).rejects.toThrow("unsafe cleanup root"); +}); + +test("mount boundaries and incomplete removal remain reported failures without reclaimed counts", async () => { + for (const failure of ["mount", "identity", "remove"]) { + const f = fixture(); const dir = path.join(f.tmpRoot, "television-web-bundle-old"); mkdirSync(dir); + let directoryReads = 0; + const probes = failure !== "remove" ? { lstat: (file: string) => { + const stat = lstatSync(file); + if (file === dir) { + directoryReads++; + if (failure === "mount") Object.defineProperty(stat, "dev", { value: stat.dev + 1 }); + if (failure === "identity" && directoryReads === 3) Object.defineProperty(stat, "ino", { value: stat.ino + 1 }); + } + return stat; + } } : { remove: () => { throw Object.assign(Error("incomplete"), { code: "EIO" }); } }; + const result = await cleanWorkerTemporaryDirectories({ tmpRoot: f.tmpRoot, current: null }, probes); + expect(result.ok).toBe(false); expect(result.removed).toEqual({}); expect(result.errors).toHaveLength(1); expect(existsSync(dir)).toBe(true); + } +}); + +test("retrieval retries after teardown without rerunning work; uncertainty never releases", async () => { + const calls: string[] = []; let attempts = 0; + const operations = { settleWrites: async () => { calls.push("writes"); }, assertOwned: () => {}, teardown: async () => { calls.push("teardown"); }, download: async () => { calls.push("download"); if (++attempts === 1) throw Error("transport"); }, release: async () => { calls.push("release"); } }; + await finishLease(operations); + expect(calls).toEqual(["writes", "teardown", "download", "download", "release"]); + for (const broken of ["teardown", "download", "ownership"] as const) { + calls.length = 0; + await expect(finishLease({ ...operations, ...(broken === "teardown" ? { teardown: async () => { throw Error("unknown dispatch"); } } : broken === "download" ? { download: async () => { throw Error("transport"); } } : { assertOwned: () => { throw Error("lost lease"); } }) })).rejects.toThrow(); + expect(calls).not.toContain("release"); + } +}); + +test("pool eligibility follows verification, and deletion reserves then unpublishes", async () => { + const calls: string[] = []; + const sandbox = { metadata: { name: "fixture", labels: { purpose: "testshards-provisioning", pool: "private" } }, wait: async () => { calls.push("wait"); } }; + const update = async (_name: string, metadata: { labels: Record }) => { calls.push(metadata.labels.purpose); }; + await publishWorker(sandbox, { provision: async () => { calls.push("provision"); }, verify: async () => { calls.push("verify"); }, update }); + expect(calls).toEqual(["wait", "provision", "verify", "testshards"]); + calls.length = 0; + await expect(publishWorker(sandbox, { provision: async () => {}, verify: async () => { throw Error("wrong architecture"); }, update })).rejects.toThrow(); + expect(calls).not.toContain("testshards"); + sandbox.metadata.labels.purpose = "testshards"; + calls.length = 0; + const ops = { reserve: async () => { calls.push("reserve"); }, update, remove: async () => { calls.push("delete"); throw Error("provider unavailable"); } }; + await expect(deleteWorker(sandbox, ops)).rejects.toThrow(); + expect(calls).toEqual(["reserve", "testshards-unavailable", "delete"]); + calls.length = 0; + await expect(deleteWorker(sandbox, { ...ops, reserve: async () => { throw Error("valid lease"); } })).rejects.toThrow(); + expect(calls).toEqual([]); +}); + + +test("renewal continues while retrieval waits and stops on ownership loss", async () => { + vi.useFakeTimers(); + try { + let lost: unknown = null; + const renew = vi.fn().mockResolvedValue(undefined); + const stop = renewLease({ renew, intervalMs: 10, onLost: (error: unknown) => { lost = error; } }); + let continueDownload!: () => void; + const waiting = new Promise(resolve => { continueDownload = resolve; }); + const release = vi.fn(); + const finishing = finishLease({ settleWrites: async () => {}, teardown: async () => {}, download: () => waiting, release, assertOwned: () => { if (lost) throw lost; } }); + await vi.advanceTimersByTimeAsync(35); + expect(renew).toHaveBeenCalledTimes(3); expect(release).not.toHaveBeenCalled(); + continueDownload(); await finishing; await stop(); + await vi.advanceTimersByTimeAsync(30); + expect(renew).toHaveBeenCalledTimes(3); expect(release).toHaveBeenCalledOnce(); + renew.mockRejectedValueOnce(Error("ownership lost")); + const stopLost = renewLease({ renew, intervalMs: 10, onLost: (error: unknown) => { lost = error; } }); + await vi.advanceTimersByTimeAsync(100); + expect(renew).toHaveBeenCalledTimes(4); expect(lost).toBeInstanceOf(Error); + await stopLost(); + } finally { vi.useRealTimers(); } +}); + +test("failed deletion of never-published workers remains retryable without their runtime", async () => { + const sandbox = { metadata: { name: "partial", labels: { purpose: "testshards-provisioning" } } }; + const ops = { reserve: vi.fn().mockRejectedValue(Error("no runtime")), update: vi.fn(), remove: vi.fn().mockRejectedValueOnce(Error("provider unavailable")).mockResolvedValueOnce(undefined) }; + await expect(deleteWorker(sandbox, ops)).rejects.toThrow("provider unavailable"); + await deleteWorker(sandbox, ops); + expect(ops.reserve).not.toHaveBeenCalled(); expect(ops.update).not.toHaveBeenCalled(); expect(ops.remove).toHaveBeenCalledTimes(2); +}); + +test("preparation evidence is downloaded before an undispatched attempt releases", async () => { + const f = fixture(); + const hooks = { scanOwners: async () => [], stopOwners: async () => [] }; + await workerLeaseOperation(f.input, hooks); + await workerLeaseOperation({ ...f.input, operation: "prepare" }, hooks); + writeFileSync(path.join(f.input.resultsRoot, "preparation.txt"), "produced before dispatch"); + const directory = path.join(f.tmpRoot, "download"); + const lease = { preparationRequested: true, executionRequested: false, teardownConfirmed: true, + shardIndex: 1, input: f.input, sandbox: { fs: { + read: async (file: string) => readFileSync(file.replace("/workspace/television/.testshards/results", f.input.resultsRoot), "utf8"), + ls: async () => { throw Error("confirmed teardown must supply its inventory"); }, + } }, + }; + await finishLease({ + settleWrites: async () => {}, assertOwned: () => {}, + teardown: async () => { expect((await workerLeaseOperation({ ...f.input, operation: "teardown" }, hooks)).ok).toBe(true); }, + download: () => retainAttemptEvidence(lease, directory), + release: async () => { + expect(existsSync(path.join(directory, "artifacts/shard-1/preparation.txt"))).toBe(true); + expect(readFileSync(path.join(directory, "artifacts/shard-1/preparation.txt"), "utf8")).toBe("produced before dispatch"); + expect((await workerLeaseOperation({ ...f.input, operation: "release", retrieved: true }, hooks)).ok).toBe(true); + }, + }); +}); diff --git a/test/repo/blaxel-pool-discovery.test.ts b/test/repo/blaxel-pool-discovery.test.ts index 7d546e46..591cc3ee 100644 --- a/test/repo/blaxel-pool-discovery.test.ts +++ b/test/repo/blaxel-pool-discovery.test.ts @@ -40,7 +40,7 @@ async function withApi(run: (discover: (deployedOnly: boolean) => Promise Promise Promise { await withApi(async (discover, requests) => { - expect(await discover(deployedOnly)).toEqual(deployedOnly ? ["a-worker", "b-worker"] : ["a-worker", "b-worker", "c-failed"]); + expect(await discover(deployedOnly)).toEqual(deployedOnly ? ["a-worker", "b-worker"] : ["a-worker", "b-worker", "c-failed", "d-provisioning", "e-unavailable"]); expect(requests).toEqual(["/sandboxes", "/sandboxes?cursor=second", "/sandboxes?cursor=third"]); }); }); diff --git a/test/repo/blaxel-stale-owner-audit.test.ts b/test/repo/blaxel-stale-owner-audit.test.ts index 5bddc464..8abb11f9 100644 --- a/test/repo/blaxel-stale-owner-audit.test.ts +++ b/test/repo/blaxel-stale-owner-audit.test.ts @@ -82,7 +82,9 @@ describe("Blaxel stale-owner audit", () => { expect(source.indexOf(later), `${later} must follow the audit branch`).toBeGreaterThan(auditBranch); } expect(source).toContain("acquireAny(sandboxes, { shardIndex: 1, slot: 0, requestedWorkers: 1, verifyArchitecture: false })"); - expect(source).toContain("const setup = await runSystemDependencySetup(lease)"); + const auditScript = source.slice(source.indexOf("const remoteScript = `${lease.input.scratch}/audit.sh`"), source.indexOf("const rawLogs =", source.indexOf("async function runStaleOwnerAudit"))); + expect(auditScript.indexOf("systemDependenciesCommand()")).toBeLessThan(auditScript.indexOf("[command, ...args]")); + expect(auditScript).toContain('workerEntryCommand({ ...lease.input, processName }, remoteScript)'); expect(source).toContain("run_step system-deps bash -lc ${shellQuote(systemDependenciesCommand())}"); expect(source).toContain("apt-get install -y --no-install-recommends python3 xauth libgtk-3-0 libxtst6 iproute2"); expect(source).not.toContain("psmisc"); diff --git a/test/repo/blaxel-worker-lease.test.ts b/test/repo/blaxel-worker-lease.test.ts new file mode 100644 index 00000000..b2c93a05 --- /dev/null +++ b/test/repo/blaxel-worker-lease.test.ts @@ -0,0 +1,194 @@ +import { spawn, spawnSync, execFile } from "node:child_process"; +import { chownSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync, existsSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, test } from "vitest"; +import { workerLeaseCommand, workerEntryCommand, shardScratchPath } from "../../scripts/test/blaxel-worker-lease.mjs"; + +// Real generated lock command, Linux flock/procfs and disposable files. The +// supplied process IDs substitute only the whole-worker audit's enumeration. +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +function fixture(unprivileged = false) { + // A root Blaxel worker must reproduce the unprivileged CI runner too. Use + // /tmp directly because the outer shard's private TMPDIR is not traversable. + const credentials = unprivileged && process.getuid?.() === 0 ? { uid: 65534, gid: 65534 } : {}; + const tmpRoot = mkdtempSync(path.join(unprivileged ? "/tmp" : os.tmpdir(), "lease-seam-")); roots.push(tmpRoot); + if (credentials.uid !== undefined) chownSync(tmpRoot, credentials.uid, credentials.gid!); + const input = { lockId: "fixture-lock", runId: "fixture-run", shardIndex: 1, shardTotal: 1, attemptId: "fixture-attempt", owner: "fixture", expiresAt: Math.floor(Date.now() / 1000) + 1200, tmpRoot, resultsRoot: path.join(tmpRoot, "results"), auditPids: [] as number[] }; + const recordPath = path.join(tmpRoot, "tv-testshard.lock", "owner.json"); + const run = (extra = {}) => spawnSync("bash", ["-c", workerLeaseCommand({ ...input, ...extra })], { encoding: "utf8", env: process.env, timeout: 15_000, ...credentials }); + return { input, recordPath, run, record: () => JSON.parse(readFileSync(recordPath, "utf8")) }; +} + +test.skipIf(process.platform !== "linux")("acquisition writes a renewable deadline and protects the lease from timestamp-only runners", () => { + const f = fixture(); + const before = Date.now(); + const acquired = f.run(); + expect(acquired.status, acquired.stderr).toBe(0); + expect(f.record()).toMatchObject({ lockId: f.input.lockId, expiresAt: 0, attemptId: f.input.attemptId }); + expect(f.record().lastRenewal).toBeGreaterThanOrEqual(before); + expect(f.record().recoverAfter - f.record().lastRenewal).toBe(1_200_000); +}); + +test.skipIf(process.platform !== "linux")("fixture audit runs without root access to unrelated process environments", async () => { + const f = fixture(true); + const ready = path.join(f.input.tmpRoot, "unrelated-ready"); + // A same-user service can be nondumpable too. It belongs outside this + // fixture's process set even though a user filter would include it. + const child = process.getuid?.() === 0 ? spawn(process.execPath, ["-e", `process.setgid(65534);process.setuid(65534);require('fs').writeFileSync(${JSON.stringify(ready)},'ready');setInterval(()=>{},1000)`], { stdio: "ignore", env: process.env }) : null; + try { + if (child) await until(() => existsSync(ready)); + const acquired = f.run(); + expect(acquired.status, acquired.stderr).toBe(0); + expect(f.record().lockId).toBe(f.input.lockId); + } finally { if (child) { child.kill("SIGKILL"); await until(() => child.signalCode !== null); } } +}); + +test.skipIf(process.platform !== "linux")("an unreadable whole-worker audit preserves the lease and its evidence", () => { + const f = fixture(true); + expect(f.run().status).toBe(0); + const before = readFileSync(f.recordPath, "utf8"); + const sentinel = path.join(f.input.tmpRoot, "television-web-bundle-preserve"); + mkdirSync(sentinel); + // No fixture scope: production must refuse reuse when ownership is unknown. + const prepared = f.run({ operation: "prepare", auditPids: undefined }); + expect(prepared.status).toBe(2); + expect(prepared.stderr).toMatch(/worker owner audit unreadable at PID \d+ \(EACCES\); worker reuse refused/); + expect(readFileSync(f.recordPath, "utf8")).toBe(before); + expect(existsSync(sentinel)).toBe(true); +}); + +test.skipIf(process.platform !== "linux" || process.getuid?.() !== 0)("fixture scope still refuses an unreadable process in its process set", async () => { + const f = fixture(true); + const ready = path.join(f.input.tmpRoot, "unreadable-ready"); + // Changing credentials after exec disables dumpability. procfs files become + // root-owned even though /proc/PID/status still identifies UID 65534. + const source = `process.setgid(65534);process.setuid(65534);require('fs').writeFileSync(${JSON.stringify(ready)},'ready');setInterval(()=>{},1000)`; + const child = spawn(process.execPath, ["-e", source], { stdio: "ignore", env: { ...process.env, TV_TEST_SHARD_OWNER: "fixture-lock" } }); + try { + await until(() => existsSync(ready)); + const acquired = f.run({ auditPids: [child.pid!] }); + expect(acquired.status).toBe(2); + expect(acquired.stderr).toContain(`worker owner audit unreadable at PID ${child.pid} (EACCES); worker reuse refused`); + expect(existsSync(f.recordPath)).toBe(false); + } finally { child.kill("SIGKILL"); await until(() => child.signalCode !== null); } +}); + +test.skipIf(process.platform !== "linux")("a quiet expired renewable lease recovers through cleanup before replacement", () => { + const f = fixture(); + mkdirSync(path.dirname(f.recordPath)); + writeFileSync(f.recordPath, JSON.stringify({ lockId: "abandoned", runId: "old-run", shardIndex: 1, attemptId: "old-attempt", expiresAt: 0, lastRenewal: 1, recoverAfter: 2, dispatch: null })); + const acquired = f.run(); + expect(acquired.status, acquired.stderr).toBe(0); + expect(f.record().lockId).toBe(f.input.lockId); + expect(f.record().recovery).toMatchObject({ previous: { lockId: "abandoned", runId: "old-run" }, cleanup: "confirmed", discardedReports: true }); +}); + +test.skipIf(process.platform !== "linux")("renewal and release reject old identities, expiry, and missing retrieval", () => { + const f = fixture(); + expect(f.run({ ttlSeconds: 60 }).status).toBe(0); + expect(f.run({ operation: "renew", lockId: "other" }).status).toBe(42); + expect(f.run({ operation: "release", retrieved: true }).status).toBe(42); + expect(f.run({ operation: "teardown" }).status).toBe(0); + expect(f.run({ operation: "release" }).status).toBe(42); + expect(f.run({ operation: "renew", ttlSeconds: 90 }).status).toBe(0); + expect(f.record().recoverAfter - f.record().lastRenewal).toBe(90_000); + writeFileSync(f.recordPath, JSON.stringify({ ...f.record(), recoverAfter: 1 })); + expect(f.run({ operation: "renew" }).status).toBe(42); + expect(f.run({ operation: "release", retrieved: true }).status).toBe(42); +}); + +const exec = promisify(execFile); +async function until(predicate: () => boolean, timeout = 5000) { + const deadline = Date.now() + timeout; + while (!predicate()) { if (Date.now() > deadline) throw Error("fixture rendezvous timeout"); await new Promise(resolve => setTimeout(resolve, 25)); } +} +function shell(value: string) { return `'${value.replaceAll("'", `'"'"'`)}'`; } + +test.skipIf(process.platform !== "linux")("competing acquisitions serialize, and killing a guard holder releases the OS guard", async () => { + const f = fixture(); + const command = (lockId: string) => exec("bash", ["-c", workerLeaseCommand({ ...f.input, lockId })], { env: process.env, timeout: 10_000 }); + const outcomes = await Promise.allSettled([command("fixture-lock"), command("new-lock")]); + expect(outcomes.filter(result => result.status === "fulfilled")).toHaveLength(1); + const owner = f.record().lockId; + rmSync(path.dirname(f.recordPath), { recursive: true }); + const ready = path.join(f.input.tmpRoot, "guard-ready"); + const child = spawn("flock", ["-F", path.join(f.input.tmpRoot, "tv-testshard.guard"), process.execPath, "-e", `require('fs').writeFileSync(${JSON.stringify(ready)}, String(process.pid));setInterval(()=>{},1000)`], { detached: true, stdio: "ignore", env: process.env }); + try { + await until(() => existsSync(ready)); + expect(Number(readFileSync(ready, "utf8"))).toBe(child.pid); + child.kill("SIGKILL"); + await command(owner); + expect(f.record().lockId).toBe(owner); + } finally { try { process.kill(-child.pid!, "SIGKILL"); } catch {} } +}); + +test.skipIf(process.platform !== "linux")("an obsolete delayed entry cannot touch checkout after recovery", async () => { + const f = fixture(); const processName = "delayed"; + expect(f.run().status).toBe(0); expect(f.run({ operation: "prepare" }).status).toBe(0); + const script = path.join(shardScratchPath(f.input), "worker.sh"); + const sentinel = path.join(f.input.tmpRoot, "checkout-touched"); + writeFileSync(script, `touch ${shell(sentinel)}\n`); + expect(f.run({ operation: "dispatch", processName }).status).toBe(0); + const entry = workerEntryCommand({ ...f.input, processName }, script); + expect(f.run({ operation: "teardown" }).status).toBe(42); + expect(f.run({ operation: "release", retrieved: true }).status).toBe(42); + // The real entry command is delayed until after the old identity is replaced. + writeFileSync(f.recordPath, JSON.stringify({ ...f.record(), recoverAfter: 1 })); + expect(f.run({ lockId: "new-lock", attemptId: "new-attempt" }).status).toBe(0); + const delayed = spawnSync("bash", ["-c", entry], { encoding: "utf8", timeout: 10_000 }); + expect(delayed.status).not.toBe(0); expect(existsSync(sentinel)).toBe(false); + expect(f.record().lockId).toBe("new-lock"); +}); + +test.skipIf(process.platform !== "linux")("recovery kills registered descendants by token, including TERM-resistant children, and holds for another owner", async () => { + const f = fixture(); const processName = "live-fixture"; + expect(f.run().status).toBe(0); expect(f.run({ operation: "prepare" }).status).toBe(0); + const script = path.join(shardScratchPath(f.input), "worker.sh"); + const ready = path.join(f.input.tmpRoot, "child-ready.json"); + const childSource = `process.on('SIGTERM',()=>{});require('fs').writeFileSync(${JSON.stringify(ready)},JSON.stringify({pid:process.pid,outer:process.env.TV_TEST_SHARD_OWNER,surface:process.env.TV_TEST_SURFACE_OWNER}));setInterval(()=>{},1000)`; + // A real nested SurfaceSupervisor replaces the surface token but preserves the + // outer attempt, even with a restricted requested child environment. + const supervisor = path.join(process.cwd(), "scripts/test/surface-supervisor.mjs"); + const parentSource = `import {SurfaceSupervisor} from ${JSON.stringify(supervisor)};const s=new SurfaceSupervisor({runId:'nested-fixture',surfaceIds:['nested']});await s.run(process.execPath,['-e',${JSON.stringify(childSource)}],{env:{PATH:process.env.PATH}});`; + writeFileSync(script, `exec ${shell(process.execPath)} --input-type=module -e ${shell(parentSource)}\n`); + expect(f.run({ operation: "dispatch", processName }).status).toBe(0); + const child = spawn("bash", ["-c", workerEntryCommand({ ...f.input, processName }, script)], { detached: true, stdio: "ignore", env: process.env }); + const unrelated = spawn(process.execPath, ["-e", "setInterval(()=>{},1000)"], { detached: true, stdio: "ignore", env: { ...process.env, TV_TEST_SHARD_OWNER: "unrelated-lock" } }); + try { + await until(() => existsSync(ready)); + const observed = JSON.parse(readFileSync(ready, "utf8")); + f.input.auditPids.push(child.pid!, observed.pid, unrelated.pid!); + expect(observed.outer).toBe(f.input.lockId); expect(observed.surface).toMatch(/^tv1\./); + expect(f.record().dispatch.pid).toBeGreaterThan(0); + writeFileSync(f.recordPath, JSON.stringify({ ...f.record(), recoverAfter: 1 })); + const held = f.run({ lockId: "new-lock", attemptId: "new-attempt" }); + expect(held.status, held.stderr).toBe(42); + expect(() => process.kill(unrelated.pid!, 0)).not.toThrow(); + const stat = existsSync(`/proc/${observed.pid}/stat`) ? readFileSync(`/proc/${observed.pid}/stat`, "utf8") : ""; + expect(stat === "" || stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")).toBe(true); + unrelated.kill("SIGKILL"); await until(() => unrelated.signalCode !== null); + const acquired = f.run({ lockId: "new-lock", attemptId: "new-attempt" }); + expect(acquired.status, acquired.stderr).toBe(0); + } finally { + try { process.kill(-child.pid!, "SIGKILL"); } catch {} + try { process.kill(-unrelated.pid!, "SIGKILL"); } catch {} + } +}, 20_000); + + +test.skipIf(process.platform !== "linux")("attempt identity and recovery evidence survive checkout resetting the result tree", () => { + const f = fixture(); + mkdirSync(path.dirname(f.recordPath)); + writeFileSync(f.recordPath, JSON.stringify({ lockId: "abandoned", runId: "old-run", attemptId: "old-attempt", expiresAt: 0, lastRenewal: 1, recoverAfter: 2 })); + expect(f.run().status).toBe(0); + expect(f.run({ operation: "prepare" }).status).toBe(0); + rmSync(f.input.resultsRoot, { recursive: true }); // checkout's real reset boundary + expect(f.run({ operation: "teardown" }).status).toBe(0); + expect(JSON.parse(readFileSync(path.join(f.input.resultsRoot, "attempt.json"), "utf8"))).toMatchObject({lockId: f.input.lockId, runId: f.input.runId}); + expect(JSON.parse(readFileSync(path.join(f.input.resultsRoot, "resources/recovery.json"), "utf8"))).toMatchObject({previous: {lockId: "abandoned"}, discardedReports: true}); + expect(JSON.parse(readFileSync(path.join(f.input.resultsRoot, "evidence.json"), "utf8")).files).toContain("temporary-cleanup.json"); + expect(existsSync(shardScratchPath(f.input))).toBe(false); +}); diff --git a/test/repo/build-config-integrity.test.ts b/test/repo/build-config-integrity.test.ts index 7b740667..aa457832 100644 --- a/test/repo/build-config-integrity.test.ts +++ b/test/repo/build-config-integrity.test.ts @@ -15,7 +15,8 @@ import os from "node:os"; import path from "node:path"; // @ts-expect-error — jsdom is intentionally installed without a declaration package import { JSDOM } from "jsdom"; -import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { afterAll, beforeAll, describe, expect } from "vitest"; +import { test } from "../helpers/vitest.ts"; // @ts-expect-error — repository build script has no declaration file import { buildCanonicalVersions } from "../../packages/canonical/scripts/build-canonical.mjs"; import { buildVersionedWebBundle } from "../helpers/versioned-web-bundle.ts"; diff --git a/test/repo/fixtures/test-runner/guidance-checkout.ts b/test/repo/fixtures/test-runner/guidance-checkout.ts new file mode 100644 index 00000000..fe82ba41 --- /dev/null +++ b/test/repo/fixtures/test-runner/guidance-checkout.ts @@ -0,0 +1,44 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import type { TestSurface } from "../../../../scripts/test/config.mjs"; + +// Authored checkout, real production CLI and native tools. No repository gate +// runs here. Each scenario owns its files and private mutex. +export function guidanceCheckout(runner: "vitest" | "playwright" = "vitest") { + const parent = path.resolve(".test-runs"); + mkdirSync(parent, { recursive: true }); + const root = mkdtempSync(path.join(parent, "guidance-checkout-")); + const home = path.join(root, "home"); + mkdirSync(home); + cpSync("scripts", path.join(root, "scripts"), { recursive: true }); + cpSync("test.prebuilt.mjs", path.join(root, "test.prebuilt.mjs")); + cpSync(".nvmrc", path.join(root, ".nvmrc")); + const manifest = JSON.parse(readFileSync("package.json", "utf8")); + writeFileSync(path.join(root, "package.json"), JSON.stringify({ type: "module", engines: manifest.engines })); + symlinkSync(path.resolve("node_modules"), path.join(root, "node_modules"), "dir"); + writeFileSync(path.join(root, ".gitignore"), "node_modules/\n.test-runs/\nhome/\nattempts.ndjson\n"); + const testsRoot = `test/runner-fixtures/guidance/${runner}`; + cpSync(testsRoot, path.join(root, testsRoot), { recursive: true }); + const surface: TestSurface = { + id: `fixture:${runner}`, runner, kind: "experiment", config: `${testsRoot}/${runner}.config.ts`, + roots: [testsRoot], excludeRoots: [], tags: ["guidance"], supports: ["file", "grep"], + preflight: [], services: [], cwd: ".", package: null, agent: false, command: null, preCommand: null, executionGroup: null, absoluteConfig: path.join(root, testsRoot, `${runner}.config.ts`), + }; + const registry = { suites: { all: { include: [] }, experiment: { include: [] } }, executionGroups: [{ id: "fixture", surfaces: [surface] }] }; + const write = (file: string, text: string) => { mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); writeFileSync(path.join(root, file), text); }; + const writeRegistry = () => write("test.config.mjs", `export default ${JSON.stringify(registry)};\n`); + writeRegistry(); + const git = (...args: string[]) => execFileSync("git", args, { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + git("init", "-q"); + git("config", "user.name", "Runner fixture"); + git("config", "user.email", "fixture@example.invalid"); + git("add", "."); + git("commit", "-qm", "Authored fixture"); + const env: NodeJS.ProcessEnv = { ...process.env, HOME: home, TV_TEST_RUNNER_SELFTEST: "1", TV_TEST_RUNNER_LOCK_PATH: path.join(root, "private-lock"), TV_GUIDANCE_ATTEMPTS: path.join(root, "attempts.ndjson") }; + delete env.TV_TEST_RUNNER_DRY_RUN; + delete env.TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT; + delete env.TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT; + const run = (args: string[], extraEnv: Record = {}) => spawnSync(process.execPath, ["scripts/test/cli.mjs", ...args], { cwd: root, env: { ...env, ...extraEnv }, encoding: "utf8", timeout: 60_000 }); + return { root, home, testsRoot, surface, registry, write, writeRegistry, git, run, env, cleanup: () => rmSync(root, { recursive: true, force: true }) }; +} diff --git a/test/repo/fixtures/test-runner/mutex-checkout.ts b/test/repo/fixtures/test-runner/mutex-checkout.ts new file mode 100644 index 00000000..5ca69768 --- /dev/null +++ b/test/repo/fixtures/test-runner/mutex-checkout.ts @@ -0,0 +1,69 @@ +import { spawn } from "node:child_process"; +import { existsSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { guidanceCheckout } from "./guidance-checkout.ts"; + +interface MutexChild { + child: ReturnType; + ready: string; + release: string; + exited: Promise; + output(): string; +} + +// Cooperative native work only: the test releases every child in finally. +export function mutexCheckout() { + const f = guidanceCheckout(); + const file = `${f.testsRoot}/one.test.ts`; + f.surface.kind = "unit"; + f.writeRegistry(); + f.write(file, `import fs from "node:fs"; +import { test, expect } from "vitest"; +test("cooperative mutex fixture", async () => { + if (process.env.TV_MUTEX_READY) fs.writeFileSync(process.env.TV_MUTEX_READY, String(process.pid)); + const deadline = Date.now() + 60000; + while (process.env.TV_MUTEX_RELEASE && !fs.existsSync(process.env.TV_MUTEX_RELEASE)) { + if (Date.now() > deadline) throw new Error("fixture was not released"); + await new Promise(resolve => setTimeout(resolve, 25)); + } + expect(process.env.TV_MUTEX_FAIL).not.toBe("1"); +}, 65000); +`); + const children: MutexChild[] = []; + function start(args = ["local", "--file", file], extra: NodeJS.ProcessEnv = {}): MutexChild { + const id = children.length; + const ready = path.join(f.root, `ready-${id}`); + const release = path.join(f.root, `release-${id}`); + const child = spawn(process.execPath, ["scripts/test/cli.mjs", ...args], { + cwd: f.root, env: { ...f.env, TV_MUTEX_READY: ready, TV_MUTEX_RELEASE: release, ...extra }, stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.on("data", chunk => { output += chunk; }); + child.stderr.on("data", chunk => { output += chunk; }); + const exited = new Promise((resolve, reject) => { + child.once("exit", code => resolve(code)); + child.once("error", reject); + }); + const entry = { child, ready, release, exited, output: () => output }; + children.push(entry); + return entry; + } + async function cleanup() { + for (const child of children) writeFileSync(child.release, "release"); + await Promise.all(children.map(child => child.exited)); + f.cleanup(); + } + return { ...f, file, start, cleanup, args: ["local", "--file", file] }; +} + +export async function waitFor(check: () => boolean, diagnostic: () => string = () => "fixture did not become ready") { + const deadline = Date.now() + 30_000; + while (!check()) { + if (Date.now() > deadline) throw new Error(diagnostic()); + await new Promise(resolve => setTimeout(resolve, 25)); + } +} + +export async function ready(child: { ready: string; output(): string }) { + await waitFor(() => existsSync(child.ready), child.output); +} diff --git a/test/repo/fixtures/test-runner/verify-mutex-checkout.ts b/test/repo/fixtures/test-runner/verify-mutex-checkout.ts new file mode 100644 index 00000000..d176cbb0 --- /dev/null +++ b/test/repo/fixtures/test-runner/verify-mutex-checkout.ts @@ -0,0 +1,43 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { mutexCheckout } from "./mutex-checkout.ts"; + +// Only downstream gate programs are authored substitutes. The verify process, +// its preflight, canonical test child, native fixture and mutex all stay real. +export function verifyMutexCheckout() { + const f = mutexCheckout(); + const manifest = JSON.parse(readFileSync(path.join(f.root, "package.json"), "utf8")); + manifest.scripts = Object.fromEntries(["lint", "type-check", "lint:package-manifests"].map(name => [name, `node phase.mjs ${name}`])); + f.write("package.json", JSON.stringify(manifest)); + f.write("phase.mjs", `import fs from 'node:fs'; import {spawnSync} from 'node:child_process'; +const phase=process.argv[2]; +const result=spawnSync(process.execPath,${JSON.stringify(["scripts/test/cli.mjs", ...f.args])},{env:{...process.env,TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT:'fixture:nested phase reached preflight'},encoding:'utf8'}); +fs.writeFileSync('phase-'+phase+'.json',JSON.stringify({status:result.status,output:result.stdout+result.stderr})); +if(process.env.TV_MUTEX_HOLD_PHASE===phase){ + fs.writeFileSync(process.env.TV_MUTEX_READY, String(process.pid)); + const deadline=Date.now()+60000; + while(!fs.existsSync(process.env.TV_MUTEX_RELEASE)){ + if(Date.now()>deadline) throw new Error('phase not released'); + await new Promise(resolve=>setTimeout(resolve,25)); + } +} +if(process.env.TV_MUTEX_FAIL_PHASE===phase) process.exit(1); +`); + f.write("scripts/test/vibe-mode-check.mjs", "process.argv[2]='vibe-mode'; await import('../../phase.mjs');"); + f.write(f.file, `import fs from 'node:fs'; import {spawnSync} from 'node:child_process'; import {test,expect} from 'vitest'; +test('canonical verify child', async()=>{ + const result=spawnSync(process.execPath,${JSON.stringify(["scripts/test/cli.mjs", ...f.args])},{env:{...process.env,TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT:'fixture:nested test reached preflight'},encoding:'utf8'}); + fs.writeFileSync('phase-tests.json',JSON.stringify({status:result.status,output:result.stdout+result.stderr})); + fs.writeFileSync(process.env.TV_MUTEX_READY, String(process.pid)); + if(!process.env.TV_MUTEX_HOLD_PHASE){ + const deadline=Date.now()+60000; + while(!fs.existsSync(process.env.TV_MUTEX_RELEASE)){ + if(Date.now()>deadline) throw new Error('native fixture not released'); + await new Promise(resolve=>setTimeout(resolve,25)); + } + } + expect(process.env.TV_MUTEX_FAIL).not.toBe('1'); +},65000); +`); + return f; +} diff --git a/test/repo/node-toolchain.test.ts b/test/repo/node-toolchain.test.ts index ff8714bb..f0d2e941 100644 --- a/test/repo/node-toolchain.test.ts +++ b/test/repo/node-toolchain.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, test } from "vitest"; @@ -9,12 +9,13 @@ import { runPreflights } from "../../scripts/test/preflight.mjs"; const repoRoot = path.resolve(import.meta.dirname, "../.."); function runCli(args: string[], env: Record = {}) { - return spawnSync(process.execPath, ["scripts/test/cli.mjs", ...args], { + const locks = mkdtempSync(path.join(os.tmpdir(), "tv-toolchain-lock-")); + try { return spawnSync(process.execPath, ["scripts/test/cli.mjs", ...args], { cwd: repoRoot, encoding: "utf8", - env: { ...process.env, FORCE_COLOR: "0", NO_COLOR: "1", ...env }, + env: { ...process.env, TV_TEST_RUNNER_SELFTEST: "1", TV_TEST_RUNNER_LOCK_PATH: path.join(locks, "mutex.sqlite3"), FORCE_COLOR: "0", NO_COLOR: "1", ...env }, timeout: 10_000, - }); + }); } finally { rmSync(locks, { recursive: true, force: true }); } } function fakeNpmBin(version: string): string { diff --git a/test/repo/resource-diagnostics-links.test.ts b/test/repo/resource-diagnostics-links.test.ts new file mode 100644 index 00000000..7938f384 --- /dev/null +++ b/test/repo/resource-diagnostics-links.test.ts @@ -0,0 +1,56 @@ +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { expect, test } from "vitest"; +import { combineAttemptReports, compactProviderSummary, summarizeInfrastructureAttempts } from "../../scripts/test/provider-outputs.mjs"; +import { normalizeProviderShardSurfaces } from "../../scripts/test/provider-normalization.mjs"; +import { buildReports } from "../../scripts/test/reporting.mjs"; +import { buildTimingEvents, serializeTimingEvents } from "../../scripts/test/timing-events.mjs"; + +const surfaces = [{ id: "e2e:fixture", runner: "playwright", roots: ["probe.test.ts"], cwd: "." }] as any; +const start = "2026-10-27T00:00:00.000Z"; +const end = "2026-10-27T00:00:01.000Z"; + +test("keeps only retained resource links across infrastructure replacement, normalization and final reporting", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "tv-resource-links-")); + try { + const attempts = [0, 1].map((index) => { + const dir = path.join(root, `attempt-${index}`); mkdirSync(dir); + const diagnostic = path.join(dir, "resources.json"); + writeFileSync(diagnostic, JSON.stringify({ identity: { attemptId: index }, start: { memoryAvailable: { value: 12345 } }, kernel: { lines: ["KERNEL-DETAIL-SENTINEL"] } })); + const native = path.join(dir, "native.json"); copyFileSync("test/repo/fixtures/test-runner/playwright-retry.json", native); + return { status: index === 0 ? "failed" : "passed", startedAt: start, completedAt: end, shardIndices: [1], outputDir: dir, shards: [{ shardIndex: 1, status: index === 0 ? "infra-failed" : "passed", resourceDiagnosticsPaths: [diagnostic], outputDir: dir, tasks: [{ surfaceId: "e2e:fixture", runner: "playwright", status: "passed", exitCode: 0, durationMs: 25, nativeResultPath: native, resourceDiagnosticsPaths: [diagnostic, path.join(dir, "missing.json")] }] }] } as any; + }); + const combined = combineAttemptReports(attempts, [1], { shardTotal: 1, outputDir: root }); + expect(combined.status).toBe("passed"); + const history = summarizeInfrastructureAttempts(attempts) as any[]; + expect(history[0].shards[0].resourceDiagnosticsPaths).toEqual([path.join(root, "attempt-0/resources.json")]); + const normalized = normalizeProviderShardSurfaces({ provider: "blaxel", providerSummary: combined, outputDir: root, surfaces, delegatedCommand: ["coordinator"], exitCode: 0, coordinatorLogPath: path.join(root, "coordinator.log") }); + expect(normalized).toHaveLength(1); + expect(normalized[0].counts.testsFlakyRecovered).toBe(1); + expect(normalized[0].failedTests).toEqual([]); + const { results, summary } = buildReports({ runId: "resource-run", provider: "blaxel", runDir: root, selection: { surfaces: ["e2e:fixture"] }, surfaces: normalized, startedAt: start, completedAt: end }); + expect(results.surfaces[0].resourceDiagnosticsPaths).toEqual(["attempt-0/resources.json", "attempt-1/resources.json"]); + for (const link of results.surfaces[0].resourceDiagnosticsPaths as string[]) expect(existsSync(path.join(root, link))).toBe(true); + expect(summary.run.status).toBe("passed"); + const events = serializeTimingEvents(buildTimingEvents({ run: { runId: "resource-run", commandProvider: "blaxel", timingProvider: "blaxel-playwright-x64-4vcpu", status: "passed", startedAt: start, completedAt: end, durationMs: 1000, selection: { surfaces: ["e2e:fixture"] }, git: { commit: null, tree: null, dirty: false }, environment: { ci: true, os: "linux", arch: "x64", node: process.version }, phases: [] }, shards: [{ index: 1, total: 1, status: "passed", phases: [], surfaces: normalized as any }] })); + for (const compact of [JSON.stringify(summary), JSON.stringify(compactProviderSummary(combined)), events]) { + expect(compact).not.toContain("KERNEL-DETAIL-SENTINEL"); + expect(compact).not.toContain("memoryAvailable"); + } + expect(readFileSync(path.join(root, "attempt-0/resources.json"), "utf8")).toContain("KERNEL-DETAIL-SENTINEL"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("baseline-only provider evidence cannot manufacture a selected-surface outcome", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "tv-resource-missing-")); + try { + const diagnostic = path.join(root, "baseline.json"); writeFileSync(diagnostic, JSON.stringify({ end: { status: "missing" } })); + const normalized = normalizeProviderShardSurfaces({ provider: "blaxel", providerSummary: { status: "failed", shards: [{ shardIndex: 1, status: "infra-failed", resourceDiagnosticsPaths: [diagnostic], tasks: [] }] }, outputDir: root, surfaces, delegatedCommand: ["coordinator"], exitCode: 1, coordinatorLogPath: path.join(root, "coordinator.log") }); + const { summary, results } = buildReports({ runId: "missing-native", provider: "blaxel", runDir: root, selection: { surfaces: ["e2e:fixture"] }, surfaces: normalized, startedAt: start, completedAt: end }); + expect(summary.run).toMatchObject({ status: "incomplete", testStatus: "unknown" }); + expect(summary.counts).toMatchObject({ surfacesRun: 0, testsFailed: 0 }); + expect(results.surfaces.map((surface) => surface.id)).toEqual(["provider:blaxel"]); + expect(results.surfaces[0].resourceDiagnosticsPaths).toEqual(["baseline.json"]); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/test/repo/resource-diagnostics.test.ts b/test/repo/resource-diagnostics.test.ts new file mode 100644 index 00000000..43e78a75 --- /dev/null +++ b/test/repo/resource-diagnostics.test.ts @@ -0,0 +1,98 @@ +import { copyFileSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { kernelLinesInInterval, startResourceDiagnostics, writeResourceDiagnostic } from "../../scripts/test/resource-diagnostics.mjs"; + +const roots: string[] = []; +const identity = { runId: "run-one", shardIndex: 1, attemptId: "lock-one", surfaceId: "e2e:fixture" }; +const startTime = "2026-10-27T00:00:00.000Z"; +const endTime = "2026-10-27T00:00:01.000Z"; +const secret = "credential-query-body-environment-argument-SENTINEL"; +const temporary = () => { const dir = mkdtempSync(path.join(os.tmpdir(), "tv-resource-contract-")); roots.push(dir); return dir; }; +const read = (file: string | null) => JSON.parse(readFileSync(file!, "utf8")); +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +test("restricts a whole-second kernel read to the observed interval and discloses missing timestamps", () => { + const line = (time: string) => `${time} Out of memory: Killed process 23 (node)`; + const within = line("2026-10-27T00:00:00,500000+00:00"); + expect(kernelLinesInInterval([ + line("2026-10-27T00:00:00,100000+00:00"), within, + line("2026-10-27T00:00:00,900000+00:00"), + "Killed process 24 (node)", "unrelated kernel message", + ], { since: "2026-10-27T00:00:00.200Z", until: "2026-10-27T00:00:00.800Z" })).toEqual({ lines: [within], omittedUnrecognizedLines: 1 }); +}); + +test("retains a baseline, honest units and reset evidence, and kernel evidence for a recovered native attempt", () => { + const root = temporary(); + const nativeResultPath = path.join(root, "native.json"); + copyFileSync("test/repo/fixtures/test-runner/playwright-retry.json", nativeResultPath); + let ended = false; + const probes: Array<[string, unknown]> = []; + const collector = startResourceDiagnostics({ + root, identity: { ...identity, env: secret, args: [secret] }, now: () => new Date(ended ? endTime : startTime), + readProbe: (name, interval) => { + probes.push([name, interval]); + if (name === "temporaryFree") throw Object.assign(new Error(secret), { code: "EACCES" }); + if (name === "oomKills") return ended ? 2 : 10; + if (name === "kernelOOM") return { lines: [`2026-10-27T00:00:00,500000+00:00 Out of memory: Killed process 23 (node) total-vm:4096kB ${secret}`] }; + return 4096; + }, + warn: () => {}, + }); + expect(read(collector.file)).toMatchObject({ identity, start: { observedAt: startTime, memoryAvailable: { value: 4096, unit: "bytes", source: "/proc/meminfo:MemAvailable" }, temporaryFree: { value: null, reason: "EACCES" } }, end: { status: "missing" } }); + ended = true; + expect(collector.finish({ runner: "playwright", nativeResultPath, exitCode: 0 })).toEqual([collector.file]); + const record = read(collector.file); + expect(record).toMatchObject({ end: { observedAt: endTime }, oomDelta: { value: null, reason: "counter-reset" }, nativeEvidence: { status: "complete", failedAttempt: true, recovered: true }, kernel: { status: "available" } }); + expect(probes.find(([name]) => name === "kernelOOM")?.[1]).toMatchObject({ since: startTime, until: endTime }); + expect(record.kernel.lines.join("\n")).toContain("Killed process 23 (node)"); + expect(JSON.stringify(record)).not.toContain(secret); +}); + +test("requests kernel evidence for missing output and preserves unavailable probes without zeroes", () => { + const root = temporary(); + const warnings: string[] = []; + const collector = startResourceDiagnostics({ root, identity, warn: (message) => warnings.push(message), readProbe: () => { throw Object.assign(new Error(secret), { code: "EPERM" }); } }); + collector.finish({ runner: "vitest", nativeResultPath: path.join(root, "absent.json"), exitCode: 17 }); + const record = read(collector.file); + expect(record).toMatchObject({ nativeEvidence: { status: "missing", failedAttempt: null }, start: { oomKills: { value: null, unit: "count", reason: "EPERM" } }, end: { memoryAvailable: { value: null } }, kernel: { status: "unavailable", reason: "EPERM" }, oomDelta: { value: null, reason: "unavailable" } }); + expect(warnings.length).toBeGreaterThan(0); + expect(warnings.join("\n")).not.toContain(secret); +}); + +test("bounds sidecars and the attempt including receipts, reserving compact observations before excerpts", () => { + const root = temporary(); + const warnings: string[] = []; + const native = path.join(root, "native.json"); + copyFileSync("test/repo/fixtures/test-runner/playwright-retry.json", native); + const collector = startResourceDiagnostics({ root, identity, warn: (message) => warnings.push(message), readProbe: (name) => name === "kernelOOM" ? { lines: Array.from({ length: 8000 }, () => "Out of memory: Killed process 23 (node) total-vm:4096kB")} : 1 }); + collector.finish({ runner: "playwright", nativeResultPath: native, exitCode: 0 }); + const record = read(collector.file); + expect(record.start.memoryAvailable.value).toBe(1); + expect(record.end.memoryAvailable.value).toBe(1); + expect(record.kernel.truncated).toBe(true); + expect(Buffer.byteLength(record.kernel.lines.join("\n"))).toBeLessThanOrEqual(64 * 1024); + expect(statSync(collector.file!).size).toBeLessThanOrEqual(128 * 1024); + const pending = startResourceDiagnostics({ root, identity, readProbe: () => 2, warn: (message) => warnings.push(message) }); + const receipt = { kind: "recovery", identity, previous: { runId: "old", lockId: "old-lock", worker: "private-worker", lastRenewal: startTime, recoverAfter: endTime, args: secret }, discardedReports: true, cleanup: "confirmed", processes: Array.from({ length: 3000 }, (_, i) => ({ pid: i + 1, startTime: "123", command: "node", outcome: "terminated", args: secret })), env: secret }; + for (let i = 0; i < 80; i++) writeResourceDiagnostic(root, `receipt-${i}`, receipt, { warn: (message) => warnings.push(message) }); + pending.finish({ runner: "vitest", nativeResultPath: null, exitCode: 17 }); + expect(read(pending.file).end.memoryAvailable.value).toBe(2); + const files = readdirSync(path.join(root, "resources")).map((name) => path.join(root, "resources", name)); + expect(files.reduce((sum, file) => sum + statSync(file).size, 0)).toBeLessThanOrEqual(8 * 1024 * 1024); + expect(files.every((file) => statSync(file).size <= 128 * 1024)).toBe(true); + expect(files.every((file) => !readFileSync(file, "utf8").includes(secret))).toBe(true); + expect(read(collector.file).end.memoryAvailable.value).toBe(1); + expect(warnings.length).toBeGreaterThan(0); +}); + +test("diagnostic write failure warns and returns no unretained link without throwing over a native outcome", () => { + const root = temporary(); + const warnings: string[] = []; + const collector = startResourceDiagnostics({ root, identity, readProbe: () => 1, write: () => { throw Object.assign(new Error(secret), { code: "ENOSPC" }); }, warn: (message) => warnings.push(message) }); + expect(collector.finish({ runner: "vitest", nativeResultPath: null, exitCode: 17 })).toEqual([]); + expect(collector.file === null || !existsSync(collector.file)).toBe(true); + expect(warnings.join("\n")).toContain("ENOSPC"); + expect(warnings.join("\n")).not.toContain(secret); +}); diff --git a/test/repo/test-runner-attestation.test.ts b/test/repo/test-runner-attestation.test.ts index 7286d458..6151cae1 100644 --- a/test/repo/test-runner-attestation.test.ts +++ b/test/repo/test-runner-attestation.test.ts @@ -11,7 +11,7 @@ import { type TestpassPolicy, } from "../../scripts/test/attestation.mjs"; -// specs/arch/test-runner/attestation.md#test-assertions — decision logic. +// specs/arch/test-runner/attestation.md#skipping-in-ci — decision logic. const TREE = "6ffb98fb8f5a062c6975441131ceddaaedfebd0a"; const NOW = 1_800_000_000; @@ -106,7 +106,7 @@ describe("policy and version domain", () => { }); }); -// specs/arch/test-runner/attestation.md#activation-policy — the trusted read +// specs/arch/test-runner/attestation.md#policy — the trusted read // crosses real Git against a temporary bare origin. describe("trusted policy read", async () => { const { execFileSync } = await import("node:child_process"); @@ -454,15 +454,36 @@ describe("verify handoff facts", async () => { ]); }); + // proofs/arch/test-runner/attestation.md#^attest-retry-options + test("parsed retry departures become noncanonical eligibility facts", () => { + for (const options of [ + { "against-test-guidance-turn-flakes-into-failures-on-broad-runs": "1" }, + { "against-test-guidance-turn-flakes-into-failures-on-broad-runs": "1", retries: "0" }, + { retries: "0" }, + { retries: "3" }, + ]) { + const facts = { + treeAtStart: TREE, treeAtEnd: TREE, testedTree: TREE, + cleanAtStart: true, cleanAtEnd: true, ignoreUncommitted: false, qualifyingRun: true, + surfaces: [{ id: "unit:root", status: "passed" }], expectedSurfaceIds: ["unit:root"], + shardSubset: false, targetedSelection: false, attestedSkipRun: false, + policyAtStart: enabledPolicy(5), policyAtEnd: enabledPolicy(5), + canonicalRetries: canonicalRetryFacts({ options, env: {} }), + }; + expect(facts.canonicalRetries, JSON.stringify(options)).toBe(false); + expect(evaluateEligibility(facts)).toEqual({ eligible: false, failures: ["canonical-retries"] }); + } + }); + test("canonical retries require the runner default AND the default flaky budget", () => { - expect(canonicalRetryFacts({ retriesOption: undefined, env: {} })).toBe(true); - expect(canonicalRetryFacts({ retriesOption: "2", env: { FLAKY_TEST_RETRIES: "5" } })).toBe(true); - expect(canonicalRetryFacts({ retriesOption: "3", env: {} })).toBe(false); - expect(canonicalRetryFacts({ retriesOption: "2", env: { FLAKY_TEST_RETRIES: "9" } })).toBe(false); + expect(canonicalRetryFacts({ options: {}, env: {} })).toBe(true); + expect(canonicalRetryFacts({ options: { retries: "2" }, env: { FLAKY_TEST_RETRIES: "5" } })).toBe(true); + expect(canonicalRetryFacts({ options: { retries: "3" }, env: {} })).toBe(false); + expect(canonicalRetryFacts({ options: { retries: "2" }, env: { FLAKY_TEST_RETRIES: "9" } })).toBe(false); }); }); -// specs/arch/test-runner/attestation.md#prune-and-retention — the full safety +// specs/arch/test-runner/attestation.md#revoking-and-pruning — the full safety // matrix against local bare remotes with the trusted policy on the fixture // remote's main. Every refusal is asserted with --apply set, and every // deletion assertion re-reads the remote namespace. @@ -685,7 +706,7 @@ describe("testpass prune", async () => { }); }); -// specs/arch/test-runner/attestation.md#skip-semantics-in-ci — the CI face: +// specs/arch/test-runner/attestation.md#skipping-in-ci — the CI face: // `decide` (attest-check) and `write` (attest-write) as real processes against // a local bare remote whose main carries a trusted policy fixture. Production // mode here crosses the actual boundary — no self-test seam involved. diff --git a/test/repo/test-runner-config.test.ts b/test/repo/test-runner-config.test.ts index b2d76a1e..73dcc8a3 100644 --- a/test/repo/test-runner-config.test.ts +++ b/test/repo/test-runner-config.test.ts @@ -242,3 +242,15 @@ function visit(dir: string, root: string, out: Set): void { else if (/\.(test|spec)\.ts$/.test(entry.name)) out.add(path.relative(root, full)); } } + +// Source contract; the private driver carries real planned/targeted transport. +test("keeps Blaxel harness fixtures separate from the driver and ordinary suites", () => { + const registry = loadTestConfig(); + for (const suite of ["all", "e2e-ci"]) expect(selectSurfaces(registry, { suite }).some(surface => surface.id.startsWith("experiment:blaxel-harness"))).toBe(false); + expect(selectSurfaces(registry, { tag: "blaxel-harness-fixture" }).map(surface => surface.id)).toEqual(["experiment:blaxel-harness-fixtures"]); + for (const surface of registry.surfaces.filter(surface => surface.id.startsWith("experiment:blaxel-harness"))) expect(surface.kind).toBe("experiment"); + const source = readFileSync("test/runner-fixtures/blaxel-harness/driver/harness.test.ts", "utf8"); + expect(source).toContain('["--tag", "blaxel-harness-fixture", "--shards", "2"'); + expect(source).toContain('["--surface", "experiment:blaxel-harness-fixtures"]'); + expect(source).not.toContain('["--surface", "experiment:blaxel-harness"]'); +}); diff --git a/test/repo/test-runner-file-selection.test.ts b/test/repo/test-runner-file-selection.test.ts new file mode 100644 index 00000000..3993b9e4 --- /dev/null +++ b/test/repo/test-runner-file-selection.test.ts @@ -0,0 +1,197 @@ +import { spawnSync } from "node:child_process"; +import { targetCommand } from "../../scripts/test/target-command.mjs"; +import { loadTestConfig } from "../../scripts/test/config.mjs"; +import { enumerateTrackedPaths, loadRegistrySnapshotAtCommit } from "../../scripts/test/file-inventory.mjs"; +import { createShardPlan } from "../../scripts/test/shard-plan.mjs"; +import { serializeTimingBaseline } from "../../scripts/test/timing-baseline.mjs"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { guidanceCheckout } from "./fixtures/test-runner/guidance-checkout.ts"; + +const dry = { TV_TEST_RUNNER_DRY_RUN: "1" }; +const zero = "--against-test-guidance-turn-flakes-into-failures-on-broad-runs"; + +// proofs/arch/test-runner/test-runner.md#^t-runner-file-boundary +describe("actual file resolution", () => { + // proofs/arch/test-runner/test-runner.md#^registry-committed-normalization + test("normalizes committed and local registry surfaces identically", async () => { + // Compares this checkout with HEAD: commit registry or package-name edits first. + const snapshot = await loadRegistrySnapshotAtCommit(); + expect(snapshot.surfaces).toEqual(loadTestConfig().surfaces); + expect(snapshot.surfaces.find((surface) => surface.id === "e2e:artifact")?.cwd).toBe("packages/artifact"); + }); + + // proofs/arch/test-runner/test-runner.md#^registry-committed-normalization + test("derives package directories from the selected commit's manifests", async () => { + const f = guidanceCheckout(); + try { + f.write("packages/fixture/package.json", JSON.stringify({ name: "@fixture/tests" })); + f.write("test.config.mjs", `export default ${JSON.stringify({ suites: f.registry.suites, surfaces: [{ ...f.surface, cwd: undefined, package: "@fixture/tests" }] })};`); + f.git("add", "."); f.git("commit", "-qm", "Package-owned fixture"); + const commit = f.git("rev-parse", "HEAD"); + f.git("mv", "packages/fixture", "packages/moved"); + f.git("commit", "-qm", "Moved package"); + expect((await loadRegistrySnapshotAtCommit({ repoRoot: f.root, commit })).surfaces[0].cwd).toBe("packages/fixture"); + expect((await loadRegistrySnapshotAtCommit({ repoRoot: f.root })).surfaces[0].cwd).toBe("packages/moved"); + } finally { f.cleanup(); } + }); + + test("uses untracked local files and the selected committed registry remotely", () => { + const f = guidanceCheckout(); + try { + const file = `${f.testsRoot}/new.test.ts`; + f.write(file, 'import { test } from "vitest"; test("new", () => {});'); + const invoke = (provider: string, extra: string[] = []) => f.run([provider, "--file", file, "--retries", "0", ...extra], dry); + expect(invoke("local").status).toBe(0); + expect(invoke("blaxel").stderr).toContain("No test files match"); + f.git("add", "."); f.git("commit", "-qm", "New file"); + const commit = f.git("rev-parse", "HEAD"); + f.surface.id = "fixture:moved-owner"; + f.writeRegistry(); + f.git("add", "."); f.git("commit", "-qm", "Changed owner"); + rmSync(path.join(f.root, file)); + expect(invoke("local").status).toBe(2); + const remote = invoke("blaxel", ["--commit", commit, "--surface", "fixture:vitest"]); + expect(remote.status, remote.stderr).toBe(0); + expect(remote.stdout).toContain("surfaces=fixture:vitest"); + expect(invoke("blaxel", ["--surface", "fixture:vitest"]).stderr).toContain("not part of the selected surfaces"); + } finally { f.cleanup(); } + }); + + test("pins the inventory commit when a selected branch name moves", async () => { + const f = guidanceCheckout(); + try { + f.git("branch", "selected"); + const commit = f.git("rev-parse", "selected"); + const snapshot = await loadRegistrySnapshotAtCommit({ repoRoot: f.root, commit: "selected" }); + f.write(`${f.testsRoot}/later.test.ts`, ""); + f.git("add", "."); f.git("commit", "-qm", "Later revision"); + f.git("branch", "-f", "selected", "HEAD"); + expect(snapshot.commit).toBe(commit); + expect(enumerateTrackedPaths({ repoRoot: f.root, commit: snapshot.commit })).not.toContain(`${f.testsRoot}/later.test.ts`); + } finally { f.cleanup(); } + }); + + test("distinguishes exact paths, broad filters, exclusions and ambiguous owners", () => { + const f = guidanceCheckout(); + try { + writeFileSync(path.join(f.home, ".tvdev-use-blaxel"), "present"); + const check = (file: string, extra: string[] = []) => f.run(["local", "--file", file, ...extra], dry); + expect(check(`${f.testsRoot}/one.test.ts`).status).toBe(0); + expect(check(f.testsRoot).stderr).toContain("one test file"); + expect(check("one.test").stderr).toContain("one test file"); + expect(check("missing.test.ts").stderr).toContain("No test files match"); + f.write("unowned.test.ts", ""); + expect(check("unowned.test.ts").stderr).toContain("No test surface owns"); + f.surface.excludeRoots.push(`${f.testsRoot}/one.test.ts`); + f.writeRegistry(); + expect(check(`${f.testsRoot}/one.test.ts`).stderr).toContain("No test surface owns"); + f.surface.excludeRoots = []; + f.registry.executionGroups[0].surfaces.push({ ...f.surface, id: "fixture:duplicate" }); + f.writeRegistry(); + expect(check(`${f.testsRoot}/one.test.ts`).stderr).toContain("multiple surfaces"); + expect(check(`${f.testsRoot}/one.test.ts`, ["--surface", "fixture:vitest"]).status).toBe(0); + rmSync(path.join(f.root, f.testsRoot, "one.test.ts.test.ts")); + expect(f.run(["local", "--surface", "fixture:vitest"], dry).stderr).toContain("one test file"); + } finally { f.cleanup(); } + }); +}); + +// proofs/arch/test-runner/test-runner.md#^t-runner-exact-native-file +// proofs/arch/test-runner/test-runner.md#^t-runner-retry-attempts +describe.each(["vitest", "playwright"] as const)("%s native selection and retries", (runner) => { + test.each(["one.test.ts", "one[1].test.ts"])("collects only the admitted file and title: %s", (name) => { + const f = guidanceCheckout(runner); + try { + writeFileSync(path.join(f.home, ".tvdev-use-blaxel"), ""); + if (name !== "one.test.ts") f.write(`${f.testsRoot}/${name}`, readFileSync(path.join(f.root, f.testsRoot, "one.test.ts"), "utf8")); + const result = f.run(["local", "--file", `${f.testsRoot}/${name}`, "--grep", "selected|sibling"]); + expect(result.status, result.stdout + result.stderr).toBe(0); + const surface = summary(f, result).surfaces[0]; + expect(surface.counts.testsPassed).toBe(1); + if (runner === "vitest") { + // Native collection is the boundary here. The remote report normalizer + // strips /workspace/television even from an authored checkout below it. + const native = JSON.parse(readFileSync(path.resolve(f.root, surface.nativeResultPath), "utf8")); + expect(native.testResults.map((file: { name: string }) => file.name)).toEqual([path.join(f.root, f.testsRoot, name)]); + } else { + expect(surface.files.map((file: { path: string }) => file.path)).toEqual([`${f.testsRoot}/${name}`]); + } + } finally { f.cleanup(); } + }); + + test.each([ + ["default", [], "1", 0, 2], ["focused zero", ["--retries", "0"], "1", 1, 1], + ["deliberate zero", [zero], "1", 1, 1], ["positive", ["--retries", "3"], "3", 0, 4], + ] as const)("%s reaches actual native attempts", (_name, flags, fails, status, attempts) => { + const f = guidanceCheckout(runner); + try { + const result = f.run(["local", "--file", `${f.testsRoot}/one.test.ts`, "--grep", "selected attempt", ...flags], { TV_GUIDANCE_FAIL_COUNT: fails }); + expect(result.status, result.stdout + result.stderr).toBe(status); + const events = readFileSync(path.join(f.root, "attempts.ndjson"), "utf8").trim().split("\n").map((line) => JSON.parse(line)); + expect(events.map((event) => event.attempt)).toEqual(Array.from({ length: attempts }, (_, i) => i)); + if (status === 0) expect(summary(f, result).surfaces[0].counts.testsFlakyRecovered).toBe(1); + } finally { f.cleanup(); } + }); + + test("targeted worker command keeps exact collection and numeric zero", () => { + const f = guidanceCheckout(runner); + try { + const file = `${f.testsRoot}/one.test.ts`; + const command = targetCommand({ runner, cwd: ".", config: f.surface.config, file, files: [file], grep: "selected|sibling", retries: "0", command: null }); + const output = path.join(f.root, "target.json"); + const result = spawnSync("bash", ["-c", command], { cwd: f.root, env: { ...f.env, TV_TARGET_NATIVE_RESULT: output }, encoding: "utf8", timeout: 60_000 }); + expect(result.status, result.stdout + result.stderr).toBe(0); + const native = readFileSync(output, "utf8"); + expect(native).toContain("selected attempt"); + expect(native).not.toContain("adversarial sibling"); + expect(readFileSync(path.join(f.root, "attempts.ndjson"), "utf8").trim().split("\n")).toHaveLength(1); + } finally { f.cleanup(); } + }); + + test.each([["planned", 0], ["planned", 2], ["diagnostic", 0], ["diagnostic", 2]] as const)("%s worker forwards budget %i even when config retries", (mode, budget) => { + const f = guidanceCheckout(runner); + try { + // This existing worker path rejects ambiguous positional filters. Its + // retry fixture needs only one file; local/targeted tests retain siblings. + rmSync(path.join(f.root, f.testsRoot, "one.test.ts.test.ts")); + f.git("add", "."); f.git("commit", "-qm", "Single worker fixture"); + const baseline = { schemaVersion: 1 as const, sourceThrough: null, files: {} }; + const plan = createShardPlan({ + inventory: [{ path: `${f.testsRoot}/one.test.ts`, surfaceId: f.surface.id, runner }], + surfaces: [f.surface], baseline, baselineBytes: serializeTimingBaseline(baseline), + timingProvider: "local-linux-x64-4cpu", testedCommit: f.git("rev-parse", "HEAD"), testedTree: f.git("rev-parse", "HEAD^{tree}"), shardTotal: 1, + }); + f.write("plan.json", JSON.stringify(plan)); + const env: NodeJS.ProcessEnv = { ...f.env, TV_TEST_TIMING_PROVIDER: "local-linux-x64-4cpu", TV_GUIDANCE_FAIL_COUNT: "1" }; + delete env.GITHUB_ACTIONS; + delete env.TEST_SHARD_CONFIG_MODULE; + const result = spawnSync(process.execPath, ["scripts/run-test-shard.mjs", ...(mode === "planned" ? ["--plan", "plan.json"] : ["--total", "1", "--surfaces", f.surface.id]), "--shard", "1", "--results-dir", ".test-runs/worker", "--test-retries", String(budget)], { cwd: f.root, env, encoding: "utf8", timeout: 60_000 }); + expect(result.status, result.stdout + result.stderr).toBe(budget === 0 ? 1 : 0); + const task = JSON.parse(readFileSync(path.join(f.root, ".test-runs/worker/shard-1.json"), "utf8")).tasks[0]; + if (mode === "planned") expect(task.retryBudget).toBe(budget); + const attempts = readFileSync(path.join(f.root, "attempts.ndjson"), "utf8").trim().split("\n"); + expect(attempts).toHaveLength(budget === 0 ? 1 : 2); + } finally { f.cleanup(); } + }); + + test("unit defaults remain zero and a per-test annotation still retries", () => { + const f = guidanceCheckout(runner); + try { + f.surface.kind = "unit"; f.writeRegistry(); + const unit = f.run(["local", "--file", `${f.testsRoot}/one.test.ts`, "--grep", "selected attempt"], { TV_GUIDANCE_FAIL_COUNT: "1" }); + expect(unit.status, unit.stdout + unit.stderr).toBe(1); + const annotated = f.run(["local", "--file", `${f.testsRoot}/one.test.ts`, "--grep", "annotated retry", "--retries", "0"]); + expect(annotated.status, annotated.stdout + annotated.stderr).toBe(0); + expect(summary(f, annotated).surfaces[0].counts.testsFlakyRecovered).toBe(1); + } finally { f.cleanup(); } + }); +}); + +function summary(f: ReturnType, result: { stdout: string }) { + const file = result.stdout.match(/summary: (.+summary.json)/)?.[1]; + expect(file).toBeTruthy(); + const summaryPath = path.resolve(f.root, file!); + return JSON.parse(readFileSync(path.join(path.dirname(summaryPath), "results.json"), "utf8")); +} diff --git a/test/repo/test-runner-guardrails.test.ts b/test/repo/test-runner-guardrails.test.ts index 4a3facfb..1a91de22 100644 --- a/test/repo/test-runner-guardrails.test.ts +++ b/test/repo/test-runner-guardrails.test.ts @@ -1,9 +1,10 @@ import { spawnSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterAll, describe, expect, test } from "vitest"; import { loadTestConfig, selectSurfaces } from "../../scripts/test/config.mjs"; +import { guidanceCheckout } from "./fixtures/test-runner/guidance-checkout.ts"; const cli = ["scripts/test/cli.mjs"]; // Real child-process homes isolate host configuration without mocking marker lookup. @@ -43,7 +44,7 @@ function selftestEnv(env: Record = {}) { const broadShortcutMessage = "Do not use this command for broad verification. Use npm run verify which also includes other critical verifications."; const blaxelGuardrailPrefix = "The default broad verification strategy should be blaxel, which requires a clean working tree and all code pushed to origin."; -const localVerifyGuardrailMessage = "~/.tvdev-use-blaxel is present. Use Blaxel for full verification: npm run verify -- blaxel. With explicit human permission under testing policy, allow local verification using npm run verify -- local --allow-extreme-inefficiency."; +const localVerifyGuardrailMessage = "~/.tvdev-use-blaxel is present. Use Blaxel for full verification: npm run verify -- blaxel. Commit and push to origin first; do not replace broader validation with serial file loops. With explicit human permission under testing policy, allow local verification using npm run verify -- local --against-test-guidance-broad-local-run."; const ciWorkflowPath = ".github/workflows/ci.yml"; function ciEndToEndShardRunCommand(workflowText: string): string { @@ -96,7 +97,7 @@ describe("test runner guardrails", () => { expect(triggers).toContain("pull_request:\n branches: [main, 'integration/**']"); }); - // specs/arch/test-runner/github-ci.md#dependency-install — the + // specs/arch/test-runner/github-ci.md#dependency-and-browser-caches — the // committed workflow encodes the exact-key cache contract. test("setup-workspace composite encodes the exact-key cache contract", () => { const action = readFileSync(setupWorkspacePath, "utf8"); @@ -116,7 +117,7 @@ describe("test runner guardrails", () => { expect(action).toMatch(/key: \$\{\{ inputs\.cache-namespace \}\}-\$\{\{ runner\.os \}\}-\$\{\{ runner\.arch \}\}-node\$\{\{ steps\.node-runtime\.outputs\.major \}\}-\$\{\{ inputs\.image-identity \}\}/); }); - // specs/arch/test-runner/github-ci.md#test-assertions — the shard + // specs/arch/test-runner/github-ci.md#dependency-and-browser-caches — the shard // readiness action binds one exact workspace+browser archive to the actual // Node and hosted-runner image selected for the job. test("setup-shard-readiness encodes the image-bound composite-cache contract", () => { @@ -177,7 +178,7 @@ describe("test runner guardrails", () => { expect(e2eJob.indexOf("setup-shard-readiness")).toBeLessThan(e2eJob.indexOf("setup-playwright")); }); - // specs/arch/test-runner/github-ci.md#test-assertions — the committed + // specs/arch/test-runner/github-ci.md — the committed // workflow encodes the browser-environment, fan-out, tool, and join contracts. test("ci.yml encodes the browser-environment, fan-out, tool-prerequisite, and join contracts", () => { const workflow = readFileSync(ciWorkflowPath, "utf8"); @@ -254,7 +255,8 @@ describe("test runner guardrails", () => { expect(joinJob).toContain("node scripts/record-test-phase.mjs"); }); - // [[arch/test-runner/github-ci.md#^gha-prebuilt-consumer|prebuilt consumer]] + // specs/arch/test-runner/github-ci.md#^gha-prebuilt-partition — a missing or + // broken archive fails the shard, with no fallback build. test("e2e shards fail loudly when the prebuilt archive is missing or extraction fails", () => { const workflow = readFileSync(ciWorkflowPath, "utf8"); const e2eJob = workflow.slice(workflow.indexOf("\n e2e:"), workflow.indexOf("\n e2e-required:")); @@ -281,7 +283,7 @@ describe("test runner guardrails", () => { expect(testJob).not.toContain("TEST_SHARD_SKIP_PRECOMMANDS"); }); - // specs/arch/test-runner/attestation.md#test-assertions — the committed + // specs/arch/test-runner/attestation.md#skipping-in-ci — the committed // workflow encodes the attestation wiring: a no-install decision job, // draft cost guards, heavy jobs that skip only on the literal 'true' // attestation output, a strict non-draft join, and a completion-bracketed @@ -383,9 +385,9 @@ describe("test runner guardrails", () => { expect(attestWrite).toContain('--captured-version "${{ needs.attest-check.outputs.version }}"'); }); - // specs/arch/test-runner/github-ci.md#ci-worker-counts — e2e:browser-app - // keeps its conservative CI worker pin at 2 (TV_PW_WORKERS is the - // measurement knob for the deferred continuous A/B). + // e2e:browser-app keeps its conservative CI worker pin at 2, a setting the + // code owns (TV_PW_WORKERS is the measurement knob for the deferred + // continuous A/B). test("e2e:browser-app pins CI Playwright workers to 2 by default", () => { const config = readFileSync("packages/web/playwright.config.ts", "utf8"); expect(config).toContain('Number(process.env.TV_PW_WORKERS ?? "2")'); @@ -429,7 +431,7 @@ describe("test runner guardrails", () => { expect(Object.keys(baseline.files).length).toBeGreaterThan(0); }); - // specs/arch/test-runner/attestation.md#prune-and-retention — argument + // specs/arch/test-runner/attestation.md#revoking-and-pruning — argument // errors are rejected before any remote operation. test("rejects malformed testpass prune arguments before any remote operation", () => { const badAge = runCli(["testpass", "prune", "--older-than", "soon"]); @@ -448,12 +450,11 @@ describe("test runner guardrails", () => { test("keeps CI-planned e2e surfaces registry-owned while desktop stays in its own job", () => { const config = loadTestConfig(); - const registrySpec = readFileSync("specs/arch/test-runner/test-registry.md", "utf8"); + const ciSpec = readFileSync("specs/arch/test-runner/github-ci.md", "utf8"); const allE2e = selectSurfaces(config, { suite: "e2e" }); const planned = selectSurfaces(config, { suite: "e2e-ci" }); expect(planned.map((surface) => surface.id)).toEqual(allE2e.filter((surface) => !surface.tags.includes("electron")).map((surface) => surface.id)); - expect(registrySpec).toContain("`e2e-ci` → `kind:e2e`, excluding `tag:electron`"); - expect(registrySpec).toContain("dedicated `e2e:desktop` job"); + expect(ciSpec).toContain("The end-to-end shards cover the `e2e-ci` suite: every end-to-end surface except Electron, which the desktop job covers."); expect(planned.some((surface) => surface.id === "e2e:desktop" || surface.tags.includes("electron"))).toBe(false); const workflow = readFileSync(ciWorkflowPath, "utf8"); expect(workflow).toContain('npm test -- local --surface e2e:desktop --run-dir-output'); @@ -505,7 +506,7 @@ describe("test runner guardrails", () => { }); test("verify plans the vibe-mode check as its last phase for both providers", () => { - for (const args of [["verify", "local", "--allow-extreme-inefficiency", "--plan"], ["verify", "blaxel", "--plan"]]) { + for (const args of [["verify", "local", "--against-test-guidance-broad-local-run", "--plan"], ["verify", "blaxel", "--plan"]]) { const result = runCli(args); expect(result.status, result.stderr).toBe(0); const phases = result.stdout.split("\n").filter((line) => line.startsWith("[verify:")); @@ -515,7 +516,7 @@ describe("test runner guardrails", () => { }); // proofs/arch/test-runner/test-runner.md#^t-marked-verify-refusal - test.each([{ args: [] }, { args: ["--allow-extreme-inefficiency"] }])("refuses failed marked-host verify preflight with extra args $args", ({ args: extraArgs }) => { + test.each([{ args: [] }, { args: ["--against-test-guidance-broad-local-run"] }])("refuses failed marked-host verify preflight with extra args $args", ({ args: extraArgs }) => { const result = runCli(["verify", "--plan", ...extraArgs], selftestEnv({ HOME: markedHome, TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT: "blaxel:working-tree:Working tree has uncommitted local changes. Commit or stash changes.", @@ -525,7 +526,7 @@ describe("test runner guardrails", () => { expect(result.stderr).toContain(blaxelGuardrailPrefix); expect(result.stderr).toContain("working-tree"); expect(result.stderr).toContain("Commit or stash changes."); - expect(result.stderr).toContain("npm run verify -- local --allow-extreme-inefficiency"); + expect(result.stderr).toContain("npm run verify -- local --against-test-guidance-broad-local-run"); expect(result.stdout).not.toContain("verify passed"); }); @@ -596,8 +597,8 @@ describe("test runner guardrails", () => { expect(result.stdout).not.toContain("verify passed"); }); - test("allows positional local verify provider with explicit extreme inefficiency acknowledgement", () => { - const result = runCli(["verify", "local", "--allow-extreme-inefficiency", "--plan"], selftestEnv({ + test("allows positional local verify provider with explicit broad-local acknowledgement", () => { + const result = runCli(["verify", "local", "--against-test-guidance-broad-local-run", "--plan"], selftestEnv({ HOME: markedHome, TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT: "blaxel:working-tree:Working tree has uncommitted local changes. Commit or stash changes.", })); @@ -655,7 +656,7 @@ describe("test runner guardrails", () => { test("verify plan includes --force for each provider's internal broad shortcut child", () => { for (const provider of ["local", "blaxel"] as const) { const args = provider === "local" - ? ["verify", provider, "--allow-extreme-inefficiency", "--plan"] + ? ["verify", provider, "--against-test-guidance-broad-local-run", "--plan"] : ["verify", provider, "--plan"]; const result = runCli(args); @@ -709,8 +710,8 @@ describe("test runner guardrails", () => { expect(result.stdout).not.toContain("verify passed"); }); - // specs/arch/test-runner/github-ci.md#retirement-record — the retired - // gha dispatch provider is rejected everywhere a provider is named. + // specs/arch/test-runner/test-runner.md#providers — GitHub CI is not a + // provider, so `gha` is rejected everywhere a provider is named. // Acceptance-shaped: the real scripts/test/cli.mjs process, real argv, // asserted exit codes and stderr. test("rejects the retired gha provider shortcut", () => { @@ -737,6 +738,39 @@ describe("test runner guardrails", () => { expect(result.stderr).toContain("--provider must be one of: local, blaxel"); }); + // These two placement cases clear the outer shard token only for an immediate + // CLI dry run; the child retains ordinary surface ownership and starts no work. + test("accepts a configured Blaxel harness coordinator and private pool without fixed names", () => { + const result = runCli(["local", "--surface", "experiment:blaxel-harness"], selftestEnv({ + TV_TEST_RUNNER_DRY_RUN: "1", + TV_TEST_SHARD_OWNER: undefined, + TV_BLAXEL_HARNESS_COORDINATOR: "dedicated-harness-coordinator", + TV_BLAXEL_HARNESS_POOL: "disposable-harness-pool", + })); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("surfaces=experiment:blaxel-harness"); + }); + + test("refuses missing Blaxel harness configuration and leased or remote placement before preflight", () => { + const configured = { + TV_TEST_RUNNER_DRY_RUN: "1", + TV_TEST_SHARD_OWNER: undefined, + TV_BLAXEL_HARNESS_COORDINATOR: "dedicated-harness-coordinator", + TV_BLAXEL_HARNESS_POOL: "disposable-harness-pool", + }; + for (const [provider, env] of [ + ["local", { ...configured, TV_BLAXEL_HARNESS_COORDINATOR: " " }], + ["local", { ...configured, TV_BLAXEL_HARNESS_POOL: undefined }], + ["local", { ...configured, TV_TEST_SHARD_OWNER: "fixture-owner" }], + ["blaxel", configured], + ] as const) { + const result = runCli([provider, "--surface", "experiment:blaxel-harness"], selftestEnv(env)); + expect(result.status, result.stderr).toBe(2); + expect(result.stderr).toContain("dedicated coordinator outside the worker pool"); + expect(result.stdout).not.toContain("[dry-run]"); + } + }); + test("rejects isolated lifecycle fixtures on developer-host local execution before preflight", () => { const result = runCli(["local", "--surface", "experiment:lifecycle-leak"], selftestEnv({ TV_TEST_RUNNER_DRY_RUN: "1", @@ -764,6 +798,62 @@ describe("test runner guardrails", () => { expect(result.stderr).not.toContain("placement guard failed"); }); + // Each refusal runs in a disposable checkout, so its own .test-runs/ shows + // whether the command made a run directory. + function refusalCheckout() { + const f = guidanceCheckout(); + const playwrightRoot = "test/runner-fixtures/guidance/playwright"; + cpSync(playwrightRoot, path.join(f.root, playwrightRoot), { recursive: true }); + f.registry.executionGroups[0]!.surfaces.push({ + ...f.surface, id: "fixture:playwright", runner: "playwright", config: `${playwrightRoot}/playwright.config.ts`, + roots: [playwrightRoot], absoluteConfig: path.join(f.root, playwrightRoot, "playwright.config.ts"), + }); + Object.assign(f.registry.suites, { all: { include: ["kind:experiment"] }, experiment: { include: ["kind:experiment"] } }); + f.writeRegistry(); + // Blaxel selection reads the registry from the tested commit. + f.git("add", "."); + f.git("commit", "-qm", "Second fixture surface"); + return f; + } + + test.each([ + [["local"], broadShortcutMessage], + [["blaxel", "--surface", "fixture:vitest,fixture:playwright"], "resolves to exactly one surface"], + [["blaxel", "--suite", "experiment"], "Blaxel suite runs require --suite all|unit|e2e"], + ])("refuses %j before remote preflight and creates no run directory", (args, message) => { + const f = refusalCheckout(); + try { + const output = path.join(f.root, "run-dir"); + const result = f.run([...args, "--run-dir-output", output], { + TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT: "blaxel:must-not-run:selection guard failed", + }); + + expect(result.status, result.stderr).toBe(2); + expect(result.stderr).toContain(message); + expect(result.stderr).not.toContain("selection guard failed"); + expect(existsSync(output)).toBe(false); + expect(existsSync(path.join(f.root, ".test-runs"))).toBe(false); + } finally { + f.cleanup(); + } + }); + + test("refuses an empty --run-dir-output before remote preflight and creates no run directory", () => { + const f = refusalCheckout(); + try { + const result = f.run(["blaxel", "--suite", "experiment", "--run-dir-output="], { + TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT: "blaxel:must-not-run:option guard failed", + }); + + expect(result.status, result.stderr).toBe(2); + expect(result.stderr).toContain("--run-dir-output requires a file path"); + expect(result.stderr).not.toContain("option guard failed"); + expect(existsSync(path.join(f.root, ".test-runs"))).toBe(false); + } finally { + f.cleanup(); + } + }); + // The architect accepted this deviation as closed pending TV-597's cause investigation and restoration. test.skip("allows isolated lifecycle fixtures only under the authorized GitHub workflow identity and opt-in", () => { const result = runCli(["local", "--surface", "experiment:lifecycle-leak"], selftestEnv({ @@ -873,3 +963,176 @@ describe("test runner guardrails", () => { } }); }); + +// proofs/arch/test-runner/test-runner.md#^t-runner-file-admission +// proofs/arch/test-runner/test-runner.md#^t-runner-local-redirection +// proofs/arch/test-runner/test-runner.md#^t-runner-retry-admission +// Downstream execution is always stopped by dry-run or verify --plan. +describe("local file and retry admission", () => { + const file = "test/repo/test-runner-guardrails.test.ts"; + const broadLocal = "--against-test-guidance-broad-local-run"; + const broadZero = "--against-test-guidance-turn-flakes-into-failures-on-broad-runs"; + const dry = (args: string[], marked = false) => runCli(args, selftestEnv({ HOME: marked ? markedHome : unmarkedHome, TV_TEST_RUNNER_DRY_RUN: "1" })); + + test.each([ + ["--surface", "unit:root"], ["--suite", "unit"], + ["--package", "@telepath-computer/television-web"], + ["--runner", "playwright"], ["--tag", "browser"], ["--file", "test/repo/test-runner"], + ])("refuses broader marked selection %s %s before execution", (flag, value) => { + const result = dry(["local", flag, value], true); + expect(result.status, result.stderr).toBe(2); + expect(result.stderr).toContain("one test file"); + expect(result.stderr).toContain("npm test -- blaxel"); + expect(result.stderr).toMatch(/commit.*push/is); + expect(result.stderr).toContain("serial"); + expect(result.stdout).not.toContain("[dry-run]"); + expect(dry(["local", flag, value]).status).toBe(0); + expect(dry(["local", flag, value, broadLocal], true).status).toBe(0); + }); + + test("marked one-file admission honors grep and co-selectors", () => { + const result = dry(["local", "--surface", "unit:root", "--file", file, "--grep", "one-file"], true); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain(`files=${file}`); + const conflict = dry(["local", "--surface", "unit:frames", "--file", file], true); + expect(conflict.status).toBe(2); + expect(conflict.stderr).toContain("not part of the selected surfaces"); + }); + + test.each([ + ["--suite", "telemetry-posthog-roundtrip"], ["--suite", "daemon-acceptance"], ["--suite", "agent"], + ["--surface", "telemetry-posthog-roundtrip:integration"], ["--surface", "daemon-acceptance:cli"], ["--surface", "agent:root"], + ["--package", "@telepath-computer/television-server"], + ])("gives a local route for %s %s", (flag, value) => { + const result = dry(["local", flag, value], true); + expect(result.status).toBe(2); + expect(result.stderr).toContain("local"); + expect(result.stderr).toContain("--file"); + expect(result.stderr).not.toContain(`blaxel ${flag} ${value}`); + }); + + test("broad-local permission does not lift the full-all shortcut guardrail", () => { + expect(dry(["local", broadLocal], true).status).toBe(2); + expect(dry(["local", broadLocal, "--force"], true).status).toBe(0); + }); + + test.each([broadLocal, broadZero])("rejects values attached to deliberate guidance flag %s", (flag) => { + for (const suffix of ["=false", "=0", "=1", "=", " false"]) { + const result = dry(["local", "--surface", "unit:root", ...`${flag}${suffix}`.split(" ")], flag === broadLocal); + expect(result.status, result.stderr).toBe(2); + expect(result.stderr).toContain(`${flag} does not take a value`); + } + }); + + test.each([ + ["telemetry-posthog-roundtrip", "packages/server/test/telemetry-posthog.integration.test.ts"], + ["daemon-acceptance", "test/node/daemon-acceptance.test.ts"], + ])("prints an executable one-file route for %s", (suite, selectedFile) => { + const result = dry(["local", "--suite", suite], true); + expect(result.status).toBe(2); + expect(result.stderr).toContain(`npm test -- local --file ${selectedFile}`); + expect(dry(["local", "--file", selectedFile], true).status).toBe(0); + }); + + test.each([["--surface", "experiment:blaxel-harness"], ["--tag", "blaxel-harness-driver"]])("routes the configured harness driver %s %s to its local one-file form", (flag, value) => { + const coordinator = selftestEnv({ HOME: markedHome, TV_TEST_RUNNER_DRY_RUN: "1", TV_TEST_SHARD_OWNER: undefined, TV_BLAXEL_HARNESS_COORDINATOR: "dedicated-harness-coordinator", TV_BLAXEL_HARNESS_POOL: "disposable-harness-pool" }); + const driver = "test/runner-fixtures/blaxel-harness/driver/harness.test.ts"; + const result = runCli(["local", flag, value], coordinator); + expect(result.status).toBe(2); + expect(result.stderr).toContain(`npm test -- local --file ${driver}`); + expect(result.stderr).not.toContain("npm test -- blaxel"); + expect(runCli(["local", "--file", driver], coordinator).status).toBe(0); + }); + + test("list and standalone preflight use the execution file resolver", () => { + const listed = runCli(["list", "--file", "runner-guardrails"], { HOME: markedHome }); + expect(listed.status, listed.stderr).toBe(0); + expect(listed.stdout).toMatch(/^unit:root\t/); + const checked = runCli(["preflight", "--provider", "local", "--file", "runner-guardrails"], selftestEnv({ HOME: markedHome, TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT: "fixture:file-resolution-reached-preflight" })); + expect(checked.status, checked.stderr).toBe(1); + expect(checked.stdout + checked.stderr).toContain("file-resolution-reached-preflight"); + expect(runCli(["list", "--file", "no-such-runner-file"]).status).toBe(2); + }); + + test("rejects the renamed option and misspelled guidance suffix", () => { + const renamed = dry(["local", "--file", file, "--allow-extreme-inefficiency"]); + expect(renamed.status).toBe(2); + expect(renamed.stderr).toContain(broadLocal); + const typo = dry(["local", "--file", file, "--against-test-guidance-turn-flakes-into-failure"]); + expect(typo.status).toBe(2); + expect(typo.stderr).toContain("Unknown"); + }); + + test.each(["local", "blaxel"])("allows focused zero but requires deliberate broader zero on %s", (provider) => { + for (const marked of [false, true]) { + const focused = dry([provider, "--file", file, "--grep", "admission", "--retries", "0"], marked); + expect(focused.status, focused.stderr).toBe(0); + const extras = provider === "local" && marked ? [broadLocal] : []; + const broad = dry([provider, "--surface", "unit:root", "--retries", "0", ...extras], marked); + expect(broad.status, broad.stderr).toBe(2); + expect(broad.stderr).toContain("default"); + expect(broad.stderr).toContain("--file"); + expect(broad.stderr).toContain(broadZero); + for (const zeroArgs of [[broadZero], [broadZero, "--retries", "0"]]) { + const admitted = dry([provider, "--surface", "unit:root", ...zeroArgs, ...extras], marked); + expect(admitted.status, admitted.stderr).toBe(0); + } + } + }); + + test.each(["local", "blaxel"])("rejects multi-file and bare-grep zero budgets on %s", (provider) => { + for (const selection of [["--file", "test/repo/test-runner"], ["--grep", "case", "--force"], ["--suite", "unit"]]) { + const result = dry([provider, ...selection, "--retries=0"]); + expect(result.status, result.stderr).toBe(2); + expect(result.stderr).toContain(broadZero); + } + }); + + test.each(["-1", "1.5", "2junk", "NaN"])("rejects invalid retry count %s", (count) => { + expect(dry(["local", "--file", file, "--retries", count]).status).toBe(2); + }); + + test("rejects conflicting retry options and preserves the unit notice", () => { + expect(dry(["local", "--file", file, broadZero, "--retries", "2"]).status).toBe(2); + const positive = dry(["local", "--file", file, "--retries", "3"]); + expect(positive.status, positive.stderr).toBe(0); + expect(positive.stderr).toContain("ignored for unit-only"); + }); + + test.each(["local", "blaxel"])("verify validates and forwards retry policy for %s", (provider) => { + const refused = runCli(["verify", provider, "--retries", "0", "--plan"]); + expect(refused.status, refused.stderr).toBe(2); + expect(refused.stderr).toContain(broadZero); + const admitted = runCli(["verify", provider, broadZero, "--plan"]); + expect(admitted.status, admitted.stderr).toBe(0); + expect(admitted.stdout).toContain("--retries 0"); + expect(admitted.stdout).toContain(broadZero); + expect(runCli(["verify", provider, "--retries", "3", "--plan"]).stdout).toContain("--retries 3"); + const marked = runCli(["verify", "local", broadLocal, "--plan"], selftestEnv({ HOME: markedHome })); + expect(marked.status, marked.stderr).toBe(0); + expect(marked.stdout).toContain(`--force ${broadLocal}`); + }); + test("redirection preserves shell arguments and explains unsupported targeting", () => { + const pattern = "name 'quoted'; $(false) `false`"; + const result = dry(["local", "--surface", "unit:root", "--grep", pattern], true); + expect(result.status).toBe(2); + const command = result.stderr.match(/then run: (.*)\. Do not replace/)?.[1]; + expect(command).toBeTruthy(); + const recorded = spawnSync("bash", ["-c", `npm() { printf '%s\\0' "$@"; }; ${command}`], { encoding: "utf8" }); + expect(recorded.status, recorded.stderr).toBe(0); + expect(recorded.stdout.split("\0").filter(Boolean)).toEqual(["test", "--", "blaxel", "--surface", "unit:root", "--grep", pattern]); + const unsupported = dry(["local", "--surface", "unit:root,unit:frames"], true); + expect(unsupported.stderr).toContain("no equivalent supported Blaxel command"); + }); + + test("help gives focused iteration, pushed remote work, and separate overrides", () => { + const result = runCli(["help"]); + expect(result.stdout).toContain("--file"); + expect(result.stdout).toContain("pushing"); + expect(result.stdout).toContain(broadLocal); + expect(result.stdout).toContain(broadZero); + expect(result.stdout).toContain("--retries 0"); + expect(result.stdout).toContain("human permission"); + }); + +}); diff --git a/test/repo/test-runner-mutex.test.ts b/test/repo/test-runner-mutex.test.ts new file mode 100644 index 00000000..3dd71048 --- /dev/null +++ b/test/repo/test-runner-mutex.test.ts @@ -0,0 +1,207 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { mutexCheckout, ready, waitFor } from "./fixtures/test-runner/mutex-checkout.ts"; + +const bypass = "--against-test-guidance-major-host-contention-and-oom-killed-processes"; +const wait = "--wait-for-local-test-mutex"; +const broad = "--against-test-guidance-broad-local-run"; + +describe("local test mutex", () => { + // proofs/arch/test-runner/test-runner.md#^t-runner-guidance-usage + test("explains mutex choices and refuses valued flags", async () => { + const f = mutexCheckout(); + try { + const help = f.run(["help"]); + for (const word of [wait, bypass, "same operating-system user", "caller", "SIGINT", "SIGTERM"]) + expect(help.stdout).toContain(word); + for (const flag of [wait, bypass]) for (const value of ["=false", "=0"]) + expect(f.run([...f.args, flag + value]).status).toBe(2); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-location + test("resolves one production path without acquiring it, across HOME and TMP overrides", async () => { + const a = mutexCheckout(); const b = mutexCheckout(); + try { + const expected = path.join(os.userInfo().homedir, ".cache/television/local-test-mutex.sqlite3"); + for (const f of [a, b]) { + const result = f.run(f.args, { TV_TEST_RUNNER_LOCK_PATH: "", TV_TEST_RUNNER_DRY_RUN: "1", TMPDIR: f.home }); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain(`mutex=${expected}`); + } + } finally { await a.cleanup(); await b.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-contention + test("refuses a contender before preflight while naming the live holder", async () => { + const f = mutexCheckout(); + try { + const holder = f.start(); await ready(holder); + const result = f.run(f.args, { TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT: "fixture:must-not-run" }); + expect(result.status, result.stdout + result.stderr).toBe(2); + for (const text of [String(holder.child.pid), f.root, "cli.mjs", wait, bypass, "OOM", "another agent"]) expect(result.stderr).toContain(text); + expect(result.stderr).not.toContain("must-not-run"); + writeFileSync(path.join(f.home, ".tvdev-use-blaxel"), ""); + const marked = f.run(f.args); + expect(marked.status).toBe(2); + expect(marked.stderr).toContain("commit and push, then use Blaxel"); + writeFileSync(holder.release, "done"); + expect(await holder.exited, holder.output()).toBe(0); + expect(f.run(f.args).status).toBe(0); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-wait + test("waits without starting native work, then acquires after release", async () => { + const f = mutexCheckout(); + try { + const holder = f.start(); await ready(holder); + const waiter = f.start([...f.args, wait]); + await waitFor(() => waiter.output().includes("Waiting for local-test mutex"), waiter.output); + expect(existsSync(waiter.ready)).toBe(false); + writeFileSync(holder.release, "done"); expect(await holder.exited).toBe(0); + await ready(waiter); writeFileSync(waiter.release, "done"); + expect(await waiter.exited, waiter.output()).toBe(0); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-wait-cancel + test.each([["SIGINT", 130], ["SIGTERM", 143]] as const)("cancels only the waiter on %s", async (signal, code) => { + const f = mutexCheckout(); + try { + const holder = f.start(); await ready(holder); + const waiter = f.start([...f.args, wait]); + await waitFor(() => waiter.output().includes("Waiting for local-test mutex"), waiter.output); + waiter.child.kill(signal); + expect(await waiter.exited, waiter.output()).toBe(code); + expect(existsSync(waiter.ready)).toBe(false); + expect(f.run(f.args).status).toBe(2); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-bypass + test("bypasses occupied ownership without replacing it, and claims a free mutex", async () => { + const f = mutexCheckout(); + try { + const holder = f.start([...f.args, bypass]); await ready(holder); + expect(f.run(f.args).status).toBe(2); + expect(f.run([...f.args, bypass]).status).toBe(0); + expect(f.run(f.args).status).toBe(2); + writeFileSync(path.join(f.home, ".tvdev-use-blaxel"), ""); + expect(f.run(["local", "--surface", f.surface.id, bypass]).stderr).toContain("one test file"); + expect(f.run(["local", "--surface", f.surface.id, broad]).stderr).toContain("mutex"); + // Independent options: even an intentional full-all shortcut must own + // the mutex. The failed-preflight sentinel bounds admitted execution. + const all = ["local", "--force", broad]; + expect(f.run([...all, "--against-test-guidance-turn-flakes-into-failures-on-broad-runs"]).stderr).toContain("Local-test mutex held"); + const admitted = f.run([...all, bypass], { TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT: "fixture:admitted after all overrides" }); + expect(admitted.status).toBe(1); + expect(admitted.stdout + admitted.stderr).toContain("admitted after all overrides"); + expect(f.run([...f.args, bypass, wait]).status).toBe(2); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-record + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-dead-owner + test.each([false, true])("reclaims a compatible record for an exited process (wait=%s)", async waiting => { + const f = mutexCheckout(); + try { + const exited = JSON.parse(execFileSync(process.execPath, ["--input-type=module", "-e", ` + import { readLinuxProcess, readMacOSProcesses } from './scripts/test/process-lifecycle.mjs'; + const p = process.platform === 'linux' ? readLinuxProcess(process.pid) : readMacOSProcesses().find(p => p.pid === process.pid); + console.log(JSON.stringify({pid:p.pid,startTime:p.startTime})); + `], { cwd: f.root, encoding: "utf8" })); + const db = new DatabaseSync(f.env.TV_TEST_RUNNER_LOCK_PATH!); + db.exec("CREATE TABLE local_test_mutex (slot INTEGER PRIMARY KEY CHECK(slot=1), record TEXT NOT NULL)"); + db.prepare("INSERT INTO local_test_mutex VALUES(1,?)").run(JSON.stringify({ version: 1, id: "exited-fixture", participants: [{ ...exited, role: "run", command: "fixture exited normally", checkout: f.root }] })); + const holder = f.start([...f.args, ...(waiting ? [wait] : [])]); await ready(holder); + const record = JSON.parse(db.prepare("SELECT record FROM local_test_mutex WHERE slot=1").get()!.record as string); + expect(record).toEqual({ version: 1, id: expect.any(String), participants: [{ pid: holder.child.pid, startTime: expect.any(String), role: "run", command: expect.stringContaining("cli.mjs local --file"), checkout: f.root }] }); + expect(record.id).not.toBe("exited-fixture"); + writeFileSync(holder.release, "done"); expect(await holder.exited).toBe(0); + expect(db.prepare("SELECT * FROM local_test_mutex").all()).toEqual([]); + db.close(); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-contention + test("admits only one of simultaneous real acquisitions", async () => { + const f = mutexCheckout(); + try { + const a = f.start(); const b = f.start(); + await waitFor(() => [a,b].some(c => c.child.exitCode === 2) && [a,b].some(c => existsSync(c.ready)), () => a.output() + b.output()); + expect([a,b].filter(c => existsSync(c.ready))).toHaveLength(1); + writeFileSync(a.release, "done"); writeFileSync(b.release, "done"); + expect((await Promise.all([a.exited,b.exited])).sort()).toEqual([0,2]); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-release + test("holds through Vite service teardown and releases after native and preflight failures", async () => { + const f = mutexCheckout(); + try { + f.write("service.config.ts", `import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; + export default { plugins: [{ name: 'observe-lock-at-close', closeBundle() { + const r = spawnSync(process.execPath, ${JSON.stringify(["scripts/test/cli.mjs", ...f.args])}, {env:{...process.env,TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT:'fixture:must-not-start'},encoding:'utf8'}); + fs.writeFileSync('service-close.json',JSON.stringify({status:r.status,stderr:r.stderr})); + }}] }; + `); + f.surface.services = [{ id: "teardown", kind: "vite", config: "service.config.ts", publishUrlEnv: "TV_MUTEX_SERVICE_URL" }]; + f.writeRegistry(); + for (const failed of [false, true]) { + const result = f.run(f.args, { TV_MUTEX_FAIL: failed ? "1" : "0" }); + expect(result.status, result.stdout + result.stderr).toBe(failed ? 1 : 0); + const close = JSON.parse(readFileSync(path.join(f.root, "service-close.json"), "utf8")); + expect(close.status, close.stderr).toBe(2); + expect(close.stderr).toContain("Local-test mutex held"); + } + expect(f.run(f.args, { TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT: "fixture:expected failure" }).status).toBe(1); + expect(f.run(f.args).status).toBe(0); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-exemptions + test("keeps informational and remote preflight paths exempt on a marked home", async () => { + const f = mutexCheckout(); + try { + writeFileSync(path.join(f.home, ".tvdev-use-blaxel"), ""); + const holder = f.start(); await ready(holder); + for (const args of [["help"], ["list", "--surface", f.surface.id], ["verify", "local", broad, "--plan"]]) expect(f.run(args).status).toBe(0); + expect(f.run(f.args, { TV_TEST_RUNNER_DRY_RUN: "1" }).status).toBe(0); + const preflight = f.run(["preflight", "--provider", "local", "--surface", f.surface.id], { TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT: "fixture:reached local check" }); + expect(preflight.status, preflight.stderr).toBe(1); + expect(preflight.stdout + preflight.stderr).toContain("reached local check"); + const remote = f.run(["blaxel", "--surface", f.surface.id], { TV_TEST_RUNNER_FAKE_REMOTE_PREFLIGHT: "blaxel:fixture:reached remote check" }); + expect(remote.status, remote.stderr).toBe(2); + expect(remote.stdout + remote.stderr).toContain("reached remote check"); + for (const args of [["local", "--surface", f.surface.id, wait], [...f.args, "--surface", "missing", wait], [...f.args, "--retries", "bad", wait], ["local", "--surface", f.surface.id, broad, "--retries", "0", wait]]) { + const result = f.run(args); + expect(result.status, result.stderr).toBe(2); + expect(result.stderr).not.toContain("Waiting for local-test mutex"); + } + f.surface.tags.push("isolated-github-only"); f.writeRegistry(); + const placement = f.run([...f.args, wait], { GITHUB_ACTIONS: "" }); + expect(placement.status, placement.stderr).toBe(2); + expect(placement.stderr).toContain("isolated-github-only"); + expect(placement.stderr).not.toContain("Waiting for local-test mutex"); + + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-selftest + test("gates private locations and isolates independent scenarios", async () => { + const a = mutexCheckout(); const b = mutexCheckout(); + try { + for (const gate of ["", "0", "true"]) expect(a.run(a.args, { TV_TEST_RUNNER_SELFTEST: gate }).status).toBe(2); + const first = a.start(); const second = b.start(); await ready(first); await ready(second); + expect(first.output()).toContain("WARNING: TV_TEST_RUNNER_LOCK_PATH"); + expect(second.output()).toContain("WARNING: TV_TEST_RUNNER_LOCK_PATH"); + const marked = a.run(a.args, { HOME: b.home }); + expect(marked.status, marked.stderr).toBe(2); + } finally { await a.cleanup(); await b.cleanup(); } + }); +}); diff --git a/test/repo/test-runner-native-entrypoints.test.ts b/test/repo/test-runner-native-entrypoints.test.ts new file mode 100644 index 00000000..ad940c09 --- /dev/null +++ b/test/repo/test-runner-native-entrypoints.test.ts @@ -0,0 +1,136 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { loadTestConfig } from "../../scripts/test/config.mjs"; +import { createShardPlan } from "../../scripts/test/shard-plan.mjs"; +import { serializeTimingBaseline } from "../../scripts/test/timing-baseline.mjs"; +import { targetCommand } from "../../scripts/test/target-command.mjs"; +import { guidanceCheckout } from "./fixtures/test-runner/guidance-checkout.ts"; +import { mutexCheckout, ready } from "./fixtures/test-runner/mutex-checkout.ts"; + +const override = "TV_TEST_AGAINST_TEST_GUIDANCE_DIRECT_NATIVE_RUN"; +const refusal = "Direct native test invocation"; + +// proofs/arch/test-runner/test-runner.md#^t-runner-guidance-usage +test("help explains deliberate native use alongside canonical guidance", () => { + const r = spawnSync(process.execPath, ["scripts/test/cli.mjs", "help"], { encoding: "utf8" }); + expect(r.status).toBe(0); + for (const text of [`${override}=1`, "native", "services", "caller", "--file", "--grep", "pushing", "human permission", "--wait-for-local-test-mutex", "--against-test-guidance-major-host-contention-and-oom-killed-processes", "--against-test-guidance-broad-local-run", "--against-test-guidance-turn-flakes-into-failures-on-broad-runs", "--retries 0", "default budget"]) expect(r.stdout).toContain(text); +}); + +function withoutContext(base: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + const env = { ...base }; + delete env.TV_TEST_RUNNER_CONTEXT; + delete env[override]; + delete env.GITHUB_ACTIONS; + delete env.TEST_SHARD_CONFIG_MODULE; + return env; +} +function native(runner: "vitest" | "playwright", config: string, args: string[], cwd: string, env: NodeJS.ProcessEnv) { + return spawnSync("npx", [runner, ...(runner === "vitest" ? ["run"] : ["test"]), "--config", config, ...args], { cwd, env, encoding: "utf8", timeout: 60_000 }); +} + +// proofs/arch/test-runner/test-runner.md#^t-runner-native-refusal +test("every registered config refuses direct invocation before service setup", () => { + const home = mkdtempSync(path.join(os.tmpdir(), "native-entrypoints-")); + const started = Date.now(); + try { + const env = withoutContext({ ...process.env, HOME: home }); + const configs = new Map(loadTestConfig().surfaces.map(s => [s.config, s])); + for (const s of configs.values()) { + // Listing cannot execute test bodies, even if the check regresses. The + // no-match filter also avoids loading the registered test modules. + const args = s.runner === "vitest" + ? ["vitest", "list", "__tv_no_matching_test_file__", "--filesOnly", "--config", s.absoluteConfig] + : ["playwright", "test", "__tv_no_matching_test_file__", "--list", "--config", s.absoluteConfig]; + const r = spawnSync("npx", args, { cwd: path.resolve(s.cwd), env, encoding: "utf8", timeout: 30_000 }); + expect(r.stderr + r.stdout, s.config).toContain(refusal); + expect(r.status, s.config).not.toBe(0); + } + } finally { + console.log(`Registered native config sweep: ${Date.now() - started}ms`); + rmSync(home, { recursive: true, force: true }); + } +}, 120_000); + +describe.each(["vitest", "playwright"] as const)("%s native guidance", runner => { + // proofs/arch/test-runner/test-runner.md#^t-runner-native-refusal + // proofs/arch/test-runner/test-runner.md#^t-runner-native-override + test("refuses absent and inexact overrides with host-appropriate next steps", () => { + const f = guidanceCheckout(runner); + try { + for (const marked of [false, true]) { + if (marked) writeFileSync(path.join(f.home, ".tvdev-use-blaxel"), ""); + for (const value of [undefined, "0", "true"]) { + const env = withoutContext(f.env); + if (value !== undefined) env[override] = value; + const r = native(runner, f.surface.config, runner === "vitest" ? ["-t", "selected|sibling"] : ["-g", "selected|sibling"], f.root, env); + const output = r.stdout + r.stderr; + expect(r.status, output).not.toBe(0); + for (const text of [refusal, "npm test -- local --file", `${override}=1`, "mutex", "file limit"]) expect(output).toContain(text); + if (marked) for (const text of ["commit", "push", "npm test -- blaxel", "npm run verify -- blaxel"]) expect(output).toContain(text); + else expect(output).not.toContain("npm test -- blaxel"); + } + } + } finally { f.cleanup(); } + }); + + test("deliberate native use skips canonical file and mutex protections", async () => { + const holder = mutexCheckout(); const f = guidanceCheckout(runner); + try { + const active = holder.start(); await ready(active); + writeFileSync(path.join(f.home, ".tvdev-use-blaxel"), ""); + const env = { ...withoutContext(f.env), [override]: "1", TV_TEST_RUNNER_LOCK_PATH: holder.env.TV_TEST_RUNNER_LOCK_PATH }; + const r = native(runner, f.surface.config, [runner === "vitest" ? "-t" : "-g", "selected|sibling", "--reporter=json"], f.root, env); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(r.stdout).toContain("selected attempt"); + expect(r.stdout).toContain("adversarial sibling"); + expect(holder.run(holder.args).status).toBe(2); + } finally { f.cleanup(); await holder.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-native-context + test.each(["local", "planned", "targeted", "diagnostic"] as const)("%s launcher supplies its own context", mode => { + const f = guidanceCheckout(runner); + try { + const file = `${f.testsRoot}/context.test.ts`; + f.write(file, `import { test, expect } from '${runner === "vitest" ? "vitest" : "@playwright/test"}'; + test('launcher context', () => { expect(process.env.TV_TEST_RUNNER_CONTEXT).toBe('1'); });`); + // Every remaining file is the one small guarded fixture, even if a + // diagnostic launcher fails to narrow. No intentional retry cases here. + rmSync(path.join(f.root, f.testsRoot, "one.test.ts")); + rmSync(path.join(f.root, f.testsRoot, "one.test.ts.test.ts")); + f.git("add", "."); f.git("commit", "-qm", "Context fixture"); + const env = withoutContext(f.env); + env.TV_TEST_TIMING_PROVIDER = "local-linux-x64-4cpu"; + let args: string[]; + if (mode === "local") args = ["scripts/test/cli.mjs", "local", "--file", file]; + else if (mode === "targeted") args = ["scripts/test/supervised-command.mjs", "--run-id", "native-context", "--surface", f.surface.id, "--result", "target-lifecycle.json", "--", "bash", "-c", targetCommand({ runner, cwd: ".", config: f.surface.config, file, files: [file], retries: "2", command: null })]; + else { + const baseline = { schemaVersion: 1 as const, sourceThrough: null, files: {} }; + const plan = createShardPlan({ inventory: [{ path: file, surfaceId: f.surface.id, runner }], surfaces: [f.surface], baseline, baselineBytes: serializeTimingBaseline(baseline), timingProvider: "local-linux-x64-4cpu", testedCommit: f.git("rev-parse", "HEAD"), testedTree: f.git("rev-parse", "HEAD^{tree}"), shardTotal: 1 }); + f.write("plan.json", JSON.stringify(plan)); + args = ["scripts/run-test-shard.mjs", ...(mode === "planned" ? ["--plan", "plan.json"] : ["--total", "1", "--surfaces", f.surface.id]), "--shard", "1", "--results-dir", ".test-runs/worker", "--test-retries", "2"]; + } + env.TV_TARGET_NATIVE_RESULT = path.join(f.root, "target.json"); + const r = spawnSync(process.execPath, args, { cwd: f.root, env, encoding: "utf8", timeout: 60_000 }); + expect(r.status, r.stdout + r.stderr).toBe(0); + const evidence = mode === "local" ? r.stdout : mode === "targeted" + ? readFileSync(env.TV_TARGET_NATIVE_RESULT, "utf8") + : readFileSync(path.join(f.root, `.test-runs/worker/native-fixture-${runner}-shard-1.json`), "utf8"); + expect(evidence).toContain(mode === "local" ? "status=passed" : "launcher context"); + } finally { f.cleanup(); } + }); +}); + +test("native override still requires the declared service setup", () => { + const env = withoutContext(process.env); + env[override] = "1"; + delete env.TV_DYNAMIC_FIRST_URL; delete env.TV_DYNAMIC_SECOND_URL; + const r = native("playwright", "test/runner-fixtures/dynamic-services/playwright.config.ts", ["--list", "__tv_no_matching_test_file__"], process.cwd(), env); + expect(r.status).not.toBe(0); + expect(r.stdout + r.stderr).toContain("TV_DYNAMIC_FIRST_URL is required"); + expect(r.stdout + r.stderr).not.toContain(refusal); +}); diff --git a/test/repo/test-runner-prebuilt.test.ts b/test/repo/test-runner-prebuilt.test.ts index 24d2147c..fab29930 100644 --- a/test/repo/test-runner-prebuilt.test.ts +++ b/test/repo/test-runner-prebuilt.test.ts @@ -9,7 +9,7 @@ import { createShardPlan } from "../../scripts/test/shard-plan.mjs"; import { serializeTimingBaseline } from "../../scripts/test/timing-baseline.mjs"; import fixtureConfig from "./fixtures/test-runner/shard-worker/config.mjs"; -// [[arch/test-runner/github-ci.md#^gha-prebuilt-cli-outputs|prebuilt output contract]] +// Prebuilt output contract: proofs/arch/test-runner/github-ci.md#^gha-prebuilt-cli-outputs function runPrebuiltCli(args: string[]) { return spawnSync(process.execPath, ["scripts/test/prebuilt.mjs", ...args], { encoding: "utf8" }); diff --git a/test/repo/test-runner-provider.test.ts b/test/repo/test-runner-provider.test.ts index 8bf3c27e..bcd0fb2b 100644 --- a/test/repo/test-runner-provider.test.ts +++ b/test/repo/test-runner-provider.test.ts @@ -5,9 +5,10 @@ import path from "node:path"; import { describe, expect, test } from "vitest"; import { combineAttemptReports, compactProviderSummary, failedTasksFromShards, formatShardLogs, infrastructureRetryShardIndices, providerCommandExitCode, readShardSummaries, runInfrastructureAttempts, summarizeInfrastructureAttempts, writeProviderReports, type AttemptReport } from "../../scripts/test/provider-outputs.mjs"; import { classifyBlaxelCoordinatorFailure, classifyBlaxelShardStatus, classifyDownloadedShardStatus, validateDownloadedShardSummary } from "../../scripts/test/blaxel-classification.mjs"; -import { acquireShardLeases, excludeSandboxesByName, leaseAcquisitionFailureShards } from "../../scripts/test/blaxel-lease-acquisition.mjs"; +import { acquireShardLeases, excludeSandboxesByName, leaseAcquisitionFailureShards, leaseRecoveryArgs } from "../../scripts/test/blaxel-lease-acquisition.mjs"; import { normalizeBlaxelSurfaces, normalizeProviderShardSurfaces, normalizeProviderTimingShards, resolveProviderRunGit } from "../../scripts/test/provider-normalization.mjs"; import { loadTestConfig, selectSurfaces } from "../../scripts/test/config.mjs"; +import { targetCommand } from "../../scripts/test/target-command.mjs"; import { normalizeProcessLeak } from "../../scripts/test/timing-events.mjs"; import { transferShardInputs } from "../../scripts/test/blaxel-shard-dispatch.mjs"; import { buildInterruptedAttemptReport, finishInterruptedRun } from "../../scripts/test/blaxel-interruption.mjs"; @@ -27,6 +28,16 @@ function writeVitestNative(file: string, total: number) { } describe("provider result normalization", () => { + test("forwards only an explicitly supplied recovery interval to the coordinator", () => { + expect(leaseRecoveryArgs({})).toEqual([]); + expect(leaseRecoveryArgs({ "lock-ttl-seconds": "60" })).toEqual(["--lock-ttl-seconds", "60"]); + expect(readFileSync("scripts/test/cli.mjs", "utf8")).toContain("args.push(...leaseRecoveryArgs(selectionOptions))"); + const coordinator = readFileSync("scripts/run-blaxel-testshards.mjs", "utf8"); + for (const profile of ["normal", "cold", "repair"]) { + expect(coordinator).toMatch(new RegExp(`${profile}: \\{ workerTimeoutSeconds: \\d+, lockTtlSeconds: 1200 \\}`)); + } + }); + test("constructs Blaxel restore/wake timing with a typed passed status", () => { const source = readFileSync("scripts/run-blaxel-testshards.mjs", "utf8"); expect(source).toContain('restorePhase = { name: "sandbox-restore-wake", status: "passed"'); @@ -55,16 +66,13 @@ describe("provider result normalization", () => { expect(source).toContain("Delete the pool before ensure so every provisioned worker is fresh."); expect(source.indexOf("const existing = await listPoolSandboxes()")) .toBeLessThan(source.indexOf("SandboxInstance.createIfNotExists")); - expect(source.indexOf("await sandbox.wait()")) - .toBeLessThan(source.indexOf("await provisionSandboxRuntime(sandbox)")); - expect(source.indexOf("await provisionSandboxRuntime(sandbox)")) - .toBeLessThan(source.indexOf("await verifySandboxArchitecture(sandbox)")); + expect(source).toContain("await publishWorker(sandbox"); expect(source).toContain("options.size ?? String(DEFAULT_BLAXEL_POOL_SIZE)"); expect(source).toContain('const memory = Number.parseInt(options.memory ?? "8192", 10)'); expect(source).toContain('const image = options.image ?? "blaxel/playwright-chromium:latest"'); expect(source).toContain('return `tv-testshard-${arch}-${pool}-${String(index).padStart(2, "0")}`'); - for (const label of ['project: "television"', 'purpose: "testshards"', "pool,", "arch,"]) { + for (const label of ['project: "television"', 'purpose: "testshards-provisioning"', "pool,", "arch,"]) { expect(source).toContain(label); } }); @@ -118,9 +126,9 @@ describe("provider result normalization", () => { }); const source = readFileSync(path.join(process.cwd(), "scripts/run-blaxel-testshards.mjs"), "utf8"); - expect(source).toContain("const dispatch = await transferShardInputs({"); + expect(source).toContain("const dispatch = await (lease.pendingWrites = transferShardInputs({"); expect(source).toContain("return dispatchFailureResult({ lease, outputDir, startedAt, logPath, dispatch });"); - expect(source.indexOf("const dispatch = await transferShardInputs({")).toBeLessThan(source.indexOf("sandbox.process.exec({", source.indexOf("async function runShard"))); + expect(source.indexOf("const dispatch = await (lease.pendingWrites = transferShardInputs({")).toBeLessThan(source.indexOf("sandbox.process.exec({", source.indexOf("async function runShard"))); }); test("preserves completed, running, and pending shard evidence before signal cleanup", async () => { @@ -234,7 +242,7 @@ describe("provider result normalization", () => { const source = readFileSync(path.join(process.cwd(), "scripts/run-blaxel-testshards.mjs"), "utf8"); expect(source).toContain("interruptedSnapshot = buildInterruptedReportSnapshot(signal)"); expect(source).toContain("releaseLeases: async () => {"); - expect(source).toContain("finalizeReport: () => writeInterruptedReportSnapshot(interruptedSnapshot)"); + expect(source).toContain("finalizeReport: () => writeInterruptedReportSnapshot(buildInterruptedReportSnapshot(signal) ?? interruptedSnapshot)"); expect(source).toContain("completedResults: activeAttempt.completedResults"); expect(source).toContain("partialResults: activeAttempt.partialResults"); expect(source).toContain("executionRequested: activeAttempt.executionRequested"); @@ -333,30 +341,19 @@ describe("provider result normalization", () => { expect(source).toContain("MAX_PARALLEL_LEASE_ACQUISITIONS = 32"); expect(source).toContain("await acquireShardLeases({"); expect(source).toContain("acquire: async ({ shardIndex, slot }) => {"); - expect(source).toContain("const lease = await acquireAny(sandboxes, { shardIndex, slot, requestedWorkers: shardIndices.length });"); - expect(source).toContain("const lockId = `${runId}-shard-${shardIndex}`"); + expect(source).toContain("const lease = await acquireAny(sandboxes, { shardIndex, slot, requestedWorkers: shardIndices.length, attempt: attempt + 1 });"); + expect(source).toContain("const lockId = `${runId}-s${shardIndex}-a${attempt}-"); expect(source).toContain("activeLeases.set(lockId, lease)"); expect(source).toContain("lease.executionRequested = true"); expect(source).toContain("lease.executionCompletionConfirmed = true"); expect(source).toContain("killProcess: lease.executionRequested === true && lease.executionCompletionConfirmed !== true"); expect(source).not.toContain("acquired.map((lease) => stopAndReleaseLease(lease, { killProcess: false }))"); expect(source).toContain("lease.logPath = logPath"); - const stopIndex = source.indexOf("await lease.sandbox.process.kill(lease.processName)"); - const ownerCleanupIndex = source.indexOf("await reapUnconfirmedShardOwners(lease)", stopIndex); - const releaseIndex = source.indexOf("await releaseLock(lease.sandbox, lease.lockId)", ownerCleanupIndex); - expect(stopIndex).toBeGreaterThan(0); - expect(ownerCleanupIndex).toBeGreaterThan(stopIndex); - expect(releaseIndex).toBeGreaterThan(ownerCleanupIndex); - expect(source).toContain("node scripts/test/stale-owner-reaper-cli.mjs --output"); - const ownerCleanupStart = source.indexOf("async function reapUnconfirmedShardOwners"); - const ownerCleanupEnd = source.indexOf("function buildInterruptedReportSnapshot", ownerCleanupStart); - const ownerCleanupSource = source.slice(ownerCleanupStart, ownerCleanupEnd); - expect(ownerCleanupSource).toContain('"set -eo pipefail"'); - expect(ownerCleanupSource).not.toContain('"set -euo pipefail"'); - expect(ownerCleanupSource).toContain('"nvm use --silent default"'); - expect(ownerCleanupSource).not.toContain(' "nvm use --silent",'); + expect(source).toContain("await finishLease({"); + expect(source).toContain('teardown: () => confirmTeardown(lease)'); + expect(source).not.toContain("release_lock()"); + const ownerCleanupSource = source.slice(source.indexOf("async function leaseOperation"), source.indexOf("function acquireAny")); expect(ownerCleanupSource.indexOf('. "$NVM_DIR/nvm.sh"')).toBeLessThan(ownerCleanupSource.indexOf('"set -u"')); - expect(ownerCleanupSource.indexOf('"set -u"')).toBeLessThan(ownerCleanupSource.indexOf("node scripts/test/stale-owner-reaper-cli.mjs --output")); const nvmFixture = mkdtempSync(path.join(os.tmpdir(), "tv-provider-nvm-nounset-")); const outsideRepository = path.join(nvmFixture, "outside-repository"); @@ -401,10 +398,7 @@ describe("provider result normalization", () => { rmSync(nvmFixture, { recursive: true, force: true }); } - expect(source).toContain("report.cleanupConfirmed === true"); - expect(source).toContain("report.liveOwners.length === 0"); - expect(source).toContain("report.remainingStaleOwners.length === 0"); - expect(source).toContain("Owner cleanup was not confirmed; retaining lease"); + }); test("plans before Blaxel leases and invokes the production explicit-file worker", () => { @@ -431,7 +425,7 @@ describe("provider result normalization", () => { expect(cliSource).toContain('--target-services-json", JSON.stringify(surface.services)'); expect(cliSource).toContain('"--target-surface-id", surface.id'); expect(source).toContain('command: parseTargetCommand(options["target-command-json"])'); - expect(source).toContain('...target.command.map(shellQuote)'); + expect(source).toContain("targetCommand(targetOptions())"); expect(source).toContain('preCommand: parseTargetPreCommand(options["target-pre-command-json"])'); expect(source).toContain('services: parseTargetServices(options["target-services-json"])'); expect(source).toContain('--services-json ${shellQuote(JSON.stringify(targetOptions().services))}'); @@ -441,11 +435,30 @@ describe("provider result normalization", () => { expect(workerSource).toContain("services: surface.services ?? []"); expect(workerSource).toContain("startSurfaceServices(task.services"); }); - test("writes targeted native reports to the supervisor-declared absolute path", () => { - const source = readFileSync(path.join(process.cwd(), "scripts/run-blaxel-testshards.mjs"), "utf8"); - expect(source).toContain('PLAYWRIGHT_JSON_OUTPUT_NAME="$TV_TARGET_NATIVE_RESULT"'); - expect(source).toContain('--outputFile "$TV_TARGET_NATIVE_RESULT"'); - expect(source).not.toContain("nativePathFromCwd"); + test("forwards the target wrapper, selectors, zero budget and absolute reporter path", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "tv-target-wrapper-")); + try { + const cwd = path.join(root, "wrapper-cwd"); + mkdirSync(cwd); + const output = path.join(root, "native.json"); + const wrapper = path.join(root, "wrapper.mjs"); + // A declared downstream wrapper records its invocation. Real native + // collection is covered by test-runner-file-selection.test.ts. + writeFileSync(wrapper, `import fs from "node:fs"; + fs.writeFileSync(process.env.PLAYWRIGHT_JSON_OUTPUT_NAME, JSON.stringify({ args: process.argv.slice(2), cwd: process.cwd(), reporter: process.env.PLAYWRIGHT_JSON_OUTPUT_NAME }));`); + const grep = "title 'quoted'; $(false)"; + const command = targetCommand({ runner: "playwright", cwd, retries: "0", command: [process.execPath, wrapper], files: ["test/file[x].test.ts"], grep }); + const result = spawnSync("bash", ["-c", command], { encoding: "utf8", env: { ...process.env, TV_TARGET_NATIVE_RESULT: output }, timeout: 10_000 }); + expect(result.status, result.stderr).toBe(0); + const recorded = JSON.parse(readFileSync(output, "utf8")); + expect(recorded.cwd).toBe(cwd); + expect(recorded.reporter).toBe(output); + expect(recorded.args.slice(1)).toEqual(["-g", grep, "--retries=0", "--reporter=json"]); + expect(recorded.args[0]).toContain(String.raw`file\[x\]\.test\.ts$`); + const nativeSource = readFileSync("scripts/test/target-native.mjs", "utf8"); + expect(nativeSource).toContain('"--outputFile", native'); + expect(nativeSource).toContain("path.resolve(process.env.TV_TARGET_NATIVE_RESULT)"); + } finally { rmSync(root, { recursive: true, force: true }); } }); test("reports only tasks that actually failed", () => { @@ -908,10 +921,9 @@ describe("provider result normalization", () => { }); test("keeps canonical stale-owner retry authority aligned with sandbox quarantine", () => { - const authority = readFileSync(path.join(process.cwd(), "specs/arch/test-runner/test-runner.md"), "utf8"); - expect(authority).not.toContain("`stale-owner` is the safety exception defined by [[blaxel-testshards.md]] and is not retried."); - expect(authority).toContain("the contaminated sandbox runs no selected test and is excluded from later attempts"); - expect(authority).toContain("a configured retry may run the unchanged shard assignment on a distinct sandbox only after that sandbox acquires its own lease and passes its owner audit"); + const authority = readFileSync(path.join(process.cwd(), "specs/arch/test-runner/blaxel-testshards.md"), "utf8"); + expect(authority).toContain("A worker that finds them runs no tests for that attempt, and the coordinator excludes it from the run's later attempts."); + expect(authority).toContain("An infrastructure retry may use a different worker once that worker has its own lease and a clean audit."); }); test("excludes every named contaminated sandbox from retry candidates", () => { @@ -1019,6 +1031,13 @@ describe("provider result normalization", () => { }; expect(validateDownloadedShardSummary(summary, plan, 1)).toBeNull(); + for (const missing of ["test execution", "final shard summary"]) { + const incomplete = { ...summary, status: "infra-failed", failureKind: "setup", tasks: [], collectedFiles: [], neverProduced: [missing] }; + expect(validateDownloadedShardSummary(incomplete, plan, 1)).toBeNull(); + expect(classifyDownloadedShardStatus({ shardSummary: incomplete, exitCode: 1 })).toBe("infra-failed"); + expect(validateDownloadedShardSummary({ ...incomplete, planId: "other" }, plan, 1)).toContain("planId mismatch"); + expect(validateDownloadedShardSummary({ ...incomplete, status: "passed" }, plan, 1)).toContain("task assignments disagree"); + } expect(validateDownloadedShardSummary({ ...summary, status: "passed" }, plan, 1)).toContain("reports passed with a skipped assigned task"); expect(validateDownloadedShardSummary({ ...summary, diff --git a/test/repo/test-runner-reporting.test.ts b/test/repo/test-runner-reporting.test.ts index f9197911..01b3222e 100644 --- a/test/repo/test-runner-reporting.test.ts +++ b/test/repo/test-runner-reporting.test.ts @@ -42,6 +42,13 @@ describe("test runner reporting", () => { expect(summary.counts.testsFailed).toBe(1); expect(summary.failedTests).toHaveLength(1); expect(JSON.stringify(summary)).not.toContain("passes"); + // specs/arch/test-runner/reporting.md#the-compact-summary pins these fields for readers of earlier runs. + expect(summary.schemaVersion).toBe(1); + expect(summary.run).toMatchObject({ id: "run", status: "failed", infraStatus: "completed", testStatus: "failed" }); + expect(summary.failedTests[0]).toMatchObject({ surfaceId: "unit:probe", file: "test/repo/fail.test.ts", title: expect.stringContaining("fails"), attempts: 1, flakyAnnotated: false }); + expect(summary.failedTests[0]).toHaveProperty("line"); + expect(summary.flakyRecoveredTests).toEqual([]); + expect(summary.processLeaks).toEqual([]); }); test("reports missing and malformed native results as infrastructure evidence failures", () => { @@ -648,6 +655,9 @@ describe("test runner reporting", () => { const { summary, results } = buildReports({ runId: "run", provider: "local", runDir: dir, selection: { surfaces: ["unit:one", "unit:two"] }, surfaces: [base("unit:one", [rawLeak, rawLeak]), base("unit:two", [])], preflights: [], startedAt: "2026-01-01T00:00:00.000Z", completedAt: "2026-01-01T00:00:01.000Z" }); expect(summary.counts.processLeaks).toBe(1); expect(summary.processLeaks).toHaveLength(1); + const [summaryLeak] = summary.processLeaks as Array<{ listeningSockets: unknown }>; + expect(summaryLeak).toMatchObject({ pid: leak.pid, command: leak.command, owningSurfaceIds: ["unit:one", "unit:two"] }); + expect(summaryLeak!.listeningSockets).toEqual(leak.listeningSockets.map(({ host, port }) => expect.objectContaining({ host, port }))); expect(summary.run.status).toBe("incomplete"); expect(summary.failedTests).toEqual([expect.objectContaining({ title: "native failure" })]); expect(summary.flakyRecoveredTests).toEqual([expect.objectContaining({ title: "recovered flake" })]); @@ -662,7 +672,10 @@ function execFileSyncResult(command: string, args: string[], env: Record JSON.parse(readFileSync(file, "utf8")); + +for (const entry of ["planned", "targeted"] as const) { + test(`${entry} parent retains real Linux observations around preparation, recovered native attempts and teardown`, { timeout: 30_000 }, () => { + const fixture = createFixture(entry, "retry"); + try { + const result = spawnSync(process.execPath, fixture.args, { encoding: "utf8", env: fixture.env, timeout: 25_000 }); + expect(result.status, result.stderr).toBe(0); + const prepared = read(path.join(fixture.dir, "preparation.json")); + expect(prepared.records).toHaveLength(1); + expect(prepared.records[0].end.status).toBe("missing"); + const record = readDiagnostics(fixture.dir)[0]; + expect(record.identity).toMatchObject({ runId: "resource-parent-run", attemptId: "resource-parent-attempt", shardIndex: 1, surfaceId: "fixture:resources" }); + expect(Date.parse(record.start.observedAt)).toBeLessThanOrEqual(Date.parse(prepared.observedAt)); + expect(Date.parse(record.end.observedAt)).toBeGreaterThanOrEqual(Date.parse(read(path.join(fixture.dir, "teardown.json")).observedAt)); + expect(record.start.memoryAvailable).toMatchObject({ unit: "bytes", source: "/proc/meminfo:MemAvailable" }); + expect(record.start.sharedMemoryUsed).toMatchObject({ unit: "bytes", source: "/proc/meminfo:Shmem" }); + expect(record.start.temporaryFree).toMatchObject({ unit: "bytes", source: "statfs", path: fixture.dir }); + expect(record.start.oomKills).toMatchObject({ unit: "count", source: "/proc/vmstat:oom_kill" }); + for (const key of ["memoryAvailable", "sharedMemoryUsed", "temporaryFree", "oomKills"]) { + const probe = record.start[key]; + if (probe.value === null) expect(probe.reason).toEqual(expect.any(String)); + else expect(probe.value).toBeGreaterThanOrEqual(0); + } + expect(record.nativeEvidence).toMatchObject({ status: "complete", failedAttempt: true, recovered: true }); + expect(["available", "unavailable"]).toContain(record.kernel.status); + if (record.kernel.status === "unavailable") { + // Expected host limitations stay unknown; malformed reader invocations + // must not masquerade as a successful probe of this installed tool. + expect(["EPERM", "EACCES", "ENOENT", "ENOTSUP", "ETIMEDOUT", "ENOBUFS"]).toContain(record.kernel.reason); + } + const resultRecord = entry === "planned" ? read(path.join(fixture.dir, "results/shard-1.json")).tasks[0] : read(path.join(fixture.dir, "results/lifecycle.json")); + expect(resultRecord.resourceDiagnosticsPaths).toHaveLength(1); + expect(existsSync(resultRecord.resourceDiagnosticsPaths[0])).toBe(true); + expect(resultRecord.processLeaks).toEqual([]); + const attemptPath = path.resolve(path.dirname(resultRecord.resourceDiagnosticsPaths[0]), record.nativeEvidence.attemptResultPath); + const attempts = readFileSync(attemptPath, "utf8").trim().split("\n").map((line) => JSON.parse(line)); + expect(attempts.map((row: any) => row.status)).toEqual(["failed", "passed"]); + } finally { fixture.cleanup(); } + }); +} + +test("targeted parent preserves a baseline after the installed native runner exits before its report", { timeout: 20_000 }, () => { + const fixture = createFixture("targeted", "exit"); + try { + const result = spawnSync(process.execPath, fixture.args, { encoding: "utf8", env: fixture.env, timeout: 15_000 }); + expect(result.status, result.stderr).toBe(17); + expect(read(path.join(fixture.dir, "preparation.json")).records).toHaveLength(1); + const record = readDiagnostics(fixture.dir)[0]; + expect(record).toMatchObject({ end: { status: "observed" }, nativeEvidence: { status: "missing" } }); + expect(existsSync(path.join(fixture.dir, "results/native.json"))).toBe(false); + } finally { fixture.cleanup(); } +}); + +test("interrupted surface parent leaves an explicit missing final observation", { timeout: 20_000 }, async () => { + const fixture = createFixture("targeted", "interrupt"); + const child = spawnOwnedProcess(process.execPath, fixture.args, { env: fixture.env, stdio: "ignore" }); + try { + await expect.poll(() => existsSync(path.join(fixture.dir, "preparation.json")), { timeout: 10_000, interval: 20 }).toBe(true); + await expect.poll(() => child.child.signalCode, { timeout: 5_000, interval: 20 }).toBe("SIGKILL"); + expect((await child.dispose()).outcome).not.toBe("survived"); + const records = readDiagnostics(fixture.dir); + expect(records).toHaveLength(1); + expect(records[0]).toMatchObject({ start: { status: "observed" }, end: { status: "missing", reason: "parent-did-not-finalize" } }); + } finally { await child.dispose(); fixture.cleanup(); } +}); + +function readDiagnostics(dir: string) { + const root = path.join(dir, "results/resources"); + return existsSync(root) ? readdirSync(root).filter((name) => name.endsWith(".json")).map((name) => read(path.join(root, name))) : []; +} + +function createFixture(entry: "planned" | "targeted", mode: string) { + mkdirSync(".test-runs", { recursive: true }); + const dir = mkdtempSync(path.join(process.cwd(), ".test-runs/resource-parent-")); + const results = path.join(dir, "results"); + mkdirSync(results); + const nativeConfig = path.join(dir, "native.config.mjs"); + writeFileSync(nativeConfig, `export default ${JSON.stringify({ test: { include: [path.resolve(sourceFile)], globalSetup: [path.resolve(`${fixtureRoot}/setup.mjs`)], pool: "forks", maxWorkers: 1, retry: 2 } })};\n`); + const preCommand = [process.execPath, path.resolve(`${fixtureRoot}/prepare.mjs`)]; + const surface = { id: "fixture:resources", runner: "vitest", kind: "e2e", package: null, config: nativeConfig, absoluteConfig: nativeConfig, roots: [sourceFile], excludeRoots: [], cwd: ".", supports: ["file"], preflight: [], tags: [], agent: false, preCommand, command: null, executionGroup: { id: "fixture:resources", name: "resource fixture", order: 1 } }; + const registryPath = path.join(dir, "registry.mjs"); + writeFileSync(registryPath, `export default ${JSON.stringify({ suites: { all: { include: ["kind:e2e"] } }, surfaces: [surface] })};\n`); + const plan = createShardPlan({ inventory: [{ path: sourceFile, surfaceId: surface.id, runner: "vitest" }], surfaces: [surface] as any, baseline: { schemaVersion: 1, sourceThrough: null, files: {} }, baselineBytes: serializeTimingBaseline({ schemaVersion: 1, sourceThrough: null, files: {} }), timingProvider: provider, testedCommit: execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(), testedTree: execFileSync("git", ["rev-parse", "HEAD^{tree}"], { encoding: "utf8" }).trim(), shardTotal: 1 }); + const planPath = path.join(dir, "plan.json"); + writeFileSync(planPath, JSON.stringify(plan)); + const env = { ...process.env, GITHUB_ACTIONS: "false", TV_TEST_RUNNER_SELFTEST: "1", TEST_SHARD_CONFIG_MODULE: registryPath, TV_TEST_TIMING_PROVIDER: provider, TV_TEST_RUN_ID: "resource-parent-run", TV_TEST_SHARD_INDEX: "1", TV_TEST_SHARD_ATTEMPT: "resource-parent-attempt", RESOURCE_FIXTURE_ROOT: dir, RESOURCE_FIXTURE_MODE: mode, TMPDIR: dir, TV_TARGET_RUNNER: "vitest", TV_TARGET_NATIVE_RESULT: path.join(results, "native.json"), TV_VITEST_ATTEMPT_FILE: path.join(results, "native.attempts.ndjson") }; + const args = entry === "planned" ? ["scripts/run-test-shard.mjs", "--plan", planPath, "--shard", "1", "--results-dir", results, "--test-retries", "2"] : ["scripts/test/supervised-command.mjs", "--run-id", "resource-parent-run", "--surface", surface.id, "--result", path.join(results, "lifecycle.json"), "--pre-command-json", JSON.stringify(preCommand), "--", "bash", "-c", targetCommand({ runner: "vitest", cwd: ".", config: nativeConfig, file: sourceFile, files: [sourceFile], retries: "2", command: null })]; + return { dir, args, env, cleanup: () => rmSync(dir, { recursive: true, force: true }) }; +} diff --git a/test/repo/test-runner-shard-worker.test.ts b/test/repo/test-runner-shard-worker.test.ts index 93e948e5..c4cfe5c7 100644 --- a/test/repo/test-runner-shard-worker.test.ts +++ b/test/repo/test-runner-shard-worker.test.ts @@ -37,9 +37,9 @@ describe("planned shard worker", () => { const fixture = workerFixture(["fixture:vitest-one", "fixture:vitest-two"], [ inventory("vitest-one.fixture.ts", "fixture:vitest-one", "vitest"), inventory("vitest-two.fixture.ts", "fixture:vitest-two", "vitest"), - ]); + ], "blaxel-playwright-x64-4vcpu"); try { - const result = runWorker(fixture, { TV_SHARD_RETRY_FIXTURE: "1" }); + const result = runWorker(fixture, { TV_SHARD_RETRY_FIXTURE: "1", TV_TEST_RUN_ID: "combined-resource-fixture", TV_TEST_SHARD_ATTEMPT: "combined-attempt", TV_TEST_SHARD_INDEX: "1" }); expect(result.status, result.stderr).toBe(0); const summary = shardSummary(fixture); expect(summary.preflights).toEqual([ @@ -73,6 +73,17 @@ describe("planned shard worker", () => { }); expect(normalized.find((surface) => surface.id === "fixture:vitest-one")?.counts.testsFlakyRecovered).toBe(0); expect(normalized.find((surface) => surface.id === "fixture:vitest-two")?.counts.testsFlakyRecovered).toBe(1); + for (const [index, task] of summary.tasks.entries()) { + expect(task.resourceDiagnosticsPaths).toHaveLength(1); + const diagnosticPath = task.resourceDiagnosticsPaths[0]; + const diagnostic = JSON.parse(readFileSync(diagnosticPath, "utf8")); + expect(diagnostic).toMatchObject({ + identity: { runId: "combined-resource-fixture", attemptId: "combined-attempt", shardIndex: 1, surfaceId: task.surfaceId }, + start: { status: "observed" }, end: { status: "observed" }, + nativeEvidence: { status: "complete", failedAttempt: index === 1, recovered: index === 1 }, + }); + expect(path.resolve(path.dirname(diagnosticPath), diagnostic.nativeEvidence.resultPath)).toBe(task.nativeResultPath); + } } finally { fixture.cleanup(); } }); diff --git a/test/repo/test-runner-verify-mutex.test.ts b/test/repo/test-runner-verify-mutex.test.ts new file mode 100644 index 00000000..8b0d2fa1 --- /dev/null +++ b/test/repo/test-runner-verify-mutex.test.ts @@ -0,0 +1,84 @@ +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { ready, waitFor } from "./fixtures/test-runner/mutex-checkout.ts"; +import { verifyMutexCheckout } from "./fixtures/test-runner/verify-mutex-checkout.ts"; + +const broad = "--against-test-guidance-broad-local-run"; +const bypass = "--against-test-guidance-major-host-contention-and-oom-killed-processes"; +const wait = "--wait-for-local-test-mutex"; + +describe("local verify mutex", () => { + // proofs/arch/test-runner/test-runner.md#^t-runner-verify-lock + test("protects phases and admits only the direct test child on a marked home", async () => { + const f = verifyMutexCheckout(); + try { + writeFileSync(path.join(f.home, ".tvdev-use-blaxel"), ""); + const verify = f.start(["verify", "local", broad]); await ready(verify); + expect(f.run(f.args).status).toBe(2); + writeFileSync(verify.release, "done"); + expect(await verify.exited, verify.output()).toBe(0); + expect(verify.output()).toContain("verify passed (local)"); + for (const phase of ["lint", "type-check", "lint:package-manifests", "tests", "vibe-mode"]) { + const result = JSON.parse(readFileSync(path.join(f.root, `phase-${phase}.json`), "utf8")); + expect(result.status, `${phase}: ${result.output}`).toBe(2); + expect(result.output).toContain("Local-test mutex held"); + } + const failedPreflight = f.run(f.args, { TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT: "fixture:released verify" }); + expect(failedPreflight.status, failedPreflight.stderr).toBe(1); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-wait + test("waits before any verify phase, then shares ownership with its child", async () => { + const f = verifyMutexCheckout(); + try { + // This command holds only the small authored native fixture. + const holder = f.start(); await ready(holder); + const verify = f.start(["verify", "local", wait]); + await waitFor(() => verify.output().includes("Waiting for local-test mutex"), verify.output); + expect(verify.output()).not.toContain("[verify:preflight]"); + writeFileSync(holder.release, "done"); expect(await holder.exited, holder.output()).toBe(0); + await ready(verify); writeFileSync(verify.release, "done"); + expect(await verify.exited, verify.output()).toBe(0); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-wait-cancel + test.each([["SIGINT",130],["SIGTERM",143]] as const)("cancels a verify wait with %s before phases", async(signal,status)=>{ + const f=verifyMutexCheckout(); + try { + const holder=f.start(); await ready(holder); + const verify=f.start(["verify","local",wait]); + await waitFor(()=>verify.output().includes("Waiting for local-test mutex"),verify.output); + verify.child.kill(signal); expect(await verify.exited).toBe(status); + expect(verify.output()).not.toContain("[verify:preflight]"); + expect(f.run(f.args).status).toBe(2); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-verify-lock + test("forwards deliberate contention and breadth options to the real test child", async()=>{ + const f=verifyMutexCheckout(); + try { + writeFileSync(path.join(f.home,".tvdev-use-blaxel"),""); + const holder=f.start(); await ready(holder); + const verify=f.start(["verify","local",broad,bypass]); await ready(verify); + writeFileSync(verify.release,"done"); expect(await verify.exited,verify.output()).toBe(0); + expect(verify.output()).toContain(bypass); + expect(f.run(f.args).status).toBe(2); + } finally { await f.cleanup(); } + }); + + // proofs/arch/test-runner/test-runner.md#^t-runner-lock-release + test("holds during a non-test phase and releases after that phase fails", async()=>{ + const f=verifyMutexCheckout(); + try { + const verify=f.start(["verify","local"],{TV_MUTEX_HOLD_PHASE:"lint",TV_MUTEX_FAIL_PHASE:"lint"}); await ready(verify); + expect(f.run(f.args).status).toBe(2); + expect(existsSync(path.join(f.root,"phase-tests.json"))).toBe(false); + writeFileSync(verify.release,"done"); expect(await verify.exited,verify.output()).toBe(1); + expect(f.run(f.args,{TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT:"fixture:released"}).status).toBe(1); + } finally { await f.cleanup(); } + }); +}); diff --git a/test/repo/vitest.build-config.config.ts b/test/repo/vitest.build-config.config.ts index b7eed3de..55ec79b1 100644 --- a/test/repo/vitest.build-config.config.ts +++ b/test/repo/vitest.build-config.config.ts @@ -1,3 +1,4 @@ +import "../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ diff --git a/test/repo/worker-builds.test.ts b/test/repo/worker-builds.test.ts new file mode 100644 index 00000000..9630b5cb --- /dev/null +++ b/test/repo/worker-builds.test.ts @@ -0,0 +1,79 @@ +import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, test } from "vitest"; +import { WorkerBuilds } from "../helpers/worker-builds.ts"; + +const roots: string[] = []; +function rendezvous() { + let resolve!: (value: T) => void; + const promise = new Promise((done) => { resolve = done; }); + return { promise, resolve }; +} +function fixture(): { root: string; builds: WorkerBuilds } { + const root = mkdtempSync(path.join(os.tmpdir(), "worker-build-contract-")); + roots.push(root); + return { root, builds: new WorkerBuilds("output-", root) }; +} +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +// proofs/arch/testing-policy.md#^testing-t-build-lifetime +test("concurrent and completed requests reuse a build while version keys remain distinct", async () => { + const { root, builds } = fixture(); + const ready = rendezvous(); + let calls = 0; + const first = builds.get("v1", async (directory) => { + calls += 1; + writeFileSync(path.join(directory, "entry.js"), "version one"); + await ready.promise; + }); + const concurrent = builds.get("v1", () => { throw new Error("prepared twice"); }); + expect(concurrent).toBe(first); + ready.resolve(); + const directory = await first; + expect(await builds.get("v1", () => { throw new Error("prepared again"); })).toBe(directory); + const second = await builds.get("v2", (output) => { writeFileSync(path.join(output, "entry.js"), "version two"); }); + expect(second).not.toBe(directory); + expect(calls).toBe(1); + expect(existsSync(path.join(directory, "entry.js"))).toBe(true); + await builds.dispose(); + expect(readdirSync(root)).toEqual([]); +}); + +test("failed preparation removes partial output and permits a fresh build", async () => { + const { root, builds } = fixture(); + let failedDirectory = ""; + await expect(builds.get("source", (directory) => { + failedDirectory = directory; + writeFileSync(path.join(directory, "partial.js"), "incomplete"); + throw new Error("asset preparation failed"); + })).rejects.toThrow("asset preparation failed"); + expect(readdirSync(root)).toEqual([]); + const directory = await builds.get("source", (output) => { writeFileSync(path.join(output, "entry.js"), "complete"); }); + expect(directory).not.toBe(failedDirectory); + expect(existsSync(path.join(directory, "entry.js"))).toBe(true); + await builds.dispose(); +}); + +test.each([false, true])("teardown waits for pending preparation, including rejection (%s)", async (reject) => { + const { root, builds } = fixture(); + const started = rendezvous(); + const ready = rendezvous(); + const build = builds.get("pending", async (directory) => { + writeFileSync(path.join(directory, "partial.js"), "pending"); + started.resolve(directory); + await ready.promise; + if (reject) throw new Error("late preparation failure"); + }); + const settled = Promise.allSettled([build]); + const directory = await started.promise; + let disposed = false; + const cleanup = builds.dispose().then(() => { disposed = true; }); + await Promise.resolve(); + expect(disposed).toBe(false); + expect(existsSync(directory)).toBe(true); + ready.resolve(); + expect((await settled)[0]!.status).toBe(reject ? "rejected" : "fulfilled"); + await cleanup; + expect(readdirSync(root)).toEqual([]); +}); diff --git a/test/runner-fixtures/blaxel-harness/README.md b/test/runner-fixtures/blaxel-harness/README.md new file mode 100644 index 00000000..46c152e4 --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/README.md @@ -0,0 +1,34 @@ +# Live Blaxel harness qualification + +Run on a dedicated credentialed Blaxel coordinator sandbox outside every worker +pool, from a committed and pushed checkout with the repository's selected Node, +dependencies and local process-lifecycle prerequisites installed. This driver must +not run on a shared developer host or a leased worker. + +Supply the coordinator's sandbox name and an exclusively owned disposable pool: + +```sh +TV_BLAXEL_HARNESS_COORDINATOR= \ +TV_BLAXEL_HARNESS_POOL= \ +npm test -- local --file test/runner-fixtures/blaxel-harness/driver/harness.test.ts +``` + +The driver deletes that pool before provisioning two fresh workers and deletes it +again after qualification. Retain any earlier reports first. Never name a shared +pool. The coordinator remains allocated for the caller to retrieve evidence and +dispose of it. Native retry defaults apply. + +The CLI requires both names, local execution and no shard owner token. The driver +also checks that the named coordinator lacks the worker `purpose=testshards` label. +These checks prevent accidental placement; the operator supplies the actual +dedicated coordinator's name. + +Cleanup fixtures and sentinels include the selected pool in their names. Each +fresh worker has one fixture-mode input, `/tmp/tv-blaxel-harness-mode.json`, read +by the committed native fixtures. It needs no additional production environment +forwarding and disappears with the private worker. + +Inspect the canonical run directory and `.test-runs/blaxel-live-*/` for the +individual commands, child run pointers, recovery evidence, completion and cleanup +records. Expected native fixture failures are checked by the driver; a passing +driver does not turn those child reports into passing suites. diff --git a/test/runner-fixtures/blaxel-harness/driver/harness.test.ts b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts new file mode 100644 index 00000000..6acff495 --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/driver/harness.test.ts @@ -0,0 +1,227 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { SandboxInstance } from "@blaxel/core"; +import { expect, test } from "vitest"; +import { listBlaxelPoolSandboxes } from "../../../../scripts/test/blaxel-pool.mjs"; + +// Acceptance: real canonical subprocesses, SDK, fresh private workers and native +// fixtures. The sole scheduling hook pauses immediately before report download. +const pool = process.env.TV_BLAXEL_HARNESS_POOL?.trim() ?? ""; +const coordinatorName = process.env.TV_BLAXEL_HARNESS_COORDINATOR?.trim() ?? ""; +const planned = ["--tag", "blaxel-harness-fixture", "--shards", "2", "--allow-shard-count-override", "1"]; +const targeted = ["--surface", "experiment:blaxel-harness-fixtures"]; +const resultsRoot = "/workspace/television/.testshards/results"; +// Each fresh worker belongs to one private pool. This fixed fixture input needs +// no additional environment forwarding through the production coordinator. +const modePath = "/tmp/tv-blaxel-harness-mode.json"; +const fixtureSuffix = `harness-${encodeURIComponent(pool)}`; +const sourceResidue = `/tmp/television-source-cli-e2e-${fixtureSuffix}`; +const webResidue = `/tmp/television-web-bundle-${fixtureSuffix}`; +const staleScratch = `/tmp/tv-testshards/${fixtureSuffix}`; +const unrelatedSentinel = `/tmp/tv-${fixtureSuffix}-unrelated/keep`; +const cacheSentinel = `/cache/tv-${fixtureSuffix}/keep`; +const repositoryRoot = path.resolve(import.meta.dirname, "../../../.."); +const cli = path.join(repositoryRoot, "scripts/test/cli.mjs"); +const children = new Set(); +const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`; +const pause = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); +async function until(probe: () => Promise, timeout = 120_000): Promise { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { const result = await probe(); if (result) return result; await pause(250); } + throw Error("live harness rendezvous timed out"); +} +function readJson(file: string) { return JSON.parse(readFileSync(file, "utf8")); } +async function remoteJson(worker: SandboxInstance, file: string) { try { return JSON.parse(await worker.fs.read(file)); } catch (error: any) { if (error?.status === 404) return null; throw error; } } + +function command(root: string, name: string, args: string[], env: NodeJS.ProcessEnv = {}) { + const log = path.join(root, `${name}.log`); + const child = spawn(process.execPath, [cli, ...args], { cwd: repositoryRoot, detached: true, env: {...process.env, ...env}, stdio: ["ignore", "pipe", "pipe"] }); + children.add(child); + const chunks: Buffer[] = []; + child.stdout!.on("data", data => chunks.push(data)); child.stderr!.on("data", data => chunks.push(data)); + const done = new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", code => { children.delete(child); writeFileSync(log, Buffer.concat(chunks)); resolve(code); }); + }); + return {child, done, log}; +} +function run(root: string, name: string, selection: string[], gate?: string, recoverySeconds?: number) { + const pointer = path.join(root, `${name}.run`); + const task = command(root, name, ["blaxel", "--pool", pool, ...selection, ...(recoverySeconds ? ["--lock-ttl-seconds", String(recoverySeconds)] : []), "--run-dir-output", pointer], gate ? {TV_TEST_RUNNER_SELFTEST: "1", TV_BLAXEL_DOWNLOAD_GATE: gate} : {}); + return {...task, directory: async () => until(async () => existsSync(pointer) ? readFileSync(pointer, "utf8").trim() : null)}; +} +async function poolCommand(root: string, name: string, operation: string) { + const code = await command(root, name, ["pool", "blaxel", operation, "--pool", pool, ...(operation === "ensure" ? ["--size", "2"] : [])]).done; + // The API accepts deletion before its listing stops returning the worker. + // Observe absence before fresh provisioning or claiming the pool is gone. + if (operation === "delete" && code === 0) await until(async () => { + if ((await listBlaxelPoolSandboxes({pool, arch: "x64"})).length === 0) return true; + await pause(1000); return null; + }); + return code; +} +async function gates(gate: string) { + // Cold worker execution is allowed 480 seconds, before teardown and retrieval. + return until(async () => [1,2].every(index => existsSync(path.join(gate, `shard-${index}.json`))) ? [1,2].map(index => readJson(path.join(gate, `shard-${index}.json`))) : null, 600_000); +} +function retained(runDir: string) { + const summary = readJson(path.join(runDir, "summary.json")); + const provider = readJson(path.join(runDir, "provider/blaxel/report.json")); + const normalized = readJson(path.join(runDir, "results.json")); + return { summary, provider, normalized }; +} +function assertDiagnostics(runDir: string, expectedRecovery = false) { + const {provider, normalized} = retained(runDir); + const sidecars = [...new Set(normalized.surfaces.flatMap((surface: any) => surface.resourceDiagnosticsPaths ?? []))]; + expect(sidecars.length).toBeGreaterThan(0); + const records = sidecars.map(file => readJson(path.resolve(runDir, file))); + for (const [index, record] of records.entries()) { + expect(provider.shards.some((shard: any) => shard.shardIndex === record.identity.shardIndex)).toBe(true); + expect(record.identity.runId).toBe(provider.runId); + expect(record.identity.attemptId).toContain(provider.runId); + if (record.kind === "surface") { + expect(record.identity.surfaceId).toBe("experiment:blaxel-harness-fixtures"); + expect(record.start.status).toBe("observed"); expect(record.end.status).toBe("observed"); + expect(record.nativeEvidence.status).toBe("complete"); + for (const reference of [record.nativeEvidence.resultPath, record.nativeEvidence.attemptResultPath].filter(Boolean)) { + expect(existsSync(path.resolve(runDir, path.dirname(sidecars[index]), reference))).toBe(true); + } + if (record.nativeEvidence.failedAttempt) expect(["available", "unavailable"]).toContain(record.kernel?.status); + if (record.kernel?.status === "unavailable") expect(record.kernel.reason).toBeTruthy(); + } + } + expect(records.some(record => record.nativeEvidence?.recovered === true)).toBe(true); + if (expectedRecovery) { + const receipts = records.filter(record => record.kind === "recovery"); + expect(receipts.length).toBe(2); + for (const receipt of receipts) { expect(receipt.discardedReports).toBe(true); expect(receipt.cleanup).toBe("confirmed"); expect(receipt.previous.runId).not.toBe(provider.runId); } + } + return records; +} + +test("qualifies exclusive retrieval, active and quiet recovery, temporary placement, and pool lifecycle", async () => { + if (!coordinatorName || !pool || process.env.TV_TEST_SHARD_OWNER) throw Error("Set TV_BLAXEL_HARNESS_COORDINATOR and TV_BLAXEL_HARNESS_POOL on the dedicated Blaxel coordinator, outside every leased worker"); + const coordinator = await SandboxInstance.get(coordinatorName); + expect(coordinator.metadata.labels?.purpose).not.toBe("testshards"); + const root = mkdtempSync(path.join(repositoryRoot, ".test-runs/blaxel-live-")); + writeFileSync(path.join(root, "identity.json"), JSON.stringify({coordinatorName, pool, revision: readFileSync(path.join(repositoryRoot, ".git/HEAD"), "utf8").trim()})); + let workers: SandboxInstance[] = []; + try { + // The caller retained earlier development reports before this fresh-pool run. + expect(await poolCommand(root, "initial-delete", "delete")).toBe(0); + expect(await poolCommand(root, "ensure", "ensure")).toBe(0); + workers = await listBlaxelPoolSandboxes({pool, arch: "x64", deployedOnly: true}); + expect(workers).toHaveLength(2); + for (const worker of workers) { + const directories = [sourceResidue, webResidue, staleScratch, path.dirname(unrelatedSentinel), path.dirname(cacheSentinel)]; + const setup = [`mkdir -p ${directories.map(quote).join(" ")}`, + ...[sourceResidue, webResidue].map(directory => `printf fixture > ${quote(`${directory}/data`)}`), + ...[unrelatedSentinel, cacheSentinel].map(file => `printf keep > ${quote(file)}`)].join("; "); + const seeded = await worker.process.exec({name: "tv-harness-seed-fixtures", command: `bash -c ${quote(setup)}`, waitForCompletion: true, timeout: 10}); + expect(seeded.exitCode).toBe(0); + await worker.fs.write(modePath, JSON.stringify({mode: "normal"})); + } + const gate = path.join(root, "normal-download"); mkdirSync(gate); + const first = run(root, "planned", planned, gate); + const waiting = await gates(gate); + const before = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); + const reportsBefore = await Promise.all(workers.map((worker, index) => worker.fs.read(`${resultsRoot}/shard-${before[index].shardIndex}.json`))); + await until(async () => { + const after = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); + return after.every((record, index) => record.lastRenewal > before[index].lastRenewal) ? after : null; + }); + const competed = run(root, "competing", planned); + expect(await competed.done).not.toBe(0); + expect(retained(await competed.directory()).summary.run.infraStatus).toBe("incomplete"); + expect(await poolCommand(root, "held-delete", "delete")).toBe(1); + for (const worker of workers) expect((await remoteJson(worker, "/tmp/tv-testshard.lock/owner.json")).runId).toBe(before[0].runId); + const reportsAfter = await Promise.all(workers.map((worker, index) => worker.fs.read(`${resultsRoot}/shard-${before[index].shardIndex}.json`))); + expect(reportsAfter).toEqual(reportsBefore); + writeFileSync(path.join(gate, "continue"), "continue"); + expect(await first.done).toBe(1); // intentional native fixture failure + const firstDir = await first.directory(); const firstResult = retained(firstDir); + expect(firstResult.summary.run.infraStatus).toBe("completed"); + expect(firstResult.summary.counts.testsFailed).toBe(1); expect(firstResult.summary.counts.testsFlakyRecovered).toBe(1); + assertDiagnostics(firstDir); + const placements: any[] = []; + for (const shard of firstResult.provider.shards) { + expect(shard.remoteScript).toBe(`${shard.scratch}/worker.sh`); + expect(shard.setupTimings.some((phase: any) => phase.name === "temporary-cleanup")).toBe(true); + const artifact = path.join(shard.outputDir, "artifacts", `shard-${shard.shardIndex}`); + const cleanup = readJson(path.join(artifact, "temporary-cleanup.json")); + expect(cleanup.removed["television-source-cli-e2e-"].directories).toBe(1); + expect(cleanup.removed["television-web-bundle-"].directories).toBe(1); + expect(cleanup.removed["stale-scratch"].directories).toBe(1); + const placement = path.join(artifact, "temporary-placement.json"); + if (existsSync(placement)) placements.push(readJson(placement)); + } + expect(placements).toHaveLength(1); + expect(placements[0].browserVersion).toMatch(/^\d+\./); + expect(placements[0].executable).toContain("/home/playwright/.cache/ms-playwright/"); + expect(placements[0].nodeDir.startsWith(placements[0].tmp + "/")).toBe(true); + expect(placements[0].descriptors.some((entry: any) => entry.path.endsWith(" (deleted)"))).toBe(true); + for (const worker of workers) { + const ownGate = waiting.find(entry => entry.sandbox === worker.metadata.name)!; + expect(await remoteJson(worker, "/tmp/tv-testshard.lock/owner.json")).toBeNull(); + await expect(worker.fs.ls(ownGate.scratch)).rejects.toMatchObject({status: 404}); + expect(await worker.fs.read(unrelatedSentinel)).toBe("keep"); + expect(await worker.fs.read(cacheSentinel)).toBe("keep"); + } + const target = run(root, "targeted", targeted); + expect(await target.done).toBe(1); + expect(retained(await target.directory()).summary.run.infraStatus).toBe("completed"); + assertDiagnostics(await target.directory()); + const targetShard = retained(await target.directory()).provider.shards[0]; + expect(existsSync(path.join(targetShard.outputDir, "artifacts/shard-1/temporary-placement.json"))).toBe(true); + + for (const scenario of ["active", "quiet"]) { + for (const worker of workers) await worker.fs.write(modePath, JSON.stringify({mode: scenario === "active" ? "active" : "normal"})); + const crashGate = path.join(root, `${scenario}-download`); mkdirSync(crashGate); + const abandoned = run(root, `${scenario}-abandoned`, planned, crashGate, 60); + let owners: any[]; + if (scenario === "quiet") { await gates(crashGate); owners = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); } + else owners = await until(async () => { + const active = await Promise.all(workers.map(worker => remoteJson(worker, `${resultsRoot}/active-fixture.json`))); + return active.every(Boolean) ? Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))) : null; + }); + const coordinatorPid = Number(owners[0].owner.split(":").at(-1)); + expect(coordinatorPid).toBeGreaterThan(1); expect(coordinatorPid).not.toBe(process.pid); + process.kill(coordinatorPid, "SIGKILL"); // actual renewer, not just its shard + expect(await abandoned.done).not.toBe(0); + for (const worker of workers) await worker.fs.write(modePath, JSON.stringify({mode: "normal"})); + owners = await until(async () => { + const current = await Promise.all(workers.map(worker => remoteJson(worker, "/tmp/tv-testshard.lock/owner.json"))); + return current.every((record, index) => record?.lockId === owners[index].lockId && Date.now() > record.recoverAfter + 500) ? current : null; + }, 120_000); + const recovered = run(root, `${scenario}-recovered`, planned); + expect(await recovered.done).toBe(1); + const directory = await recovered.directory(); + expect(retained(directory).summary.run.infraStatus).toBe("completed"); + const records = assertDiagnostics(directory, true); + for (const receipt of records.filter(record => record.kind === "recovery")) { + const previous = owners.find(owner => owner.lockId === receipt.previous.lockId); + expect(previous).toBeTruthy(); + expect(receipt.previous.lastRenewal).toBe(previous.lastRenewal); + expect(receipt.previous.recoverAfter).toBe(previous.recoverAfter); + } + if (scenario === "active") expect(records.filter(record => record.kind === "recovery").every(record => record.processes.length > 0)).toBe(true); + writeFileSync(path.join(root, `${scenario}-recovery-evidence.json`), JSON.stringify({owners, recoveredRun: directory})); + } + expect(await poolCommand(root, "final-delete", "delete")).toBe(0); + expect(await listBlaxelPoolSandboxes({pool, arch: "x64"})).toHaveLength(0); + const absent = run(root, "absent-pool", targeted); expect(await absent.done).not.toBe(0); + writeFileSync(path.join(root, "completed.json"), JSON.stringify({status: "passed", runs: readdirSync(root).filter(file => file.endsWith(".run"))})); + } finally { + const pending = [...children]; + const stopped = Promise.all(pending.map(child => new Promise(resolve => child.once("close", () => resolve())))); + for (const child of pending) { try { process.kill(-child.pid!, "SIGTERM"); } catch {} } + await Promise.race([stopped, pause(10_000)]); + for (const child of children) { try { process.kill(-child.pid!, "SIGKILL"); } catch {} } + // A valid lease is never bypassed. With our coordinators stopped, the same + // manager may delete an expired private worker, which is itself cleanup. + let cleanup = await poolCommand(root, "finally-delete", "delete"); + if (cleanup !== 0) { await pause(65_000); cleanup = await poolCommand(root, "finally-delete-expired", "delete"); } + writeFileSync(path.join(root, "cleanup.json"), JSON.stringify({exitCode: cleanup})); + } +}); diff --git a/test/runner-fixtures/blaxel-harness/driver/vitest.config.ts b/test/runner-fixtures/blaxel-harness/driver/vitest.config.ts new file mode 100644 index 00000000..7b116a66 --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/driver/vitest.config.ts @@ -0,0 +1,3 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; +import { defineConfig } from "vitest/config"; +export default defineConfig({ root: import.meta.dirname, test: { include: ["*.test.ts"], maxWorkers: 1, fileParallelism: false, testTimeout: 1_200_000, hookTimeout: 120_000 } }); diff --git a/test/runner-fixtures/blaxel-harness/fixtures/outcomes.test.ts b/test/runner-fixtures/blaxel-harness/fixtures/outcomes.test.ts new file mode 100644 index 00000000..39b8894d --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/fixtures/outcomes.test.ts @@ -0,0 +1,7 @@ +import { expect, test } from "vitest"; +import { activeFixture, mode } from "./support.ts"; + +test("bounded active owned process", activeFixture); +let attempt = 0; +test("recovers a deliberate first attempt", () => { if (mode() === "normal") expect(++attempt).toBeGreaterThan(1); }); +test("retains a deliberate native failure", () => { if (mode() === "normal") expect("fixture failure").toBe("fixture success"); }); diff --git a/test/runner-fixtures/blaxel-harness/fixtures/support.ts b/test/runner-fixtures/blaxel-harness/fixtures/support.ts new file mode 100644 index 00000000..b1786840 --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/fixtures/support.ts @@ -0,0 +1,20 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { spawnOwnedProcess } from "../../../helpers/owned-process.ts"; + +export const results = "/workspace/television/.testshards/results"; +export function mode() { + const file = "/tmp/tv-blaxel-harness-mode.json"; + return existsSync(file) ? JSON.parse(readFileSync(file, "utf8")).mode : "normal"; +} +export function evidence(name: string, value: unknown) { + mkdirSync(results, { recursive: true }); + writeFileSync(path.join(results, name), JSON.stringify(value)); +} +export async function activeFixture() { + if (mode() !== "active") return; + const child = spawnOwnedProcess(process.execPath, ["-e", "process.on('SIGTERM',()=>{});setTimeout(()=>process.exit(0),180000);setInterval(()=>{},1000)"], { stdio: "ignore" }); + evidence("active-fixture.json", { pid: child.pid, supervisor: process.pid, token: process.env.TV_TEST_SHARD_OWNER }); + try { await new Promise(resolve => setTimeout(resolve, 180_000)); } + finally { await child.dispose(); } +} diff --git a/test/runner-fixtures/blaxel-harness/fixtures/temporary.test.ts b/test/runner-fixtures/blaxel-harness/fixtures/temporary.test.ts new file mode 100644 index 00000000..78edc8ea --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/fixtures/temporary.test.ts @@ -0,0 +1,33 @@ +import { expect, test } from "vitest"; +import { mkdtempSync, readdirSync, readFileSync, readlinkSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { chromium } from "playwright"; +import { activeFixture, evidence } from "./support.ts"; + +test("bounded active owned process", activeFixture); +test("Node and Chromium place real temporary files under the shard attempt", async () => { + const tmp = os.tmpdir(); + expect(tmp).toMatch(/^\/tmp\/tv-testshards\/[^/]+\/tmp$/); + const nodeDir = mkdtempSync(path.join(tmp, "node-placement-")); + writeFileSync(path.join(nodeDir, "created"), "node temporary file"); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + await page.setContent(''); + await page.screenshot(); + const descriptors: Array<{pid: number; fd: string; path: string}> = []; + for (const pid of readdirSync("/proc").filter(value => /^\d+$/.test(value))) { + try { + const env = readFileSync(`/proc/${pid}/environ`, "utf8"); + const exe = readlinkSync(`/proc/${pid}/exe`); + if (!env.split("\0").includes(`TV_TEST_SHARD_OWNER=${process.env.TV_TEST_SHARD_OWNER}`) || !/chrome|chromium/.test(exe)) continue; + for (const fd of readdirSync(`/proc/${pid}/fd`)) { + try { const target = readlinkSync(`/proc/${pid}/fd/${fd}`); if (target.startsWith(tmp + "/")) descriptors.push({pid: Number(pid), fd, path: target}); } catch {} + } + } catch {} + } + evidence("temporary-placement.json", { tmp, nodeDir, browserVersion: browser.version(), executable: chromium.executablePath(), token: process.env.TV_TEST_SHARD_OWNER, surface: process.env.TV_TEST_SURFACE_OWNER, descriptors }); + expect(descriptors.some(entry => entry.path.includes("Chromium") && entry.path.endsWith(" (deleted)"))).toBe(true); + } finally { await browser.close(); rmSync(nodeDir, {recursive: true, force: true}); } +}); diff --git a/test/runner-fixtures/blaxel-harness/fixtures/vitest.config.ts b/test/runner-fixtures/blaxel-harness/fixtures/vitest.config.ts new file mode 100644 index 00000000..6781e592 --- /dev/null +++ b/test/runner-fixtures/blaxel-harness/fixtures/vitest.config.ts @@ -0,0 +1,3 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; +import { defineConfig } from "vitest/config"; +export default defineConfig({ root: import.meta.dirname, test: { include: ["*.test.ts"], maxWorkers: 1, fileParallelism: false, testTimeout: 240_000 } }); diff --git a/test/runner-fixtures/dynamic-services/playwright.config.ts b/test/runner-fixtures/dynamic-services/playwright.config.ts index 60762c71..915616bb 100644 --- a/test/runner-fixtures/dynamic-services/playwright.config.ts +++ b/test/runner-fixtures/dynamic-services/playwright.config.ts @@ -1,3 +1,4 @@ +import "../../../scripts/test/native-entrypoint.mjs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { defineConfig } from "@playwright/test"; diff --git a/test/runner-fixtures/guidance/playwright/one.test.ts b/test/runner-fixtures/guidance/playwright/one.test.ts new file mode 100644 index 00000000..7b211f07 --- /dev/null +++ b/test/runner-fixtures/guidance/playwright/one.test.ts @@ -0,0 +1,16 @@ +import { appendFileSync } from "node:fs"; +import { expect, test } from "@playwright/test"; + +test("selected attempt", async ({}, info) => { + if (process.env.TV_GUIDANCE_ATTEMPTS) appendFileSync(process.env.TV_GUIDANCE_ATTEMPTS, `${JSON.stringify({ title: "selected attempt", attempt: info.retry })}\n`); + expect(info.retry).toBeGreaterThanOrEqual(Number(process.env.TV_GUIDANCE_FAIL_COUNT ?? "0")); +}); +test("other title", async () => {}); +// Live handoff probe: default retries recover; blanket zero deliberately fails. +test("deliberate first-attempt failure", async ({}, info) => { + expect(info.retry).toBeGreaterThanOrEqual(1); +}); +test.describe("annotated", () => { + test.describe.configure({ retries: 1 }); + test("annotated retry", async ({}, info) => { expect(info.retry).toBeGreaterThanOrEqual(1); }); +}); diff --git a/test/runner-fixtures/guidance/playwright/one.test.ts.test.ts b/test/runner-fixtures/guidance/playwright/one.test.ts.test.ts new file mode 100644 index 00000000..d9a1083e --- /dev/null +++ b/test/runner-fixtures/guidance/playwright/one.test.ts.test.ts @@ -0,0 +1,2 @@ +import { test } from "@playwright/test"; +test("adversarial sibling must not be selected by an exact file", async () => {}); diff --git a/test/runner-fixtures/guidance/playwright/playwright.config.ts b/test/runner-fixtures/guidance/playwright/playwright.config.ts new file mode 100644 index 00000000..4fdc985c --- /dev/null +++ b/test/runner-fixtures/guidance/playwright/playwright.config.ts @@ -0,0 +1,3 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; +import { defineConfig } from "@playwright/test"; +export default defineConfig({ testDir: ".", testMatch: "*.test.ts", retries: 7, workers: 1 }); diff --git a/test/runner-fixtures/guidance/vitest/one.test.ts b/test/runner-fixtures/guidance/vitest/one.test.ts new file mode 100644 index 00000000..759d3072 --- /dev/null +++ b/test/runner-fixtures/guidance/vitest/one.test.ts @@ -0,0 +1,16 @@ +import { appendFileSync } from "node:fs"; +import { expect, test } from "vitest"; + +test("selected attempt", (context) => { + const attempt = context.task.result?.retryCount ?? 0; + if (process.env.TV_GUIDANCE_ATTEMPTS) appendFileSync(process.env.TV_GUIDANCE_ATTEMPTS, `${JSON.stringify({ title: "selected attempt", attempt })}\n`); + expect(attempt).toBeGreaterThanOrEqual(Number(process.env.TV_GUIDANCE_FAIL_COUNT ?? "0")); +}); +test("other title", () => {}); +// Live handoff probe: default retries recover; blanket zero deliberately fails. +test("deliberate first-attempt failure", (context) => { + expect(context.task.result?.retryCount ?? 0).toBeGreaterThanOrEqual(1); +}); +test("annotated retry", { retry: 1 }, (context) => { + expect(context.task.result?.retryCount ?? 0).toBeGreaterThanOrEqual(1); +}); diff --git a/test/runner-fixtures/guidance/vitest/one.test.ts.test.ts b/test/runner-fixtures/guidance/vitest/one.test.ts.test.ts new file mode 100644 index 00000000..a531e2df --- /dev/null +++ b/test/runner-fixtures/guidance/vitest/one.test.ts.test.ts @@ -0,0 +1,2 @@ +import { test } from "vitest"; +test("adversarial sibling must not be selected by an exact file", () => {}); diff --git a/test/runner-fixtures/guidance/vitest/vitest.config.ts b/test/runner-fixtures/guidance/vitest/vitest.config.ts new file mode 100644 index 00000000..f3c23278 --- /dev/null +++ b/test/runner-fixtures/guidance/vitest/vitest.config.ts @@ -0,0 +1,5 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; +import { defineConfig } from "vitest/config"; +import { fileURLToPath } from "node:url"; + +export default defineConfig({ root: fileURLToPath(new URL(".", import.meta.url)), test: { include: ["*.test.ts"], retry: 7, maxWorkers: 1 } }); diff --git a/test/runner-fixtures/process-lifecycle/active/vitest.config.ts b/test/runner-fixtures/process-lifecycle/active/vitest.config.ts index cbbdd127..c61d03c3 100644 --- a/test/runner-fixtures/process-lifecycle/active/vitest.config.ts +++ b/test/runner-fixtures/process-lifecycle/active/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ root: import.meta.dirname, test: { include: ["active.fixture.test.ts"], testTimeout: 60_000, retry: 0 } }); diff --git a/test/runner-fixtures/process-lifecycle/clean/vitest.config.ts b/test/runner-fixtures/process-lifecycle/clean/vitest.config.ts index ec57d717..e76bc7dc 100644 --- a/test/runner-fixtures/process-lifecycle/clean/vitest.config.ts +++ b/test/runner-fixtures/process-lifecycle/clean/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ root: import.meta.dirname, test: { include: ["clean.fixture.test.ts"], retry: 0 } }); diff --git a/test/runner-fixtures/process-lifecycle/late-spawn/vitest.config.ts b/test/runner-fixtures/process-lifecycle/late-spawn/vitest.config.ts index 17674837..d7457add 100644 --- a/test/runner-fixtures/process-lifecycle/late-spawn/vitest.config.ts +++ b/test/runner-fixtures/process-lifecycle/late-spawn/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ root: import.meta.dirname, test: { include: ["late-spawn.fixture.test.ts"], retry: 0 } }); diff --git a/test/runner-fixtures/process-lifecycle/leak/vitest.config.ts b/test/runner-fixtures/process-lifecycle/leak/vitest.config.ts index 2ae9ae40..be56cfb2 100644 --- a/test/runner-fixtures/process-lifecycle/leak/vitest.config.ts +++ b/test/runner-fixtures/process-lifecycle/leak/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ root: import.meta.dirname, test: { include: ["leak.fixture.test.ts"], retry: 0 } }); diff --git a/test/runner-fixtures/process-lifecycle/orchestrator/mutex.test.ts b/test/runner-fixtures/process-lifecycle/orchestrator/mutex.test.ts new file mode 100644 index 00000000..79b7346b --- /dev/null +++ b/test/runner-fixtures/process-lifecycle/orchestrator/mutex.test.ts @@ -0,0 +1,156 @@ +import { existsSync, readFileSync, utimesSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, test } from "vitest"; +import { decodeOwnerToken, processIdentityMatches, readLinuxProcess, scanLinuxProcesses } from "../../../../scripts/test/process-lifecycle.mjs"; +import { mutexCheckout, ready, waitFor } from "../../../repo/fixtures/test-runner/mutex-checkout.ts"; +import { verifyMutexCheckout } from "../../../repo/fixtures/test-runner/verify-mutex-checkout.ts"; + +// proofs/arch/test-runner/test-runner.md#^t-runner-lock-faults +// This file belongs only to the isolated-github-only orchestrator surface. +// Never invoke these forced-death fixtures on a developer host or Blaxel. +const waitFlag = "--wait-for-local-test-mutex"; +type Identity = { pid: number; startTime: string }; + +function fixture(verify = false) { + const f = verify ? verifyMutexCheckout() : mutexCheckout(); + const owners = new Map(); + const remember = (p: Identity) => owners.set(p.pid, { pid: p.pid, startTime: p.startTime }); + const record = () => { + const db = new DatabaseSync(f.env.TV_TEST_RUNNER_LOCK_PATH!); + try { + const row = db.prepare("SELECT record FROM local_test_mutex WHERE slot=1").get(); + const participants: (Identity & { role: string })[] = row ? JSON.parse(row.record as string).participants : []; + participants.forEach(remember); + return participants; + } finally { db.close(); } + }; + const originalStart = f.start; + f.start = (args = f.args, env = {}) => { + const child = originalStart(args, env); + remember(readLinuxProcess(child.child.pid!)); + return child; + }; + const listenerReady = path.join(f.root, "listener.json"); + const termCheck = path.join(f.root, "term-check.json"); + f.write("owned-listener.mjs", ` +import { createServer } from 'node:http'; import fs from 'node:fs'; import { spawnSync } from 'node:child_process'; +const server = createServer((_req,res)=>res.end('mutex fixture')); +process.on('SIGTERM',()=>{ + const r=spawnSync(process.execPath,${JSON.stringify(["scripts/test/cli.mjs", ...f.args])},{env:{...process.env,TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT:'fixture:cleanup contention probe'},encoding:'utf8',timeout:5000}); + fs.writeFileSync(${JSON.stringify(termCheck)},JSON.stringify({status:r.status,output:r.stdout+r.stderr})); + if(process.env.TV_MUTEX_RESIST_TERM!=='1') server.close(()=>process.exit(0)); +}); +setTimeout(()=>process.exit(1),60000).unref(); +server.listen(0,'127.0.0.1',()=>fs.writeFileSync(${JSON.stringify(listenerReady)},JSON.stringify({pid:process.pid,ownerToken:process.env.TV_TEST_SURFACE_OWNER,port:server.address().port}))); +`); + f.write(f.file, ` +import fs from 'node:fs'; import {spawn} from 'node:child_process'; import {test} from 'vitest'; +test('owned listener until released',async()=>{ + fs.rmSync(${JSON.stringify(listenerReady)},{force:true}); + const child=spawn(process.execPath,['owned-listener.mjs'],{detached:true,stdio:'ignore',env:process.env}); + const exited=new Promise(resolve=>child.once('exit',resolve)); child.unref(); + const deadline=Date.now()+60000; + while(!fs.existsSync(${JSON.stringify(listenerReady)})){ + if(Date.now()>deadline) throw new Error('listener readiness timeout'); + await new Promise(resolve=>setTimeout(resolve,20)); + } + fs.writeFileSync(process.env.TV_MUTEX_READY,String(process.pid)); + while(!fs.existsSync(process.env.TV_MUTEX_RELEASE)){ + if(Date.now()>deadline) throw new Error('native release timeout'); + await new Promise(resolve=>setTimeout(resolve,20)); + } + child.kill('SIGTERM'); await exited; +},65000); +`); + async function cleanup() { + // Independent of the mechanism under test. Capture only this fixture's + // record, then revalidate exact PID/start identities and owner tokens. + if (existsSync(f.env.TV_TEST_RUNNER_LOCK_PATH!)) record(); + for (const p of owners.values()) signal(p, "SIGKILL"); + const remaining = () => scanLinuxProcesses().filter(p => { + const owner = decodeOwnerToken(p.ownerToken); + const expected = owner && owners.get(owner.supervisorPid); + return expected && expected.startTime === owner.supervisorStartTime; + }); + await waitFor(() => { + const children = remaining(); + for (const child of children) signal(child, "SIGKILL"); + return children.length === 0; + }); + await f.cleanup(); + } + return { ...f, record, cleanup, listenerReady, termCheck }; +} + +function signal(p: Identity, name: NodeJS.Signals) { + try { + const current = readLinuxProcess(p.pid); + if (processIdentityMatches(current, p) && current.state !== "Z") process.kill(p.pid, name); + } catch (error) { if (!["ESRCH", "ENOENT"].includes((error as NodeJS.ErrnoException).code!)) throw error; } +} +function live(p: Identity) { + try { const current = readLinuxProcess(p.pid); return current.state !== "Z" && processIdentityMatches(current, p); } + catch (error) { if (["ESRCH", "ENOENT"].includes((error as NodeJS.ErrnoException).code!)) return false; throw error; } +} +function listener(f: ReturnType) { + return JSON.parse(readFileSync(f.listenerReady, "utf8")) as { pid: number; ownerToken: string; port: number }; +} + +// Linux evidence only: the isolated workflow uses an ephemeral Ubuntu runner. +describe("local mutex crash recovery", () => { + test("holds ownership while interrupted descendants shut down", async () => { + const f = fixture(); + try { + const holder = f.start(); await ready(holder); f.record(); + const owned = listener(f); + holder.child.kill("SIGTERM"); + expect(await holder.exited, holder.output()).toBe(143); + const check = JSON.parse(readFileSync(f.termCheck, "utf8")); + expect(check.status, check.output).toBe(2); + expect(check.output).toContain("Local-test mutex held"); + expect(scanLinuxProcesses().filter(p => p.ownerToken === owned.ownerToken)).toEqual([]); + expect(f.run(f.args, { TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT: "fixture:released" }).status).toBe(1); + } finally { await f.cleanup(); } + }); + + test("keeps a surviving verify test child protected, regardless of record age", async () => { + const f = fixture(true); + try { + const verify = f.start(["verify", "local"]); await ready(verify); + const child = f.record().find(p => p.role === "run")!; + expect(child).toBeDefined(); + utimesSync(f.env.TV_TEST_RUNNER_LOCK_PATH!, new Date(0), new Date(0)); + verify.child.kill("SIGKILL"); await verify.exited; + expect(live(child)).toBe(true); + expect(f.run(f.args).status).toBe(2); + const waiter = f.start([...f.args, waitFlag]); + await waitFor(() => waiter.output().includes("Waiting for local-test mutex"), waiter.output); + expect(existsSync(waiter.ready)).toBe(false); + writeFileSync(verify.release, "done"); + await waitFor(() => !live(child)); + await ready(waiter); writeFileSync(waiter.release, "done"); + expect(await waiter.exited, waiter.output()).toBe(0); + } finally { await f.cleanup(); } + }); + + test.each([false, true])("reaps abandoned owned processes and fails before new work (waiting=%s)", async waiting => { + const f = fixture(); + try { + const holder = f.start(f.args, { TV_MUTEX_RESIST_TERM: "1" }); await ready(holder); f.record(); + const owned = listener(f); + const waiter = waiting ? f.start([...f.args, waitFlag]) : null; + if (waiter) await waitFor(() => waiter.output().includes("Waiting for local-test mutex"), waiter.output); + holder.child.kill("SIGKILL"); await holder.exited; + const recovery = waiter ?? f.start(); + expect(await recovery.exited, recovery.output()).toBe(1); + expect(recovery.output()).toContain("stale owner process"); + expect(existsSync(recovery.ready)).toBe(false); + expect(scanLinuxProcesses().filter(p => p.ownerToken === owned.ownerToken)).toEqual([]); + const later = f.run(f.args, { TV_TEST_RUNNER_FAKE_LOCAL_PREFLIGHT: "fixture:clean invocation admitted" }); + expect(later.status, later.stdout + later.stderr).toBe(1); + expect(later.stderr).toContain("clean invocation admitted"); + expect(later.stdout + later.stderr).not.toContain("Detected and reaped"); + } finally { await f.cleanup(); } + }); +}); diff --git a/test/runner-fixtures/process-lifecycle/orchestrator/orchestrator.test.ts b/test/runner-fixtures/process-lifecycle/orchestrator/orchestrator.test.ts index 8e6ec8f6..098a2003 100644 --- a/test/runner-fixtures/process-lifecycle/orchestrator/orchestrator.test.ts +++ b/test/runner-fixtures/process-lifecycle/orchestrator/orchestrator.test.ts @@ -293,6 +293,8 @@ function innerEnv(fixtureDir: string, readyFile: string): NodeJS.ProcessEnv { GITHUB_ACTIONS: "true", GITHUB_WORKFLOW: "Process Lifecycle Fault Injection", TV_TEST_ISOLATED_GITHUB: "1", + TV_TEST_RUNNER_SELFTEST: "1", + TV_TEST_RUNNER_LOCK_PATH: path.join(fixtureDir, "mutex.sqlite3"), TV_TEST_TIMING_PROVIDER: "github-ubuntu-24.04-x64-2vcpu-vm", TV_LIFECYCLE_FIXTURE_DIR: fixtureDir, TV_LIFECYCLE_READY_FILE: readyFile, diff --git a/test/runner-fixtures/process-lifecycle/orchestrator/vitest.config.ts b/test/runner-fixtures/process-lifecycle/orchestrator/vitest.config.ts index fd7dcbea..1da02d58 100644 --- a/test/runner-fixtures/process-lifecycle/orchestrator/vitest.config.ts +++ b/test/runner-fixtures/process-lifecycle/orchestrator/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; -export default defineConfig({ root: import.meta.dirname, test: { include: ["orchestrator.test.ts"], testTimeout: 60_000, hookTimeout: 15_000, retry: 0 } }); +export default defineConfig({ root: import.meta.dirname, test: { include: ["*.test.ts"], testTimeout: 60_000, hookTimeout: 15_000, retry: 0 } }); diff --git a/test/runner-fixtures/process-lifecycle/precommand-leak/vitest.config.ts b/test/runner-fixtures/process-lifecycle/precommand-leak/vitest.config.ts index 90d26b62..1556fe17 100644 --- a/test/runner-fixtures/process-lifecycle/precommand-leak/vitest.config.ts +++ b/test/runner-fixtures/process-lifecycle/precommand-leak/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ root: import.meta.dirname, test: { include: ["precommand.fixture.test.ts"], retry: 0 } }); diff --git a/test/runner-fixtures/process-lifecycle/recovered-leak/vitest.config.ts b/test/runner-fixtures/process-lifecycle/recovered-leak/vitest.config.ts index 14ebc96d..bce5762e 100644 --- a/test/runner-fixtures/process-lifecycle/recovered-leak/vitest.config.ts +++ b/test/runner-fixtures/process-lifecycle/recovered-leak/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ root: import.meta.dirname, test: { include: ["recovered-leak.fixture.test.ts"], retry: 1 } }); diff --git a/test/runner-fixtures/process-lifecycle/term-resistant/vitest.config.ts b/test/runner-fixtures/process-lifecycle/term-resistant/vitest.config.ts index 394402d8..a58b703c 100644 --- a/test/runner-fixtures/process-lifecycle/term-resistant/vitest.config.ts +++ b/test/runner-fixtures/process-lifecycle/term-resistant/vitest.config.ts @@ -1,3 +1,4 @@ +import "../../../../scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({ root: import.meta.dirname, test: { include: ["term-resistant.fixture.test.ts"], retry: 0 } }); diff --git a/vitest.config.ts b/vitest.config.ts index d94ec75b..4d9a1f0a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,3 +1,4 @@ +import "./scripts/test/native-entrypoint.mjs"; import { defineConfig } from "vitest/config"; export default defineConfig({